From e3aefe821dc889dfb0cbde400c7c2a8eebe5b6fe Mon Sep 17 00:00:00 2001 From: tallison Date: Thu, 12 Feb 2026 14:27:00 -0500 Subject: [PATCH 1/2] TIKA-4659 -- tika-eval-lite --- docs/modules/ROOT/nav.adoc | 1 + docs/modules/ROOT/pages/advanced/index.adoc | 1 + .../ROOT/pages/advanced/tika-eval.adoc | 294 ++++++++++ .../tika/textquality/TextQualityResult.java | 84 +++ .../tika/textquality/TextQualityScorer.java | 105 ++++ tika-eval/pom.xml | 1 + tika-eval/tika-eval-lite/pom.xml | 66 +++ .../textquality/BigramProfileGenerator.java | 291 ++++++++++ .../textquality/BigramTextQualityScorer.java | 339 ++++++++++++ ....apache.tika.textquality.TextQualityScorer | 1 + .../tika/eval/textquality/bigram_profiles/afr | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/amh | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ara | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/asm | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ast | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/aze | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/bak | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ban | 450 ++++++++++++++++ .../tika/eval/textquality/bigram_profiles/bel | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ben | 504 ++++++++++++++++++ .../eval/textquality/bigram_profiles/ben-rom | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/bih | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/bos | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/bre | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/bul | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/cat | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ceb | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ces | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/che | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ckb | 321 +++++++++++ .../tika/eval/textquality/bigram_profiles/cmn | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/cym | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/dan | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/deu | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/div | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ekk | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ell | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/eng | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/epo | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/est | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/eus | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/fao | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/fas | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/fin | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/fra | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/fry | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ful | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/gla | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/gle | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/glg | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/gom | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/gsw | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/gug | 425 +++++++++++++++ .../tika/eval/textquality/bigram_profiles/guj | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/hat | 490 +++++++++++++++++ .../tika/eval/textquality/bigram_profiles/hau | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/heb | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/hin | 504 ++++++++++++++++++ .../eval/textquality/bigram_profiles/hin-rom | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/hrv | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/hun | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/hye | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ibo | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ind | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/isl | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ita | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/jav | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/jpn | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/kan | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/kat | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/kaz | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/khm | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/kin | 406 ++++++++++++++ .../tika/eval/textquality/bigram_profiles/kir | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/knn | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/kor | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/kur | 504 ++++++++++++++++++ .../textquality/bigram_profiles/languages.lst | 148 +++++ .../tika/eval/textquality/bigram_profiles/lao | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/lat | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/lav | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/lim | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/lin | 424 +++++++++++++++ .../tika/eval/textquality/bigram_profiles/lit | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ltz | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/lug | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/lvs | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/mal | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/mar | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/mhr | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/min | 477 +++++++++++++++++ .../tika/eval/textquality/bigram_profiles/mkd | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/mlg | 413 ++++++++++++++ .../tika/eval/textquality/bigram_profiles/mlt | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/mon | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/mri | 412 ++++++++++++++ .../tika/eval/textquality/bigram_profiles/msa | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/mya | 504 ++++++++++++++++++ .../eval/textquality/bigram_profiles/mya-zaw | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/nan | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/nds | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/nep | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/new | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/nld | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/nno | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/nob | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/nso | 414 ++++++++++++++ .../tika/eval/textquality/bigram_profiles/oci | 503 +++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ori | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/orm | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/pan | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/pes | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/pnb | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/pol | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/por | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/pus | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/quz | 483 +++++++++++++++++ .../tika/eval/textquality/bigram_profiles/roh | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ron | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/rus | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/san | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/sin | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/slk | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/slv | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/snd | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/som | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/spa | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/sqi | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/srd | 224 ++++++++ .../tika/eval/textquality/bigram_profiles/srp | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ssw | 197 +++++++ .../tika/eval/textquality/bigram_profiles/swa | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/swe | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/tam | 504 ++++++++++++++++++ .../eval/textquality/bigram_profiles/tam-rom | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/tat | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/tel | 504 ++++++++++++++++++ .../eval/textquality/bigram_profiles/tel-rom | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/tgk | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/tgl | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/tha | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/tsn | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/tuk | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/tur | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/uig | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/ukr | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/urd | 504 ++++++++++++++++++ .../eval/textquality/bigram_profiles/urd-rom | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/uzb | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/vie | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/vol | 415 ++++++++++++++ .../tika/eval/textquality/bigram_profiles/war | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/wol | 480 +++++++++++++++++ .../tika/eval/textquality/bigram_profiles/xho | 463 ++++++++++++++++ .../tika/eval/textquality/bigram_profiles/yid | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/yor | 367 +++++++++++++ .../eval/textquality/bigram_profiles/zho-simp | 504 ++++++++++++++++++ .../eval/textquality/bigram_profiles/zho-trad | 504 ++++++++++++++++++ .../tika/eval/textquality/bigram_profiles/zul | 504 ++++++++++++++++++ .../BigramTextQualityScorerTest.java | 261 +++++++++ 160 files changed, 74476 insertions(+) create mode 100644 docs/modules/ROOT/pages/advanced/tika-eval.adoc create mode 100644 tika-core/src/main/java/org/apache/tika/textquality/TextQualityResult.java create mode 100644 tika-core/src/main/java/org/apache/tika/textquality/TextQualityScorer.java create mode 100644 tika-eval/tika-eval-lite/pom.xml create mode 100644 tika-eval/tika-eval-lite/src/main/java/org/apache/tika/eval/textquality/BigramProfileGenerator.java create mode 100644 tika-eval/tika-eval-lite/src/main/java/org/apache/tika/eval/textquality/BigramTextQualityScorer.java create mode 100644 tika-eval/tika-eval-lite/src/main/resources/META-INF/services/org.apache.tika.textquality.TextQualityScorer create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/afr create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/amh create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ara create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/asm create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ast create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/aze create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bak create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ban create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bel create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ben create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ben-rom create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bih create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bos create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bre create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bul create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/cat create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ceb create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ces create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/che create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ckb create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/cmn create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/cym create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/dan create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/deu create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/div create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ekk create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ell create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/eng create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/epo create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/est create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/eus create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fao create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fas create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fin create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fra create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fry create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ful create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gla create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gle create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/glg create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gom create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gsw create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gug create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/guj create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hat create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hau create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/heb create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hin create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hin-rom create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hrv create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hun create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hye create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ibo create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ind create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/isl create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ita create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/jav create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/jpn create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kan create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kat create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kaz create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/khm create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kin create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kir create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/knn create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kor create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kur create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/languages.lst create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lao create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lat create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lav create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lim create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lin create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lit create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ltz create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lug create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lvs create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mal create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mar create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mhr create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/min create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mkd create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mlg create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mlt create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mon create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mri create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/msa create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mya create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mya-zaw create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nan create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nds create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nep create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/new create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nld create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nno create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nob create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nso create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/oci create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ori create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/orm create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pan create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pes create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pnb create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pol create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/por create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pus create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/quz create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/roh create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ron create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/rus create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/san create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/sin create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/slk create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/slv create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/snd create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/som create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/spa create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/sqi create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/srd create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/srp create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ssw create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/swa create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/swe create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tam create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tam-rom create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tat create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tel create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tel-rom create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tgk create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tgl create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tha create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tsn create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tuk create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tur create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/uig create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ukr create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/urd create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/urd-rom create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/uzb create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/vie create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/vol create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/war create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/wol create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/xho create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/yid create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/yor create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/zho-simp create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/zho-trad create mode 100644 tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/zul create mode 100644 tika-eval/tika-eval-lite/src/test/java/org/apache/tika/eval/textquality/BigramTextQualityScorerTest.java diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index eaeddf383ff..40eafd13248 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -37,6 +37,7 @@ ** xref:advanced/setting-limits.adoc[Setting Limits] ** xref:advanced/spooling.adoc[Spooling] ** xref:advanced/embedded-documents.adoc[Embedded Document Metadata] +** xref:advanced/tika-eval.adoc[Text Quality Scoring] * xref:developers/index.adoc[Developers] ** xref:developers/serialization.adoc[Serialization and Configuration] * xref:faq.adoc[FAQ] diff --git a/docs/modules/ROOT/pages/advanced/index.adoc b/docs/modules/ROOT/pages/advanced/index.adoc index 35469f0e88c..95b97f97633 100644 --- a/docs/modules/ROOT/pages/advanced/index.adoc +++ b/docs/modules/ROOT/pages/advanced/index.adoc @@ -26,6 +26,7 @@ This section covers advanced usage and internals of Apache Tika. * xref:advanced/spooling.adoc[TikaInputStream and Spooling] - Understanding how TikaInputStream handles buffering, caching, and spooling to disk * xref:advanced/embedded-documents.adoc[Embedded Document Metadata] - Understanding how Tika tracks embedded documents and their paths * xref:advanced/zip-detection.adoc[ZIP Detection and Salvaging] - How Tika detects and recovers truncated ZIP-based files +* xref:advanced/tika-eval.adoc[Text Quality Scoring] - Measuring extracted text quality using character bigram profiles == Integration Testing diff --git a/docs/modules/ROOT/pages/advanced/tika-eval.adoc b/docs/modules/ROOT/pages/advanced/tika-eval.adoc new file mode 100644 index 00000000000..00a086ec59f --- /dev/null +++ b/docs/modules/ROOT/pages/advanced/tika-eval.adoc @@ -0,0 +1,294 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + += Text Quality Scoring (tika-eval-lite) + +The `tika-eval-lite` module provides a lightweight text quality scorer +that measures how well extracted text matches known language patterns. +It uses character bigram frequency profiles derived from language corpora +and requires no external dependencies beyond `tika-core`. + +== Overview + +The scorer computes the average log~2~-likelihood per character bigram +against language-specific profiles. Higher (less negative) scores indicate +text that better matches known language patterns. The score is naturally +normalized by text length, so short and long texts produce comparable +values. + +Scores are designed for *comparison*, not absolute thresholds. Compare +two text variants (e.g., forward vs reversed, charset A vs charset B) +against the same language -- the higher score wins. + +== Use Cases + +* **RTL text direction detection** -- Score both the original and + reversed text against the same language profile. The higher score + indicates the correct reading order. +* **Charset detection** -- Score text decoded under candidate charsets. + The highest-scoring charset is most likely correct. +* **Mojibake / junk detection** -- Compare the extracted text's score + against known-good text in the same language. A significantly lower + score suggests garbled or wrong-charset text. + +== Maven Dependency + +[source,xml] +---- + + org.apache.tika + tika-eval-lite + ${tika.version} + +---- + +The module depends only on `tika-core` (provided scope). When +`tika-eval-lite` is on the classpath, `TextQualityScorer.getDefault()` +returns the `BigramTextQualityScorer` via the ServiceLoader/SPI +mechanism. When it is absent, a no-op scorer is returned. + +== Basic Usage + +[source,java] +---- +import org.apache.tika.textquality.TextQualityScorer; +import org.apache.tika.textquality.TextQualityResult; + +TextQualityScorer scorer = TextQualityScorer.getDefault(); + +TextQualityResult result = scorer.score("The quick brown fox..."); + +double score = result.getScore(); // e.g. -8.03 +String language = result.getLanguage(); // e.g. "eng" +double confidence = result.getConfidence(); // gap to 2nd-best +int bigramCount = result.getBigramCount(); // bigrams analyzed +---- + +=== Scoring Against a Specific Language + +When comparing two variants of the same text (e.g., for RTL detection +or charset selection), score both against the same language profile: + +[source,java] +---- +TextQualityResult forward = scorer.score(text, "ara"); +TextQualityResult reversed = scorer.score(reversedText, "ara"); + +// Higher score wins +if (forward.getScore() > reversed.getScore()) { + // forward is the correct reading order +} +---- + +=== Configuring Maximum Text Length + +By default, only the first 10,000 characters are analyzed. Beyond this +length, additional text adds negligible precision. To change: + +[source,java] +---- +BigramTextQualityScorer scorer = new BigramTextQualityScorer(); +scorer.setMaxTextLength(20_000); +---- + +== How It Works + +=== Character Bigram Extraction + +Input text is first normalized with NFKD decomposition and combining +mark (diacritic) removal. This matches the ICU folding applied to the +Wikipedia and Leipzig corpus data used to build the profiles. It also +prevents Arabic tashkeel (fatha, kasra, shadda, etc.) from breaking +the bigram chain. + +The scorer then extracts consecutive pairs of lowercase letters. Non-letter +characters (digits, punctuation, whitespace) act as word boundaries. For +each word, three types of bigrams are emitted: + +* **Internal bigrams**: consecutive letter pairs (`he`, `el`, `ll`, `lo`) +* **Word-start bigram**: `_h` (boundary marker + first letter) +* **Word-end bigram**: `o_` (last letter + boundary marker) + +For example, `"Hello, World!"` produces: `_h`, `he`, `el`, `ll`, `lo`, +`o_`, `_w`, `wo`, `or`, `rl`, `ld`, `d_`. + +Word-boundary bigrams are critical for RTL detection because word-initial +and word-final character patterns are highly directional. For example, +Hebrew final forms (sofit letters like ך ,ם ,ן) appear at word ends in +forward text but at word starts when reversed. + +=== Language Profiles + +Each language profile contains the top 500 character bigrams (including +word-boundary bigrams) and their log~2~-probabilities. Profiles are +derived from the `common_tokens` data in `tika-eval-core`, which was +built from Wikipedia and Leipzig corpus data processed through ICU +folding (NFKC normalization, case folding, diacritic removal). The +profiles cover 148 languages. Profiles from corpora with fewer than +500,000 total bigram occurrences are excluded to avoid unreliable +probability estimates. + +Each profile file includes precomputed statistics in its header: + +[cols="1,3"] +|=== +| Header | Description + +| `TOTAL_BIGRAMS` +| Total bigram occurrences in the source corpus. + +| `UNIQUE_BIGRAMS` +| Distinct bigram types observed in the corpus. + +| `UNSEEN_LOG_PROB` +| Estimated log~2~-probability for bigrams not in the profile + (see <>). + +| `EXPECTED_SCORE` +| Expected average log~2~-likelihood for perfect text drawn from + this language's distribution (negative entropy). Stored for + reference; not used by the scorer at runtime. +|=== + +=== Scoring Algorithm + +For each language profile, the scorer computes: + +[stem] +++++ +\text{score} = \frac{1}{N} \sum_{i=1}^{N} \log_2 P(b_i) +++++ + +where _N_ is the total bigram count and _P(b~i~)_ is the probability +of bigram _b~i~_ under the profile. Bigrams not in the profile receive +the profile's unseen log-probability. + +The language with the highest score is selected as the best match. The +*confidence* is the score difference between the best and second-best +language. + +[[unseen-estimation]] +=== Unseen Bigram Estimation + +Rather than using an arbitrary fixed penalty for bigrams not in the +profile, the scorer uses held-out estimation from the corpus statistics: + +[stem] +++++ +P_{\text{unseen}} = \frac{1 - \sum_{j=1}^{K} P(b_j)}{U - K} +++++ + +where _K_ is the number of bigrams in the profile (500), _U_ is the +total number of unique bigrams in the corpus, and the numerator is the +remaining probability mass not covered by the profile. + +This produces per-language calibrated penalties: + +* *English* (606 unique bigrams, top-500 covers ~99%): unseen + log~2~-prob = -19.4 (harsh -- almost all bigrams are known) +* *Arabic* (835 unique bigrams, top-500 covers ~99%): unseen + log~2~-prob = -15.2 +* *Chinese* (29,673 unique bigrams, top-500 covers ~14.5%): unseen + log~2~-prob = -15.4 + +Note that Arabic and Chinese have similar per-bigram unseen penalties +despite very different coverage. This is because the ratio of remaining +mass to unseen count converges. The practical difference is captured by +how *often* unseen bigrams are hit, which is reflected in the expected +score. + +== Known Limitations + +=== CJK Coverage + +The top-500 bigram profiles cover only ~14.5% of Chinese character +bigrams (compared to ~99% for English). This means most CJK bigrams +in the input text will hit the unseen floor penalty, compressing the +score range and reducing discrimination between good and garbled CJK +text. + +For CJK mojibake detection, complement bigram scoring with +script-level checks: replacement characters (U+FFFD), unexpected +script mixing, and CID/GID fallback patterns are more reliable +signals than bigram scores for CJK text. + +=== Arabic Alphabet Symmetry + +Arabic has a small alphabet (28 letters). Approximately 82% of bigrams +in the Arabic profile have their reverse also present in the profile. +This means the forward/reverse score difference for Arabic text is +modest (~0.6 bits/bigram), compared to Hebrew (~1.2 bits/bigram) +which benefits from distinctive sofit (final-form) letters at word +boundaries. + +Arabic RTL detection still works -- the signal is real, just smaller. +Word-boundary bigrams help significantly (Arabic word-start and +word-end patterns are more asymmetric than internal bigrams). + +=== Not a Language Detector + +While the scorer identifies the best-matching language profile, it +is not designed as a general-purpose language detector. It lacks +the sophistication of dedicated tools (e.g., language priors, +n-gram interpolation, script-based shortcuts). Use it for quality +scoring and comparison, not language identification. + +=== Raw Scores Are Language-Dependent + +Raw scores vary by language (e.g., English ~-8.0 vs Chinese ~-13.7 +for good text) because languages differ in character inventory size +and bigram entropy. Each profile's header includes an `EXPECTED_SCORE` +(the negative entropy of the language model) for reference, but +the scorer does not use it at runtime. All three intended use cases +-- RTL detection, charset detection, and mojibake detection -- work +by comparing two variants, so absolute score normalization is +unnecessary. + +== Regenerating Profiles + +Profiles are generated from `tika-eval-core`'s `common_tokens` data. +The generation logic is documented and reproducible via +`BigramProfileGenerator`: + +[source,bash] +---- +java -cp tika-eval-lite.jar \ + org.apache.tika.eval.textquality.BigramProfileGenerator \ + path/to/common_tokens \ + path/to/output/bigram_profiles \ + 500 +---- + +The generator reads each language's token file, decomposes words into +character bigrams weighted by term frequency, selects the top-N, and +writes profile files with all precomputed statistics. + +== Architecture + +The interface and result class live in `tika-core` to allow scoring +without pulling in additional dependencies: + +* `org.apache.tika.textquality.TextQualityScorer` -- abstract class + with SPI discovery +* `org.apache.tika.textquality.TextQualityResult` -- immutable result + +The implementation lives in `tika-eval-lite`: + +* `org.apache.tika.eval.textquality.BigramTextQualityScorer` -- + bigram-based scorer (discovered via `META-INF/services`) +* `org.apache.tika.eval.textquality.BigramProfileGenerator` -- + profile generation and documentation of formulas diff --git a/tika-core/src/main/java/org/apache/tika/textquality/TextQualityResult.java b/tika-core/src/main/java/org/apache/tika/textquality/TextQualityResult.java new file mode 100644 index 00000000000..f9e12d5b441 --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/textquality/TextQualityResult.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.textquality; + +/** + * Result of scoring text quality against language models. + * The score is the average log-likelihood per character bigram + * under the best-matching language profile. Higher (less negative) + * scores indicate text that better matches known language patterns. + * + *

Scores are designed for comparison, not absolute thresholds. + * Compare two text variants (e.g., forward vs reversed, charset A + * vs charset B) against the same language profile — the higher + * score indicates the better variant.

+ */ +public class TextQualityResult { + + private final double score; + private final String language; + private final double confidence; + private final int bigramCount; + + public TextQualityResult(double score, String language, + double confidence, int bigramCount) { + this.score = score; + this.language = language; + this.confidence = confidence; + this.bigramCount = bigramCount; + } + + /** + * Average log2-likelihood per bigram under the best-matching + * language profile. Typical range: -6 to -10 for good text, + * below -12 for garbled/reversed text. Language-dependent. + */ + public double getScore() { + return score; + } + + /** + * ISO 639-3 language code of the best-matching profile. + */ + public String getLanguage() { + return language; + } + + /** + * Score difference between best and second-best language match. + * Higher values indicate more confident language identification. + */ + public double getConfidence() { + return confidence; + } + + /** + * Number of character bigrams extracted from the input text. + * Low counts (below ~20) produce unreliable scores. + */ + public int getBigramCount() { + return bigramCount; + } + + @Override + public String toString() { + return String.format(java.util.Locale.ROOT, + "TextQualityResult[score=%.4f, lang=%s, " + + "confidence=%.4f, bigrams=%d]", + score, language, confidence, bigramCount); + } +} diff --git a/tika-core/src/main/java/org/apache/tika/textquality/TextQualityScorer.java b/tika-core/src/main/java/org/apache/tika/textquality/TextQualityScorer.java new file mode 100644 index 00000000000..ad5009b60ff --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/textquality/TextQualityScorer.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.textquality; + +import java.util.List; + +import org.apache.tika.config.ServiceLoader; +import org.apache.tika.utils.CompareUtils; + +/** + * Scores text quality by measuring how well character bigram frequencies + * match known language profiles. Uses average log-likelihood per bigram, + * which is naturally normalized across different text lengths. + * + *

Implementations are discovered via the ServiceLoader/SPI mechanism. + * When no implementation is on the classpath, {@link #getDefault()} returns + * a no-op scorer that always returns a neutral result.

+ * + *

Use cases include: + *

+ */ +public abstract class TextQualityScorer { + + private static final ServiceLoader DEFAULT_SERVICE_LOADER = + new ServiceLoader(); + + /** + * Get the default TextQualityScorer. If no implementation is available + * on the classpath, returns a no-op scorer. + */ + public static TextQualityScorer getDefault() { + List scorers = getScorers(); + if (scorers.isEmpty()) { + return NoOpTextQualityScorer.INSTANCE; + } + return scorers.get(0); + } + + public static List getScorers() { + return getScorers(DEFAULT_SERVICE_LOADER); + } + + public static List getScorers(ServiceLoader loader) { + List scorers = + loader.loadStaticServiceProviders(TextQualityScorer.class); + scorers.sort(CompareUtils::compareClassName); + return scorers; + } + + /** + * Score the quality of the given text. Returns the average + * log2-likelihood per character bigram under the best-matching + * language profile, along with the detected language. + */ + public abstract TextQualityResult score(CharSequence text); + + /** + * Score text against a specific language profile. Use this for + * comparison use cases (RTL direction, charset detection) where + * two text variants should be scored against the same language. + * + * @param text text to score + * @param language ISO 639-3 language code + * @return score against that language, or neutral result if + * the language is not available + */ + public abstract TextQualityResult score(CharSequence text, + String language); + + /** + * No-op implementation returned when no real scorer is on the classpath. + */ + private static class NoOpTextQualityScorer extends TextQualityScorer { + static final NoOpTextQualityScorer INSTANCE = + new NoOpTextQualityScorer(); + + @Override + public TextQualityResult score(CharSequence text) { + return new TextQualityResult(0.0, "unk", 0.0, 0); + } + + @Override + public TextQualityResult score(CharSequence text, String language) { + return new TextQualityResult(0.0, "unk", 0.0, 0); + } + } +} diff --git a/tika-eval/pom.xml b/tika-eval/pom.xml index dcddfc34b71..488cb3d437d 100644 --- a/tika-eval/pom.xml +++ b/tika-eval/pom.xml @@ -35,6 +35,7 @@ pom tika-eval-core + tika-eval-lite tika-eval-app diff --git a/tika-eval/tika-eval-lite/pom.xml b/tika-eval/tika-eval-lite/pom.xml new file mode 100644 index 00000000000..485b9b71820 --- /dev/null +++ b/tika-eval/tika-eval-lite/pom.xml @@ -0,0 +1,66 @@ + + + + + tika-eval + org.apache.tika + 4.0.0-SNAPSHOT + + 4.0.0 + tika-eval-lite + Apache Tika eval lite + + + + ${project.groupId} + tika-core + ${project.version} + provided + + + + ${project.groupId} + tika-core + ${project.version} + test-jar + test + + + + + + + org.apache.rat + apache-rat-plugin + + + src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/* + + + + + + + + 3.0.0-rc1 + + diff --git a/tika-eval/tika-eval-lite/src/main/java/org/apache/tika/eval/textquality/BigramProfileGenerator.java b/tika-eval/tika-eval-lite/src/main/java/org/apache/tika/eval/textquality/BigramProfileGenerator.java new file mode 100644 index 00000000000..a89603ecc13 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/java/org/apache/tika/eval/textquality/BigramProfileGenerator.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.eval.textquality; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; + +/** + * Generates character bigram profile files from tika-eval-core's + * common_tokens data. Each profile contains the top-N bigrams with + * log2-probabilities, plus precomputed statistics: + * + *
    + *
  • {@code #TOTAL_BIGRAMS} — total bigram occurrences in corpus
    • + *
  • {@code #UNIQUE_BIGRAMS} — distinct bigram types in corpus
    • + *
  • {@code #UNSEEN_LOG_PROB} — estimated log2-probability for + * bigrams not in the profile (see {@link #computeUnseenLogProb})
    • + *
  • {@code #EXPECTED_SCORE} — expected average log2-likelihood + * for perfect text (see {@link #computeExpectedScore})
    • + *
+ * + *

Usage: {@code java BigramProfileGenerator + * [topN]}

+ * + */ +public class BigramProfileGenerator { + + private static final int DEFAULT_TOP_N = 500; + private static final double FALLBACK_UNSEEN_LOG_PROB = -20.0; + + /** + * Estimate log2-probability for bigrams not in the top-N profile, + * using held-out corpus statistics. + * + *

The profile contains the top-N bigrams covering most of the + * probability mass. The remaining mass is distributed among all + * other unique bigrams observed in the corpus:

+ * + * 
+     *   coveredMass = sum(2^logprob for each profile bigram)
+     *   remainingMass = 1.0 - coveredMass
+     *   unseenCount = uniqueBigrams - profileSize
+     *   unseenLogProb = log2(remainingMass / unseenCount)
+     *
+ * + *

This gives a per-language calibrated penalty. Languages with + * large character inventories (e.g., CJK with ~30K unique bigrams) + * get a moderate unseen penalty because the remaining mass is + * spread over many bigrams. Languages with small alphabets + * (e.g., English with ~600 unique bigrams) get a harsh penalty + * because there are few unseen bigrams and little remaining mass.

+ * + * @param profileLogProbs log2-probabilities of the top-N bigrams + * @param uniqueBigrams total unique bigram types in the corpus + * @return estimated log2-probability for unseen bigrams + */ + public static double computeUnseenLogProb( + Map profileLogProbs, long uniqueBigrams) { + if (uniqueBigrams <= profileLogProbs.size()) { + // Profile covers all known bigrams; use min-seen as floor + double minLogProb = FALLBACK_UNSEEN_LOG_PROB; + for (double lp : profileLogProbs.values()) { + if (lp < minLogProb) { + minLogProb = lp; + } + } + return minLogProb; + } + + double coveredMass = 0.0; + for (double lp : profileLogProbs.values()) { + coveredMass += Math.pow(2.0, lp); + } + double remainingMass = Math.max(0.0, 1.0 - coveredMass); + long unseenCount = uniqueBigrams - profileLogProbs.size(); + double avgUnseenProb = remainingMass / unseenCount; + + if (avgUnseenProb <= 0.0) { + return FALLBACK_UNSEEN_LOG_PROB; + } + return Math.log(avgUnseenProb) / Math.log(2.0); + } + + /** + * Compute the expected average log2-likelihood per bigram for + * perfect text drawn from this language's distribution. + * + * 
+     *   expectedScore = sum(p * log2(p) for each profile bigram)
+     *                 + remainingMass * unseenLogProb
+     *
+ * + *

This is the negative entropy of the language model. Stored + * in profile files for reference but not used at scoring time + * (all use cases compare two text variants, so normalization + * is unnecessary).

+ * + * @param profileLogProbs log2-probabilities of the top-N bigrams + * @param unseenLogProb log2-probability for unseen bigrams + * @return expected score (negative, since log-probs are negative) + */ + public static double computeExpectedScore( + Map profileLogProbs, double unseenLogProb) { + double score = 0.0; + double coverageMass = 0.0; + for (double logProb : profileLogProbs.values()) { + double p = Math.pow(2.0, logProb); + score += p * logProb; + coverageMass += p; + } + double unseenMass = Math.max(0.0, 1.0 - coverageMass); + score += unseenMass * unseenLogProb; + return score; + } + + /** + * Word-boundary marker. Must match + * {@link BigramTextQualityScorer#BOUNDARY}. + */ + static final char BOUNDARY = '_'; + + /** + * Extract character bigrams from a word, including word-boundary + * bigrams ({@code _x} for word-start, {@code x_} for word-end). + * Only alphabetic characters are included, after lowercasing. + */ + static Map extractBigramsFromWord( + String word, long termFreq) { + Map bigrams = new HashMap<>(); + List chars = new ArrayList<>(); + for (int i = 0; i < word.length(); i++) { + char c = word.charAt(i); + if (Character.isLetter(c)) { + chars.add(Character.toLowerCase(c)); + } + } + if (chars.isEmpty()) { + return bigrams; + } + // Word-start boundary bigram + bigrams.merge(String.valueOf(BOUNDARY) + chars.get(0), + termFreq, Long::sum); + // Internal bigrams + for (int i = 0; i < chars.size() - 1; i++) { + String bg = String.valueOf(chars.get(i)) + chars.get(i + 1); + bigrams.merge(bg, termFreq, Long::sum); + } + // Word-end boundary bigram + bigrams.merge(String.valueOf(chars.get(chars.size() - 1)) + BOUNDARY, + termFreq, Long::sum); + return bigrams; + } + + public static void main(String[] args) throws IOException { + if (args.length < 2) { + System.err.println("Usage: BigramProfileGenerator " + + " [topN]"); + System.exit(1); + } + Path tokensDir = Paths.get(args[0]); + Path outputDir = Paths.get(args[1]); + int topN = args.length >= 3 + ? Integer.parseInt(args[2]) : DEFAULT_TOP_N; + + Files.createDirectories(outputDir); + int profileCount = 0; + + try (DirectoryStream stream = + Files.newDirectoryStream(tokensDir)) { + for (Path langFile : stream) { + if (!Files.isRegularFile(langFile)) { + continue; + } + String lang = langFile.getFileName().toString(); + if (generateProfile(langFile, outputDir, lang, topN)) { + profileCount++; + } + } + } + System.out.printf(Locale.ROOT, + "Generated %d profiles in %s%n", profileCount, outputDir); + } + + private static boolean generateProfile( + Path langFile, Path outputDir, String lang, int topN) + throws IOException { + Map bigramCounts = new HashMap<>(); + long totalBigrams = 0; + + try (BufferedReader reader = Files.newBufferedReader( + langFile, StandardCharsets.UTF_8)) { + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (line.isEmpty() || line.startsWith("#") + || line.startsWith("___")) { + continue; + } + String[] parts = line.split("\t"); + String token = parts[0]; + long termFreq; + if (parts.length >= 3) { + termFreq = Long.parseLong(parts[2]); + } else if (parts.length >= 2) { + termFreq = Long.parseLong(parts[1]); + } else { + termFreq = 1; + } + + Map wordBigrams = + extractBigramsFromWord(token, termFreq); + for (Map.Entry e : wordBigrams.entrySet()) { + bigramCounts.merge(e.getKey(), e.getValue(), Long::sum); + totalBigrams += e.getValue(); + } + } + } + + if (totalBigrams == 0) { + return false; + } + + // Sort by frequency descending, take top-N + List> sorted = new ArrayList<>( + bigramCounts.entrySet()); + sorted.sort((a, b) -> Long.compare(b.getValue(), a.getValue())); + List> top = sorted.subList( + 0, Math.min(topN, sorted.size())); + + // Compute log2-probabilities + Map logProbs = new TreeMap<>(); + for (Map.Entry e : top) { + double prob = (double) e.getValue() / totalBigrams; + logProbs.put(e.getKey(), + Math.log(prob) / Math.log(2.0)); + } + + long uniqueBigrams = bigramCounts.size(); + double unseenLogProb = computeUnseenLogProb( + logProbs, uniqueBigrams); + double expectedScore = computeExpectedScore( + logProbs, unseenLogProb); + + // Write profile + Path outPath = outputDir.resolve(lang); + try (BufferedWriter writer = Files.newBufferedWriter( + outPath, StandardCharsets.UTF_8)) { + writer.write(String.format(Locale.ROOT, + "#TOTAL_BIGRAMS\t%d%n", totalBigrams)); + writer.write(String.format(Locale.ROOT, + "#UNIQUE_BIGRAMS\t%d%n", uniqueBigrams)); + writer.write(String.format(Locale.ROOT, + "#UNSEEN_LOG_PROB\t%.6f%n", unseenLogProb)); + writer.write(String.format(Locale.ROOT, + "#EXPECTED_SCORE\t%.6f%n", expectedScore)); + for (Map.Entry e : top) { + double prob = (double) e.getValue() / totalBigrams; + double lp = Math.log(prob) / Math.log(2.0); + writer.write(String.format(Locale.ROOT, + "%s\t%.6f%n", e.getKey(), lp)); + } + } + return true; + } +} diff --git a/tika-eval/tika-eval-lite/src/main/java/org/apache/tika/eval/textquality/BigramTextQualityScorer.java b/tika-eval/tika-eval-lite/src/main/java/org/apache/tika/eval/textquality/BigramTextQualityScorer.java new file mode 100644 index 00000000000..1c0faccbfa1 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/java/org/apache/tika/eval/textquality/BigramTextQualityScorer.java @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.eval.textquality; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.text.Normalizer; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.tika.textquality.TextQualityResult; +import org.apache.tika.textquality.TextQualityScorer; + +/** + * Scores text quality using character bigram profiles derived from + * language corpora. For each language, the profile contains the top 200 + * character bigrams and their log2-probabilities. Scoring computes the + * average log2-likelihood per bigram against each language profile and + * returns the best match. + * + *

Bigram profiles were generated from the common_tokens data in + * tika-eval-core by decomposing words into character bigrams weighted + * by term frequency.

+ * + *

This should not be used as a language detector. It is more lightweight + * and is meant to give an indication of "text quality".

+ */ +public class BigramTextQualityScorer extends TextQualityScorer { + + private static final String PROFILES_PATH = + "/org/apache/tika/eval/textquality/bigram_profiles/"; + + // Fallback unseen logprob used when a profile lacks corpus + // statistics needed for held-out estimation. + private static final double FALLBACK_UNSEEN_LOG_PROB = -20.0; + + // Minimum total bigram count for a language profile to be loaded. + // Profiles from tiny corpora (srd=80K, ssw=44K) have unreliable + // probability estimates that cause false language matches. + private static final long MIN_CORPUS_BIGRAMS = 500_000; + + // Default maximum characters to analyze. Beyond this, additional + // text adds negligible precision to the score. + private static final int DEFAULT_MAX_TEXT_LENGTH = 10_000; + + // Loaded lazily on first use + private volatile Map profiles; + + private int maxTextLength = DEFAULT_MAX_TEXT_LENGTH; + + public int getMaxTextLength() { + return maxTextLength; + } + + public void setMaxTextLength(int maxTextLength) { + this.maxTextLength = maxTextLength; + } + + @Override + public TextQualityResult score(CharSequence text) { + Map profs = getProfiles(); + if (profs.isEmpty()) { + return new TextQualityResult(0.0, "unk", 0.0, 0); + } + + int len = Math.min(text.length(), maxTextLength); + // Extract bigrams from input text + Map textBigrams = extractBigrams(text, len); + int totalBigrams = 0; + for (int count : textBigrams.values()) { + totalBigrams += count; + } + + if (totalBigrams == 0) { + return new TextQualityResult(0.0, "unk", 0.0, 0); + } + + // Score against each language profile + String bestLang = "unk"; + double bestScore = Double.NEGATIVE_INFINITY; + double secondBest = Double.NEGATIVE_INFINITY; + + for (Map.Entry entry : profs.entrySet()) { + double langScore = scoreAgainstProfile(textBigrams, + totalBigrams, entry.getValue()); + if (langScore > bestScore) { + secondBest = bestScore; + bestScore = langScore; + bestLang = entry.getKey(); + } else if (langScore > secondBest) { + secondBest = langScore; + } + } + + double confidence = (secondBest == Double.NEGATIVE_INFINITY) + ? 0.0 : bestScore - secondBest; + + return new TextQualityResult(bestScore, bestLang, + confidence, totalBigrams); + } + + @Override + public TextQualityResult score(CharSequence text, String language) { + Map profs = getProfiles(); + LanguageProfile profile = profs.get(language); + if (profile == null) { + return new TextQualityResult(0.0, language, 0.0, 0); + } + + int len = Math.min(text.length(), maxTextLength); + Map textBigrams = extractBigrams(text, len); + int totalBigrams = 0; + for (int count : textBigrams.values()) { + totalBigrams += count; + } + + if (totalBigrams == 0) { + return new TextQualityResult(0.0, language, 0.0, 0); + } + + double langScore = scoreAgainstProfile(textBigrams, + totalBigrams, profile); + return new TextQualityResult(langScore, language, + 0.0, totalBigrams); + } + + // Word-boundary marker used in boundary bigrams (_x for word-start, + // x_ for word-end). Underscore is not a letter, so it won't collide + // with any real character bigram. + static final char BOUNDARY = '_'; + + /** + * Extract character bigrams from text, including word-boundary + * bigrams. Input is first normalized to match the language + * profiles: NFKD decomposition with combining mark removal + * (equivalent to ICU folding used to generate common_tokens). + * + *

For each word (consecutive letters), emits:

+ *
    + *
  • Internal bigrams: consecutive lowercased letter pairs
    • + *
  • Word-start bigram: {@code _x} (boundary + first letter)
    • + *
  • Word-end bigram: {@code x_} (last letter + boundary)
    • + *
+ * + *

Word-boundary bigrams are critical for RTL detection because + * word-initial and word-final character patterns are highly + * directional. For example, Hebrew final forms (sofit letters) + * appear at word ends in forward text but word starts when + * reversed.

+ */ + static Map extractBigrams(CharSequence text) { + return extractBigrams(text, text.length()); + } + + static Map extractBigrams(CharSequence text, int len) { + // Normalize to match profile generation: NFKD decomposition + // strips compatibility differences, then we skip combining + // marks (diacritics) in the loop below. This matches ICU + // folding used to build common_tokens. + String normalized = Normalizer.normalize( + text.subSequence(0, len), Normalizer.Form.NFKD); + + Map counts = new HashMap<>(); + char prev = 0; + boolean prevIsLetter = false; + + for (int i = 0; i < normalized.length(); i++) { + char c = normalized.charAt(i); + + // Skip combining marks (diacritics). After NFKD, + // accented chars are decomposed to base + combining + // mark(s). Arabic tashkeel (fatha, kasra, etc.) are + // also combining marks. Skipping them prevents them + // from breaking the bigram chain. + if (Character.getType(c) == Character.NON_SPACING_MARK) { + continue; + } + + boolean isLetter = Character.isLetter(c); + if (isLetter) { + c = Character.toLowerCase(c); + if (prevIsLetter) { + String bigram = String.valueOf(prev) + c; + counts.merge(bigram, 1, Integer::sum); + } else { + String startBigram = String.valueOf(BOUNDARY) + c; + counts.merge(startBigram, 1, Integer::sum); + } + prev = c; + } else if (prevIsLetter) { + String endBigram = String.valueOf(prev) + BOUNDARY; + counts.merge(endBigram, 1, Integer::sum); + } + prevIsLetter = isLetter; + } + if (prevIsLetter) { + String endBigram = String.valueOf(prev) + BOUNDARY; + counts.merge(endBigram, 1, Integer::sum); + } + return counts; + } + + /** + * Compute average log2-likelihood of the text bigrams under a + * language profile. Unseen bigrams get the profile's floor + * log-probability. + */ + private static double scoreAgainstProfile( + Map textBigrams, int totalBigrams, + LanguageProfile profile) { + double totalLogProb = 0.0; + + for (Map.Entry entry : textBigrams.entrySet()) { + String bigram = entry.getKey(); + int count = entry.getValue(); + Double logProb = profile.bigramLogProbs.get(bigram); + if (logProb == null) { + logProb = profile.unseenLogProb; + } + totalLogProb += count * logProb; + } + + return totalLogProb / totalBigrams; + } + + private Map getProfiles() { + if (profiles == null) { + synchronized (this) { + if (profiles == null) { + profiles = loadAllProfiles(); + } + } + } + return profiles; + } + + private Map loadAllProfiles() { + Map result = new ConcurrentHashMap<>(); + // Load the language list from the index file + try (InputStream is = getClass().getResourceAsStream( + PROFILES_PATH + "languages.lst")) { + if (is == null) { + return Collections.emptyMap(); + } + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(is, StandardCharsets.UTF_8))) { + String lang; + while ((lang = reader.readLine()) != null) { + lang = lang.trim(); + if (lang.isEmpty() || lang.startsWith("#")) { + continue; + } + LanguageProfile profile = loadProfile(lang); + if (profile != null) { + result.put(lang, profile); + } + } + } + } catch (IOException e) { + // Log and return whatever we have + } + return result; + } + + private LanguageProfile loadProfile(String lang) { + try (InputStream is = getClass().getResourceAsStream( + PROFILES_PATH + lang)) { + if (is == null) { + return null; + } + Map logProbs = new HashMap<>(); + long totalBigrams = 0; + double unseenLogProb = FALLBACK_UNSEEN_LOG_PROB; + + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(is, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("#TOTAL_BIGRAMS\t")) { + totalBigrams = Long.parseLong( + line.substring("#TOTAL_BIGRAMS\t".length())); + } else if (line.startsWith("#UNSEEN_LOG_PROB\t")) { + unseenLogProb = Double.parseDouble( + line.substring( + "#UNSEEN_LOG_PROB\t".length())); + } else if (line.startsWith("#")) { + continue; + } else { + int tab = line.indexOf('\t'); + if (tab > 0) { + String bigram = line.substring(0, tab); + double logProb = Double.parseDouble( + line.substring(tab + 1)); + logProbs.put(bigram, logProb); + } + } + } + } + + if (totalBigrams < MIN_CORPUS_BIGRAMS) { + return null; + } + + return new LanguageProfile(logProbs, unseenLogProb); + } catch (IOException | NumberFormatException e) { + return null; + } + } + + static class LanguageProfile { + final Map bigramLogProbs; + final double unseenLogProb; + + LanguageProfile(Map bigramLogProbs, + double unseenLogProb) { + this.bigramLogProbs = bigramLogProbs; + this.unseenLogProb = unseenLogProb; + } + } +} diff --git a/tika-eval/tika-eval-lite/src/main/resources/META-INF/services/org.apache.tika.textquality.TextQualityScorer b/tika-eval/tika-eval-lite/src/main/resources/META-INF/services/org.apache.tika.textquality.TextQualityScorer new file mode 100644 index 00000000000..1c6cb4a6f63 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/META-INF/services/org.apache.tika.textquality.TextQualityScorer @@ -0,0 +1 @@ +org.apache.tika.eval.textquality.BigramTextQualityScorer diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/afr b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/afr new file mode 100644 index 00000000000..e2f6fd8b863 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/afr @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 301011292 +#UNIQUE_BIGRAMS 624 +#UNSEEN_LOG_PROB -19.047809 +#EXPECTED_SCORE -7.281920 +e_ -4.414793 +ie -4.988141 +_d -5.178774 +er -5.464496 +di -5.473532 +t_ -5.567416 +an -5.748243 +r_ -5.860147 +_v -5.913157 +n_ -5.940426 +s_ -5.941296 +aa -6.040364 +ge -6.338015 +_h -6.375940 +_w -6.386796 +en -6.456510 +ee -6.467207 +_s -6.514351 +_g -6.535599 +de -6.551562 +te -6.586864 +et -6.596289 +el -6.679942 +_m -6.733175 +oo -6.797802 +d_ -6.836122 +es -6.860356 +_b -6.882001 +in -6.887588 +g_ -6.888236 +_o -6.900871 +k_ -6.914975 +ar -6.918913 +_n -6.978683 +on -7.041264 +or -7.062373 +va -7.066803 +at -7.069977 +nd -7.075776 +le -7.092265 +he -7.164843 +st -7.186050 +l_ -7.187633 +oe -7.190673 +_a -7.194907 +wa -7.241504 +_k -7.274680 +ni -7.320841 +me -7.361469 +it -7.368547 +we -7.371671 +re -7.391118 +da -7.394243 +ng -7.424969 +be -7.425168 +ns -7.428911 +_t -7.461813 +se -7.462006 +li -7.584215 +rd -7.585553 +ke -7.608115 +ve -7.662045 +al -7.691443 +ek -7.757239 +rs -7.878633 +ik -7.885149 +ma -7.910578 +si -7.937814 +ri -7.954383 +ro -7.979406 +_p -7.992450 +_l -7.996314 +ui -8.014013 +ei -8.067926 +_e -8.072925 +ig -8.087025 +ou -8.113317 +la -8.113709 +ed -8.131326 +ka -8.132812 +m_ -8.152312 +nt -8.153451 +ra -8.156008 +as -8.182505 +ko -8.213239 +ne -8.228636 +om -8.234681 +is -8.277802 +ll -8.302292 +vo -8.353410 +to -8.385376 +sk -8.412491 +so -8.430410 +ho -8.438622 +vi -8.438637 +_j -8.444302 +wo -8.459198 +ul -8.504466 +_r -8.538562 +eg -8.555789 +u_ -8.586028 +em -8.626428 +ta -8.639169 +ur -8.644530 +ag -8.700330 +ak -8.729556 +sa -8.732179 +ir -8.768658 +id -8.813732 +hu -8.821918 +ti -8.826889 +ew -8.868447 +p_ -8.905331 +na -8.912441 +mo -8.930023 +no -8.931407 +ol -8.932969 +f_ -8.934772 +ha -8.936003 +eu -8.951191 +op -8.975966 +pe -8.994987 +us -9.018251 +lo -9.028579 +pr -9.056726 +rt -9.057709 +ot -9.058120 +rk -9.090583 +tr -9.111722 +am -9.131956 +_i -9.133215 +os -9.154307 +gr -9.242130 +a_ -9.256014 +ok -9.271363 +od -9.290924 +wi -9.299440 +ad -9.300567 +ba -9.302063 +il -9.342522 +_u -9.353394 +ki -9.355603 +ru -9.403026 +ry -9.410443 +tu -9.436236 +ga -9.456819 +gt -9.470901 +mi -9.492762 +do -9.506762 +nk -9.519739 +af -9.532961 +sp -9.535779 +hi -9.538913 +jo -9.543065 +bo -9.557746 +og -9.579073 +kr -9.580724 +go -9.585492 +pa -9.609249 +ev -9.695382 +y_ -9.696260 +kl -9.698489 +eb -9.701131 +br -9.702173 +ld -9.749076 +sl -9.796821 +i_ -9.834829 +vr -9.838238 +_f -9.841436 +ep -9.849620 +dr -9.849646 +nn -9.865601 +pl -9.879402 +gs -9.879515 +ap -9.894819 +ts -9.905573 +ef -9.919536 +rl -9.938694 +gi -9.965973 +ss -9.985537 +rg -10.003402 +ks -10.004844 +bl -10.029393 +yd -10.042374 +un -10.054433 +ja -10.071531 +ai -10.073141 +su -10.117160 +kk -10.123289 +ty -10.170226 +du -10.176419 +lu -10.180334 +ds -10.184322 +rm -10.233587 +po -10.243727 +eh -10.258270 +bi -10.303114 +tw -10.334678 +lk -10.371506 +wy -10.385715 +lf -10.407071 +of -10.410166 +rw -10.416073 +fr -10.439324 +ls -10.451507 +_c -10.487199 +ly -10.513735 +uu -10.522330 +kt -10.538542 +uk -10.559929 +yk -10.577443 +ys -10.580816 +lg -10.602568 +ku -10.614608 +fe -10.643197 +rv -10.662240 +rb -10.676100 +nu -10.749798 +rh -10.751424 +rn -10.751755 +ju -10.791137 +ch -10.821328 +ud -10.836594 +by -10.854594 +mm -10.882217 +o_ -10.900745 +ug -10.902936 +tt -10.915555 +th -10.915962 +oi -10.918492 +lt -10.930734 +ms -10.938990 +ue -11.006140 +ia -11.008725 +mp -11.032672 +pi -11.049030 +yn -11.052426 +bu -11.096034 +io -11.108499 +yf -11.114044 +im -11.126023 +fi -11.153098 +fo -11.159859 +je -11.212080 +ae -11.257507 +ut -11.265366 +pp -11.270827 +uw -11.363799 +ji -11.374041 +nl -11.391369 +ob -11.410851 +um -11.416525 +sw -11.427427 +ky -11.447981 +gl -11.452245 +tl -11.468221 +fs -11.473785 +fd -11.503283 +nv -11.510634 +vl -11.517573 +tg -11.533568 +ea -11.552144 +rr -11.575401 +mb -11.603552 +ab -11.633526 +md -11.657076 +dd -11.685715 +gh -11.701590 +gg -11.730989 +ip -11.733027 +rp -11.734692 +pu -11.737488 +nb -11.740059 +lm -11.778167 +tj -11.784409 +ow -11.786490 +if -11.792377 +kw -11.803196 +ps -11.804138 +mu -11.843138 +hr -11.866893 +pt -11.977079 +iv -11.997432 +lp -12.013076 +sm -12.032241 +fa -12.083593 +pg -12.099931 +au -12.119419 +aw -12.164406 +ff -12.169586 +ub -12.175737 +eo -12.175914 +co -12.179956 +h_ -12.200834 +nh -12.216191 +fg -12.247870 +sd -12.257671 +ye -12.297240 +sb -12.306072 +yl -12.311470 +tv -12.337007 +sh -12.355374 +nw -12.383166 +dw -12.432042 +sv -12.442648 +ov -12.456805 +ic -12.491906 +yw -12.504689 +rf -12.516772 +ft -12.533468 +kh -12.542561 +sg -12.564023 +gu -12.566899 +ua -12.591418 +kb -12.603298 +lw -12.657417 +nr -12.699262 +fl -12.756714 +td -12.813716 +lb -12.835481 +oh -12.847194 +gd -12.852856 +ca -12.855162 +av -12.878179 +gb -12.900617 +vu -12.961474 +sn -12.987940 +ac -12.995983 +dh -12.998412 +yb -13.072650 +mg -13.076910 +nm -13.100794 +b_ -13.103660 +lj -13.149261 +sr -13.154059 +py -13.171729 +sy -13.196891 +mt -13.240430 +ey -13.251792 +tm -13.253897 +vy -13.268579 +nc -13.279212 +my -13.293145 +ib -13.300089 +kn -13.305077 +yg -13.327516 +iw -13.328847 +ce -13.330969 +db -13.342224 +tb -13.344964 +lh -13.348959 +nj -13.400785 +gm -13.433976 +sc -13.444410 +dv -13.449120 +dg -13.467297 +iu -13.503520 +c_ -13.508426 +ck -13.528108 +tk -13.533781 +gn -13.594438 +dj -13.596039 +ay -13.607240 +mn -13.617866 +_z -13.672263 +w_ -13.706387 +yt -13.714675 +up -13.743506 +ph -13.759901 +dt -13.800492 +yp -13.806179 +ny -13.809203 +dm -13.878324 +_y -13.878829 +v_ -13.900800 +ah -13.909845 +fh -13.919318 +ec -13.940788 +ml -13.948118 +dp -13.958457 +bb -13.963961 +lv -13.989147 +oj -13.989225 +gv -13.989849 +gk -13.996805 +nf -14.031378 +bs -14.066388 +wu -14.069104 +pv -14.082844 +dk -14.108097 +fu -14.164978 +cl -14.167181 +ln -14.204697 +ze -14.212501 +yv -14.222270 +ct -14.247404 +dl -14.261266 +ci -14.278879 +np -14.279165 +oc -14.289685 +ya -14.352163 +pb -14.354570 +sf -14.374487 +tz -14.419673 +pd -14.441475 +mh -14.455835 +sj -14.482248 +iz -14.486862 +ym -14.494918 +fw -14.535089 +tn -14.622211 +yh -14.641435 +kg -14.667765 +zi -14.670012 +pm -14.692678 +rc -14.706836 +x_ -14.719452 +zu -14.723984 +za -14.728921 +yo -14.773730 +ej -14.824002 +wr -14.842469 +z_ -14.863032 +fk -14.876232 +fb -14.901680 +tp -14.924451 +pk -14.954571 +kp -14.956398 +az -14.986578 +cr -15.020903 +bw -15.066552 +oy -15.098985 +kv -15.126495 +tc -15.139411 +mf -15.164186 +pn -15.166299 +dy -15.228604 +ex -15.241358 +fn -15.250670 +qu -15.251044 +oa -15.282599 +hn -15.291798 +ww -15.322303 +mw -15.322499 +fv -15.329784 +mv -15.345263 +ht -15.352062 +uf -15.359096 +fy -15.360103 +uc -15.372452 +pw -15.391485 +ii -15.443716 +rj -15.449924 +aj -15.454436 +vs -15.484224 +mr -15.503242 +cu -15.506808 +ws -15.527485 +lr -15.530885 +fm -15.652996 +hw -15.684452 +gp -15.717643 +kd -15.780459 +ux -15.842469 +ao -15.860321 +fp -15.909329 +uy -15.956094 +wh -16.032743 +dn -16.083093 +tf -16.099490 +hy -16.100500 +wn -16.116755 +bt -16.128382 +zo -16.168063 +df -16.206690 +gw -16.207415 +vg -16.246007 +ez -16.297964 +mk -16.340284 +uv -16.353065 +_q -16.387576 +km -16.400785 +hs -16.417889 +zy -16.530431 +bj -16.608736 +ax -16.688496 +yr -16.741601 +hl -16.826506 +nz -16.833206 +xi -16.886212 +cy -16.914352 +cc -16.990941 +_x -17.023135 +xa -17.042414 +wk -17.073146 +cq -17.119483 +ix -17.158215 +hm -17.288725 +uz -17.333935 +ox -17.401200 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/amh b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/amh new file mode 100644 index 00000000000..b6e1cb00461 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/amh @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 4436420 +#UNIQUE_BIGRAMS 8102 +#UNSEEN_LOG_PROB -14.518163 +#EXPECTED_SCORE -10.479302 +ት_ -5.330140 +_የ -5.365740 +_በ -5.529902 +_አ -5.806059 +ን_ -5.842784 +ር_ -6.305303 +ል_ -6.414352 +_እ -6.484746 +ው_ -6.488274 +ም_ -6.494828 +_መ -6.651885 +ች_ -6.890523 +_ከ -7.101584 +ስ_ -7.265581 +_ይ -7.337287 +እን -7.392933 +_ነ -7.527755 +_ተ -7.528776 +ያ_ -7.573913 +ድ_ -7.625637 +_ወ -7.752219 +ነበ -7.825715 +_ም -7.963403 +ብ_ -8.066595 +መን -8.107357 +ኛ_ -8.110409 +የሚ -8.119515 +_ለ -8.145155 +_ማ -8.243632 +ንድ -8.289496 +በር -8.325242 +_ዘ -8.335500 +_ሲ -8.360506 +አን -8.384323 +_ስ -8.391512 +ህ_ -8.394026 +ይ_ -8.437334 +ንግ -8.448537 +ና_ -8.465335 +ንደ -8.487223 +ዊ_ -8.505189 +ቸው -8.514435 +ሆን -8.518245 +የአ -8.541563 +ነት -8.571808 +_ያ -8.616802 +ጥ_ -8.623455 +_ደ -8.626537 +ዚህ -8.635044 +ለው -8.652473 +_ው -8.663771 +የተ -8.683156 +ዘመ -8.684494 +_ክ -8.799469 +ትዮ -8.829334 +ኢት -8.835709 +ክ_ -8.845100 +ዮጵ -8.854101 +ጵያ -8.857114 +_ግ -8.864977 +ቱ_ -8.874408 +ሉ_ -8.878688 +_ት -8.880986 +ዎች -8.944454 +ላ_ -8.997984 +ለት -8.998150 +ስጥ -9.000980 +እስ -9.014875 +_ዓ -9.038792 +_ድ -9.045994 +ያን -9.048402 +ገር -9.058250 +ከተ -9.085374 +_ሰ -9.085551 +ውስ -9.092457 +በአ -9.095656 +_ን -9.124770 +ደ_ -9.130409 +ነ_ -9.135155 +ውን -9.159124 +ነገ -9.160426 +_ታ -9.167140 +ሲሆ -9.171821 +አገ -9.184821 +በመ -9.192602 +ራ_ -9.239400 +ሌ_ -9.259589 +ንጉ -9.267384 +ማር -9.268186 +ሩ_ -9.271799 +ቅ_ -9.328793 +ተማ -9.352406 +ማ_ -9.360293 +ለም -9.388785 +_ጥ -9.395559 +በተ -9.404346 +_ብ -9.420523 +ፍ_ -9.422530 +ወይ -9.435756 +ታ_ -9.469018 +_ባ -9.493421 +ሳሌ -9.497412 +ስት -9.504244 +ላል -9.506844 +ሞ_ -9.522544 +ረ_ -9.526855 +ምሳ -9.530699 +ስከ -9.543746 +ርስ -9.550071 +ኑ_ -9.555199 +ሆነ -9.559119 +አማ -9.562311 +በኋ -9.578381 +ኋላ -9.585359 +ይም -9.588861 +ንዲ -9.594381 +አይ -9.597653 +_ኢ -9.606498 +_ሁ -9.616674 +ለ_ -9.624867 +በረ -9.630784 +_ቀ -9.647379 +ድረ -9.652343 +ሪያ -9.652866 +በዚ -9.670248 +የመ -9.689990 +ግሥ -9.698340 +ቶች -9.701857 +ሪክ -9.711367 +ከ_ -9.726164 +ንዳ -9.729197 +ርኛ -9.741393 +ሮ_ -9.741671 +ጀመ -9.784048 +ድር -9.786631 +ግ_ -9.790370 +አለ -9.812423 +ሥት -9.824166 +ማለ -9.831555 +ዓመ -9.836303 +ላይ -9.851244 +ተኛ -9.851845 +ባለ -9.859075 +ስተ -9.863310 +ደግ -9.872425 +መር -9.880986 +ተለ -9.897019 +አስ -9.899502 +ግሞ -9.900434 +_ል -9.915429 +በት -9.916686 +ረስ -9.945576 +ምን -9.946538 +ሆኑ -9.946859 +ሁለ -9.952648 +ከዚ -9.956843 +ኝ_ -9.960727 +መስ -9.964296 +ዝ_ -9.964621 +_ገ -9.975056 +ደረ -9.980959 +ክል -9.982274 +መት -9.982603 +ታሪ -9.997153 +ክፍ -9.999481 +_ጀ -10.001813 +ሱ_ -10.015548 +ረት -10.018581 +ክር -10.023311 +ሥ_ -10.040332 +ተ_ -10.056171 +ንም -10.061374 +ታት -10.069038 +ርያ -10.078150 +ናቸ -10.078853 +ሳይ -10.089088 +ጉሥ -10.119153 +ደር -10.122412 +ሪካ -10.123862 +ለመ -10.125678 +ዳን -10.139550 +_ቅ -10.139917 +ሚገ -10.146536 +ታወ -10.154669 +ሮች -10.161356 +_ሥ -10.169947 +ንት -10.171446 +_ና -10.178589 +መነ -10.197558 +ለማ -10.213300 +የነ -10.216778 +ባል -10.226875 +አካ -10.231169 +ዱ_ -10.237436 +ሪ_ -10.238221 +ያው -10.238614 +መሆ -10.241761 +ዚያ -10.244520 +ሎች -10.244914 +የኢ -10.250845 +በሩ -10.253622 +አባ -10.254813 +መጀ -10.261185 +ካ_ -10.261584 +ተመ -10.268386 +_ፈ -10.276027 +ይነ -10.280469 +ትር -10.310301 +ወን -10.313608 +_ካ -10.327330 +ግሊ -10.330676 +ችን -10.330676 +ትን -10.335710 +ይገ -10.339919 +ኛው -10.346255 +ሽ_ -10.361576 +ያለ -10.362860 +ባ_ -10.368867 +ብር -10.383563 +_ሕ -10.386607 +_ቁ -10.390094 +ረው -10.391840 +ተባ -10.393151 +ስለ -10.409866 +አመ -10.410308 +ዝብ -10.412523 +አብ -10.414741 +በእ -10.437559 +_ህ -10.456626 +_ፍ -10.457540 +በኢ -10.458455 +ርት -10.472710 +ሊዝ -10.487573 +መል -10.493187 +ዋል -10.501649 +ቶ_ -10.507317 +ትም -10.507791 +በ_ -10.513959 +_ሚ -10.521587 +ገኝ -10.524938 +በላ -10.534070 +ግብ -10.542291 +ራዊ -10.544232 +_ሳ -10.547148 +ቃ_ -10.553488 +መታ -10.560346 +ንያ -10.561328 +ይታ -10.580123 +በማ -10.580123 +ህር -10.583612 +ሚያ -10.586109 +ይህ -10.593627 +ናት -10.596142 +መሪ -10.597149 +ተሰ -10.598661 +ተወ -10.603712 +_ዕ -10.607766 +ምክ -10.611832 +ቋን -10.617951 +ምሮ -10.622558 +ንስ -10.624610 +ንቋ -10.628723 +ብረ -10.629238 +ድን -10.635432 +ባቢ -10.637503 +ስራ -10.638021 +ዘን -10.639058 +ራት -10.639058 +ልበ -10.651035 +ህን -10.654176 +ጥር -10.655224 +ጽ_ -10.655224 +ቁጥ -10.658374 +አዲ -10.659952 +ጥን -10.666808 +ጀም -10.671574 +_ሃ -10.682221 +ሕዝ -10.683825 +ክን -10.685430 +_ሆ -10.685966 +ቅር -10.688110 +ጠር -10.690258 +ሎ_ -10.691871 +ስቲ -10.693486 +ዲስ -10.709732 +_ጉ -10.711913 +ወቅ -10.713550 +ንዱ -10.716283 +ምህ -10.761857 +እር -10.761857 +ዩ_ -10.772057 +ህም -10.781757 +_ቋ -10.782329 +ሁን -10.782902 +ከል -10.786918 +አሁ -10.795562 +ናል -10.801934 +_ጠ -10.806004 +የሆ -10.810085 +በዘ -10.823577 +ባላ -10.824756 +የእ -10.826526 +ሉት -10.831851 +ሌሎ -10.835412 +ላት -10.841963 +ቲያ -10.849743 +ይሆ -10.854552 +ማሪ -10.859982 +አም -10.860586 +ከአ -10.868468 +ይን -10.871511 +ጊዜ -10.878228 +_ቢ -10.881292 +መጽ -10.895470 +አል -10.897329 +መረ -10.901677 +_ሊ -10.904791 +ተገ -10.911666 +ነው -10.912292 +_ሌ -10.922355 +ጎር -10.922355 +ገድ -10.922986 +ዛት -10.924249 +ጃ_ -10.927412 +ርጎ -10.928680 +ካከ -10.929948 +መካ -10.933124 +ዝኛ -10.935669 +ግን -10.936306 +መና -10.937581 +ገኘ -10.943333 +ፊት -10.945255 +ድም -10.958137 +ታዊ -10.959431 +ነዚ -10.960079 +ባት -10.965921 +_ጽ -10.974402 +_ኤ -10.978989 +ሰዎ -10.986227 +ጣጠ -10.986227 +መጠ -10.988207 +ግር -10.990190 +ተከ -10.991514 +ቆጣ -10.994165 +እነ -10.996156 +ዓክ -10.999481 +ልክ -11.007492 +ስታ -11.007492 +ጨረ -11.014875 +የቀ -11.015548 +ገኛ -11.021620 +ስል -11.022296 +ፍል -11.025004 +ዲሁ -11.025682 +ልጣ -11.037254 +ያም -11.037937 +ጦር -11.039305 +ምር -11.042046 +ለያ -11.045478 +ርጉ -11.055825 +ሰው -11.055825 +ዳ_ -11.055825 +_ሀ -11.057210 +ንን -11.057210 +ሻ_ -11.064853 +አበ -11.076042 +ክት -11.080965 +ቢ_ -11.083785 +_ቤ -11.083785 +ማስ -11.084491 +የት -11.088734 +ንገ -11.091570 +እግ -11.091570 +ህዝ -11.092280 +ሴ_ -11.097258 +መለ -11.100111 +ዓለ -11.107985 +ታል -11.110859 +ረጃ -11.116624 +ፍተ -11.120963 +ያት -11.124588 +ግድ -11.124588 +_ፕ -11.126041 +ርነ -11.131138 +ቦች -11.131867 +ሜሪ -11.133328 +የግ -11.136252 +ውም -11.136985 +ቱን -11.141385 +አሜ -11.145799 +ለዚ -11.152446 +አቆ -11.156152 +ከፍ -11.156152 +ማዊ -11.158380 +አው -11.158380 +ግል -11.162101 +ንታ -11.162847 +ይባ -11.165085 +ጅ_ -11.165832 +ራል -11.167327 +ኔ_ -11.174826 +ደቡ -11.178589 +ሁ_ -11.180852 +ይል -11.184632 +_ዝ -11.186147 +ብዛ -11.186905 +ሁም -11.191461 +ግስ -11.194506 +ተው -11.206752 +ሙ_ -11.208290 +ገራ -11.212914 +ምስ -11.215231 +በሚ -11.216005 +_ሠ -11.217553 +ዲህ -11.223761 +ዝር -11.226096 +ብሔ -11.229996 +_ኃ -11.233124 +ገሥ -11.235474 +የር -11.240187 +ዳር -11.241761 +ካባ -11.242548 +ብለ -11.244125 +ወለ -11.245704 +የወ -11.247284 +ገኙ -11.249657 +በስ -11.252828 +መሰ -11.255211 +በባ -11.259191 +ቋ_ -11.268787 +ሌላ -11.272000 +ቤተ -11.272000 +ቹ_ -11.272000 +ቡብ -11.276834 +ጣን -11.281683 +ሰብ -11.293062 +ይች -11.296330 +አድ -11.297966 +በግ -11.299605 +ሩት -11.311953 +ሃይ -11.312780 +ሥራ -11.316922 +ምና -11.319413 +ሰራ -11.325242 +ላቸ -11.327748 +ሠረ -11.332772 +ጽሐ -11.337813 +አፍ -11.341184 +የሰ -11.342028 +መሠ -11.344563 +ቅም -11.347102 +ጠቀ -11.348797 +ረገ -11.351344 +ግዛ -11.353044 +ተቀ -11.356451 +ጫ_ -11.359865 +ሳ_ -11.366719 +በብ -11.366719 +መሳ -11.367578 +ደብ -11.368438 +ተር -11.371019 +ረግ -11.372743 +ኃይ -11.374469 +ራሱ -11.376196 +መጨ -11.377061 +ፈረ -11.377061 +ተዋ -11.380525 +ጣል -11.380525 +ማው -11.382260 +የም -11.386607 +ላን -11.387478 +ግሪ -11.391840 +ተብ -11.403245 +ሚለ -11.405007 +ስር -11.405890 +ኘው -11.406772 +በደ -11.407655 +ከነ -11.407655 +አደ -11.408539 +ገሮ -11.410308 +ውያ -11.412966 +ረሻ -11.420077 +ገን -11.420969 +ሚባ -11.426328 +_ሙ -11.429016 +ገል -11.432607 +እያ -11.436207 +ጭ_ -11.443434 +ወረ -11.444340 +መድ -11.444340 +ራው -11.446153 +ይወ -11.450697 +ምድ -11.452519 +በፊ -11.460745 +ፍለ -11.463497 +ሞች -11.463497 +ታቸ -11.463497 +ታላ -11.464416 +ሀገ -11.466255 +ችላ -11.469940 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ara b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ara new file mode 100644 index 00000000000..34ca2e5d905 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ara @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 246081411 +#UNIQUE_BIGRAMS 936 +#UNSEEN_LOG_PROB -15.055621 +#EXPECTED_SCORE -7.743507 +_ا -3.903151 +ال -3.969388 +ه_ -4.759996 +ي_ -5.663143 +لا -6.056700 +_و -6.086896 +ا_ -6.158838 +ن_ -6.335228 +لي -6.383404 +يه -6.431010 +لم -6.458141 +ل_ -6.536670 +_م -6.587846 +ر_ -6.636912 +ان -6.638761 +ت_ -6.647955 +_ب -6.820778 +م_ -6.840100 +_ع -6.928301 +وا -6.964820 +د_ -6.990614 +_ت -7.046730 +ات -7.086067 +ام -7.172899 +ار -7.221081 +ري -7.225254 +ما -7.256595 +_ل -7.260524 +را -7.348250 +عل -7.450706 +لت -7.467463 +_ي -7.521862 +ين -7.553748 +يا -7.619252 +ول -7.628639 +اي -7.668990 +اس -7.698052 +با -7.712609 +ب_ -7.732552 +ير -7.814471 +ها -7.836602 +لع -7.915732 +بي -7.918838 +عا -7.933603 +دي -7.957961 +له -7.999408 +نا -8.035111 +اد -8.045896 +ني -8.048991 +ره -8.050226 +ع_ -8.063735 +_ح -8.149406 +_ك -8.149660 +اب -8.168668 +لك -8.192728 +نه -8.208628 +اع -8.233050 +لس -8.272510 +قا -8.295620 +دا -8.321916 +من -8.323062 +مي -8.339467 +_ف -8.343099 +كا -8.367043 +ور -8.373143 +_س -8.385620 +ون -8.396178 +لق -8.436896 +سي -8.439602 +لل -8.447542 +لح -8.455094 +ف_ -8.457408 +ق_ -8.465491 +تي -8.467639 +_ق -8.467785 +مه -8.476508 +ء_ -8.488260 +مو -8.489151 +لب -8.495040 +_ن -8.497920 +ده -8.516886 +اء -8.531351 +وي -8.558261 +س_ -8.570149 +ست -8.576155 +وم -8.610258 +لو -8.611021 +ك_ -8.628576 +في -8.631126 +يد -8.647046 +لد -8.651688 +مر -8.656416 +سا -8.658189 +اح -8.659765 +عد -8.662254 +_ه -8.671126 +مل -8.696934 +اف -8.738305 +يم -8.741752 +يي -8.746606 +يل -8.749558 +مع -8.816854 +اق -8.827626 +لف -8.846082 +لن -8.847416 +_ج -8.871561 +دو -8.875507 +نت -8.890568 +جا -8.918354 +او -8.931233 +بر -8.939919 +تا -8.945863 +وق -8.959119 +لج -8.959837 +بع -8.965564 +مس -8.975245 +_د -8.979831 +تم -9.000744 +حا -9.021266 +حي -9.023949 +يو -9.028360 +اه -9.035068 +هذ -9.040395 +حد -9.047686 +رو -9.071877 +شر -9.080529 +رب -9.081624 +تح -9.084161 +يق -9.087286 +ذا -9.111705 +اج -9.112746 +لش -9.119376 +اك -9.121242 +رك -9.125418 +قد -9.129815 +تو -9.133367 +به -9.142622 +قي -9.154500 +عم -9.155105 +تر -9.162571 +عه -9.170897 +مت -9.180070 +فا -9.184839 +_ر -9.200172 +بل -9.205831 +تق -9.208780 +عر -9.231955 +يس -9.253557 +اخ -9.272793 +_ش -9.274753 +_خ -9.283257 +تع -9.292668 +طا -9.299538 +يع -9.302504 +هم -9.328080 +لر -9.333725 +ته -9.346473 +كو -9.366530 +فه -9.369772 +قو -9.370261 +عي -9.370587 +دم -9.383067 +نو -9.385540 +اص -9.393944 +ح_ -9.396644 +در -9.404712 +مد -9.435387 +شا -9.458031 +ض_ -9.467326 +وت -9.475228 +و_ -9.480671 +سل -9.481916 +وب -9.506834 +قه -9.513688 +كن -9.528493 +لذ -9.530234 +وع -9.559526 +سو -9.562794 +ود -9.568116 +مج -9.568891 +جم -9.569247 +يت -9.578805 +خل -9.607056 +ث_ -9.610434 +سب -9.611099 +مح -9.620750 +وس -9.637192 +لص -9.659720 +وف -9.671700 +ضا -9.678266 +فر -9.680044 +صا -9.690920 +ذي -9.713015 +اط -9.723276 +حت -9.726600 +قر -9.729678 +لخ -9.742016 +نس -9.743688 +وه -9.747196 +كي -9.750419 +زي -9.756887 +اض -9.765485 +جه -9.766623 +_ص -9.772718 +كل -9.775783 +ز_ -9.776441 +سم -9.782278 +يف -9.794222 +تل -9.798789 +تب -9.805462 +خا -9.808682 +عب -9.812414 +ط_ -9.823362 +ج_ -9.833698 +ذه -9.840831 +لط -9.855372 +بد -9.887074 +حر -9.896934 +كم -9.915894 +يك -9.918406 +يب -9.918513 +كر -9.923856 +ند -9.928654 +بن -9.944545 +سه -9.959936 +عن -9.968915 +لغ -9.998805 +قت -10.000311 +وج -10.023421 +خر -10.030057 +تخ -10.036920 +رت -10.064855 +هو -10.066005 +تص -10.066118 +قب -10.067331 +قل -10.070587 +_ط -10.088374 +وك -10.096978 +جي -10.105568 +صر -10.127528 +مك -10.128492 +اث -10.141505 +ذل -10.161093 +جد -10.162598 +حم -10.164090 +كه -10.165349 +رج -10.166723 +عت -10.182027 +تن -10.184085 +مش -10.184112 +لث -10.185579 +كت -10.201286 +اش -10.207151 +صل -10.208635 +تج -10.211955 +طر -10.216251 +مق -10.217842 +هر -10.218122 +نف -10.222094 +بح -10.225529 +غي -10.227474 +رف -10.240927 +جل -10.246165 +_ذ -10.271079 +نظ -10.273297 +رض -10.280265 +زا -10.282111 +حو -10.282505 +تف -10.283345 +ضي -10.291911 +بو -10.292404 +سر -10.293183 +طي -10.317995 +رس -10.346915 +ثا -10.351604 +مص -10.353520 +صو -10.364206 +هي -10.372148 +ثل -10.402901 +_غ -10.419916 +تش -10.430986 +جو -10.434364 +طو -10.443512 +جر -10.443757 +مب -10.444149 +از -10.445677 +طل -10.446478 +حق -10.473089 +كب -10.476652 +هد -10.477763 +وز -10.479236 +يج -10.501534 +تس -10.504009 +فق -10.506787 +حل -10.514168 +كث -10.517713 +حك -10.535276 +بق -10.541126 +سن -10.543740 +فت -10.554694 +وح -10.559977 +حه -10.562147 +عو -10.578639 +صح -10.588346 +رق -10.598245 +جن -10.598481 +وض -10.627671 +عض -10.629221 +دد -10.634477 +هن -10.641334 +شي -10.646644 +يث -10.652369 +يح -10.653483 +يز -10.666822 +مث -10.671920 +صي -10.680766 +ص_ -10.687547 +شك -10.708368 +خد -10.719930 +سط -10.721276 +قع -10.723157 +ثر -10.728370 +تك -10.745881 +مم -10.746798 +سع -10.763027 +يط -10.768519 +فل -10.782918 +فع -10.791673 +رن -10.801779 +طه -10.805908 +تط -10.805929 +شه -10.813379 +نق -10.815723 +سك -10.828556 +تت -10.832474 +بت -10.837925 +اذ -10.841826 +رد -10.849334 +قط -10.860311 +يض -10.870439 +تد -10.878904 +ثي -10.914774 +نط -10.916586 +ظا -10.922149 +وط -10.926433 +طن -10.942023 +طب -10.951488 +ذ_ -10.960818 +خت -10.963227 +غر -10.986828 +بم -11.002451 +يخ -11.004208 +فس -11.017235 +يش -11.021872 +شع -11.032210 +نب -11.039065 +غا -11.039620 +طق -11.076215 +رح -11.083372 +نم -11.085715 +جت -11.085855 +_ز -11.088470 +خط -11.092381 +صد -11.102349 +صف -11.102684 +نش -11.120234 +بش -11.128781 +خص -11.133409 +وص -11.148010 +صه -11.150393 +مخ -11.162557 +وو -11.173240 +_ث -11.183458 +فض -11.189538 +حس -11.191703 +خي -11.213423 +فو -11.213771 +دف -11.239778 +نح -11.251022 +دن -11.264530 +بط -11.293242 +دع -11.296232 +رع -11.307347 +خ_ -11.339315 +عق -11.364662 +صب -11.375026 +ضو -11.388338 +نج -11.394764 +_ض -11.402123 +حص -11.420084 +دل -11.427849 +لز -11.438451 +جز -11.447117 +ش_ -11.483686 +ظه -11.507692 +غ_ -11.511264 +صن -11.549624 +ظم -11.559429 +زه -11.561323 +ضع -11.561571 +دت -11.589303 +ضم -11.601457 +زم -11.602076 +ضر -11.628247 +دث -11.630252 +نص -11.635447 +كز -11.649202 +شخ -11.659748 +اغ -11.664117 +ذك -11.670066 +ضه -11.700571 +بب -11.715064 +جب -11.726521 +عش -11.733012 +بس -11.741739 +ظر -11.758399 +قص -11.777701 +شب -11.787927 +ثه -11.789109 +خب -11.792287 +مط -11.794617 +تغ -11.826478 +شم -11.832709 +فن -11.838666 +لض -11.853949 +بك -11.857382 +مز -11.870625 +نع -11.905534 +حف -11.911732 +هل -11.932802 +دب -11.941008 +زو -11.941723 +قض -11.962958 +رم -11.970726 +سف -11.983987 +قف -11.992610 +دخ -12.002655 +وذ -12.008996 +يص -12.009528 +مف -12.013643 +نذ -12.014055 +زر -12.024838 +كس -12.032628 +فك -12.036859 +سج -12.041596 +شت -12.049254 +تض -12.064290 +هه -12.067986 +رغ -12.074773 +جع -12.082634 +حز -12.083372 +ثم -12.112282 +شو -12.125581 +ثق -12.128571 +كد -12.129832 +سس -12.136521 +دس -12.137419 +وظ -12.153969 +تز -12.209946 +حب -12.213952 +ضل -12.216740 +ظي -12.217465 +طف -12.219058 +يذ -12.256978 +ثن -12.275054 +فظ -12.294390 +قن -12.298696 +نك -12.300351 +شف -12.312497 +وخ -12.322492 +ضح -12.343057 +رص -12.343270 +قم -12.364785 +غل -12.368997 +خو -12.373439 +دق -12.398289 +وث -12.405777 +ثو -12.407114 +صم -12.418014 +حث -12.448656 +وش -12.450197 +خم -12.454304 +عس -12.474582 +غم -12.495791 +رط -12.504861 +شد -12.540671 +نز -12.545886 +هج -12.558916 +مض -12.563735 +عز -12.586668 +زب -12.597854 +تظ -12.602514 +كذ -12.641353 +طع -12.642627 +هت -12.653238 +خف -12.656565 +وغ -12.665183 +رر -12.685469 +غه -12.686744 +عظ -12.700454 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/asm b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/asm new file mode 100644 index 00000000000..443e306921b --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/asm @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 3343189 +#UNIQUE_BIGRAMS 1387 +#UNSEEN_LOG_PROB -14.595388 +#EXPECTED_SCORE -7.682060 +ৰ_ -3.664945 +ত_ -4.948266 +_ক -4.949066 +ল_ -5.226356 +_আ -5.288364 +কৰ -5.330563 +_ব -5.670506 +ন_ -5.703317 +_প -5.734408 +ক_ -6.037841 +_ত -6.166579 +_স -6.177907 +আৰ -6.319785 +পৰ -6.486447 +_ম -6.508122 +ছল -6.622435 +_চ -6.623243 +_হ -6.637135 +_ন -6.661697 +_দ -6.848779 +_এ -6.858962 +_অ -6.945511 +_ল -6.980668 +ব_ -7.060443 +ট_ -7.133453 +চন -7.170272 +ই_ -7.197123 +ছ_ -7.203979 +তও -7.257778 +ৱ_ -7.266057 +_ভ -7.279738 +_গ -7.301493 +তৰ -7.363388 +নত -7.365664 +নৰ -7.378675 +বল -7.388620 +ও_ -7.440972 +_ৰ -7.454230 +_জ -7.528693 +ম_ -7.625755 +হছ -7.715601 +কল -7.716327 +সম -7.717325 +ৰত -7.724700 +গত -7.808607 +মৰ -7.814618 +_য -7.830149 +আছ -7.871287 +ৰৰ -7.953512 +লক -7.979851 +হ_ -8.043892 +_শ -8.077186 +জন -8.080453 +খন -8.081739 +বব -8.101396 +লগ -8.112102 +মন -8.122167 +বৰ -8.140681 +গ_ -8.155616 +ৰছ -8.179689 +নম -8.189356 +ৰণ -8.206845 +ণ_ -8.210291 +হৰ -8.281818 +বহ -8.360478 +প_ -8.380040 +জ_ -8.426647 +লৰ -8.467306 +ধৰ -8.477267 +_থ -8.504121 +থক -8.511819 +গৰ -8.515762 +পছ -8.519558 +ৱৰ -8.527817 +ৰব -8.550774 +ৰক -8.579376 +মহ -8.583343 +সক -8.636792 +জৰ -8.639715 +_উ -8.652334 +খ_ -8.716054 +এট -8.718779 +দ_ -8.765715 +হত -8.786144 +_খ -8.801273 +চৰ -8.802043 +তক -8.816563 +খত -8.824366 +অস -8.858612 +ৰজ -8.866848 +মজ -8.902956 +ভৰ -8.908751 +ছত -8.918115 +দন -8.929432 +লত -8.930695 +শ_ -8.951053 +দৰ -8.983888 +কম -8.985418 +হৱ -8.991994 +মত -9.004795 +কন -9.028712 +মল -9.029163 +কত -9.036622 +_ফ -9.052574 +_ধ -9.069631 +_ছ -9.108644 +পত -9.111267 +দশ -9.111506 +তখ -9.143119 +নক -9.143363 +আহ -9.154140 +হচ -9.165990 +ওৰ -9.170708 +জত -9.188222 +_ঘ -9.188474 +বছ -9.199341 +ৰল -9.212082 +ছৰ -9.238687 +ভ_ -9.259959 +চপ -9.263933 +চল -9.284776 +সৰ -9.285045 +ৰপ -9.292332 +টৰ -9.297482 +সই -9.320750 +নজ -9.323789 +ষ_ -9.332109 +তল -9.333779 +থ_ -9.337682 +দখ -9.340757 +তন -9.348894 +ছব -9.385370 +ৰচ -9.409818 +_ই -9.439473 +_ও -9.453928 +ভগ -9.459385 +লভ -9.474348 +অন -9.478037 +ফল -9.478344 +পন -9.490089 +জল -9.492574 +জৱ -9.499741 +ধন -9.529410 +খল -9.533882 +যগ -9.542545 +গৱ -9.570490 +এখ -9.574761 +শষ -9.582019 +লন -9.593309 +তত -9.595310 +শৰ -9.617850 +আদ -9.621585 +লখ -9.622265 +বশ -9.631477 +কছ -9.638682 +এক -9.730646 +ওল -9.758035 +স_ -9.770418 +ধ_ -9.784431 +নহ -9.792062 +সত -9.798965 +ৱন -9.803585 +ভত -9.815979 +ৰই -9.828872 +ৰৱ -9.835166 +ৰখ -9.835560 +পল -9.888568 +ভষ -9.899242 +হন -9.899242 +_ড -9.909996 +শত -9.909996 +চ_ -9.915820 +কব -9.920831 +নট -9.956403 +পৱ -9.969322 +কই -9.974957 +উপ -9.984543 +পথ -9.986293 +বধ -10.020397 +এজ -10.042982 +মক -10.046172 +লৱ -10.048455 +অধ -10.049369 +বট -10.051657 +দল -10.065001 +হট -10.089711 +চক -10.119164 +টক -10.119164 +_ট -10.122528 +কথ -10.123009 +ইৰ -10.132665 +ঘৰ -10.135090 +সগ -10.141900 +যদ -10.162029 +ঠ_ -10.168471 +নৱ -10.182946 +ইত -10.185958 +ণৰ -10.186964 +লই -10.193013 +সধ -10.197060 +সহ -10.201626 +বদ -10.214900 +বত -10.216953 +সন -10.238165 +হল -10.240774 +ৰম -10.243910 +দৱ -10.251781 +ৰগ -10.295041 +কৱ -10.304287 +সভ -10.321302 +মট -10.331831 +_ঠ -10.340757 +ঘট -10.353686 +ওপ -10.367872 +লচ -10.373585 +কক -10.380472 +নই -10.383351 +আগ -10.387969 +ধক -10.407765 +নস -10.416585 +ৱত -10.418946 +ইব -10.438575 +নগ -10.445779 +শক -10.446381 +নব -10.448189 +চত -10.451206 +গছ -10.456654 +ণত -10.463949 +বজ -10.467000 +পদ -10.470670 +গঠ -10.471282 +তই -10.497868 +টই -10.499741 +আন -10.501617 +দও -10.502868 +আম -10.505375 +ভৱ -10.513553 +আল -10.517975 +নদ -10.562310 +বক -10.562963 +এন -10.564269 +ইট -10.567540 +কট -10.573446 +ৎস -10.601331 +ওক -10.614125 +গল -10.618868 +যৰ -10.619546 +নল -10.622945 +ৰহ -10.623626 +ৱল -10.626352 +লছ -10.627716 +সব -10.651814 +চহ -10.656682 +অশ -10.664365 +ইছ -10.672089 +ষত -10.678440 +দব -10.678440 +খৰ -10.678440 +হক -10.679147 +ঠই -10.686952 +গণ -10.689800 +বখ -10.694798 +ৱই -10.704127 +ষণ -10.713516 +তথ -10.732480 +এই -10.744275 +অভ -10.751696 +কশ -10.757661 +হস -10.758408 +e_ -10.762900 +জক -10.766654 +ঠন -10.777976 +ভল -10.795509 +বচ -10.800888 +হম -10.821044 +ৰও -10.824171 +হজ -10.836743 +ৱহ -10.842278 +ইন -10.843071 +সৱ -10.845451 +বগ -10.845451 +টন -10.854211 +টত -10.865439 +ষৰ -10.867050 +নছ -10.880817 +শল -10.893895 +লপ -10.900479 +আক -10.906265 +যন -10.937237 +লম -10.965434 +পট -10.968890 +উৎ -10.976695 +পই -10.983669 +পব -10.989799 +শৱ -11.018157 +বস -11.031645 +খব -11.043437 +ছম -11.048912 +দট -11.063615 +অত -11.071023 +বভ -11.078469 +ঙৰ -11.084079 +তহ -11.092535 +তপ -11.094421 +দত -11.097254 +কহ -11.101041 +মছ -11.134605 +উঠ -11.162029 +আধ -11.176938 +গন -11.187971 +ৰষ -11.203152 +যৱ -11.213362 +লব -11.215413 +সজ -11.239208 +শখ -11.261282 +কষ -11.262342 +জপ -11.269781 +শন -11.292332 +চব -11.307565 +পচ -11.310850 +পহ -11.315241 +ফট -11.318544 +ভম -11.326280 +জই -11.327388 +গট -11.329608 +ৰট -11.339638 +সল -11.378173 +ঙ_ -11.381623 +টল -11.400164 +অৱ -11.417765 +শস -11.446381 +ভব -11.475577 +লও -11.492884 +দক -11.496620 +মঠ -11.505375 +গদ -11.519241 +দছ -11.520509 +লজ -11.524317 +এব -11.533242 +কস -11.538367 +আই -11.543510 +লহ -11.543510 +নন -11.552556 +ৰধ -11.560354 +মণ -11.566885 +বন -11.569506 +ৱস -11.570818 +_a -11.572131 +an -11.576078 +জব -11.576078 +ইখ -11.594643 +ৱক -11.595978 +মখ -11.595978 +দহ -11.597314 +থৱ -11.601331 +ৰঙ -11.605359 +লল -11.614802 +মই -11.620225 +অহ -11.621585 +এগ -11.636620 +_t -11.643506 +নথ -11.644887 +পঠ -11.653203 +মদ -11.655985 +ঠক -11.662965 +কৎ -11.679855 +ইক -11.695514 +তম -11.695514 +থৰ -11.721509 +ডঙ -11.724426 +জগ -11.724426 +তব -11.736156 +নপ -11.743535 +বম -11.749466 +d_ -11.750953 +nd -11.768912 +আট -11.776461 +th -11.779492 +ওচ -11.779492 +পজ -11.793210 +ইজ -11.793210 +ষক -11.816368 +on -11.825736 +আজ -11.833590 +কপ -11.833590 +ড_ -11.838322 +যক -11.839903 +দই -11.843071 +গহ -11.849426 +মধ -11.849426 +উল -11.851020 +শশ -11.859012 +he -11.862222 +ফক -11.862222 +লট -11.868662 +ভন -11.883260 +n_ -11.888159 +ৱশ -11.894716 +অক -11.894716 +সদ -11.896360 +কঠ -11.898006 +ভপ -11.906265 +পখ -11.907922 +সফ -11.919577 +গও -11.938084 +যব -11.956831 +ঠত -11.958548 +মগ -11.961987 +ডল -11.975826 +ৰদ -11.979307 +দউ -11.986293 +ফৰ -11.995074 +ঘষ -11.996836 +ৱষ -12.003909 +সচ -12.011015 +এল -12.014582 +খক -12.016369 +তজ -12.047085 +হব -12.054408 +সয -12.058084 +উত -12.065463 +লদ -12.074741 +গই -12.084079 +ভজ -12.099146 +হদ -12.114373 +_ঢ -12.116287 +ছন -12.131697 +খৱ -12.141412 +er -12.143363 +বই -12.168968 +খছ -12.168968 +ডম -12.180940 +_i -12.186964 +বপ -12.188978 +কচ -12.195035 +ডক -12.201118 +ৱদ -12.201118 +নচ -12.207227 +দস -12.209269 +নখ -12.213362 +টব -12.231924 +ইল -12.231924 +ছৱ -12.250729 +গব -12.254941 +টছ -12.278331 +_p -12.284776 +আত -12.299928 +ti -12.302106 +ৎপ -12.304287 +জহ -12.317442 +এৰ -12.332943 +নভ -12.335172 +y_ -12.353121 +ওৱ -12.375877 +টও -12.385081 +in -12.389705 +io -12.398998 +হই -12.401330 +ৰন -12.427241 +ঘল -12.429620 +উৰ -12.434389 +ছদ -12.439174 +l_ -12.448792 +যত -12.458474 +ফ_ -12.468222 +at -12.468222 +সঘ -12.478037 +গজ -12.478037 +মচ -12.480501 +r_ -12.480501 +দঘ -12.490399 +টৱ -12.500366 +_s -12.500366 +ভট -12.502868 +s_ -12.510402 +ৰস -12.510402 +হৎ -12.512922 +চম -12.517975 +ঠৰ -12.520509 +ঙল -12.528135 +অৰ -12.530686 +ৎ_ -12.535802 +অপ -12.538367 +পক -12.540937 +ষদ -12.543510 +ধৱ -12.546089 +মঘ -12.569506 +or -12.569506 +গভ -12.580036 +ইদ -12.587985 +থব -12.593309 +যজ -12.593309 +জধ -12.598652 +ছপ -12.598652 +te -12.614802 +আশ -12.617511 +কণ -12.622945 +গম -12.622945 +তট -12.625670 +থত -12.628399 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ast b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ast new file mode 100644 index 00000000000..c8837f66468 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ast @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 7967596 +#UNIQUE_BIGRAMS 533 +#UNSEEN_LOG_PROB -19.322649 +#EXPECTED_SCORE -7.307004 +s_ -4.841320 +es -5.092409 +a_ -5.233881 +u_ -5.634884 +e_ -5.807157 +en -5.851969 +_c -5.863150 +_p -6.016465 +os -6.058929 +n_ -6.087186 +on -6.191572 +er -6.211038 +co -6.489246 +l_ -6.507218 +nt -6.553596 +re -6.558332 +te -6.577296 +_l -6.577694 +ue -6.636595 +ie -6.643837 +ci -6.647245 +_e -6.668988 +el -6.682093 +_d -6.687308 +ra -6.689755 +_n -6.708816 +ar -6.723915 +an -6.744658 +_m -6.749033 +de -6.780378 +_s -6.793394 +or -6.798666 +r_ -6.802481 +le -6.807502 +ta -6.899319 +_a -6.911126 +_t -6.925427 +un -6.929593 +al -6.940338 +ne -6.940471 +na -6.956641 +ma -6.984528 +ri -7.035354 +_f -7.049484 +lo -7.058700 +ol -7.108055 +ti -7.162812 +ca -7.184009 +o_ -7.187225 +la -7.194102 +st -7.197181 +me -7.209724 +io -7.238748 +qu -7.265717 +da -7.280646 +in -7.299919 +ia -7.323478 +ll -7.464841 +am -7.488156 +ic -7.528473 +tr -7.533194 +mi -7.534067 +po -7.539110 +ro -7.550131 +as -7.589032 +se -7.593467 +pe -7.599354 +si -7.624038 +li -7.652027 +tu -7.685406 +_q -7.703031 +so -7.722250 +ac -7.748994 +pa -7.781453 +om -7.792089 +cu -7.792571 +nu -7.810791 +di -7.847020 +no -7.851279 +_r -7.869795 +ec -7.916322 +to -7.929593 +nd -7.939960 +_u -7.966798 +id -8.118610 +pr -8.127241 +is -8.145122 +ni -8.275278 +i_ -8.287617 +mo -8.318267 +nc -8.337584 +ru -8.342101 +em -8.344218 +_v -8.384798 +ad -8.385948 +iu -8.412417 +do -8.414393 +gu -8.422819 +at -8.446249 +_b -8.451944 +rt -8.487051 +pu -8.489327 +_i -8.503254 +il -8.528573 +_g -8.532390 +au -8.535746 +it -8.557002 +ab -8.582389 +fo -8.585515 +ce -8.612830 +ve -8.624789 +_o -8.625003 +lu -8.625504 +vi -8.668473 +sa -8.680383 +mu -8.722901 +im -8.732727 +ur -8.745493 +mp -8.772636 +ba -8.772874 +ga -8.818006 +fi -8.843065 +gr -8.866200 +fa -8.883883 +ed -8.892806 +su -8.895391 +xe -8.918949 +sp -8.922547 +_h -8.969155 +ns -9.011141 +sc -9.016195 +pi -9.032791 +ae -9.052077 +_x -9.079047 +br -9.092329 +du -9.111732 +ul -9.115443 +oc -9.191744 +rm -9.201306 +et -9.206110 +eg -9.210180 +ch -9.212756 +bi -9.232008 +ua -9.299091 +bl -9.307672 +rr -9.312154 +iv -9.350883 +ot -9.375447 +ye -9.388980 +fe -9.423006 +ct -9.435991 +ir -9.446564 +va -9.452388 +us -9.499317 +op -9.535678 +ob -9.548638 +be -9.560348 +ap -9.582666 +oi -9.628654 +pl -9.644363 +ig -9.650025 +od -9.667883 +um -9.685071 +ut -9.743319 +cr -9.747671 +ay -9.748294 +ui -9.751724 +rd -9.794821 +xu -9.837090 +ib -9.849064 +za -9.862149 +ha -9.873655 +lt -9.887995 +ax -9.894012 +ai -9.949865 +mb -9.955607 +ex -9.984299 +uc -9.993868 +ip -10.010394 +ng -10.025979 +ey -10.031085 +he -10.031464 +bo -10.042688 +go -10.048237 +ho -10.053808 +fu -10.055156 +ep -10.105734 +ev -10.117553 +xi -10.162293 +vo -10.192274 +_y -10.193334 +ea -10.222242 +rg -10.235933 +hu -10.262155 +eu -10.269735 +ud -10.277580 +rs -10.351120 +yo -10.351829 +rc -10.371844 +if -10.374245 +xo -10.401661 +rn -10.422136 +av -10.423378 +bu -10.428111 +iz -10.433359 +sm -10.438375 +ao -10.439884 +cl -10.454038 +fr -10.464234 +ez -10.474244 +dr -10.487441 +ix -10.489002 +z_ -10.510764 +ag -10.543089 +xa -10.577262 +up -10.626505 +lm -10.641178 +ub -10.644073 +eo -10.719920 +yu -10.759236 +nf -10.760178 +ya -10.793856 +eb -10.795464 +vu -10.803532 +ei -10.804180 +og -10.879612 +az -10.926418 +ef -11.030895 +uy -11.034309 +hi -11.035449 +nv -11.052654 +fl -11.056119 +af -11.074354 +ug -11.075136 +cc -11.092032 +t_ -11.126431 +_j -11.171661 +zo -11.179619 +iq -11.203332 +ox -11.262600 +ov -11.288635 +y_ -11.331855 +th -11.343571 +d_ -11.355857 +rb -11.412479 +nq -11.432358 +zu -11.450993 +ee -11.501022 +oy -11.532323 +rp -11.543629 +eq -11.602658 +lg -11.611696 +of -11.674823 +gl -11.681944 +pt -11.683730 +gn -11.752661 +nx -11.755162 +lv -11.760178 +yi -11.862318 +rv -11.865017 +ou -12.058434 +_w -12.073184 +oo -12.085722 +uv -12.159184 +aq -12.176682 +lc -12.292718 +m_ -12.303661 +ge -12.309164 +jo -12.323014 +ld -12.418910 +_z -12.433860 +oe -12.433860 +nn -12.443914 +nz -12.453022 +sl -12.465257 +c_ -12.557207 +ii -12.560484 +oz -12.563769 +pc -12.573670 +ux -12.599284 +k_ -12.624217 +g_ -12.624217 +h_ -12.712609 +rl -12.723589 +_k -12.743319 +wa -12.767103 +rq -12.862318 +uz -12.929946 +rf -13.021831 +ja -13.027868 +sq -13.041543 +gi -13.059980 +ry -13.067732 +oh -13.089663 +ph -13.123197 +ke -13.136179 +tt -13.144353 +rx -13.152574 +oa -13.160841 +sh -13.182562 +ss -13.201199 +lb -13.220081 +ck -13.242718 +wi -13.289088 +sf -13.305493 +uo -13.325800 +gc -13.447955 +ls -13.480698 +ow -13.503648 +x_ -13.516322 +ly -13.526969 +rz -13.544170 +uf -13.555026 +ju -13.612830 +hn -13.635694 +wo -13.684922 +bt -13.716260 +w_ -13.740838 +lr -13.745804 +rk -13.778508 +oq -13.791287 +my -13.799009 +ps -13.822425 +je -13.854251 +nr -13.942719 +we -13.968611 +cn -14.015820 +uu -14.021831 +bs -14.021831 +tl -14.099165 +ty -14.108729 +ew -14.137811 +nm -14.137811 +lf -14.160841 +p_ -14.228746 +zs -14.260377 +ka -14.303661 +ah -14.303661 +hr -14.318383 +zi -14.382681 +gh -14.406077 +ts -14.437873 +gm -14.474502 +tc -14.491085 +mn -14.491085 +mm -14.499448 +nl -14.516322 +ki -14.524834 +ms -14.550674 +yl -14.550674 +ny -14.559391 +yw -14.608300 +lp -14.626505 +dw -14.677786 +b_ -14.687308 +aw -14.716260 +sb -14.770895 +ds -14.822425 +ys -14.865017 +sg -14.892290 +yr -14.903345 +dy -14.948433 +ok -14.977346 +nk -15.073964 +sn -15.105534 +pp -15.105534 +ak -15.118358 +hy -15.137811 +tm -15.144353 +yc -15.150926 +f_ -15.191003 +xv -15.267502 +yn -15.325800 +pg -15.425867 +aj -15.441897 +yd -15.482770 +ji -15.533396 +bn -15.542009 +bc -15.559391 +ko -15.594796 +ff -15.621932 +zc -15.658927 +bd -15.677786 +ze -15.687308 +by -15.696894 +cy -15.745804 +sy -15.755788 +lq -15.775966 +uq -15.838250 +wr -15.859624 +dn -15.870431 +uh -15.870431 +nh -15.870431 +eh -15.870431 +ks -15.925713 +wn -15.937028 +ht -15.937028 +tw -15.937028 +tn -15.937028 +bf -15.959929 +bb -15.959929 +sr -15.983199 +dq -16.055348 +ft -16.131297 +kl -16.225273 +nb -16.239213 +gt -16.386554 +pb -16.417918 +ik -16.433860 +v_ -16.466281 +dv -16.568161 +sv -16.603785 +tb -16.677786 +nj -16.696894 +rh -16.696894 +ku -16.696894 +ln -16.755788 +wl -16.755788 +ej -16.775966 +yp -16.796430 +dg -16.796430 +iy -16.817189 +aa -16.817189 +sk -16.817189 +wh -16.838250 +lz -16.881319 +py -16.881319 +lk -16.903345 +ws -16.903345 +ky -16.903345 +hl -16.925713 +lx -16.948433 +oj -16.948433 +ih -16.994976 +qi -16.994976 +cb -17.018822 +xh -17.018822 +bx -17.092823 +yg -17.118358 +zq -17.118358 +xt -17.144353 +cs -17.170826 +dd -17.197793 +mw -17.311003 +uk -17.311003 +zz -17.371124 +sw -17.433860 +bm -17.499448 +kn -17.499448 +dc -17.568161 +mg -17.640311 +wt -17.640311 +sd -17.640311 +ww -17.640311 +mc -17.640311 +tz -17.640311 +xf -17.755788 +ym -17.755788 +gb -17.796430 +tg -17.838250 +cd -17.881319 +dm -17.881319 +sz -17.971517 +zt -17.971517 +gg -18.018822 +yb -18.018822 +wy -18.018822 +xp -18.067732 +dl -18.067732 +tp -18.067732 +wu -18.067732 +hb -18.118358 +dt -18.118358 +km -18.118358 +wf -18.118358 +cq -18.170826 +mt -18.170826 +tv -18.170826 +vd -18.170826 +kf -18.170826 +gw -18.170826 +kh -18.225273 +xw -18.340751 +cm -18.340751 +lh -18.402151 +dz -18.402151 +xx -18.466281 +bv -18.466281 +rw -18.533396 +yt -18.533396 +pz -18.603785 +gs -18.603785 +wk -18.838250 +iw -18.838250 +bh -18.838250 +kg -18.838250 +fb -18.925713 +mr -18.925713 +xy -18.925713 +kd -18.925713 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/aze b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/aze new file mode 100644 index 00000000000..3a18edb51e4 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/aze @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 270717474 +#UNIQUE_BIGRAMS 655 +#UNSEEN_LOG_PROB -18.152648 +#EXPECTED_SCORE -7.361449 +a_ -5.000420 +ar -5.115149 +la -5.284750 +in -5.355184 +i_ -5.356809 +n_ -5.395419 +an -5.602642 +da -5.662673 +r_ -5.950922 +ma -6.031426 +ir -6.210297 +as -6.236561 +ri -6.245650 +_a -6.280867 +si -6.306467 +al -6.311084 +ra -6.334153 +_b -6.349082 +di -6.414670 +il -6.428798 +sa -6.438848 +ya -6.489434 +li -6.523807 +ni -6.551045 +at -6.620850 +_i -6.665257 +_m -6.686816 +_s -6.730408 +_o -6.764254 +un -6.774727 +ta -6.802786 +is -6.821661 +na -6.864012 +ba -6.937654 +ay -7.006018 +_d -7.039056 +ad -7.050697 +nd -7.077710 +ti -7.117086 +ha -7.141149 +am -7.220377 +mi -7.247413 +_h -7.255857 +iy -7.266882 +ol -7.282414 +ca -7.313232 +bi -7.314698 +_q -7.360063 +_t -7.404434 +ak -7.432397 +_e -7.434432 +az -7.494136 +qa -7.548277 +_y -7.552214 +ki -7.552971 +_g -7.603067 +yi -7.725171 +ur -7.784424 +b_ -7.806529 +za -7.826361 +im -7.856448 +t_ -7.874291 +on -7.875588 +aq -7.928689 +k_ -7.963361 +mu -7.965313 +ib -7.967236 +_k -8.002028 +ah -8.008372 +_c -8.022712 +q_ -8.034381 +ci -8.044179 +va -8.080533 +zi -8.129786 +ka -8.158151 +er -8.166917 +ab -8.167826 +nu -8.172586 +m_ -8.197238 +_v -8.197296 +st -8.213583 +u_ -8.221400 +s_ -8.246394 +us -8.267794 +_n -8.276872 +or -8.296576 +ed -8.305054 +lu -8.319127 +rd -8.323297 +lm -8.338980 +du -8.382054 +l_ -8.415655 +rl -8.423756 +ik -8.431522 +iz -8.443401 +ey -8.444054 +id -8.448017 +nl -8.501244 +_u -8.526289 +ac -8.556211 +ld -8.573328 +iq -8.622745 +_p -8.684991 +de -8.688606 +ga -8.689950 +bu -8.699826 +et -8.704292 +ru -8.717850 +ax -8.739824 +fa -8.776168 +_r -8.779040 +go -8.849912 +ag -8.856777 +gu -8.861680 +um -8.863662 +xa -8.879008 +z_ -8.902785 +af -8.905353 +su -8.922579 +yy -8.975653 +ll -8.979879 +ye -8.998040 +so -9.004622 +ul -9.015000 +ec -9.032330 +_x -9.046203 +tu -9.047988 +av -9.054326 +cu -9.081204 +ov -9.083310 +gi -9.091962 +sl -9.094276 +tl -9.112230 +rm -9.172806 +rb -9.189686 +_f -9.258862 +ve -9.285246 +el -9.313610 +en -9.313983 +qi -9.328063 +ub -9.390443 +pa -9.425963 +xi -9.494270 +uz -9.494655 +do -9.495974 +ic -9.502187 +uc -9.525287 +fi -9.552341 +yc -9.556312 +yu -9.566442 +ns -9.577593 +d_ -9.583513 +td -9.583906 +ox -9.601956 +sd -9.623897 +it -9.636524 +yo -9.648181 +rt -9.661295 +oz -9.695224 +re -9.702377 +ro -9.707059 +ql -9.763118 +tm -9.778645 +se -9.789970 +oy -9.800624 +lk -9.834706 +if -9.855893 +ut -9.865500 +pr -9.889117 +nc -9.904071 +ig -9.913163 +ai -9.934933 +nt -9.952261 +rk -9.952514 +te -9.956305 +os -9.956485 +hi -9.984577 +rs -9.989807 +bl -10.025854 +_z -10.037384 +es -10.040130 +vi -10.045048 +he -10.049153 +ev -10.053717 +qu -10.068719 +ko -10.073851 +uk -10.096625 +_l -10.114592 +zu -10.148359 +ke -10.149285 +be -10.172228 +x_ -10.208466 +ml -10.211856 +co -10.218121 +v_ -10.222323 +to -10.233135 +em -10.240143 +ix -10.252175 +gl -10.253371 +yd -10.258435 +ud -10.281093 +bo -10.296727 +ap -10.315395 +ua -10.338158 +ne -10.338862 +ug -10.362620 +ge -10.365345 +kt -10.370648 +vl -10.371975 +kl -10.376488 +zl -10.380382 +dd -10.382237 +c_ -10.390082 +ku -10.409934 +me -10.440319 +ia -10.452821 +uq -10.454345 +nm -10.484293 +le -10.486187 +dl -10.544010 +mo -10.546989 +sm -10.548473 +nk -10.584794 +nr -10.612242 +om -10.648848 +xt -10.668264 +ss -10.696409 +ks -10.727416 +f_ -10.758236 +xs -10.760370 +qo -10.769560 +mm -10.771319 +yn -10.847375 +ek -10.865704 +hu -10.874531 +ok -10.893515 +sk -10.909478 +kd -10.913924 +qe -10.915014 +vr -10.937256 +aa -10.938626 +pu -10.957354 +no -10.992036 +ob -11.009969 +ez -11.055678 +lq -11.125016 +tr -11.148898 +xu -11.170010 +qd -11.176125 +og -11.183308 +ot -11.218914 +vu -11.255735 +po -11.282645 +iv -11.319121 +vv -11.380070 +cl -11.382259 +ny -11.385007 +kr -11.396885 +yr -11.401909 +uy -11.412736 +ls -11.428469 +qq -11.431513 +lo -11.475252 +y_ -11.487319 +hb -11.490106 +yl -11.492884 +rc -11.508480 +fo -11.528271 +bs -11.573518 +hs -11.578823 +pi -11.592055 +ux -11.597957 +op -11.601347 +tt -11.603120 +md -11.603966 +zd -11.614471 +rg -11.624768 +eh -11.625744 +qr -11.686609 +pl -11.692911 +xd -11.717072 +mk -11.734145 +h_ -11.744311 +sq -11.763674 +tb -11.781536 +sp -11.818822 +br -11.851790 +e_ -11.856228 +oh -11.867415 +pe -11.872342 +ho -11.882467 +dr -11.883634 +ef -11.891691 +qt -11.912208 +ze -11.922162 +dm -11.941044 +rq -11.969485 +uh -11.969955 +lt -11.971238 +sr -11.982731 +sc -11.983832 +hl -11.984415 +rn -11.988458 +ym -11.991580 +p_ -12.047669 +io -12.054136 +ck -12.099651 +ih -12.103468 +xl -12.118153 +sg -12.132606 +g_ -12.141391 +hk -12.143270 +uv -12.170108 +ft -12.170550 +ji -12.171656 +ly -12.221712 +xm -12.227348 +rz -12.246357 +cm -12.251961 +ng -12.279860 +ce -12.297726 +nq -12.335760 +rp -12.342553 +lx -12.353750 +cr -12.370415 +up -12.406357 +oq -12.411835 +qs -12.417597 +rh -12.418413 +zr -12.443930 +fl -12.459154 +ms -12.463872 +ht -12.477515 +od -12.483667 +db -12.487151 +lc -12.523933 +mp -12.542953 +nf -12.570792 +hr -12.604646 +lh -12.606639 +fr -12.616043 +rr -12.617884 +yt -12.650509 +vq -12.651126 +fe -12.653218 +uf -12.657688 +gr -12.672954 +bd -12.676613 +lb -12.702594 +ct -12.719172 +ip -12.724500 +ea -12.732675 +_j -12.742684 +ln -12.768704 +zm -12.781629 +fu -12.794861 +zv -12.819535 +tc -12.869188 +o_ -12.876503 +ii -12.876704 +ju -12.880034 +rx -12.883453 +mc -12.903607 +nv -12.909745 +ts -12.909909 +hd -12.932228 +yg -12.943854 +kc -12.946965 +mt -12.949028 +oc -12.958160 +ep -12.990148 +bb -13.029086 +eg -13.051487 +km -13.063252 +nz -13.097968 +xo -13.125421 +hm -13.132103 +sv -13.145127 +lg -13.169764 +ej -13.212830 +nb -13.266749 +cd -13.268746 +vb -13.281212 +fd -13.281530 +ps -13.304530 +xe -13.334273 +sb -13.337910 +vt -13.355509 +ex -13.379161 +xc -13.385933 +ty -13.456545 +hn -13.456725 +mr -13.485469 +hz -13.508139 +zo -13.513551 +rv -13.526006 +hv -13.656449 +fq -13.692188 +mb -13.696851 +nn -13.707505 +sh -13.734489 +vc -13.765621 +cb -13.798653 +mz -13.812157 +kk -13.832615 +vs -13.859453 +kv -13.866679 +xn -13.870904 +rj -13.921604 +bh -13.940834 +qc -13.968929 +dc -13.975009 +tk -13.991146 +xr -14.000025 +vz -14.001860 +of -14.101476 +gd -14.235266 +oj -14.239898 +bm -14.250869 +by -14.255865 +zy -14.276551 +vo -14.306580 +tn -14.308958 +au -14.321233 +sy -14.321996 +gm -14.328122 +zg -14.349544 +vd -14.355118 +eo -14.460985 +sf -14.550861 +yf -14.551757 +sn -14.575501 +ff -14.579019 +yh -14.622447 +ch -14.646300 +lv -14.714580 +rf -14.753058 +cp -14.761914 +zn -14.810701 +yv -14.845736 +yk -14.916321 +yq -14.940248 +dn -14.959135 +dq -15.022641 +aj -15.022641 +ds -15.043546 +eu -15.062204 +eq -15.064576 +bk -15.111535 +pm -15.127661 +hq -15.165743 +mn -15.194430 +ei -15.206670 +zb -15.231058 +eb -15.254198 +ry -15.270956 +dv -15.276445 +zz -15.316984 +mq -15.369033 +lz -15.382401 +ui -15.425608 +dy -15.426311 +cq -15.432425 +zs -15.432661 +gv -15.434784 +kb -15.443544 +ae -15.448063 +vg -15.450448 +bc -15.474752 +sx -15.494297 +qn -15.509878 +j_ -15.511122 +cc -15.561775 +my -15.578627 +zc -15.610800 +tv -15.624195 +qy -15.628239 +ja -15.718741 +fs -15.733473 +th -15.791230 +oo -15.806420 +lf -15.837599 +yx -15.856446 +qh -15.858661 +ys -15.868830 +mh -15.982235 +pk -15.983961 +cn -15.988805 +bt -15.996449 +ou -15.999239 +gz -16.004835 +vn -16.028507 +tq -16.075575 +pd -16.084435 +dx -16.109461 +qb -16.368807 +kz -16.379218 +gc -16.408123 +bx -16.417888 +xb -16.431954 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bak b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bak new file mode 100644 index 00000000000..611c9fb4d76 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bak @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 12525813 +#UNIQUE_BIGRAMS 1452 +#UNSEEN_LOG_PROB -14.315482 +#EXPECTED_SCORE -8.388636 +н_ -5.396894 +а_ -5.528839 +_б -5.888157 +ын -5.946520 +ар -6.169565 +ан -6.270169 +_т -6.293505 +_и -6.317169 +ы_ -6.359081 +_к -6.377244 +ла -6.453553 +та -6.462138 +е_ -6.479084 +га -6.568255 +ен -6.628244 +р_ -6.631369 +ә_ -6.705870 +_а -6.759507 +ыл -6.807621 +ал -6.956893 +әр -6.974731 +нд -6.975949 +лә -7.025072 +да -7.104220 +на -7.127077 +_м -7.186771 +ны -7.235423 +ер -7.243098 +ән -7.261854 +ра -7.338299 +ры -7.359913 +ул -7.372837 +лг -7.390102 +к_ -7.408437 +_һ -7.408906 +и_ -7.409572 +иы -7.469754 +лы -7.485664 +ба -7.507798 +ел -7.557746 +бе -7.558875 +ат -7.586835 +п_ -7.599825 +ре -7.601255 +тә -7.625341 +м_ -7.631746 +ма -7.671111 +гы -7.671628 +ле -7.673672 +_с -7.675013 +те -7.726579 +ил -7.751306 +_я -7.763043 +рә -7.794840 +ит -7.832569 +әм -7.845439 +ст -7.863082 +ка -7.883281 +дә -7.955520 +_ҡ -7.956864 +л_ -7.961364 +_д -7.962857 +_х -7.996994 +гә -8.006559 +бу -8.012912 +ин -8.019143 +_у -8.027834 +т_ -8.033950 +ор -8.042883 +аи -8.056310 +аш -8.063718 +һы -8.076066 +ки -8.077528 +һә -8.098084 +аг -8.098273 +ге -8.109045 +әк -8.130026 +нә -8.180258 +ты -8.204346 +мә -8.212557 +не -8.231333 +_р -8.232892 +әт -8.300314 +лд -8.352252 +ур -8.365373 +ик -8.391474 +_ө -8.398492 +се -8.410356 +аһ -8.421528 +ке -8.433504 +ам -8.437491 +ми -8.458649 +сә -8.460190 +рг -8.474048 +де -8.481521 +ыр -8.512017 +рт -8.512943 +әи -8.533964 +ти -8.552785 +сы -8.578929 +ҡа -8.579017 +он -8.579326 +кә -8.601434 +ия -8.608475 +за -8.617087 +са -8.637399 +шы -8.637858 +ме -8.638684 +_г -8.683015 +әл -8.687994 +ис -8.694994 +то -8.701117 +_ш -8.717459 +ас -8.728507 +ир -8.738951 +би -8.743041 +өн -8.787340 +әс -8.789428 +ды -8.794328 +ди -8.825707 +ак -8.833672 +еш -8.868994 +_ә -8.883336 +ек -8.916011 +лл -8.923373 +ха -8.937986 +өс -8.947451 +ни -8.956750 +ез -8.958639 +a_ -8.959557 +ри -9.007952 +_ү -9.009732 +кү -9.023093 +ем -9.028135 +_э -9.106916 +кы -9.115324 +_о -9.121469 +рл -9.155679 +тө -9.171864 +я_ -9.181462 +ун -9.185681 +әг -9.191730 +яр -9.193887 +n_ -9.200106 +ту -9.208735 +у_ -9.209349 +че -9.218171 +нг -9.224703 +кө -9.224841 +ше -9.226495 +_п -9.229327 +зе -9.238064 +ли -9.239456 +тт -9.252674 +ҡу -9.256614 +ш_ -9.263181 +ау -9.274834 +үз -9.290328 +_н -9.297268 +ып -9.303877 +аз -9.305698 +им -9.313299 +шл -9.315572 +нн -9.337908 +ыш -9.357343 +ык -9.358023 +ar -9.381184 +ыу -9.385877 +уш -9.391126 +an -9.400904 +з_ -9.405192 +шу -9.408398 +ет -9.412238 +ко -9.414437 +_в -9.422950 +_ч -9.441170 +ид -9.450326 +бә -9.469631 +_b -9.488785 +_ф -9.496086 +ва -9.506268 +ыг -9.542829 +la -9.543860 +зә -9.554214 +әү -9.562202 +си -9.569972 +тл -9.574883 +ро -9.588829 +ав -9.600495 +ҡ_ -9.606408 +ҡы -9.611445 +кт -9.615053 +ио -9.617313 +_е -9.620936 +иг -9.622297 +_ж -9.623841 +кл -9.637262 +бы -9.654238 +уы -9.656281 +хә -9.656932 +шә -9.657211 +аҡ -9.672544 +иә -9.675461 +еп -9.686332 +ол -9.692991 +өр -9.698818 +шҡ -9.708325 +ыи -9.724409 +рз -9.731246 +нс -9.735069 +ап -9.753045 +ад -9.768432 +рн -9.772254 +үл -9.816746 +зы -9.818097 +чы -9.831257 +ру -9.834408 +ес -9.842210 +ta -9.842633 +со -9.851651 +ос -9.862225 +ыҡ -9.872985 +нт -9.898371 +ү_ -9.900461 +ча -9.901783 +i_ -9.910513 +ус -9.913287 +һу -9.918182 +ип -9.922870 +ь_ -9.922982 +с_ -9.925668 +шт -9.939852 +рд -9.940304 +из -9.951209 +ҡо -9.953377 +ег -9.969800 +in -9.992969 +ян -10.033074 +тк -10.044828 +_t -10.061839 +әш -10.068750 +шк -10.078929 +мо -10.100137 +нл -10.109904 +e_ -10.114366 +рм -10.118329 +рс -10.120122 +әб -10.132610 +пр -10.138051 +уг -10.140779 +ша -10.142600 +лм -10.155416 +ga -10.171133 +эш -10.171399 +r_ -10.172993 +_a -10.179657 +аб -10.189845 +уи -10.196452 +ку -10.207169 +ль -10.217828 +лк -10.220711 +ом -10.222362 +һа -10.228429 +үр -10.236465 +әз -10.240361 +_з -10.269351 +о_ -10.273337 +da -10.286655 +ыт -10.301695 +тр -10.304605 +ск -10.308982 +тү -10.341789 +al -10.375665 +зи -10.408319 +at -10.417899 +ов -10.421212 +вы -10.436453 +әп -10.440449 +өл -10.466450 +үг -10.483498 +ыз -10.500584 +ло -10.504594 +ие -10.518718 +жи -10.536229 +мө -10.537598 +en -10.538282 +әһ -10.547389 +мы -10.575059 +ци -10.586879 +ыс -10.594873 +уз -10.596120 +сө -10.599690 +_s -10.602911 +һе -10.605421 +as -10.605780 +нч -10.639740 +рк -10.640843 +_ю -10.650993 +в_ -10.666822 +еү -10.734284 +be -10.758421 +_i -10.761417 +өи -10.767227 +өч -10.783376 +_k -10.795198 +nd -10.818721 +чә -10.820594 +ве -10.826020 +ra -10.829998 +па -10.849843 +ев -10.851119 +үт -10.858798 +ым -10.864155 +бү -10.868671 +мб -10.877961 +ni -10.882738 +_m -10.892119 +мм -10.895626 +na -10.898481 +_y -10.904209 +пл -10.904871 +фа -10.915510 +нь -10.922870 +уа -10.927798 +_л -10.931167 +ө_ -10.932968 +зг -10.937704 +el -10.942456 +ду -10.943363 +үп -10.945633 +ук -10.956120 +er -10.957265 +яз -10.958410 +ог -10.970839 +үс -10.977559 +фи -10.991092 +ҡт -11.001680 +ba -11.004517 +үе -11.013061 +зл -11.032471 +иб -11.032954 +il -11.039969 +үм -11.047263 +ья -11.048970 +ma -11.058519 +зм -11.066895 +sa -11.075072 +го -11.079053 +ач -11.079302 +ги -11.104442 +тн -11.113855 +по -11.119994 +бө -11.123845 +кс -11.124102 +p_ -11.127190 +өм -11.129768 +li -11.137532 +өз -11.148733 +ок -11.152136 +гр -11.160812 +зу -11.161867 +бо -11.163187 +до -11.164509 +су -11.166625 +сс -11.168480 +сл -11.179657 +ед -11.180192 +яҡ -11.182867 +si -11.183670 +лт -11.186620 +ul -11.186889 +ri -11.187426 +di -11.189576 +их -11.199564 +еи -11.203090 +ош -11.211805 +ya -11.229120 +иө -11.232164 +өт -11.237716 +яһ -11.250006 +як -11.263818 +ят -11.273194 +яш -11.273764 +дү -11.283206 +ши -11.286942 +әх -11.288382 +am -11.292710 +хе -11.307814 +аф -11.310151 +_x -11.316894 +m_ -11.318658 +жа -11.322487 +лу -11.326623 +лө -11.338802 +qa -11.361049 +жә -11.361352 +уб -11.364689 +яс -11.372608 +фә -11.375359 +бр -11.384260 +му -11.386416 +ут -11.395076 +но -11.405348 +эл -11.424532 +ir -11.427384 +сп -11.430560 +от -11.437571 +it -11.440129 +пу -11.448152 +bu -11.462057 +рш -11.462707 +шм -11.464659 +кк -11.474130 +оз -11.477082 +пе -11.486966 +_d -11.494257 +ае -11.499583 +мл -11.505598 +үш -11.506603 +_q -11.513321 +зн -11.524814 +le -11.534007 +өш -11.538454 +бл -11.538796 +ay -11.539825 +te -11.544978 +уч -11.547734 +ti -11.549459 +фе -11.558810 +яг -11.573127 +оп -11.579106 +нк -11.586525 +рб -11.590071 +ял -11.597547 +ть -11.600406 +is -11.608655 +өк -11.617675 +ич -11.619486 +хи -11.624568 +t_ -11.629304 +ur -11.638454 +ki -11.643973 +әд -11.646555 +re -11.647664 +_c -11.648773 +_u -11.653960 +д_ -11.666260 +ое -11.676780 +кр -11.678666 +ив -11.679044 +юл -11.691561 +ви -11.715376 +se -11.719254 +оч -11.720031 +mi -11.736051 +рү -11.759020 +кч -11.785611 +l_ -11.786017 +иж -11.806911 +еб -11.815189 +de -11.817681 +рө -11.818097 +жы -11.834408 +aq -11.839042 +чи -11.839886 +az -11.844962 +ну -11.845386 +on -11.846233 +ge -11.848355 +ум -11.867594 +иф -11.868025 +об -11.872337 +xa -11.874065 +дө -11.878395 +ag -11.885350 +es -11.899801 +ҡл -11.903326 +гъ -11.906418 +уҡ -11.913509 +ha -11.924660 +кш -11.927798 +ci -11.933643 +tu -11.934996 +k_ -11.935898 +иа -11.952236 +әж -11.954977 +su -11.955891 +ч_ -11.966915 +әч -11.968760 +_h -11.971071 +сү -11.973384 +ip -11.978023 +һө -11.982677 +or -11.985010 +пт -11.991092 +мн -11.991561 +лү -11.992030 +во -12.008071 +зө -12.009970 +ая -12.013299 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ban b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ban new file mode 100644 index 00000000000..15f5c5ef0c4 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ban @@ -0,0 +1,450 @@ +#TOTAL_BIGRAMS 2837703 +#UNIQUE_BIGRAMS 446 +#UNSEEN_LOG_PROB -18.114364 +#EXPECTED_SCORE -7.095144 +an -4.344092 +n_ -4.860561 +a_ -5.269894 +ng -5.446499 +da -5.635418 +_p -5.659371 +_d -5.863562 +i_ -5.907197 +en -5.914385 +ka -5.955975 +er -5.990889 +_s -6.161441 +_m -6.208939 +ar -6.261055 +la -6.291076 +pe -6.338013 +ra -6.395617 +ta -6.397159 +in -6.471320 +al -6.554652 +at -6.556996 +ga -6.557762 +di -6.603550 +_k -6.617760 +as -6.618159 +si -6.624065 +ya -6.650789 +pa -6.662123 +_t -6.662174 +me -6.703436 +g_ -6.728232 +ak -6.755109 +ah -6.770346 +_b -6.786250 +se -6.799271 +sa -6.817276 +h_ -6.819743 +ma -6.821697 +na -6.841210 +un -7.045787 +_a -7.076131 +te -7.079153 +k_ -7.107828 +ba -7.134725 +ik -7.135654 +ad -7.161696 +tu -7.182888 +em -7.183775 +am -7.185846 +ti -7.272249 +t_ -7.357975 +ri -7.367850 +ke -7.400892 +ha -7.448229 +nt -7.454636 +be -7.469246 +u_ -7.490940 +_i -7.493686 +l_ -7.536274 +uk -7.542327 +ia -7.599453 +nd -7.615216 +el -7.653089 +_l -7.705291 +m_ -7.716476 +li -7.717866 +ni -7.758242 +is -7.765193 +s_ -7.784007 +_y -7.809442 +r_ -7.818021 +ua -7.846290 +il -7.876198 +ku -7.886026 +ja -7.886508 +id -7.898831 +ai -7.915919 +eb -7.943688 +it -7.970216 +_h -8.008848 +ek -8.077641 +on -8.079703 +_u -8.080115 +ru -8.148868 +ap -8.206272 +us -8.215460 +ny -8.290519 +mi -8.309265 +mb -8.310394 +ur -8.318487 +bi -8.320923 +le -8.333004 +de -8.355142 +um -8.370371 +bu -8.373740 +es -8.428389 +gg -8.451518 +gi -8.466186 +ag -8.483005 +_j -8.486648 +ko -8.486830 +ca -8.511294 +mu -8.550024 +ih -8.580061 +ut -8.614319 +ne -8.625520 +su -8.641064 +et -8.649206 +ju -8.664390 +wa -8.676196 +hu -8.693351 +ol -8.695247 +_n -8.727855 +gk -8.738890 +mp -8.794918 +lu -8.825037 +pu -8.871428 +gu -8.886508 +aa -8.936695 +du -8.949206 +ay -8.952225 +up -8.957775 +or -8.981993 +st -8.991795 +re -9.041562 +ki -9.090610 +pr -9.091719 +ge -9.095051 +rt -9.102857 +ul -9.162788 +p_ -9.210783 +_r -9.215612 +ro -9.277682 +om -9.294824 +eg -9.358141 +io -9.359477 +im -9.384404 +aj -9.435940 +ep -9.437349 +_g -9.448318 +eh -9.481732 +_o -9.506664 +rs -9.508514 +ud -9.523777 +ec -9.534294 +hi -9.538447 +lo -9.547549 +nj -9.578700 +_c -9.592371 +pi -9.657804 +au -9.670177 +nu -9.670177 +ir -9.679736 +rn -9.683912 +_e -9.713912 +no -9.736286 +ks -9.806480 +os -9.826652 +rk -9.887952 +kt -9.889880 +uh -9.898588 +ed -9.995942 +ib -9.996461 +ns -10.002185 +ab -10.004272 +po -10.008979 +tr -10.033813 +_w -10.052048 +ip -10.172436 +aw -10.188365 +rj -10.197887 +og -10.242767 +rb -10.285910 +ok -10.290360 +rg -10.293547 +do -10.314111 +lm -10.338260 +_f -10.367514 +rl -10.381010 +to -10.404936 +so -10.425764 +e_ -10.451874 +rd -10.464030 +ot -10.482096 +nc -10.521160 +rm -10.521907 +uj -10.530905 +kn -10.557475 +if -10.572106 +ae -10.587669 +gs -10.587669 +ug -10.614518 +gr -10.640253 +je -10.701583 +sy -10.733254 +ji -10.738456 +ub -10.871190 +tk -10.896164 +sw -10.905886 +f_ -10.919607 +ej -10.974813 +od -10.975836 +rh -10.987144 +mo -10.988176 +ye -11.074348 +fa -11.122275 +fo -11.194309 +ve -11.248940 +go -11.251417 +cu -11.260119 +fi -11.291634 +wi -11.292909 +kr -11.298020 +b_ -11.386444 +iv -11.397373 +d_ -11.419484 +kh -11.451874 +nf -11.470508 +ew -11.471951 +sk -11.471951 +fe -11.490848 +ea -11.567470 +op -11.569013 +ui -11.579867 +ig -11.590802 +sn -11.595514 +sm -11.622511 +rp -11.628937 +ci -11.681405 +ob -11.684748 +pk -11.732389 +sp -11.734119 +vi -11.737588 +np -11.756812 +ij -11.769181 +hk -11.774514 +ce -11.792436 +ac -11.872143 +nk -11.922565 +hw -11.930481 +o_ -11.956512 +va -12.031151 +eo -12.043975 +sl -12.137084 +we -12.162497 +pl -12.167165 +af -12.171850 +ry -12.195501 +nn -12.202672 +lk -12.234168 +pt -12.261366 +of -12.278945 +tn -12.284007 +gh -12.330384 +bo -12.367514 +bl -12.381010 +eu -12.453299 +ov -12.517429 +ev -12.535425 +ef -12.550596 +_v -12.619309 +lt -12.625721 +ml -12.635392 +he -12.688099 +co -12.718616 +ie -12.767407 +rc -12.788834 +mn -12.806936 +dm -12.847578 +kk -12.889398 +kl -12.889398 +fu -12.893260 +ho -12.901017 +ei -12.904911 +dk -12.904911 +ht -12.952476 +iw -12.997500 +ps -13.005840 +yu -13.056914 +wo -13.132511 +oh -13.141671 +wu -13.231721 +hl -13.251417 +gl -13.271385 +ow -13.281474 +rw -13.354143 +ii -13.354143 +iu -13.364830 +ll -13.381010 +oa -13.381010 +ln -13.464749 +gn -13.493778 +ft -13.505555 +mk -13.517429 +y_ -13.523403 +ic -13.523403 +rv -13.715193 +dh -13.742805 +yi -13.770956 +th -13.778081 +sc -13.806936 +hn -13.904911 +ky -13.912730 +uw -13.936446 +pn -14.001664 +oo -14.010027 +ss -14.010027 +oc -14.018440 +fl -14.069970 +za -14.087564 +zi -14.114364 +ms -14.123409 +br -14.132511 +iy -14.160168 +nv -14.178904 +dr -14.178904 +nl -14.256383 +rf -14.256383 +iz -14.276421 +bk -14.286545 +dp -14.327768 +vo -14.370203 +bd -14.447607 +oy -14.529402 +bs -14.541474 +fr -14.590802 +ch -14.590802 +yk -14.616113 +kd -14.616113 +kp -14.616113 +jo -14.654932 +yo -14.694825 +_z -14.708372 +tm -14.778081 +sd -14.821582 +bj -14.836379 +sr -14.866437 +km -14.866437 +pb -14.912730 +ee -15.026901 +ue -15.061253 +ty -15.096442 +c_ -15.096442 +mm -15.188365 +ou -15.188365 +ff -15.207473 +az -15.226839 +uu -15.246468 +sf -15.246468 +oi -15.286545 +ts -15.286545 +dn -15.307009 +ls -15.348829 +gy -15.348829 +tp -15.348829 +py -15.370203 +by -15.370203 +w_ -15.370203 +pd -15.391898 +ys -15.436292 +sb -15.436292 +hr -15.482096 +wh -15.505555 +uf -15.553649 +ao -15.603402 +pp -15.654932 +gm -15.708372 +bp -15.708372 +v_ -15.735852 +ax -15.735852 +tt -15.735852 +sh -15.735852 +gj -15.792436 +ey -15.881703 +dw -15.912730 +tl -15.912730 +kb -15.944439 +uc -15.976861 +x_ -16.043975 +mr -16.043975 +bt -16.114364 +kw -16.114364 +ds -16.114364 +dl -16.188365 +mf -16.266367 +bb -16.266367 +tg -16.348829 +dy -16.348829 +tb -16.348829 +hm -16.391898 +td -16.436292 +sj -16.436292 +md -16.529402 +dg -16.628937 +ws -16.681405 +ph -16.681405 +xy -16.735852 +ly -16.792436 +gt -16.976861 +mt -16.976861 +vr -16.976861 +oe -16.976861 +_x -17.043975 +xl -17.043975 +lv -17.043975 +pg -17.043975 +jr -17.114364 +wt -17.114364 +fk -17.114364 +ck -17.114364 +rr -17.114364 +wn -17.266367 +uo -17.266367 +cc -17.348829 +zo -17.348829 +tv -17.348829 +ez -17.348829 +lr -17.436292 +dv -17.436292 +ux -17.436292 +gd -17.436292 +tw -17.436292 +wl -17.436292 +uy -17.436292 +lh -17.529402 +bm -17.529402 +db -17.529402 +tj -17.628937 +hy -17.628937 +ld -17.628937 +pj -17.628937 +js -17.628937 +bn -17.735852 +hd -17.735852 +ex -17.851330 +xt -17.851330 +oj -17.851330 +iq -17.851330 +qi -17.851330 +nw -17.976861 +cr -17.976861 +bh -18.114364 +pm -18.114364 +_q -18.114364 +qu -18.114364 +bg -18.114364 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bel b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bel new file mode 100644 index 00000000000..cfd06dde0cd --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bel @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 136282708 +#UNIQUE_BIGRAMS 1084 +#UNSEEN_LOG_PROB -16.179990 +#EXPECTED_SCORE -8.053851 +а_ -5.526851 +ра -5.822615 +_п -5.878096 +на -6.048155 +і_ -6.076426 +у_ -6.149300 +ка -6.234378 +та -6.346259 +я_ -6.418592 +_с -6.436484 +ы_ -6.471085 +е_ -6.553392 +ва -6.567851 +ад -6.568267 +ар -6.670677 +ал -6.682243 +па -6.717605 +_а -6.769841 +ь_ -6.905918 +ас -6.912323 +пр -6.981471 +ла -6.986715 +ан -6.998724 +лі -6.999941 +_б -7.019615 +_г -7.033732 +ам -7.033942 +_к -7.078621 +_в -7.086905 +ст -7.111042 +дз -7.113503 +_м -7.120023 +_н -7.127005 +ць -7.127366 +ав -7.145393 +ры -7.174098 +ау -7.188171 +ма -7.204920 +кі -7.213495 +_д -7.216820 +_т -7.221581 +да -7.258671 +м_ -7.366619 +ны -7.378876 +ні -7.379688 +и_ -7.388014 +га -7.388064 +_з -7.433867 +ац -7.509781 +о_ -7.543902 +ак -7.567857 +ус -7.572865 +ль -7.585175 +то -7.633558 +_р -7.679714 +_у -7.701612 +аг -7.719072 +аи -7.729472 +ты -7.730881 +вы -7.764550 +х_ -7.774227 +ск -7.832010 +не -7.834786 +ча -7.836477 +ле -7.840788 +за -7.883633 +ьн -7.915861 +ін -7.936923 +_я -7.946841 +ат -7.961535 +зе -7.999364 +ен -8.026959 +ел -8.032311 +ыя -8.048719 +са -8.052686 +ве -8.059433 +мі -8.072457 +ым -8.073977 +од -8.081467 +аз -8.100143 +ру -8.103802 +_ш -8.110469 +аб -8.112969 +чы -8.114380 +цы -8.125781 +ае -8.138298 +сь -8.167649 +ня -8.183411 +ля -8.183513 +эт -8.195885 +го -8.211653 +рэ -8.217364 +ро -8.228176 +ца -8.252373 +ік -8.304003 +тр -8.319013 +ер -8.319151 +шт -8.391213 +ку -8.391341 +оу -8.425044 +ун -8.463420 +бе -8.465873 +ся -8.468537 +ол -8.475615 +гэ -8.488774 +ту -8.498542 +ме -8.511244 +_л -8.529137 +ко -8.541403 +ых -8.561311 +іц -8.562956 +як -8.592400 +ві -8.614859 +к_ -8.619257 +бы -8.628792 +ба -8.651915 +_ч -8.666091 +ю_ -8.670661 +ул -8.682825 +ов -8.692638 +пе -8.697614 +эн -8.700967 +іс -8.703780 +ды -8.726173 +се -8.731707 +цц -8.733724 +ап -8.742050 +нь -8.763351 +во -8.767737 +ая -8.782830 +уд -8.785649 +дн -8.813264 +зі -8.822930 +ын -8.841598 +т_ -8.876971 +_і -8.881387 +ор -8.894648 +ач -8.895267 +мо -8.899305 +ці -8.947295 +сп -8.965532 +сі -8.968807 +вя -8.977537 +ян -8.993911 +ьц -9.035563 +но -9.046756 +ія -9.087116 +ім -9.100645 +кр -9.103403 +нн -9.109711 +бо -9.111402 +ыл -9.116928 +ур -9.125198 +д_ -9.130309 +еи -9.149040 +ах -9.166751 +ыц -9.168444 +іл -9.175925 +тв -9.177021 +ло -9.177953 +ду -9.187680 +жа -9.206294 +су -9.214314 +ык -9.235413 +тэ -9.259678 +нт -9.273284 +р_ -9.294074 +му -9.296708 +аш -9.327581 +шы -9.329419 +ец -9.341345 +яр -9.358441 +ес -9.361140 +зн -9.379256 +чн -9.406915 +св -9.412567 +зя -9.430454 +бу -9.456940 +аю -9.459539 +ыі -9.460493 +зь -9.468788 +эр -9.470409 +нс -9.474282 +ук -9.477387 +сц -9.479955 +ну -9.480031 +іт -9.483020 +до -9.509148 +яг -9.520649 +іх -9.525103 +ыс -9.531827 +ед -9.536591 +с_ -9.545685 +ыт -9.551551 +іч -9.561467 +_х -9.569955 +_ц -9.579837 +ша -9.580866 +це -9.586723 +он -9.600792 +гі -9.624194 +кл -9.627690 +ха -9.667236 +ця -9.675176 +нк -9.685120 +хо -9.688278 +ут -9.713182 +ои -9.717640 +жы -9.732819 +уп -9.733918 +кт -9.745957 +по -9.753805 +лю -9.763719 +ог -9.764280 +ую -9.779718 +ек -9.786593 +ос -9.788660 +ьк -9.797960 +н_ -9.798346 +юц -9.817342 +_ж -9.822777 +гр -9.827117 +дэ -9.830081 +дл -9.856188 +мя -9.857680 +шч -9.862048 +ял -9.868703 +бл -9.882696 +ем -9.890663 +сл -9.898502 +жн -9.906557 +гу -9.907065 +ыв -9.911575 +ву -9.925950 +яд -9.939119 +лу -9.940481 +ыу -9.952158 +уц -9.954446 +ум -9.961859 +ыч -9.982958 +аі -10.015077 +бі -10.033731 +яц -10.035763 +кс -10.044401 +_ф -10.099640 +нд -10.101419 +тк -10.101756 +ин -10.107654 +сн -10.107958 +зв -10.111759 +пі -10.112380 +об -10.123213 +ож -10.123662 +яу -10.134117 +др -10.136295 +уч -10.147070 +_э -10.176121 +ір -10.178462 +эс -10.198971 +рт -10.211380 +ет -10.221279 +бр -10.225443 +ш_ -10.261320 +ьв -10.275237 +ям -10.286180 +чэ -10.290788 +рн -10.299807 +ом -10.310307 +зы -10.312984 +тн -10.316044 +рш -10.336772 +уж -10.364039 +нц -10.370191 +пл -10.370247 +мы -10.371550 +ув -10.376214 +аж -10.401607 +ьм -10.410513 +із -10.441666 +з_ -10.449460 +ис -10.463382 +дк -10.492231 +шэ -10.505850 +ьш -10.536308 +ыд -10.555059 +б_ -10.557737 +эк -10.561683 +уз -10.568274 +рм -10.583024 +іу -10.592784 +пу -10.599338 +ыи -10.607293 +іі -10.609424 +гл -10.610450 +сы -10.635053 +цэ -10.635558 +ош -10.650302 +ід -10.653180 +лы -10.660460 +ів -10.670812 +л_ -10.730677 +зу -10.743251 +пы -10.772525 +ят -10.773192 +рг -10.775825 +эм -10.776976 +ыр -10.782708 +нш -10.791069 +юд -10.793979 +ое -10.799383 +ев -10.813774 +ч_ -10.818182 +жо -10.848038 +ыб -10.870495 +рк -10.881357 +яс -10.918330 +уб -10.923851 +мн -10.931832 +еж -10.945671 +цк -10.952305 +от -10.956001 +яз -10.960846 +иш -10.963507 +яш -10.969396 +ок -10.973455 +фі -10.973902 +рс -10.974158 +со -10.979898 +еу -10.996650 +зр -10.998858 +фа -11.025201 +яв -11.033520 +_е -11.061437 +мэ -11.064903 +ез -11.096148 +эу -11.106101 +уг -11.112204 +дж -11.136093 +зд -11.148391 +дв -11.167329 +э_ -11.167841 +цт -11.176635 +ге -11.181323 +аф -11.190275 +эл -11.191240 +ыз -11.192552 +дп -11.206919 +дс -11.216157 +эб -11.227738 +чу -11.230890 +эд -11.231450 +хт -11.234890 +зм -11.244644 +пэ -11.296608 +ьл -11.303013 +оз -11.321425 +ух -11.353946 +мл -11.361502 +эз -11.382206 +ые -11.388039 +уе -11.392272 +яч -11.412791 +шн -11.415073 +мп -11.428461 +рж -11.451223 +ыю -11.452409 +дч -11.489397 +шк -11.492200 +дб -11.493878 +яп -11.495253 +цо -11.505681 +іж -11.506481 +оч -11.507036 +ып -11.513149 +шл -11.555409 +оп -11.557864 +яб -11.566925 +ея -11.586390 +эч -11.600028 +кц -11.605230 +эи -11.613698 +яе -11.628436 +уш -11.632967 +кн -11.662449 +бн -11.672677 +бя -11.675548 +г_ -11.725936 +кв -11.729418 +нф -11.734784 +юч -11.743287 +яи -11.750929 +ьс -11.752645 +пя -11.778407 +гч -11.791256 +ху -11.800704 +іг -11.808202 +цу -11.841924 +ыг -11.854962 +мк -11.855354 +жу -11.856492 +еч -11.876414 +іш -11.884236 +чо -11.895403 +дм -11.896210 +ях -11.900777 +зк -11.922926 +эа -11.938756 +ьб -11.943876 +нэ -11.947008 +рч -11.950774 +ег -12.014606 +ке -12.017324 +мс -12.024980 +нч -12.039747 +шу -12.044121 +шм -12.048554 +ыш -12.050439 +фо -12.054216 +еп -12.071244 +ик -12.086816 +рв -12.089397 +ьт -12.116405 +зб -12.140053 +хі -12.141391 +ж_ -12.158373 +рц -12.171352 +эг -12.177371 +оц -12.203295 +яж -12.205194 +эц -12.227256 +бс -12.233462 +зг -12.244464 +жэ -12.254877 +уа -12.261775 +еб -12.279560 +еі -12.313752 +еш -12.324571 +дд -12.345629 +тл -12.376087 +гн -12.395235 +сэ -12.396033 +ію -12.418053 +рд -12.433020 +ню -12.439180 +дт -12.449919 +іб -12.462053 +хв -12.465342 +эп -12.468098 +лк -12.468518 +юб -12.486387 +аа -12.489063 +ц_ -12.518077 +зо -12.539282 +ья -12.570172 +сю -12.584145 +ех -12.596615 +фр -12.620081 +фе -12.624553 +им -12.636300 +іи -12.642649 +ср -12.651204 +сх -12.670467 +еа -12.672055 +хн -12.682943 +рх -12.683361 +_ю -12.685312 +сс -12.695598 +фэ -12.698058 +зл -12.699043 +ьп -12.724609 +нг -12.729562 +шо -12.750127 +i_ -12.791707 +ьд -12.818526 +пш -12.818909 +юз -12.832743 +иц -12.850538 +ид -12.863575 +жк -12.917019 +п_ -12.922186 +аэ -12.946214 +вэ -12.993949 +иг -13.013686 +уу -13.026702 +зэ -13.037787 +яю -13.052641 +іе -13.063203 +см -13.068558 +эх -13.098143 +хр -13.111478 +эф -13.144935 +пн -13.151566 +рл -13.165796 +іп -13.217291 +_о -13.236779 +ют -13.239029 +уи -13.239643 +эж -13.246726 +ье -13.285520 +чв -13.323757 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ben b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ben new file mode 100644 index 00000000000..677ef00b9e9 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ben @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 8419510 +#UNIQUE_BIGRAMS 1425 +#UNSEEN_LOG_PROB -14.757583 +#EXPECTED_SCORE -7.792979 +র_ -3.932620 +ন_ -4.754840 +_ক -4.855875 +কর -5.281331 +_ব -5.674861 +ল_ -5.751024 +_স -5.776439 +ক_ -5.830486 +ত_ -5.933319 +_এ -5.942184 +_আ -6.024005 +ব_ -6.055797 +_ত -6.089869 +_প -6.345936 +_হ -6.392192 +য_ -6.430519 +_দ -6.447587 +_ম -6.525367 +ট_ -6.712031 +_ন -6.755562 +পর -6.867153 +_জ -6.950976 +ছ_ -6.998299 +তর -7.155191 +_থ -7.196542 +বল -7.227202 +থক -7.247993 +দর -7.290818 +_অ -7.361280 +_শ -7.397050 +নর -7.411826 +_র -7.426401 +ম_ -7.453537 +ই_ -7.457868 +_য -7.517716 +মন -7.599000 +শ_ -7.624573 +জন -7.639324 +এব -7.659726 +এক -7.689872 +রত -7.745966 +বর -7.755747 +রর -7.814979 +রন -7.829288 +তন -7.832252 +জ_ -7.877955 +_ভ -7.897025 +কট -7.905505 +লন -7.907520 +_চ -7.918754 +_গ -7.952312 +সর -8.082117 +সব -8.082720 +স_ -8.152727 +নয -8.173255 +লর -8.231650 +সল -8.240174 +আম -8.250678 +সম -8.264312 +কন -8.270384 +_ল -8.284740 +শর -8.288860 +_ছ -8.289021 +মর -8.307903 +দশ -8.314652 +খন -8.342525 +যছ -8.366925 +গ_ -8.366982 +দ_ -8.374071 +রক -8.385773 +ছল -8.450836 +হয -8.452756 +_খ -8.459194 +ণ_ -8.478806 +যর -8.505957 +হব -8.512826 +কল -8.523442 +রণ -8.547155 +দন -8.549978 +_উ -8.565279 +_ই -8.597506 +পন -8.603492 +ষ_ -8.611713 +রব -8.639871 +_ফ -8.642060 +ধর -8.645349 +জর -8.648922 +কজ -8.658583 +নক -8.674178 +ও_ -8.684998 +নত -8.700241 +ড_ -8.724462 +মল -8.739543 +বন -8.744018 +হস -8.749317 +লক -8.756265 +রছ -8.770413 +ওয -8.786666 +কছ -8.803717 +খ_ -8.816561 +তদ -8.847563 +যগ -8.858498 +নম -8.877552 +বশ -8.899069 +থ_ -8.905382 +ছন -8.935939 +নন -8.963817 +অন -8.965529 +চল -9.023916 +হল -9.104061 +আপ -9.107081 +রজ -9.118941 +কম -9.130420 +গল -9.131861 +হত -9.135326 +_ধ -9.147324 +হর -9.147324 +দয -9.165019 +দল -9.168367 +মদ -9.185824 +বদ -9.188721 +লত -9.226304 +ময -9.256483 +লম -9.262995 +জল -9.268692 +সন -9.278129 +গর -9.336198 +টক -9.360435 +লদ -9.365625 +_ঘ -9.385429 +ধ_ -9.407020 +শষ -9.427404 +যন -9.430948 +বস -9.433315 +মত -9.439489 +দখ -9.459254 +_ট -9.461548 +ভব -9.461790 +পল -9.469908 +_ড -9.471488 +তক -9.477217 +খল -9.487143 +আর -9.508451 +উপ -9.511700 +তল -9.528559 +টর -9.535154 +চর -9.574722 +আস -9.582189 +রপ -9.585081 +সক -9.586530 +প_ -9.612987 +ফল -9.647890 +বড -9.649678 +_ও -9.653261 +রয -9.663230 +জব -9.689023 +ভর -9.694408 +এট -9.700241 +দক -9.736032 +তব -9.741596 +রখ -9.761238 +রচ -9.763173 +নজ -9.777238 +বহ -9.784020 +সহ -9.809471 +মস -9.813629 +ইন -9.835536 +ধন -9.855716 +রল -9.856669 +নব -9.868634 +কব -9.874573 +মক -9.874734 +বছ -9.875700 +শন -9.876183 +ছর -9.879730 +হ_ -9.879730 +এস -9.893517 +মট -9.899069 +বক -9.901525 +নষ -9.936862 +বধ -9.939720 +এখ -9.941741 +আগ -9.957499 +বব -9.960569 +মহ -9.969303 +আল -9.973776 +বষ -9.983284 +লয -9.985193 +জত -9.986583 +শহ -9.987800 +সই -9.990238 +হন -9.995826 +হজ -10.038619 +কথ -10.039701 +চন -10.040242 +আব -10.044036 +কত -10.044036 +হম -10.048929 +পত -10.052382 +যত -10.060593 +লখ -10.069403 +ভগ -10.081049 +দব -10.081792 +যয -10.094475 +নট -10.112762 +ঘট -10.125913 +বই -10.196542 +রম -10.200569 +আদ -10.216179 +নই -10.216994 +পশ -10.223330 +সথ -10.228050 +বত -10.230106 +বচ -10.232784 +টন -10.258791 +ফর -10.271866 +_ঢ -10.272078 +অভ -10.283992 +ইস -10.284847 +আছ -10.291489 +ভয -10.307468 +নস -10.311600 +আই -10.319680 +যদ -10.329568 +ঢক -10.336641 +বজ -10.343749 +সদ -10.355151 +ইর -10.360998 +মজ -10.365738 +সভ -10.418465 +ছড -10.426932 +ষয -10.439489 +লগ -10.442109 +উন -10.445211 +কয -10.466873 +লশ -10.471732 +খব -10.474899 +চ_ -10.477095 +খত -10.480273 +যক -10.482723 +কই -10.485177 +চত -10.497758 +নগ -10.498501 +ঠন -10.520482 +তয -10.527294 +যব -10.536172 +এম -10.537445 +সধ -10.539483 +পজ -10.540759 +ইউ -10.560031 +জম -10.560290 +গত -10.561843 +ভল -10.567033 +পড -10.570156 +রস -10.573546 +তহ -10.574068 +রহ -10.576160 +ভষ -10.583766 +নদ -10.584029 +গয -10.584292 +রধ -10.585608 +রই -10.587452 +ভ_ -10.587980 +দও -10.589299 +ডন -10.593264 +রও -10.595914 +লছ -10.625926 +কদ -10.642265 +এল -10.643087 +তই -10.652158 +লই -10.658791 +যম -10.664898 +নল -10.675789 +লও -10.696397 +ঠক -10.699243 +গঠ -10.702380 +শক -10.709536 +দত -10.731218 +দই -10.731509 +মম -10.749686 +কশ -10.752935 +ষর -10.760941 +খর -10.763024 +তম -10.764215 +পথ -10.781605 +অধ -10.785229 +ষণ -10.787953 +ইট -10.803793 +মব -10.810548 +গছ -10.820429 +ইল -10.831003 +সয -10.836946 +অপ -10.867353 +নও -10.879246 +ধক -10.879246 +পয -10.884095 +লভ -10.884419 +সত -10.892865 +রগ -10.896454 +গন -10.919500 +t_ -10.928823 +অব -10.932502 +বভ -10.932837 +এন -10.946298 +এর -10.947313 +পদ -10.955797 +সগ -10.963646 +ণর -10.969131 +ডট -10.971194 +অশ -10.979127 +ঠ_ -10.982244 +হদ -11.008831 +সস -11.010951 +ৎস -11.014492 +ফস -11.032325 +শধ -11.039160 +শখ -11.051472 +মশ -11.055843 +ওপ -11.062424 +চক -11.063157 +শত -11.070508 +লব -11.074937 +st -11.092416 +রদ -11.092416 +লজ -11.093164 +উজ -11.109351 +নউ -11.116942 +r_ -11.121516 +গণ -11.123044 +চয -11.143055 +রট -11.160991 +হক -11.162562 +লচ -11.162954 +তত -11.164527 +ফ_ -11.166889 +সপ -11.182336 +সট -11.186323 +উৎ -11.196743 +জয -11.201577 +রশ -11.208860 +লল -11.216179 +জক -11.219035 +জদ -11.219444 +আন -11.219444 +টল -11.224765 +যখ -11.236707 +হও -11.259210 +সচ -11.260051 +লট -11.271019 +নপ -11.275259 +গম -11.277384 +ছব -11.309207 +সখ -11.329348 +আহ -11.332438 +ইত -11.335534 +থব -11.337306 +গব -11.352460 +ছট -11.368227 +ণত -11.386002 +এত -11.387837 +ডর -11.391055 +বয -11.392897 +চই -11.408649 +তখ -11.413782 +জট -11.423633 +সড -11.447363 +পঠ -11.454558 +পক -11.455039 +সছ -11.458410 +ডক -11.465661 +ইক -11.474411 +ডয -11.486652 +বগ -11.504463 +দম -11.513452 +ভম -11.521993 +ঘষ -11.523001 +_b -11.530585 +কউ -11.534138 +_f -11.541780 +শল -11.547924 +ডত -11.585871 +এদ -11.586398 +ঘর -11.588507 +_m -11.590620 +গড -11.592206 +_ঠ -11.598568 +উ_ -11.599632 +e_ -11.601228 +যট -11.603892 +যজ -11.620521 +রষ -11.630809 +ধব -11.648303 +বম -11.649954 +আশ -11.656577 +bd -11.658791 +ds -11.658791 +ডল -11.658791 +শয -11.675509 +মড -11.697535 +রফ -11.701524 +ষক -11.703809 +মখ -11.704380 +বএ -11.712983 +কভ -11.718747 +ফট -11.728017 +অফ -11.733258 +দট -11.736178 +মঝ -11.742623 +পও -11.744385 +শশ -11.755599 +ভত -11.755599 +ডআ -11.759752 +টত -11.762131 +সজ -11.763322 +টই -11.783114 +এগ -11.804406 +যস -11.805019 +থন -11.805632 +উঠ -11.809318 +সফ -11.827885 +ষদ -11.834128 +কঠ -11.858100 +খছ -11.870237 +ঝ_ -11.876666 +শপ -11.885067 +বঝ -11.896780 +আয -11.902673 +মছ -11.903986 +কক -11.911887 +পছ -11.926487 +যও -11.930494 +কষ -11.957499 +পট -11.960911 +আক -11.964330 +নহ -11.971194 +ভই -11.972570 +পচ -11.978090 +ভট -11.982244 +শম -11.982937 +মই -11.987105 +এছ -11.990587 +দপ -11.994777 +আও -12.033043 +বট -12.050382 +যল -12.054020 +পব -12.058399 +ঠত -12.059861 +কপ -12.061325 +es -12.081235 +পস -12.118084 +অত -12.128788 +তশ -12.131861 +শদ -12.140345 +অথ -12.141119 +ne -12.148879 +মধ -12.151995 +ফন -12.166101 +শস -12.184328 +আজ -12.185126 +লফ -12.186723 +উই -12.204405 +জহ -12.210889 +লড -12.223945 +মণ -12.224765 +ভপ -12.232166 +ডও -12.233815 +জপ -12.238776 +জড -12.251252 +নভ -12.253761 +_t -12.286772 +টব -12.302266 +গই -12.340858 +ৎপ -12.346201 +মঠ -12.348880 +জই -12.350669 +বপ -12.354253 +কৎ -12.362351 +জধ -12.366869 +আট -12.379596 +লপ -12.391515 +নচ -12.398900 +উর -12.398900 +y_ -12.403534 +যথ -12.411914 +ar -12.414718 +ur -12.443062 +হট -12.448799 +ফই -12.459375 +খজ -12.462273 +চধ -12.464208 +শট -12.471975 +ma -12.475874 +_d -12.481743 +চব -12.514454 +পহ -12.531599 +ভড -12.534646 +দহ -12.535663 +l_ -12.557189 +কখ -12.562361 +জগ -12.570677 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ben-rom b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ben-rom new file mode 100644 index 00000000000..6bebab325bd --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ben-rom @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 130572748 +#UNIQUE_BIGRAMS 862 +#UNSEEN_LOG_PROB -17.708397 +#EXPECTED_SCORE -7.640105 +e_ -5.116164 +a_ -5.592684 +r_ -5.725336 +i_ -5.747615 +_a -5.918300 +_b -5.943839 +ar -5.990629 +_k -5.991709 +o_ -6.115455 +an -6.157462 +_s -6.165839 +ha -6.175051 +ho -6.331890 +or -6.346149 +am -6.346950 +n_ -6.377058 +_m -6.438531 +ch -6.447154 +_t -6.515341 +ma -6.535198 +on -6.570967 +_d -6.583335 +_p -6.767560 +la -6.822057 +ta -6.854644 +_c -6.870122 +ba -6.884285 +ra -6.904120 +sh -6.949303 +ko -7.032162 +he -7.037241 +_h -7.084991 +ng -7.131804 +re -7.182359 +_n -7.256128 +kh -7.256882 +er -7.261231 +m_ -7.282408 +al -7.288389 +in -7.292304 +at -7.330214 +na -7.330698 +to -7.330993 +bo -7.374399 +_j -7.387689 +l_ -7.407452 +lo -7.408379 +y_ -7.455731 +_g -7.478101 +th -7.480175 +t_ -7.485959 +hi -7.510969 +te -7.516692 +as -7.519701 +ol -7.524787 +ka -7.585324 +di -7.592203 +de -7.636621 +ri -7.649595 +sa -7.650194 +ak -7.657649 +mi -7.659439 +le -7.689290 +s_ -7.697009 +mo -7.708334 +ot -7.709753 +_r -7.725475 +ai -7.734727 +hu -7.764959 +d_ -7.781069 +u_ -7.840486 +ne -7.845769 +ek -7.853803 +ro -7.856929 +ke -7.882284 +_e -7.886529 +en -7.890651 +ad -7.897832 +ni -7.909372 +po -7.915580 +pa -7.938838 +ti -7.941171 +be -7.941790 +_l -7.981903 +h_ -8.011757 +da -8.018303 +ye -8.027856 +ab -8.101939 +ur -8.112010 +om -8.207166 +it -8.213889 +no -8.218950 +es -8.222127 +gl -8.237098 +is -8.256253 +so -8.280895 +il -8.289920 +ja -8.294873 +k_ -8.298939 +se -8.306825 +st -8.309514 +_f -8.316274 +ul -8.319377 +li -8.341212 +ah -8.383787 +_o -8.399901 +dh -8.431412 +el -8.465751 +me -8.472988 +aa -8.491655 +tu -8.506469 +ir -8.507699 +ob -8.516056 +ki -8.519693 +g_ -8.532548 +un -8.544809 +ay -8.549504 +od -8.555486 +ok -8.559683 +nd -8.567374 +os -8.597967 +ag -8.637998 +_v -8.642292 +go -8.667957 +ud -8.674118 +_i -8.687576 +nt -8.690221 +et -8.705222 +ga -8.727314 +jo -8.735928 +ic -8.764041 +iy -8.825451 +si -8.843102 +ac -8.864274 +do -8.867697 +bi -8.874516 +ll -8.895841 +ap -8.897674 +su -8.898116 +ge -8.926651 +oy -8.972881 +mu -8.992568 +du -8.998185 +um -9.039054 +ya -9.078782 +bh -9.086642 +us -9.090624 +_u -9.155984 +ut -9.209846 +im -9.213594 +ed -9.231813 +rt -9.231866 +b_ -9.235445 +ik -9.240128 +pr -9.268154 +id -9.274024 +ru -9.278821 +ey -9.279136 +je -9.300617 +va -9.331278 +ou -9.354322 +em -9.380270 +aj -9.394626 +vi -9.395802 +ea -9.398677 +kt -9.419848 +ec -9.424669 +pu -9.430009 +ei -9.430772 +bu -9.452190 +uk -9.474622 +ee -9.491830 +ss -9.516704 +ia -9.573843 +gu -9.638791 +oi -9.639654 +ow -9.671737 +pe -9.690403 +nn -9.720773 +gh -9.728851 +lp -9.827257 +_w -9.830452 +ib -9.834542 +ub -9.845459 +ie -9.861835 +hh -9.872878 +co -9.873272 +ve -9.880940 +nu -9.909835 +wa -9.919625 +fi -9.935507 +tt -9.936936 +oo -9.937759 +up -9.957573 +oa -10.005832 +io -10.062528 +hm -10.064206 +ip -10.131448 +tr -10.133432 +eb -10.151742 +p_ -10.166167 +az -10.181970 +ca -10.204113 +nl -10.223005 +oh -10.225463 +ry -10.237044 +ku -10.238005 +gi -10.239393 +rl -10.247134 +ov -10.255699 +fa -10.259676 +op -10.271537 +ce -10.278079 +ji -10.308218 +au -10.315408 +fo -10.316732 +za -10.385123 +og -10.403563 +w_ -10.451865 +eo -10.464049 +fe -10.475141 +yo -10.487905 +ex -10.512453 +_y -10.517335 +rb -10.526204 +ph -10.526465 +pi -10.529859 +af -10.534145 +rs -10.540055 +ig -10.540253 +ht -10.544768 +ct -10.549264 +mr -10.566352 +sl -10.567995 +pl -10.577519 +ck -10.586632 +dd -10.589659 +br -10.590511 +oc -10.608649 +mm -10.612969 +nk -10.626005 +fu -10.632972 +x_ -10.639513 +gr -10.641983 +wn -10.683173 +j_ -10.725274 +ew -10.743268 +lu -10.748732 +fr -10.775016 +lt -10.797376 +ju -10.806766 +gs -10.811365 +qu -10.817773 +pn -10.840729 +ty -10.841865 +bl -10.844016 +jh -10.860931 +uj -10.867274 +dr -10.877340 +rc -10.889277 +ao -10.892214 +ly -10.926856 +eg -10.932804 +ij -10.940123 +ts -10.964247 +ua -10.967075 +c_ -10.994855 +ui -11.001771 +av -11.003222 +we -11.009267 +oj -11.027464 +nc -11.062717 +vo -11.068210 +mb -11.077445 +cr -11.088325 +aw -11.103464 +hr -11.105069 +ue -11.126647 +ci -11.148302 +_z -11.158671 +ny -11.162364 +rj -11.174622 +f_ -11.182356 +rk -11.228853 +iv -11.259460 +if -11.306370 +ev -11.314733 +ep -11.315634 +ns -11.326545 +rg -11.347353 +kk -11.386099 +uc -11.419424 +eh -11.437513 +rd -11.450875 +cc -11.463612 +iq -11.470681 +of -11.478290 +pt -11.507973 +kr -11.508102 +sc -11.541767 +rn -11.542097 +mp -11.572059 +z_ -11.580968 +hl -11.617926 +hy -11.658925 +nj -11.663792 +wi -11.682192 +yi -11.710202 +gy -11.728607 +zi -11.735749 +rp -11.743078 +lk -11.747250 +ps -11.762524 +bd -11.876301 +rm -11.885134 +iz -11.927286 +ms -11.949576 +v_ -11.967340 +ff -11.972926 +wo -11.975148 +cl -12.003358 +bs -12.018222 +sm -12.029079 +lb -12.047857 +ls -12.061961 +sp -12.062622 +aq -12.075631 +yu -12.094932 +by -12.095319 +sw -12.098319 +ug -12.127092 +ld -12.135419 +pp -12.140998 +rr -12.170134 +ih -12.211876 +cu -12.238752 +kb -12.244746 +sk -12.249365 +bb -12.254106 +sy -12.262225 +sr -12.268208 +mn -12.330125 +iu -12.338799 +yr -12.346029 +uy -12.362692 +ae -12.377549 +kl -12.378960 +oe -12.383440 +ks -12.418577 +lm -12.434391 +ze -12.451741 +eu -12.457695 +kd -12.474324 +dy -12.496243 +uh -12.572396 +hc -12.649808 +wd -12.697156 +zu -12.732813 +hf -12.736880 +gn -12.760760 +my -12.761680 +tc -12.774939 +ws -12.775868 +hb -12.801669 +ky -12.845398 +hn -12.877464 +qb -12.881961 +lc -12.892340 +dl -12.902287 +uv -12.935831 +tk -12.942252 +dp -12.965748 +md -12.992872 +cs -13.015201 +pd -13.028433 +rh -13.032315 +vu -13.041602 +gg -13.064135 +lh -13.078209 +mk -13.082611 +ej -13.106774 +ft -13.139103 +zz -13.142096 +gt -13.164645 +xe -13.208944 +jj -13.216286 +mh -13.237684 +_q -13.238111 +sn -13.244102 +ds -13.260056 +hs -13.281018 +ii -13.287632 +ix -13.300729 +sb -13.303854 +ax -13.330694 +nm -13.335026 +ef -13.339830 +df -13.355262 +kn -13.368990 +lz -13.371330 +ox -13.377196 +mt -13.380491 +zh -13.389237 +xy -13.393987 +fl -13.395415 +wh -13.398036 +yl -13.430116 +ys -13.451741 +bn -13.456204 +py -13.459437 +wr -13.466173 +db -13.471434 +zm -13.480751 +tl -13.500079 +kj -13.536900 +jp -13.546122 +uu -13.551949 +jk -13.556734 +uf -13.571723 +dt -13.576304 +rv -13.624888 +gc -13.638350 +tm -13.668532 +yn -13.688816 +_x -13.699947 +nv -13.713835 +ez -13.742321 +xx -13.742927 +tv -13.751435 +gb -13.769219 +zo -13.781925 +bc -13.792233 +mc -13.798832 +kw -13.859945 +yt -13.862246 +rf -13.874474 +nh -13.890158 +dg -13.901949 +xt -13.902625 +sg -13.916226 +hk -13.938952 +uz -13.941731 +ln -13.944864 +yp -13.965572 +dm -13.971949 +q_ -13.975504 +bt -14.002451 +mg -14.018864 +oz -14.031390 +nb -14.054140 +js -14.073820 +bz -14.101520 +nf -14.108139 +ww -14.119894 +km -14.137708 +ym -14.144095 +tw -14.194992 +uo -14.196651 +tn -14.249472 +dn -14.334113 +gp -14.348332 +dz -14.358044 +lg -14.369692 +hd -14.375786 +ml -14.384267 +nr -14.404252 +dv -14.441625 +xi -14.476463 +cy -14.479993 +tp -14.514746 +xp -14.535587 +yw -14.537688 +gd -14.550888 +lv -14.576574 +sq -14.595597 +zr -14.603276 +xa -14.622099 +np -14.647963 +lw -14.665658 +uw -14.682991 +oq -14.717700 +bk -14.726060 +jn -14.733264 +lf -14.741110 +sd -14.822007 +rw -14.840041 +nw -14.865541 +jt -14.885468 +vt -14.949750 +yg -14.973015 +pg -15.033983 +hw -15.037694 +wl -15.045146 +xc -15.065461 +yc -15.099192 +vh -15.099968 +dw -15.104631 +hp -15.111656 +yd -15.170745 +vn -15.176462 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bih b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bih new file mode 100644 index 00000000000..6ed92d666be --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bih @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 194084 +#UNIQUE_BIGRAMS 549 +#UNSEEN_LOG_PROB -14.106890 +#EXPECTED_SCORE -7.181464 +ल_ -4.084396 +र_ -4.777196 +_म -4.887281 +म_ -5.023290 +_ज -5.163043 +न_ -5.376806 +_ब -5.413086 +_स -5.416892 +_क -5.432858 +_ए -5.662816 +ग_ -5.830766 +जल -5.961768 +ट_ -6.127010 +त_ -6.144257 +_ह -6.160648 +_भ -6.347153 +बट -6.358087 +स_ -6.393894 +_र -6.417210 +कल -6.422301 +एग -6.424853 +कर -6.431254 +_ग -6.479522 +ह_ -6.521928 +_अ -6.574800 +_न -6.799793 +हल -6.819807 +व_ -6.869354 +_प -6.911686 +क_ -6.942440 +_ल -6.976671 +रत -7.019427 +य_ -7.093631 +सल -7.170788 +_द -7.184779 +इल -7.214278 +भर -7.262541 +द_ -7.336301 +लड -7.348364 +डर -7.360528 +एक -7.376497 +रण -7.482842 +ण_ -7.496201 +धर -7.523294 +_व -7.523294 +रह -7.543954 +ब_ -7.555094 +गव -7.600537 +पर -7.601981 +वस -7.653432 +_इ -7.719264 +सध -7.722401 +लग -7.728694 +_श -7.758967 +वल -7.788245 +हर -7.793182 +वन -7.816452 +_आ -7.865882 +जन -7.900986 +_च -8.004079 +तर -8.038845 +प_ -8.066476 +हव -8.140057 +जव -8.150580 +नव -8.274000 +पन -8.369105 +रक -8.383927 +सब -8.388902 +सर -8.388902 +सम -8.398904 +_त -8.452579 +हस -8.463034 +_ख -8.500232 +नक -8.591907 +शह -8.594778 +रप -8.594778 +गइ -8.603426 +नग -8.635584 +मन -8.644481 +चन -8.659431 +ख_ -8.662440 +मल -8.686738 +बद -8.702136 +बन -8.705235 +कह -8.730271 +थ_ -8.775159 +बह -8.778419 +लक -8.784962 +नल -8.801450 +वर -8.865882 +ज_ -8.886842 +भग -8.897437 +अध -8.922465 +ई_ -8.936965 +_उ -8.936965 +रस -8.962695 +धन -8.992674 +हत -8.996466 +रज -9.031046 +नद -9.074469 +दन -9.098716 +कब -9.110994 +खत -9.144257 +गर -9.152694 +इह -9.174004 +गण -9.178304 +णन -9.178304 +अप -9.182617 +सन -9.199999 +इस -9.230931 +गल -9.253439 +_य -9.257983 +ष_ -9.294859 +सभ -9.361751 +सक -9.361751 +दस -9.406450 +बर -9.426770 +धव -9.452579 +मय -9.452579 +भइ -9.473565 +च_ -9.484173 +कन -9.500232 +मह -9.516473 +रब -9.527403 +पह -9.560697 +बस -9.641509 +ध_ -9.653432 +लल -9.683679 +रन -9.683679 +खल -9.702136 +कइ -9.714573 +लख -9.727118 +अल -9.733432 +भज -9.752540 +हद -9.765422 +नम -9.798137 +श_ -9.831612 +_फ -9.831612 +नह -9.838401 +हख -9.845222 +हन -9.886842 +जप -9.915270 +ईल -9.922465 +मद -9.936965 +पल -9.958991 +दह -9.966409 +बत -9.966409 +जत -9.966409 +सग -9.966409 +नय -9.988893 +भष -9.996466 +टन -10.004079 +नस -10.011733 +कछ -10.019427 +छ_ -10.019427 +दर -10.019427 +मर -10.027163 +चर -10.050622 +नप -10.066476 +ए_ -10.074469 +मत -10.074469 +सथ -10.082506 +भ_ -10.115111 +_ऊ -10.123378 +लह -10.140057 +बल -10.148469 +अउ -10.156931 +उर -10.156931 +भस -10.165442 +_ध -10.182617 +जध -10.199999 +एह -10.199999 +तय -10.208770 +दश -10.208770 +दप -10.217594 +_घ -10.217594 +यर -10.235405 +गग -10.244394 +लम -10.244394 +जह -10.262541 +लय -10.262541 +उन -10.262541 +रल -10.262541 +तम -10.299535 +गत -10.318394 +गन -10.327917 +अत -10.337503 +सट -10.337503 +तह -10.337503 +आद -10.356868 +यक -10.386413 +बक -10.396397 +लव -10.396397 +तथ -10.406450 +वध -10.406450 +गध -10.406450 +गभ -10.426770 +अइ -10.437039 +पज -10.447381 +जय -10.447381 +कट -10.457797 +जग -10.468290 +बच -10.500232 +पव -10.511039 +कम -10.555094 +दल -10.566322 +सह -10.589042 +टर -10.589042 +जब -10.600537 +दय -10.600537 +दख -10.600537 +अन -10.623807 +कय -10.635584 +रव -10.635584 +एश -10.683679 +शय -10.683679 +गय -10.683679 +हम -10.683679 +चल -10.683679 +धय -10.695957 +मट -10.708341 +कत -10.720832 +धक -10.746143 +टत -10.758967 +_झ -10.758967 +शर -10.758967 +मज -10.758967 +वट -10.771906 +आध -10.784962 +_छ -10.798137 +छट -10.798137 +शक -10.798137 +सव -10.811434 +लन -10.811434 +आप -10.824855 +गह -10.852076 +ड_ -10.852076 +दव -10.852076 +ऊच -10.865882 +नत -10.865882 +रई -10.879821 +कई -10.879821 +_औ -10.893896 +तन -10.908110 +सत -10.922465 +जद -10.936965 +_ढ -10.966409 +एव -10.981359 +नर -10.981359 +फट -10.981359 +_ओ -10.981359 +गई -10.996466 +तल -11.027163 +इन -11.027163 +इत -11.042760 +बध -11.042760 +रम -11.042760 +आव -11.058527 +कस -11.074469 +रच -11.090588 +जक -11.123378 +लए -11.140057 +घट -11.140057 +लत -11.140057 +अम -11.156931 +ढर -11.191282 +मड -11.191282 +पस -11.191282 +रय -11.191282 +ङ_ -11.208770 +लब -11.208770 +दम -11.226472 +झल -11.226472 +यह -11.244394 +शस -11.262541 +नच -11.280919 +भई -11.318394 +खन -11.318394 +फर -11.318394 +जर -11.318394 +वक -11.337503 +हब -11.337503 +हज -11.337503 +फल -11.356868 +अश -11.396397 +रग -11.396397 +नइ -11.416575 +चह -11.437039 +ऊप -11.437039 +जस -11.437039 +यन -11.437039 +उस -11.437039 +वश -11.437039 +अर -11.500232 +मग -11.543954 +रख -11.543954 +नष -11.566322 +तप -11.566322 +यग -11.612125 +नऊ -11.635584 +ऊ_ -11.635584 +आस -11.635584 +उद -11.635584 +पत -11.635584 +ओक -11.659431 +सद -11.683679 +और -11.683679 +पच -11.683679 +जए -11.708341 +जम -11.708341 +कव -11.733432 +भत -11.733432 +शख -11.758967 +टल -11.758967 +बग -11.784962 +आक -11.784962 +तक -11.784962 +यम -11.811434 +उह -11.838401 +हच -11.838401 +लस -11.865882 +वज -11.865882 +अद -11.893896 +टब -11.922465 +जइ -11.922465 +_ऐ -11.922465 +चज -11.981359 +यल -11.981359 +चई -11.981359 +पद -12.011733 +अछ -12.042760 +छम -12.042760 +पम -12.042760 +खट -12.074469 +टङ -12.074469 +सघ -12.074469 +घ_ -12.074469 +अग -12.074469 +मक -12.074469 +आज -12.106890 +औस -12.140057 +पट -12.140057 +कथ -12.140057 +चट -12.140057 +भव -12.140057 +डल -12.174004 +हआ -12.174004 +आ_ -12.174004 +घर -12.174004 +उप -12.174004 +सस -12.174004 +कठ -12.208770 +ठम -12.208770 +तव -12.208770 +चक -12.208770 +इट -12.208770 +तब -12.208770 +इख -12.244394 +मस -12.244394 +_ठ -12.244394 +ठक -12.244394 +गम -12.244394 +गउ -12.280919 +उ_ -12.280919 +शम -12.280919 +वद -12.280919 +हय -12.280919 +हथ -12.318394 +आइ -12.356868 +सप -12.356868 +रङ -12.356868 +शअ -12.356868 +थर -12.356868 +अह -12.356868 +रर -12.356868 +हई -12.356868 +हए -12.396397 +वभ -12.396397 +घष -12.437039 +रआ -12.437039 +आत -12.437039 +सय -12.437039 +भम -12.437039 +हउ -12.478859 +उव -12.478859 +यण -12.478859 +_ऋ -12.478859 +यज -12.478859 +थल -12.478859 +आग -12.521928 +वय -12.521928 +मछ -12.521928 +छप -12.521928 +कग -12.521928 +खव -12.521928 +आम -12.566322 +ऐत -12.566322 +शव -12.566322 +बब -12.566322 +यत -12.566322 +बई -12.612125 +झप -12.612125 +टड -12.612125 +रइ -12.612125 +_थ -12.612125 +खर -12.659431 +यप -12.708341 +लई -12.708341 +अभ -12.708341 +पय -12.708341 +शष -12.708341 +रथ -12.708341 +तज -12.708341 +शल -12.758967 +कश -12.758967 +गज -12.811434 +हट -12.811434 +वह -12.811434 +_r -12.865882 +ri -12.865882 +iv -12.865882 +ve -12.865882 +er -12.865882 +r_ -12.865882 +चथ -12.865882 +पठ -12.865882 +ठर -12.865882 +सफ -12.865882 +हक -12.865882 +जई -12.922465 +आर -12.922465 +षत -12.981359 +पध -12.981359 +खक -12.981359 +इड -12.981359 +डय -12.981359 +नड -13.042760 +आय -13.042760 +अव -13.042760 +रभ -13.042760 +कप -13.106890 +वत -13.174004 +इक -13.174004 +ऋत -13.174004 +णस -13.174004 +अफ -13.244394 +गड -13.244394 +डम -13.244394 +मई -13.244394 +हग -13.244394 +_ई -13.244394 +ईस -13.244394 +_ट -13.244394 +टव -13.244394 +बज -13.244394 +अथ -13.318394 +थव -13.318394 +ओह -13.318394 +ऐस -13.396397 +कफ -13.396397 +फ_ -13.396397 +णम -13.396397 +गद -13.396397 +वष -13.396397 +षय -13.396397 +वई -13.396397 +पख -13.396397 +लप -13.396397 +कभ -13.396397 +ओड -13.478859 +डश -13.478859 +खज -13.478859 +दक -13.478859 +नब -13.478859 +सख -13.478859 +मउ -13.478859 +उट -13.478859 +कज -13.478859 +णच -13.566322 +खस -13.566322 +आब -13.659431 +आल -13.659431 +मच -13.659431 +आऊ -13.659431 +ऊर -13.659431 +रद -13.659431 +मख -13.659431 +खय -13.659431 +बव -13.758967 +मण -13.758967 +णप -13.758967 +मश -13.758967 +ढल -13.758967 +मथ -13.758967 +शत -13.758967 +एय -13.758967 +अब -13.865882 +फइ -13.865882 +बइ -13.865882 +इठ -13.865882 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bos b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bos new file mode 100644 index 00000000000..df5de4c685d --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bos @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 49577358 +#UNIQUE_BIGRAMS 576 +#UNSEEN_LOG_PROB -18.983564 +#EXPECTED_SCORE -7.586789 +a_ -4.970608 +e_ -5.430702 +je -5.650983 +i_ -5.682602 +_p -5.820811 +o_ -5.931594 +_s -6.012771 +st -6.160805 +ra -6.269473 +na -6.283139 +ko -6.421300 +ni -6.490307 +_n -6.545880 +u_ -6.552983 +_k -6.569526 +ij -6.597752 +an -6.611859 +ta -6.796966 +_o -6.819813 +pr -6.828151 +no -6.869051 +po -6.875881 +re -6.941938 +en -6.983280 +_d -6.987391 +ja -6.997355 +ov -7.006631 +ti -7.017448 +os -7.022948 +li -7.029757 +ka -7.039823 +ci -7.068506 +ri -7.115207 +m_ -7.151005 +ma -7.159575 +la -7.166515 +in -7.177105 +_b -7.183563 +od -7.188758 +va -7.189552 +nj -7.223011 +_m -7.227853 +ed -7.237688 +ne -7.252246 +_i -7.257352 +oj -7.263259 +vi -7.358325 +_t -7.400015 +vo -7.403755 +is -7.409255 +da -7.445408 +im -7.447687 +ic -7.451528 +ad -7.472321 +av -7.488786 +om -7.511247 +al -7.541358 +to -7.547202 +ro -7.553278 +ak -7.562578 +di -7.576964 +or -7.577125 +_v -7.586189 +ar -7.598290 +_z -7.622125 +og -7.656047 +bi -7.682937 +aj -7.699252 +sk -7.700353 +_r -7.715491 +ve -7.724756 +at -7.728361 +il -7.744140 +ce -7.751490 +sa -7.768915 +lj -7.769486 +za -7.796436 +ji -7.832814 +ju -7.840264 +_u -7.861777 +er -7.878683 +ih -7.902507 +on -7.907417 +te -7.930779 +_g -7.963418 +h_ -7.971882 +go -7.979251 +as -8.002659 +ac -8.045486 +it -8.063932 +lo -8.075966 +em -8.081308 +me -8.089988 +ik -8.101946 +tr -8.102874 +et -8.151866 +mo -8.182886 +de -8.198864 +n_ -8.227238 +el -8.228442 +do -8.233574 +zi -8.241831 +dn -8.245050 +es -8.246225 +ca -8.266575 +az -8.272853 +am -8.273600 +iz -8.275908 +_j -8.296712 +ek -8.313241 +io -8.332304 +g_ -8.351603 +mi -8.376996 +ol -8.399027 +_a -8.406888 +_c -8.419346 +ob -8.424736 +se -8.440552 +ec -8.448165 +ru -8.490239 +ao -8.534107 +sv -8.584568 +iv -8.588405 +ze -8.590894 +sl -8.591904 +t_ -8.594410 +si -8.594871 +ki -8.609550 +bo -8.618260 +tu -8.621282 +k_ -8.632857 +ok -8.636535 +j_ -8.686877 +sn -8.718080 +dr -8.723384 +le -8.761154 +r_ -8.762657 +ke -8.782549 +vr -8.795472 +ku -8.812288 +gr -8.819948 +du -8.868657 +us -8.880979 +tv -8.895512 +d_ -8.903294 +uc -8.926115 +nu -8.950007 +ot -8.952831 +vn -8.966173 +pa -8.966406 +eg -8.994524 +ut -9.010164 +ev -9.029438 +_l -9.046616 +ir -9.060191 +br -9.061605 +ba -9.062912 +_f -9.064376 +ga -9.076516 +zn -9.088712 +ez -9.101683 +_h -9.116240 +oz -9.142379 +oc -9.143613 +op -9.178091 +nt -9.182041 +uz -9.222128 +c_ -9.262540 +kr -9.262754 +s_ -9.279818 +lu -9.316790 +up -9.365117 +ud -9.368825 +su -9.384708 +ns -9.385233 +ug -9.418301 +_e -9.420832 +sp -9.450840 +vl -9.539684 +pi -9.592421 +gl -9.596628 +ck -9.621420 +un -9.629995 +ap -9.630317 +cn -9.652863 +pu -9.674102 +rs -9.682279 +tn -9.703910 +rv -9.747395 +rz -9.791147 +ln -9.808525 +ur -9.852856 +zv -9.855711 +pe -9.867189 +mj -9.894875 +uk -9.900899 +sr -9.903684 +ds -9.910221 +gu -9.962539 +ul -9.968212 +rn -9.978039 +jo -9.979978 +ig -9.993411 +be -9.994777 +eb -10.004847 +kt -10.023111 +vj -10.069010 +so -10.076312 +cu -10.078103 +mu -10.096687 +dj -10.124809 +kl -10.152627 +sm -10.158868 +zb -10.165437 +fi -10.179271 +gi -10.189022 +zl -10.191333 +vu -10.201577 +uj -10.223398 +nd -10.224860 +um -10.225243 +sj -10.230512 +id -10.233767 +ps -10.267409 +ub -10.267875 +pl -10.279005 +rt -10.307449 +ge -10.330270 +ag -10.335750 +bl -10.349732 +v_ -10.385486 +hi -10.422828 +nc -10.460382 +bu -10.479616 +zu -10.502355 +dv -10.536698 +ep -10.563134 +ip -10.626632 +ho -10.639433 +zo -10.676481 +js -10.679628 +jn -10.720828 +z_ -10.757736 +fe -10.786539 +ts -10.787413 +au -10.814723 +eo -10.865179 +tk -10.866645 +rc -10.885073 +ml -10.907368 +nk -10.923046 +l_ -10.927007 +rm -10.932740 +he -10.942729 +bj -10.946687 +ha -10.976456 +fo -10.979449 +kv -11.058545 +rd -11.078481 +rg -11.132334 +jv -11.134164 +ab -11.165838 +zm -11.193104 +rb -11.194808 +dl -11.275899 +kc -11.278932 +mb -11.293905 +mn -11.301450 +zd -11.324251 +hr -11.386226 +zg -11.421470 +uv -11.427549 +lt -11.445048 +ks -11.468770 +mp -11.487114 +dz -11.516310 +rk -11.534840 +ah -11.569886 +fa -11.639479 +db -11.656287 +bn -11.663632 +pj -11.679962 +zr -11.717492 +pc -11.755622 +ms -11.756729 +p_ -11.765213 +cl -11.769372 +vs -11.773543 +rl -11.806205 +ua -11.813832 +co -11.815824 +rp -11.825825 +sc -11.831117 +tp -11.839837 +pt -11.865233 +cj -11.872416 +ef -11.947548 +oi -11.980095 +ea -12.014115 +lm -12.038881 +cr -12.041700 +ng -12.041946 +dm -12.062212 +gd -12.087698 +fr -12.091122 +km -12.141639 +pn -12.151402 +jm -12.155512 +mr -12.164969 +kn -12.181635 +af -12.185425 +ej -12.220964 +ia -12.232541 +eh -12.329259 +ib -12.368421 +of -12.371039 +hv -12.382181 +fu -12.455471 +dg -12.519638 +uh -12.543761 +ou -12.559309 +uo -12.569001 +zj -12.570594 +ai -12.573784 +eu -12.603901 +rh -12.633180 +tj -12.662123 +nz -12.667981 +jb -12.676148 +jk -12.705586 +b_ -12.751001 +th -12.759854 +nf -12.778134 +if -12.790657 +y_ -12.808708 +lb -12.890753 +sd -12.964195 +vd -12.995222 +ae -13.026445 +hu -13.082893 +vc -13.104771 +fb -13.128289 +ht -13.144535 +hn -13.170324 +_w -13.186781 +tl -13.231701 +oh -13.232541 +sh -13.243788 +rj -13.267409 +gn -13.283277 +ch -13.306969 +ll -13.361054 +ei -13.392940 +f_ -13.437765 +oe -13.496080 +ls -13.522203 +lk -13.600282 +jc -13.624334 +nv -13.641709 +ue -13.694741 +mm -13.721614 +vk -13.821711 +bz -13.921578 +oa -14.015320 +ft -14.026931 +mc -14.115578 +ss -14.118163 +ii -14.123347 +ld -14.128029 +oo -14.130114 +jt -14.143745 +sf -14.167644 +wi -14.289382 +cv -14.302847 +mx -14.343405 +ay -14.345826 +dp -14.352507 +jd -14.376445 +xc -14.381405 +ee -14.452695 +wa -14.462516 +ew -14.502482 +ry -14.511969 +yo -14.515373 +_y -14.524259 +ow -14.575914 +jp -14.576625 +w_ -14.578760 +nr -14.589481 +ph -14.598837 +np -14.601728 +tz -14.666846 +tm -14.715338 +hd -14.717688 +ie -14.739811 +rr -14.763087 +fs -14.773644 +ui -14.784280 +hm -14.809126 +uf -14.873180 +nn -14.958625 +ty -15.002845 +tb -15.056375 +hl -15.059352 +tt -15.075338 +md -15.087445 +jl -15.147436 +lf -15.148493 +nl -15.174085 +jz -15.219992 +x_ -15.221103 +nb -15.234503 +we -15.243506 +td -15.266262 +pk -15.280090 +sb -15.331957 +ws -15.337971 +tf -15.351290 +mf -15.405831 +fl -15.583038 +gm -15.591635 +lc -15.594511 +wo -15.671394 +bd -15.672914 +ex -15.919322 +jr -16.008589 +lg -16.018214 +_x -16.199043 +ly -16.201234 +cc -16.232261 +ey -16.291715 +mk -16.294051 +aa -16.301083 +bb -16.310513 +hk -16.315251 +xi -16.331957 +vg -16.356164 +fg -16.365961 +ya -16.365961 +pd -16.390751 +gt -16.393253 +kp -16.393253 +ff -16.395760 +ny -16.423627 +bv -16.457270 +jj -16.577336 +ct -16.600282 +yj -16.608982 +zk -16.659296 +yt -16.659296 +nh -16.683595 +pp -16.711429 +rf -16.714555 +oy -16.720828 +hs -16.755823 +uu -16.788391 +gh -16.852372 +lp -16.883698 +lv -16.904967 +mt -16.912126 +bc -16.955848 +jg -16.963265 +yl -16.970721 +dh -16.978216 +ix -17.031797 +wn -17.047478 +bs -17.051425 +dy -17.111967 +bh -17.145326 +ox -17.145326 +tc -17.205626 +qu -17.245765 +lz -17.250295 +aw -17.254839 +gv -17.268557 +iu -17.277776 +yp -17.324773 +ux -17.378303 +tw -17.439057 +my -17.444237 +dk -17.459890 +zz -17.465146 +xv -17.497089 +ax -17.513330 +sw -17.546370 +bk -17.551951 +gw -17.574493 +bm -17.580184 +sy -17.632441 +wh -17.644315 +jh -17.711429 +ye -17.794994 +pb -17.794994 +mv -17.835258 +gz -17.862738 +dd -17.862738 +_q -17.890753 +hy -17.897842 +xe -17.912126 +lh -17.912126 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bre b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bre new file mode 100644 index 00000000000..bd8f51eebc9 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bre @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 5762475 +#UNIQUE_BIGRAMS 550 +#UNSEEN_LOG_PROB -18.461868 +#EXPECTED_SCORE -7.401378 +t_ -5.049704 +an -5.262866 +n_ -5.460846 +ou -5.633634 +et -5.788943 +en -5.844007 +ez -5.859013 +_d -5.988107 +re -6.064816 +_g -6.091218 +r_ -6.144736 +_b -6.233879 +_e -6.305774 +ve -6.361460 +er -6.379502 +nt -6.385496 +zh -6.456321 +ar -6.513682 +e_ -6.529392 +z_ -6.557414 +s_ -6.557862 +_k -6.570299 +_s -6.586400 +_v -6.597607 +_a -6.638502 +nn -6.674978 +l_ -6.767059 +ne -6.819086 +u_ -6.880386 +ga -6.895299 +de -6.899986 +d_ -6.908142 +_h -6.922314 +eu -6.928247 +ro -7.001742 +_m -7.011788 +be -7.029668 +h_ -7.050790 +el -7.051388 +_r -7.180933 +ad -7.184061 +_p -7.184716 +st -7.337533 +g_ -7.342604 +di -7.345573 +ev -7.351122 +ur -7.353699 +ha -7.363808 +_t -7.401323 +le -7.426685 +on -7.430783 +_l -7.433550 +al -7.434373 +la -7.435456 +me -7.483529 +as -7.486759 +ke -7.489815 +ri -7.490130 +ra -7.512813 +ma -7.602367 +in -7.613110 +us -7.613748 +ta -7.732358 +tr -7.739350 +he -7.769024 +ll -7.775537 +ag -7.777348 +iz -7.781088 +ed -7.781529 +ze -7.797816 +na -7.837694 +te -7.841306 +ka -7.850869 +li -7.872590 +em -7.904508 +ao -7.904928 +vi -7.916918 +ge -7.922071 +br -7.954369 +sk -8.008786 +k_ -8.040207 +_i -8.042053 +o_ -8.054247 +vo -8.082267 +ie -8.095902 +wa -8.110636 +io -8.120287 +oe -8.126500 +pe -8.137034 +da -8.157404 +ia -8.162416 +do -8.163924 +av -8.194254 +es -8.198220 +is -8.200869 +ue -8.201385 +ol -8.214562 +_w -8.218957 +eg -8.219778 +lo -8.222841 +ud -8.225611 +a_ -8.227261 +sa -8.245913 +oa -8.263577 +nv -8.286377 +va -8.292251 +ho -8.317587 +or -8.342726 +go -8.375692 +it -8.385287 +ni -8.389899 +iv -8.396717 +v_ -8.396970 +am -8.408153 +_n -8.448166 +ko -8.452808 +no -8.469306 +ae -8.471526 +i_ -8.493194 +ei -8.508339 +to -8.531405 +ba -8.563439 +pa -8.616988 +dr -8.618069 +_o -8.634392 +rz -8.697849 +at -8.714685 +se -8.717633 +uz -8.786384 +_f -8.789705 +il -8.811136 +hi -8.821972 +ch -8.836434 +ab -8.884255 +ti -8.913293 +kr -8.930047 +un -8.940342 +we -8.952074 +zi -8.962028 +m_ -9.016091 +du -9.039614 +ek -9.043836 +vr -9.052052 +az -9.057778 +mo -9.072125 +bl -9.135202 +om -9.168815 +ua -9.182424 +ea -9.183588 +za -9.188984 +bo -9.206923 +gw -9.226136 +ut -9.278193 +rv -9.320465 +_c -9.416598 +po -9.433637 +tu -9.447030 +oz -9.454915 +mm -9.472593 +ul -9.477582 +ak -9.488871 +mi -9.540699 +ep -9.573896 +_u -9.575423 +je -9.588856 +rn -9.601443 +ir -9.617479 +mp -9.643675 +ik -9.672600 +ru -9.683264 +_z -9.695252 +so -9.700659 +ij -9.705877 +im -9.714895 +ig -9.724818 +zo -9.738013 +pr -9.744656 +gr -9.779657 +nd -9.781198 +fe -9.795144 +ki -9.804293 +_y -9.808776 +ob -9.816431 +um -9.838725 +pl -9.846310 +si -9.861369 +p_ -9.870714 +iw -9.888402 +id -9.918129 +eb -9.933960 +eo -9.941818 +os -9.952198 +rt -10.009625 +mb -10.024150 +_j -10.029635 +od -10.039877 +ug -10.101530 +sp -10.125661 +rr -10.218360 +nk -10.224038 +ns -10.238181 +ui -10.339965 +rm -10.398575 +ye -10.402636 +rd -10.430697 +mu -10.440405 +fr -10.449478 +bi -10.503697 +fi -10.533816 +gi -10.542378 +kl -10.560790 +og -10.563818 +mz -10.566853 +gn -10.585967 +ot -10.638078 +b_ -10.658571 +wi -10.680591 +ng -10.706713 +bu -10.733317 +uh -10.767823 +ok -10.783623 +ic -10.789372 +ai -10.810799 +ec -10.815754 +ku -10.897924 +ee -10.973938 +rg -11.025715 +lu -11.039350 +ja -11.043572 +aa -11.055778 +rl -11.072395 +gl -11.074013 +rk -11.093575 +jo -11.120078 +gu -11.125661 +oi -11.132952 +th -11.135766 +ap -11.141976 +ib -11.193815 +op -11.224038 +fa -11.272143 +lz -11.274622 +lh -11.310416 +ub -11.318066 +j_ -11.332198 +pi -11.343214 +ac -11.343864 +ca -11.357595 +vu -11.369469 +zu -11.374778 +lv -11.398237 +ya -11.451931 +y_ -11.494639 +fo -11.527520 +ii -11.530479 +hu -11.549865 +tv -11.561168 +ef -11.573324 +lm -11.575614 +co -11.604948 +nu -11.624576 +aj -11.632503 +vl -11.640474 +su -11.692557 +nc -11.728636 +sh -11.782300 +ji -11.785832 +ih -11.877057 +mg -11.880828 +lt -11.940588 +ip -12.011174 +tl -12.050989 +ju -12.073473 +ov -12.097410 +_x -12.101805 +af -12.117294 +au -12.179808 +if -12.216274 +pu -12.227036 +ss -12.236670 +nj -12.258585 +of -12.299647 +hp -12.311052 +rs -12.364179 +ce -12.366822 +kt -12.409770 +rb -12.459667 +zn -12.551367 +ci -12.583276 +lf -12.646080 +nr -12.663841 +ls -12.685118 +wr -12.735450 +uj -12.735450 +lb -12.839872 +ej -12.884610 +xv -12.884610 +ms -12.888402 +x_ -12.899836 +vn -12.934695 +ck -12.992691 +ld -13.007046 +f_ -13.013242 +oh -13.034091 +fu -13.046746 +xi -13.057378 +ay -13.083218 +oj -13.118407 +rf -13.118407 +iu -13.152195 +oc -13.161341 +ry -13.170545 +pk -13.179808 +ph -13.217466 +rc -13.217466 +tt -13.224637 +yo -13.280838 +c_ -13.290839 +ix -13.365500 +nz -13.376108 +sc -13.400265 +ds -13.411133 +nw -13.444237 +pt -13.463904 +md -13.478118 +eh -13.501155 +fl -13.509890 +up -13.557390 +ny -13.560412 +ew -13.563439 +ks -13.575614 +dk -13.594071 +oo -13.650902 +uk -13.667094 +ah -13.693386 +lg -13.710064 +dd -13.720165 +sl -13.757817 +hn -13.847232 +aw -13.862067 +rp -13.884610 +uc -13.888402 +w_ -13.958411 +nl -14.027805 +cl -14.048866 +qu -14.078879 +sv -14.083218 +tk -14.096313 +sr -14.109529 +ow -14.154476 +hr -14.159049 +nh -14.163636 +ly -14.177486 +zv -14.339316 +ex -14.349733 +gh -14.354969 +ct -14.365500 +lw -14.370794 +kn -14.376108 +dw -14.408409 +yu -14.413863 +sm -14.413863 +yn -14.521619 +sb -14.569514 +wo -14.631709 +ey -14.631709 +ht -14.644476 +hv -14.644476 +pp -14.696706 +nb -14.703370 +oy -14.744012 +mr -14.757817 +nf -14.764770 +cu -14.785832 +uf -14.814401 +uo -14.843547 +lk -14.888402 +ys -14.888402 +dv -14.903668 +ts -14.934695 +dl -14.966404 +hs -14.990652 +cr -14.998826 +lj -15.023629 +tc -15.057378 +wy -15.065940 +nm -15.091935 +tn -15.091935 +ty -15.118407 +zr -15.127340 +yd -15.154476 +_q -15.200869 +vy -15.210330 +kh -15.308510 +uy -15.349733 +zs -15.402975 +dy -15.424834 +xa -15.458257 +bb -15.469572 +lp -15.480977 +kw -15.515743 +zk -15.551367 +dg -15.563439 +rh -15.587892 +rj -15.638078 +sf -15.663841 +ff -15.663841 +vv -15.676897 +my -15.703370 +zz -15.744012 +cq -15.757817 +cc -15.757817 +dt -15.800046 +yw -15.828901 +xx -15.858344 +mh -15.873295 +tm -15.903668 +zl -15.919098 +mv -15.934695 +dj -15.934695 +ux -15.966404 +yl -15.966404 +iq -15.982524 +hy -15.982524 +rw -15.998826 +zd -15.998826 +jk -16.031992 +ym -16.031992 +ax -16.065940 +wn -16.083218 +gg -16.100705 +ft -16.100705 +lr -16.118407 +tw -16.118407 +cy -16.136329 +sn -16.154476 +ml -16.172855 +pd -16.210330 +by -16.229438 +zg -16.248804 +kj -16.268433 +bn -16.268433 +uv -16.288332 +yv -16.328974 +gb -16.349733 +ox -16.349733 +jp -16.370794 +pg -16.370794 +tb -16.370794 +ln -16.370794 +kv -16.392168 +fy -16.435889 +fg -16.435889 +pv -16.480977 +bd -16.480977 +sd -16.480977 +sw -16.480977 +hm -16.504061 +mc -16.527520 +mn -16.551367 +qi -16.575614 +hw -16.575614 +zf -16.575614 +ps -16.575614 +td -16.625367 +bs -16.703370 +xu -16.730337 +sy -16.730337 +vs -16.757817 +zt -16.757817 +dh -16.814401 +rx -16.843547 +wh -16.843547 +gt -16.873295 +pf -16.903668 +cs -16.966404 +gd -16.966404 +ky -16.998826 +vd -17.031992 +dn -17.065940 +ws -17.100705 +mf -17.172855 +kk -17.172855 +db -17.172855 +gs -17.210330 +tz -17.288332 +hl -17.288332 +xf -17.328974 +wu -17.328974 +mk -17.413863 +df -17.458257 +tg -17.504061 +dp -17.551367 +sg -17.551367 +vj -17.650902 +kf -17.650902 +yr -17.703370 +kg -17.757817 +fk -17.757817 +pm -17.757817 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bul b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bul new file mode 100644 index 00000000000..5bb4976ab81 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/bul @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 150820373 +#UNIQUE_BIGRAMS 1120 +#UNSEEN_LOG_PROB -16.439097 +#EXPECTED_SCORE -7.825372 +а_ -4.883279 +и_ -5.435210 +о_ -5.706651 +_п -5.805176 +_с -6.026362 +то -6.066270 +та -6.098721 +е_ -6.124522 +ни -6.161210 +ат -6.181810 +на -6.212127 +ст -6.236355 +ра -6.309533 +ва -6.438713 +ен -6.518529 +те -6.518590 +_к -6.543981 +ит -6.605132 +пр -6.638424 +ре -6.671335 +т_ -6.746591 +ка -6.831692 +ан -6.838472 +ко -6.913204 +но -6.939078 +ри -6.944621 +_н -6.951646 +_в -7.005662 +ия -7.018653 +_т -7.039153 +_о -7.069231 +_д -7.084064 +ов -7.090910 +я_ -7.150161 +по -7.163509 +ед -7.190487 +ин -7.191788 +_м -7.196008 +ли -7.206678 +_и -7.209882 +ет -7.317712 +от -7.406870 +ти -7.413757 +ор -7.442204 +ве -7.455711 +_б -7.487976 +ав -7.489286 +ро -7.518560 +ма -7.522761 +н_ -7.589386 +не -7.607767 +ар -7.613821 +ле -7.625910 +ди -7.648148 +_р -7.650833 +ел -7.661509 +ме -7.676248 +ал -7.694340 +во -7.700253 +ос -7.734540 +од -7.751640 +за -7.765696 +ви -7.772176 +ис -7.797699 +ол -7.802463 +_г -7.802762 +де -7.830789 +ск -7.856570 +да -7.885860 +из -7.886786 +ки -7.899474 +го -7.927577 +ер -7.929511 +_з -7.952998 +ла -7.989820 +ят -8.002728 +ес -8.022767 +об -8.037339 +ци -8.067240 +ър -8.094091 +тр -8.107244 +ои -8.181425 +ие -8.186864 +че -8.189962 +га -8.198473 +съ -8.204189 +ми -8.234407 +тв -8.247185 +се -8.262425 +он -8.266190 +аз -8.281538 +ас -8.287423 +ак -8.314797 +ил -8.315269 +ад -8.326747 +д_ -8.350386 +ло -8.378563 +ем -8.387857 +ик -8.409314 +мо -8.423123 +им -8.434979 +ез -8.453638 +_а -8.464528 +до -8.487347 +ам -8.504368 +си -8.522390 +сл -8.599925 +въ -8.650368 +_е -8.651133 +па -8.684090 +лн -8.700181 +ич -8.723581 +ек -8.764678 +л_ -8.773442 +в_ -8.779986 +ом -8.786903 +р_ -8.807906 +зи -8.816640 +_у -8.824780 +ча -8.826099 +ир -8.848337 +би -8.863544 +ог -8.867953 +бо -8.871591 +дн -8.874389 +нт -8.881522 +ив -8.884937 +_ч -8.940946 +гр -8.950242 +м_ -8.956182 +к_ -8.992570 +аи -9.008478 +иц -9.021503 +пе -9.042479 +же -9.064728 +ок -9.066152 +ъл -9.079551 +ру -9.085395 +ев -9.120461 +са -9.141003 +нс -9.149502 +з_ -9.149545 +_л -9.160221 +с_ -9.185051 +кр -9.194379 +сп -9.210882 +тн -9.213674 +ът -9.231641 +вр -9.235384 +тъ -9.276490 +_ф -9.283597 +ап -9.287041 +чи -9.293023 +зв -9.306848 +пъ -9.313783 +бъ -9.356004 +яв -9.390609 +бе -9.393424 +ля -9.403177 +оп -9.403793 +кт -9.410384 +бр -9.423488 +ба -9.451872 +со -9.462759 +пи -9.476165 +ня -9.477528 +къ -9.488597 +св -9.493457 +дв -9.539084 +ид -9.546827 +ио -9.552353 +_х -9.561489 +ож -9.582890 +ач -9.592023 +оч -9.604300 +ег -9.607983 +ур -9.628666 +ги -9.633140 +ус -9.644380 +дъ -9.678197 +ръ -9.678682 +вн -9.686811 +сн -9.706209 +рн -9.706737 +вс -9.707168 +уч -9.708464 +жи -9.711886 +оя -9.722910 +жд -9.739659 +еж -9.742598 +_ц -9.751055 +рт -9.752193 +це -9.756133 +аб -9.758948 +иг -9.763431 +еи -9.764661 +ту -9.770289 +ще -9.770907 +фи -9.781956 +ъд -9.782360 +ии -9.797637 +чн -9.805088 +еч -9.814196 +ъс -9.860824 +рс -9.868672 +аг -9.869281 +др -9.874470 +оз -9.877787 +ща -9.884001 +ше -9.886426 +зн -9.907823 +жа -9.944697 +лу -9.944792 +ое -9.972499 +ду -9.979126 +рв -9.980981 +ац -9.985067 +ха -9.991499 +ср -10.008927 +ца -10.013309 +_ж -10.029614 +бл -10.042116 +лк -10.048211 +ащ -10.053944 +хо -10.066763 +лг -10.087064 +щи -10.103135 +пл -10.108257 +мн -10.111999 +кл -10.121356 +ку -10.122070 +що -10.161799 +ъв -10.173372 +ещ -10.191379 +дс -10.193931 +ря -10.200084 +гл -10.204671 +ъщ -10.216049 +нд -10.235845 +ъм -10.318839 +зп -10.327687 +еш -10.336207 +ъз -10.357672 +ям -10.377246 +вл -10.382369 +ув -10.400744 +кв -10.402193 +вя -10.404318 +нн -10.408498 +уп -10.410189 +ге -10.450740 +еп -10.455044 +як -10.464123 +ум -10.472769 +рг -10.474647 +рм -10.492864 +иа -10.500285 +у_ -10.505296 +зд -10.510184 +бщ -10.511761 +аж -10.521655 +ян -10.530457 +ши -10.537633 +уб -10.545761 +ец -10.580127 +ощ -10.603109 +тк -10.608087 +зо -10.609402 +зе -10.610733 +уг -10.614852 +см -10.669438 +чк -10.677593 +пу -10.683166 +лю -10.697534 +оф -10.703024 +ул -10.714535 +нц -10.733172 +ъп -10.756513 +ък -10.781097 +зл -10.803284 +вт -10.814041 +фо -10.820410 +ун -10.891314 +уд -10.896592 +мп -10.908862 +тя -10.917717 +рж -10.950563 +тт -10.971961 +_ш -10.972480 +иш -10.978181 +х_ -10.984311 +рк -10.990253 +ут -11.007223 +шн -11.007518 +аш -11.014823 +ук -11.020077 +кс -11.036922 +ищ -11.037927 +жн -11.065214 +нк -11.066116 +ае -11.068148 +ях -11.082721 +мя -11.096668 +яс -11.105409 +лс -11.106547 +фе -11.111407 +бя -11.114076 +ая -11.116898 +зк -11.118812 +бу -11.131782 +мъ -11.135995 +чв -11.143722 +еб -11.149274 +рд -11.165595 +ша -11.175827 +ах -11.180504 +ео -11.182036 +зб -11.186979 +уж -11.215083 +бв -11.220437 +ип -11.249742 +ън -11.258738 +вк -11.262470 +их -11.268994 +гу -11.271096 +_я -11.274504 +ъж -11.295846 +тб -11.402711 +лт -11.409642 +оц -11.420168 +зм -11.426473 +ех -11.434499 +мв -11.438981 +зр -11.442411 +г_ -11.453635 +яб -11.481783 +ял -11.482412 +гн -11.496273 +бс -11.509654 +дж -11.509961 +нг -11.528520 +уш -11.532963 +_ю -11.554783 +хи -11.559308 +п_ -11.595023 +юч -11.595733 +лъ -11.609088 +иж -11.611960 +рх -11.612319 +ся -11.625919 +му -11.634561 +фа -11.652495 +хр -11.663530 +ке -11.668410 +су -11.671246 +лж -11.713346 +еф -11.725377 +ну -11.730732 +ъб -11.736889 +тс -11.751557 +лз -11.771584 +мл -11.786848 +чо -11.798557 +дя -11.800193 +рш -11.806210 +бн -11.822885 +аф -11.870623 +ъо -11.874389 +уа -11.885602 +зг -11.898216 +ея -11.911421 +хн -11.929366 +щ_ -11.938123 +фр -11.940791 +дп -11.980981 +рл -11.986871 +кц -11.995711 +зу -12.006298 +рб -12.037525 +ху -12.039537 +уз -12.068004 +ьо -12.082659 +б_ -12.089563 +кн -12.091774 +щу -12.108236 +чу -12.110899 +тл -12.113991 +дм -12.117302 +ц_ -12.145108 +дл -12.168961 +ж_ -12.195231 +ъч -12.200039 +ош -12.208209 +еа -12.216516 +пт -12.230790 +лб -12.234842 +зс -12.257660 +иб -12.271120 +зт -12.315580 +e_ -12.316556 +вз -12.318851 +рз -12.320514 +ъг -12.324041 +яд -12.338236 +дк -12.351922 +ох -12.356982 +ау -12.362009 +нф -12.394653 +нъ -12.395581 +ця -12.443796 +рц -12.455622 +ш_ -12.481537 +лм -12.499925 +яр -12.524062 +шв -12.528463 +ч_ -12.533502 +гъ -12.556367 +фу -12.591831 +щн -12.617149 +ву -12.670342 +_щ -12.685574 +ъц -12.691763 +тд -12.699314 +жк -12.708056 +пя -12.710234 +жб -12.739830 +мс -12.745337 +хв -12.754628 +яз -12.772722 +шк -12.773124 +тг -12.774329 +вд -12.785970 +ух -12.832517 +пс -12.843794 +цъ -12.844146 +чл -12.879246 +хе -12.895335 +нч -12.935311 +ф_ -12.962539 +ее -13.020177 +вг -13.026229 +юб -13.054677 +чр -13.062102 +км -13.072529 +юн -13.088188 +иф -13.090522 +цв -13.104777 +вч -13.110857 +мб -13.142511 +юл -13.160090 +зъ -13.166320 +зх -13.242053 +шо -13.254806 +ъи -13.260990 +оу -13.265410 +уц -13.272870 +юз -13.287047 +оа -13.314947 +ют -13.315239 +юд -13.323943 +вш -13.335762 +чт -13.363218 +нь -13.375771 +жо -13.379742 +яг -13.385974 +вм -13.395735 +зя -13.405459 +бю -13.429742 +мр -13.448760 +пк -13.495831 +ящ -13.551592 +ii -13.552971 +ъю -13.573465 +юж -13.583646 +мк -13.589765 +da -13.604583 +уи -13.611032 +сг -13.611391 +фл -13.642980 +рп -13.643714 +тп -13.656256 +сц -13.658481 +id -13.665922 +нв -13.684818 +ae -13.695058 +бх -13.707288 +оо -13.713443 +пн -13.716272 +нз -13.734410 +лв -13.742909 +сд -13.761918 +жу -13.777550 +жъ -13.777550 +сб -13.792131 +нш -13.803848 +вп -13.810979 +юс -13.840984 +_t -13.849713 +хм -13.882565 +уе -13.892714 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/cat b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/cat new file mode 100644 index 00000000000..c195e3daa62 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/cat @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 306604083 +#UNIQUE_BIGRAMS 616 +#UNSEEN_LOG_PROB -19.859341 +#EXPECTED_SCORE -7.339162 +s_ -4.669279 +a_ -5.211799 +es -5.257275 +_p -5.822096 +er -5.848049 +en -5.938368 +re -6.008661 +t_ -6.054420 +_c -6.124646 +r_ -6.147442 +_d -6.250042 +_a -6.279493 +ta -6.284377 +e_ -6.287337 +nt -6.315404 +ar -6.349894 +an -6.366954 +_s -6.474488 +de -6.490214 +_e -6.502905 +ra -6.543665 +el -6.570536 +on -6.638967 +al -6.672351 +_m -6.739536 +qu -6.758498 +ca -6.773929 +at -6.786607 +ci -6.837498 +me -6.840129 +ue -6.842143 +se -6.846113 +l_ -6.865180 +co -6.880601 +or -6.933689 +pe -6.951256 +st -6.965965 +na -6.984037 +te -7.008219 +le -7.037306 +_t -7.054149 +ri -7.074256 +n_ -7.157375 +ls -7.165927 +in -7.179828 +ia -7.221414 +io -7.249503 +_f -7.264526 +_q -7.266511 +it -7.319133 +am -7.364423 +tr -7.368952 +ti -7.445290 +ns -7.456891 +_l -7.458340 +o_ -7.466149 +ic -7.497957 +ma -7.509142 +ac -7.531158 +om -7.590497 +pr -7.608164 +_r -7.614778 +la -7.614820 +ll -7.657820 +un -7.669083 +ro -7.711274 +pa -7.711870 +di -7.766866 +li -7.791185 +po -7.792361 +is -7.809809 +_v -7.866721 +ne -7.882007 +da -7.899485 +so -7.905306 +ts -7.913322 +os -7.967963 +ad -7.984955 +va -7.994293 +mb -7.995189 +ve -7.997415 +si -8.006339 +ol -8.017582 +ni -8.037458 +vi -8.081979 +nc -8.091373 +sa -8.169946 +_u -8.244290 +rt -8.268955 +to -8.282478 +ec -8.296649 +i_ -8.332434 +_g -8.356214 +em -8.361074 +_n -8.369517 +et -8.375096 +_i -8.402271 +mi -8.407183 +fi -8.432931 +ir -8.441687 +eg -8.448668 +ur -8.455023 +u_ -8.461550 +rs -8.472286 +gu -8.519685 +as -8.522834 +mo -8.546029 +il -8.576377 +_b -8.582046 +ba -8.607120 +_h -8.618134 +b_ -8.622504 +no -8.626765 +ct -8.641352 +ua -8.641684 +im -8.661108 +ob -8.670707 +ix -8.685878 +ce -8.705220 +ie -8.705892 +eu -8.713192 +ot -8.718316 +mp -8.730014 +ei -8.742738 +ss -8.745801 +m_ -8.760749 +do -8.778792 +fe -8.780719 +av -8.819942 +rr -8.855062 +us -8.884975 +ny -8.893200 +lo -8.901796 +lt -8.909811 +c_ -8.918400 +fo -8.936646 +nd -8.938739 +oc -8.940118 +ui -8.953693 +tu -8.971338 +ge -8.972799 +bl -8.995978 +br -9.002665 +iv -9.017327 +id -9.054761 +gr -9.072992 +_o -9.080233 +sp -9.159508 +ev -9.170727 +ab -9.194324 +rm -9.271760 +ha -9.290734 +ut -9.297694 +be -9.320143 +ul -9.323721 +ga -9.326313 +ap -9.333232 +ig -9.392590 +ai -9.401981 +du -9.404792 +cu -9.491237 +sc -9.498897 +aq -9.508084 +su -9.517920 +od -9.526497 +ed -9.535286 +pl -9.566792 +_j -9.575414 +gi -9.589654 +bi -9.641979 +cr -9.644098 +rc -9.644753 +iu -9.645555 +pi -9.702291 +fa -9.729411 +op -9.765766 +ip -9.790972 +ex -9.793851 +ov -9.799191 +au -9.838065 +go -9.846669 +cl -9.889752 +pu -9.915509 +ep -9.944560 +lu -9.955274 +ib -9.962429 +vo -9.969828 +x_ -9.985507 +mu -10.019228 +up -10.020019 +ng -10.027597 +rd -10.028874 +if -10.028987 +ys -10.062478 +dr -10.068625 +p_ -10.080406 +ru -10.087034 +rn -10.088309 +d_ -10.142187 +uc -10.145951 +xe -10.158927 +ou -10.173237 +xi -10.222286 +ju -10.262601 +og -10.271670 +cc -10.272106 +ea -10.354931 +eb -10.360767 +jo -10.386316 +um -10.390093 +ud -10.394355 +ag -10.400912 +ja -10.420441 +cs -10.427014 +ub -10.454422 +nv -10.468367 +tg -10.559566 +fr -10.569907 +af -10.572793 +lm -10.606047 +ho -10.609594 +tz -10.615244 +za -10.646400 +xa -10.716068 +bo -10.728070 +ef -10.735463 +rg -10.774303 +ya -10.775944 +fu -10.799266 +y_ -10.818410 +nu -10.853413 +g_ -10.855417 +bu -10.881535 +hi -11.055642 +nf -11.064392 +rv -11.074088 +gl -11.096889 +pt -11.123475 +iq -11.201088 +ug -11.235755 +lg -11.278281 +of -11.294783 +ps -11.295398 +xo -11.386058 +tj -11.508574 +aj -11.527841 +_x -11.566949 +he -11.593909 +rq -11.672442 +gn -11.703789 +eo -11.738338 +je -11.765602 +sm -11.824950 +xp -11.850352 +eq -11.868888 +fl -11.878688 +bs -11.884192 +rl -11.900307 +ms -12.190086 +xt -12.197824 +ii -12.247707 +rb -12.313607 +th -12.350606 +ch -12.361608 +yo -12.369256 +pc -12.413416 +_z -12.437775 +nj -12.529634 +_w -12.573617 +oe -12.600572 +nn -12.618184 +zo -12.626372 +ee -12.636793 +tm -12.649586 +ld -12.655554 +ze -12.690307 +rp -12.692298 +hu -12.718286 +oi -12.780489 +vu -12.787425 +lc -12.796770 +nq -12.823875 +oa -12.850839 +bt -12.864003 +oj -12.901099 +h_ -12.922456 +rf -12.927029 +k_ -12.956760 +z_ -12.963735 +mm -13.016954 +_k -13.023482 +rx -13.127817 +sf -13.141060 +ax -13.150143 +cn -13.169521 +sl -13.261111 +sq -13.267639 +ej -13.272938 +ox -13.300872 +sh -13.340785 +ye -13.345920 +sd -13.347635 +mn -13.356097 +ao -13.356590 +oq -13.357429 +ds -13.379424 +nz -13.386108 +bj -13.393380 +tx -13.413571 +lv -13.434933 +tl -13.457410 +ae -13.462340 +ez -13.534148 +xc -13.536774 +sb -13.549750 +nr -13.563413 +oo -13.592296 +oh -13.609483 +gm -13.618983 +wa -13.672534 +ah -13.762656 +dm -13.821046 +xv -13.842451 +mt -13.863968 +lb -13.867832 +ck -13.876298 +wi -13.963885 +f_ -13.988148 +we -14.014927 +zi -14.039755 +ke -14.040151 +lf -14.062680 +ay -14.080911 +nm -14.144848 +sg -14.153054 +yi -14.153397 +nl -14.161393 +gs -14.162860 +bd -14.173688 +ry -14.223495 +uj -14.247639 +rk -14.254612 +tc -14.382536 +eh -14.382636 +_y -14.411057 +az -14.445287 +ki -14.472841 +uf -14.544343 +tt -14.549411 +tb -14.582854 +ey -14.649737 +ka -14.693077 +kh -14.694948 +ow -14.730450 +mf -14.752750 +iz -14.757825 +ak -14.801498 +ux -14.807692 +v_ -14.884316 +lp -14.888592 +w_ -14.894026 +hn -14.904522 +oy -14.919463 +uv -14.930882 +gh -14.948033 +bn -14.952203 +xu -14.966896 +dv -14.998122 +uo -15.021407 +ph -15.044279 +lh -15.048418 +ty -15.051291 +dq -15.125712 +ew -15.135672 +ly -15.276482 +sr -15.278351 +bm -15.326455 +wo -15.328971 +ok -15.384447 +nk -15.385252 +aw -15.444028 +ik -15.462818 +lz -15.516727 +ff -15.558352 +hr -15.569750 +gd -15.625272 +dd -15.627652 +gg -15.630275 +ko -15.646596 +nx -15.757695 +ij -15.759260 +dj -15.791189 +sn -15.808367 +oz -15.871847 +hm -15.879486 +tn -15.894312 +ln -15.951607 +ht -15.961781 +pd -15.974753 +rj -15.977179 +sv -15.977786 +gt -15.978394 +pp -16.085239 +ks -16.151169 +ji -16.164932 +dy -16.245992 +lk -16.262913 +ku -16.280035 +aa -16.280410 +sk -16.334209 +nh -16.347488 +dw -16.353780 +kl -16.363665 +uh -16.434828 +my -16.468994 +bb -16.487033 +uz -16.512322 +dn -16.514082 +ft -16.529133 +rz -16.530469 +bc -16.584009 +bv -16.609660 +fs -16.614373 +tw -16.626700 +tp -16.651674 +cd -16.666770 +yd -16.750933 +dh -16.752490 +gc -16.765537 +wn -16.792524 +yl -16.815676 +ek -16.815676 +sy -16.829310 +wh -16.865372 +zz -16.867059 +dg -16.878352 +ws -16.912191 +yn -16.987231 +by -17.006926 +xx -17.027523 +nb -17.044597 +yu -17.070916 +hy -17.137197 +cy -17.137876 +xh -17.234700 +gb -17.237605 +zh -17.276669 +ky -17.322208 +yr -17.374019 +ym -17.394951 +uu -17.447809 +uk -17.475840 +xs -17.536271 +hs -17.550653 +vd -17.592819 +cq -17.632425 +kb -17.647803 +pn -17.701954 +pb -17.707986 +sw -17.745752 +wr -17.769737 +rh -17.804861 +zu -17.829858 +ml -17.880053 +yw -17.894885 +mr -17.953397 +dl -17.954592 +wl -17.959381 +fg -17.986008 +ih -17.989678 +rw -18.006926 +tv -18.009407 +yc -18.023130 +hl -18.035719 +bg -18.042055 +kr -18.045870 +dt -18.095087 +pm -18.114986 +cb -18.159756 +uq -18.179177 +vy -18.187581 +uy -18.205960 +gp -18.274430 +mc -18.274430 +zq -18.284911 +mv -18.306105 +wt -18.333821 +xf -18.357331 +cm -18.425273 +dc -18.431913 +ww -18.438585 +py -18.440258 +yp -18.465584 +lx -18.531806 +tf -18.533590 +df -18.535377 +kn -18.575253 +hw -18.593749 +bh -18.599345 +vl -18.620049 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ceb b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ceb new file mode 100644 index 00000000000..542d7f7bcd8 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ceb @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 72558904 +#UNIQUE_BIGRAMS 648 +#UNSEEN_LOG_PROB -19.311138 +#EXPECTED_SCORE -6.941439 +an -4.644323 +ng -4.884744 +_k -5.127317 +_a -5.266914 +a_ -5.313310 +g_ -5.387547 +n_ -5.416213 +ga -5.676493 +o_ -5.746707 +ka -5.759663 +ta -5.788815 +_m -5.821782 +_p -5.913514 +ad -5.949738 +t_ -6.065750 +ba -6.128974 +pa -6.134850 +al -6.199028 +y_ -6.243907 +s_ -6.318454 +ib -6.354542 +la -6.356676 +at -6.374293 +li -6.396515 +_n -6.402778 +ag -6.489641 +ay -6.533842 +as -6.581388 +in -6.582580 +ma -6.605774 +ki -6.622423 +na -6.665128 +wa -6.686122 +ab -6.755235 +bo -6.773554 +da -6.826401 +un -6.832898 +to -6.888334 +me -6.918823 +ot -6.924201 +ro -6.929438 +_g -7.037183 +on -7.070790 +ra -7.080706 +et -7.084695 +tr -7.098253 +ha -7.196957 +aw -7.242384 +ao -7.246457 +_d -7.255716 +it -7.400428 +yo -7.420204 +ul -7.453782 +ih -7.455181 +os -7.458492 +pu -7.484296 +ap -7.498547 +_i -7.513706 +gi -7.600349 +_u -7.628995 +i_ -7.668633 +_l -7.679284 +ni -7.697927 +w_ -7.745043 +am -7.777416 +il -7.792686 +_h -7.859268 +nt -7.905177 +pi -7.930696 +lo -7.951778 +sa -7.992173 +ub -8.015883 +ak -8.074116 +_s -8.083756 +_t -8.109983 +om -8.175362 +si -8.211069 +ut -8.218157 +iw -8.220213 +do -8.272898 +hu -8.325907 +l_ -8.346560 +oy -8.381629 +ar -8.396955 +dr -8.404583 +po -8.411458 +e_ -8.430869 +gw -8.433417 +dt -8.443280 +_y -8.447924 +or -8.457188 +kw -8.490004 +bu -8.491291 +yu -8.492079 +el -8.502043 +du -8.509400 +le -8.518653 +er -8.557019 +mi -8.607016 +mg -8.611877 +op -8.636868 +id -8.661857 +be -8.662993 +sy -8.813119 +hi -8.843084 +wo -8.852961 +_b -8.882817 +mo -8.911310 +dl -8.967364 +dp -8.989690 +eb -9.045971 +_c -9.091973 +gt -9.267331 +ol -9.273261 +re -9.287156 +_r -9.346289 +r_ -9.437244 +en -9.438526 +lu -9.474694 +us -9.485899 +is -9.523203 +se -9.545971 +_w -9.563858 +ri -9.624778 +ac -9.676507 +ch -9.726416 +ll -9.728861 +ir -9.736236 +ne -9.834236 +d_ -9.863721 +ns -9.900153 +te -9.927716 +es -9.928452 +de -9.942646 +ai -9.974338 +ed -9.988245 +py -9.990853 +up -10.023820 +ti -10.100156 +ou -10.103127 +k_ -10.130948 +uy -10.134788 +ke -10.136600 +aa -10.137518 +uk -10.167023 +ek -10.167205 +st -10.234479 +ah -10.292221 +ca -10.297992 +bi -10.342941 +c_ -10.414732 +kt -10.446509 +nd -10.455638 +h_ -10.465556 +gp -10.473563 +he -10.504654 +dy -10.518937 +ko -10.524930 +ic -10.543387 +di -10.563888 +co -10.568892 +ho -10.569164 +rt -10.569950 +ia -10.630125 +im -10.632964 +rr -10.662985 +ie -10.713705 +pe -10.732087 +_f -10.765547 +oi -10.787309 +th -10.823531 +mu -10.835653 +ce -10.879591 +m_ -10.892989 +og -10.893708 +iy -10.946290 +eh -10.959097 +gu -10.975618 +no -11.019892 +ur -11.047949 +od -11.103564 +ge -11.109702 +oc -11.113574 +_e -11.163142 +gs -11.185566 +_j -11.193367 +au -11.220201 +_o -11.226191 +rg -11.252338 +so -11.257586 +rd -11.262316 +ea -11.326482 +ep -11.328475 +ei -11.337142 +gh -11.342244 +go -11.343225 +io -11.377675 +tu -11.397921 +ph -11.416932 +ck -11.442159 +ru -11.453378 +ci -11.457063 +u_ -11.508270 +_v -11.534041 +nu -11.573672 +sc -11.575431 +rk -11.604731 +wi -11.675092 +ee -11.682262 +ve -11.686581 +ss -11.695852 +br -11.716781 +va -11.739716 +nn -11.798845 +vi -11.808441 +gr -11.824937 +ik -11.848793 +p_ -11.859984 +ig -11.871485 +tt -11.884432 +ws -11.900305 +eo -11.944918 +ue -11.945859 +nc -11.964252 +um -11.987801 +su -12.012151 +oh -12.013877 +cr -12.037170 +sh -12.049591 +sd -12.067316 +ja -12.104133 +eu -12.113090 +fr -12.113883 +cu -12.113971 +em -12.138414 +ec -12.150657 +ae -12.154097 +sp -12.186075 +f_ -12.218779 +rn -12.229624 +kp -12.260705 +ry -12.300071 +rs -12.351306 +oa -12.357031 +gl -12.359851 +ya -12.369498 +rm -12.383772 +rl -12.385048 +ud -12.400337 +uo -12.409286 +ob -12.419380 +mp -12.420797 +ii -12.453211 +ip -12.455443 +ov -12.468455 +ey -12.507401 +iu -12.525223 +jo -12.543862 +av -12.588108 +pt -12.597566 +pr -12.605350 +gn -12.612803 +oo -12.616669 +je -12.617919 +rh -12.624307 +eg -12.628708 +ju -12.633753 +qu -12.639578 +ui -12.653858 +we -12.704850 +kr -12.717785 +fi -12.722346 +lt -12.723152 +ld -12.723556 +oe -12.735574 +ow -12.766690 +ls -12.784676 +uc -12.809011 +mb -12.809725 +kh -12.839874 +ev -12.842500 +ct -12.842500 +ua -12.863240 +hr -12.865760 +fe -12.870964 +sl -12.874693 +rc -12.898938 +ps -12.910526 +bl -12.936164 +iv -13.016593 +z_ -13.022206 +pl -13.029502 +nj -13.030001 +x_ -13.043367 +ly -13.048917 +yl -13.066037 +ku -13.070306 +cl -13.080604 +rp -13.094623 +b_ -13.140387 +_z -13.153915 +fo -13.163735 +fa -13.170501 +hy -13.188765 +ok -13.208015 +gb -13.210840 +ys -13.230197 +hn -13.266571 +vo -13.268925 +ds -13.276204 +ef -13.282728 +ts -13.293269 +ug -13.296466 +ff -13.306100 +yr -13.322097 +ze -13.389202 +ye -13.431850 +ks -13.476478 +ew -13.496560 +dj -13.510182 +nk -13.528626 +nr -13.541845 +sm -13.558540 +hl -13.581268 +ht -13.589823 +yn -13.617294 +yi -13.618544 +tl -13.621798 +_q -13.667117 +af -13.690322 +v_ -13.713638 +mm -13.720869 +if -13.741690 +dw -13.765859 +lb -13.782293 +ty -13.787344 +za -13.792977 +sk -13.798916 +lm -13.806873 +hm -13.829850 +uh -13.843523 +gg -13.886538 +ex -13.896207 +ez -13.914511 +rv -13.942411 +lp -13.956566 +rb -13.959097 +ax -14.001840 +yc -14.004452 +ny -14.012644 +uv -14.103871 +yp -14.113354 +nb -14.144343 +zo -14.197890 +aj -14.207638 +my -14.211405 +ms -14.214048 +nh -14.226191 +iz -14.231918 +dg -14.233449 +pp -14.255835 +lh -14.256613 +lk -14.259730 +aq -14.290477 +ym -14.332930 +oz -14.337039 +fl -14.397545 +uf -14.400982 +tz -14.461598 +lc -14.463842 +cc -14.488768 +lv -14.496100 +lf -14.509487 +dd -14.532391 +xi -14.535221 +ix -14.573006 +uu -14.575431 +kn -14.576888 +zi -14.583219 +kl -14.603369 +az -14.621798 +cy -14.638436 +fu -14.647593 +nz -14.670743 +tc -14.690585 +yt -14.711236 +mc -14.720869 +lg -14.743597 +zh -14.845277 +xa -14.889854 +ox -14.895903 +sb -14.944604 +bb -15.003472 +bs -15.063481 +ux -15.092364 +uz -15.108429 +vr -15.131796 +yg -15.147587 +of -15.203506 +mt -15.207262 +yb -15.212537 +nl -15.215560 +_x -15.233832 +wh -15.242285 +xo -15.261681 +zu -15.265592 +tw -15.282927 +yd -15.292470 +pf -15.300472 +vu -15.310941 +qa -15.325563 +ky -15.327197 +by -15.356093 +sw -15.358597 +sn -15.446425 +rf -15.452653 +cq -15.542794 +ji -15.542794 +sf -15.554229 +hw -15.597935 +wy -15.624809 +tk -15.627826 +gy -15.658350 +tn -15.667634 +xy -15.723556 +bd -15.756197 +nf -15.773913 +tm -15.802037 +wn -15.818029 +hs -15.834200 +wl -15.842354 +iq -15.848207 +nw -15.882629 +ij -15.925297 +ft -15.966717 +uj -15.973098 +j_ -16.079226 +rz -16.115470 +cs -16.123965 +kb -16.133939 +mn -16.199760 +gm -16.201257 +wr -16.210274 +rq -16.222385 +zy -16.226953 +cn -16.245371 +df -16.305294 +dh -16.396687 +xe -16.477838 +dm -16.523935 +sq -16.589087 +qo -16.600897 +sz -16.628834 +hb -16.659379 +yx -16.661438 +wu -16.667634 +gk -16.716045 +dn -16.731106 +uw -16.741962 +lz -16.768353 +ln -16.795237 +sr -16.852906 +pb -16.940222 +cz -16.955302 +nm -16.988528 +ej -17.014617 +mr -17.049254 +yk -17.054658 +rj -17.068255 +zz -17.087510 +qi -17.112649 +kk -17.118296 +db -17.143983 +oj -17.152647 +rw -17.170135 +fs -17.202756 +hk -17.264026 +lj -17.311749 +cg -17.357762 +cd -17.381330 +sj -17.394973 +sv -17.433169 +vs -17.454438 +q_ -17.458013 +zl -17.501625 +sg -17.509023 +bw -17.589087 +hc -17.669706 +oq -17.673857 +yz -17.690585 +nv -17.724632 +lw -17.728945 +xt -17.741962 +ml -17.750706 +eq -17.755097 +vl -17.777259 +bk -17.790721 +wf -17.818029 +tj -17.818029 +gf -17.841186 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ces b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ces new file mode 100644 index 00000000000..70464b96497 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ces @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 269926106 +#UNIQUE_BIGRAMS 602 +#UNSEEN_LOG_PROB -18.063579 +#EXPECTED_SCORE -7.723768 +e_ -5.255617 +i_ -5.336592 +_p -5.580805 +st -6.114907 +u_ -6.129568 +ni -6.147330 +ne -6.235416 +_s -6.277823 +a_ -6.345268 +pr -6.361902 +y_ -6.383067 +_n -6.538652 +te -6.578715 +_v -6.586467 +o_ -6.617732 +en -6.680258 +ro -6.722055 +ra -6.737127 +po -6.755107 +na -6.829107 +_z -6.882810 +le -6.884589 +ch -6.939244 +_t -6.959647 +_k -6.961504 +_d -6.987236 +_m -6.999754 +re -7.006900 +_j -7.012002 +m_ -7.016670 +ce -7.060640 +ve -7.082552 +al -7.108190 +la -7.122945 +ta -7.169316 +t_ -7.201313 +ov -7.229351 +ic -7.268490 +ri -7.285012 +ou -7.327300 +ko -7.329991 +od -7.336914 +va -7.352711 +an -7.363954 +em -7.367435 +er -7.380715 +li -7.410111 +de -7.413052 +ho -7.430900 +ti -7.433819 +je -7.443466 +to -7.449877 +_r -7.453592 +at -7.458104 +_o -7.475451 +ak -7.493795 +ed -7.497861 +el -7.500298 +me -7.504162 +_b -7.523108 +ci -7.538213 +os -7.571230 +_c -7.601073 +no -7.636238 +l_ -7.637744 +av -7.652531 +h_ -7.661486 +vy -7.690801 +ad -7.733428 +et -7.757742 +in -7.764032 +es -7.764161 +za -7.797275 +it -7.797807 +si -7.801852 +do -7.826102 +sk -7.850963 +ka -7.864457 +ob -7.874003 +tr -7.876677 +ze -7.896866 +as -7.906421 +ji -7.908293 +lo -7.940128 +is -7.959051 +dn -7.965464 +or -8.030395 +mi -8.040969 +_a -8.043626 +ec -8.044224 +k_ -8.054780 +by -8.068452 +zi -8.069505 +ac -8.070726 +se -8.072784 +ol -8.075266 +il -8.083722 +ku -8.111753 +da -8.120599 +ar -8.132319 +ej -8.153395 +oz -8.172118 +ky -8.188783 +vi -8.222498 +_u -8.242044 +im -8.309096 +ek -8.332594 +ke -8.341772 +vo -8.344933 +ny -8.356955 +kt -8.363666 +ez -8.369704 +di -8.381095 +ot -8.382733 +ma -8.398639 +am -8.400119 +ru -8.400274 +_l -8.444828 +tu -8.453846 +on -8.459924 +n_ -8.482862 +az -8.491953 +_h -8.492241 +so -8.495127 +om -8.539244 +oc -8.541965 +ik -8.551978 +mo -8.567057 +pa -8.607137 +eh -8.641023 +ok -8.680312 +sl -8.688705 +us -8.690795 +d_ -8.692154 +zn -8.700682 +yc -8.775798 +z_ -8.798738 +mu -8.805258 +ud -8.829577 +uz -8.835301 +sp -8.862425 +tn -8.909564 +vn -8.913546 +ja -8.919276 +iz -8.924902 +dy -8.941117 +du -8.979298 +js -9.001548 +be -9.037024 +iv -9.044292 +cn -9.048689 +yl -9.060275 +pe -9.076302 +bo -9.092542 +op -9.103155 +sa -9.124439 +aj -9.141806 +eb -9.161556 +nu -9.168955 +ck -9.170934 +nt -9.174268 +hl -9.189569 +ev -9.204221 +oh -9.205713 +uj -9.211820 +ln -9.219962 +s_ -9.221691 +id -9.223693 +zd -9.232495 +ry -9.240849 +bu -9.268050 +lu -9.271293 +sv -9.278641 +ty -9.281967 +_f -9.293244 +ys -9.302161 +kl -9.309190 +dl -9.315486 +ca -9.344299 +r_ -9.393675 +ut -9.408632 +hr -9.424892 +oj -9.430697 +vs -9.440735 +kr -9.463621 +ha -9.494393 +up -9.497693 +ly -9.521681 +tv -9.549807 +nc -9.550721 +pl -9.589190 +dr -9.590380 +ba -9.617186 +uc -9.630109 +sn -9.647361 +kd -9.701011 +ap -9.714172 +c_ -9.723183 +rn -9.727812 +br -9.768843 +ur -9.771676 +ab -9.776215 +ah -9.802888 +pi -9.813372 +bi -9.826638 +ir -9.836348 +_e -9.864193 +_i -9.888623 +vr -9.889530 +pu -9.893698 +yt -9.918110 +ym -9.956764 +um -9.958018 +hu -10.032757 +ih -10.048859 +ns -10.050281 +ul -10.057923 +ie -10.064799 +rv -10.073158 +vu -10.075371 +fi -10.080252 +uv -10.141302 +ep -10.173623 +un -10.185667 +vl -10.211411 +nk -10.254518 +bl -10.281878 +ls -10.306669 +rm -10.334377 +co -10.334963 +tk -10.349701 +sm -10.366742 +ub -10.379886 +rt -10.382758 +mn -10.443929 +lk -10.444294 +he -10.444681 +my -10.449711 +uh -10.467581 +zv -10.485132 +hy -10.487102 +ia -10.492505 +fo -10.492621 +vz -10.518580 +uk -10.523426 +zo -10.527089 +yz -10.542287 +zp -10.545112 +ct -10.545128 +dv -10.550841 +zk -10.562392 +bn -10.575814 +ip -10.580816 +su -10.584504 +hn -10.639551 +rs -10.663364 +j_ -10.666410 +jn -10.678727 +zu -10.688917 +ts -10.695744 +v_ -10.701732 +ds -10.732810 +yb -10.801099 +yr -10.804345 +jm -10.857716 +ps -10.895804 +nd -10.935522 +yv -11.005229 +rz -11.042273 +rc -11.057536 +au -11.059748 +kv -11.078997 +ij -11.079934 +kc -11.107417 +ii -11.108774 +zs -11.122925 +tl -11.133872 +cu -11.148394 +bc -11.166548 +sy -11.168035 +yn -11.174877 +dk -11.201213 +dp -11.216025 +rd -11.219029 +io -11.234399 +zh -11.239456 +zb -11.251759 +nn -11.269211 +og -11.291264 +gi -11.329032 +cl -11.353619 +ml -11.357695 +yd -11.448612 +zr -11.453670 +ex -11.477156 +dc -11.477735 +lm -11.479444 +eg -11.489413 +ga -11.499931 +_g -11.508175 +ib -11.518282 +yp -11.525654 +jd -11.525969 +ef -11.570611 +yk -11.577586 +rk -11.592165 +sc -11.624690 +gr -11.669148 +b_ -11.676600 +ea -11.679420 +ge -11.691266 +fe -11.735706 +pn -11.738004 +zm -11.757469 +dm -11.796158 +vc -11.804450 +kn -11.809563 +p_ -11.810139 +fa -11.837771 +yh -11.847133 +vk -11.847310 +nf -11.858302 +ht -11.873543 +nz -11.884434 +rg -11.930068 +ng -11.931737 +rh -11.964577 +jv -11.973534 +hi -11.993707 +ss -12.056636 +ag -12.096223 +zl -12.154485 +go -12.173000 +bs -12.175124 +ua -12.216928 +zy -12.222690 +fr -12.223687 +bj -12.235675 +eu -12.239986 +mp -12.273465 +of -12.304762 +lz -12.306087 +db -12.321407 +hc -12.342709 +vd -12.360784 +ig -12.389432 +jo -12.393941 +mc -12.455200 +zt -12.458716 +ju -12.488445 +fu -12.488568 +lt -12.509265 +xi -12.548013 +af -12.553786 +_w -12.570692 +yu -12.570985 +mr -12.575676 +sr -12.644915 +mz -12.701785 +lb -12.731829 +eo -12.749055 +hv -12.750491 +gu -12.796976 +rl -12.916761 +py -12.922185 +vb -12.922807 +mk -12.946238 +mt -12.948898 +sd -12.969113 +ms -12.983802 +sh -12.996631 +pt -13.005482 +yj -13.006317 +zc -13.007417 +pc -13.062454 +rp -13.083687 +lc -13.088893 +bd -13.100629 +nv -13.176088 +mb -13.180895 +kz -13.194158 +jl -13.216776 +f_ -13.219832 +g_ -13.241460 +vh -13.261527 +ai -13.272063 +ki -13.336393 +ax -13.342098 +gl -13.342653 +hd -13.350113 +jc -13.374881 +rb -13.377893 +bv -13.391785 +jt -13.413723 +if -13.477400 +tm -13.490443 +tc -13.494879 +we -13.511385 +cs -13.548622 +xt -13.560196 +zj -13.586911 +th -13.610916 +oo -13.612455 +tb -13.645840 +sb -13.769211 +wa -13.777894 +dh -13.800670 +hz -13.801585 +ao -13.807091 +ll -13.827614 +gy -13.845362 +xp -13.893678 +tz -13.978702 +gn -13.984062 +cm -14.033485 +bk -14.067676 +oa -14.100347 +w_ -14.105614 +x_ -14.119246 +iu -14.149134 +dd -14.163872 +ae -14.280388 +xu -14.305708 +dz -14.339326 +nl -14.340102 +fl -14.401931 +ks -14.425965 +lh -14.431977 +ow -14.446821 +cc -14.451004 +lv -14.488967 +dt -14.504164 +ph -14.505033 +jp -14.560002 +ft -14.606843 +uf -14.614063 +sf -14.648240 +ue -14.661060 +hk -14.666470 +wi -14.669671 +hm -14.679174 +ld -14.695815 +ee -14.705350 +cy -14.712364 +ew -14.743400 +fy -14.867319 +yo -14.888724 +km -15.046901 +jb -15.050524 +cv -15.053248 +tt -15.156045 +_y -15.182235 +ay -15.214589 +td -15.256027 +jr -15.309936 +cr -15.324995 +lf -15.327850 +tw -15.373405 +pk -15.387082 +jk -15.423262 +nb -15.458686 +ug -15.462783 +ux -15.472714 +xe -15.498957 +ox -15.505655 +tf -15.558583 +gh -15.571278 +ff -15.640575 +oi -15.644128 +ei -15.646868 +xa -15.647965 +mm -15.680437 +cz -15.691142 +pp -15.695957 +rr -15.711933 +cb -15.716819 +bt -15.719412 +wo -15.740617 +vt -15.751781 +oe -15.766006 +sj -15.766900 +uo -15.789729 +mf -15.802196 +xn -15.820019 +jz -15.831504 +rf -15.866840 +bh -15.892620 +jh -15.934852 +lg -15.956101 +kk -16.042566 +bz -16.049074 +dj -16.072088 +ey -16.089126 +nh -16.098846 +kh -16.100347 +ws -16.126875 +md -16.144577 +nr -16.304086 +ww -16.374994 +mv -16.403436 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/che b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/che new file mode 100644 index 00000000000..52efab4a470 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/che @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 6798735 +#UNIQUE_BIGRAMS 727 +#UNSEEN_LOG_PROB -16.786981 +#EXPECTED_SCORE -6.770835 +н_ -3.723948 +ан -4.300874 +ин -5.036212 +а_ -5.064677 +ла -5.158767 +та -5.258652 +_к -5.845318 +ра -5.870237 +ро -5.981919 +ер -6.067279 +ос -6.145292 +ии -6.146299 +вл -6.158585 +ог -6.161985 +_и -6.193195 +гӏ -6.221006 +ст -6.221086 +_э -6.238364 +_ю -6.256679 +эв -6.275789 +у_ -6.280549 +_р -6.282794 +къ -6.284943 +ио -6.311596 +юк -6.315579 +ъа -6.318354 +ош -6.335199 +ӏу -6.335971 +ӏо -6.336485 +шт -6.345533 +кӏ -6.348625 +_о -6.350823 +ци -6.362090 +де -6.377516 +си -6.389332 +ед -6.394642 +ас -6.398593 +сс -6.411017 +_ф -6.417279 +ац -6.428750 +фе -6.431953 +об -6.508768 +бл -6.521968 +ка -6.727224 +ов -6.967931 +ск -7.150302 +ва -7.386684 +на -7.904807 +ар -7.946285 +_м -7.979373 +ал -8.071012 +ол -8.101461 +но -8.205483 +ни -8.259993 +ор -8.269129 +_с -8.277862 +во -8.301702 +_п -8.303847 +ко -8.308885 +да -8.331333 +_в -8.411939 +ло -8.432319 +_т -8.510953 +ен -8.567391 +_б -8.636983 +_г -8.638759 +и_ -8.676984 +_н -8.755145 +ма -8.761486 +ли -8.767669 +ве -8.828495 +о_ -8.839729 +ел -8.848114 +ки -8.929892 +ме -8.979480 +ле -9.013402 +ха -9.014718 +_а -9.016695 +мо -9.034723 +нс -9.040298 +_д -9.139611 +го -9.207363 +ре -9.212138 +за -9.264815 +_л -9.269915 +аи -9.287046 +ль -9.303981 +хь -9.428001 +ис -9.432392 +га -9.440479 +ев -9.441216 +ад -9.459326 +ах -9.484947 +ми -9.485859 +ет -9.492569 +ти -9.500848 +ех -9.504388 +ур -9.506702 +од -9.537279 +ав -9.611694 +ир -9.616850 +ь_ -9.632429 +ам -9.710460 +ри -9.731772 +ом -9.732494 +ич -9.763513 +вс -9.770724 +ьн -9.785256 +е_ -9.785817 +пе -9.789381 +бо -9.820510 +то -9.823006 +са -9.850169 +не -9.862759 +_ш -9.863945 +хо -9.922254 +лу -9.943827 +оь -9.957687 +_х -9.960433 +рх -9.965304 +ба -9.979158 +ои -10.000519 +ӏа -10.018235 +сл -10.033054 +тв -10.051627 +гд -10.069529 +тр -10.088811 +до -10.096922 +ку -10.105313 +_у -10.121059 +вк -10.126030 +ил -10.138175 +по -10.140329 +_я -10.142486 +ьр -10.143926 +нг -10.153561 +рт -10.154045 +па -10.160345 +чо -10.163019 +рс -10.171804 +пс -10.193009 +гу -10.210251 +он -10.212515 +ша -10.220595 +ат -10.230503 +гр -10.233821 +ик -10.268475 +ян -10.291427 +те -10.297290 +хк -10.298626 +ьс -10.305860 +кр -10.308818 +кх -10.317457 +яр -10.326420 +се -10.334891 +сн -10.339833 +со -10.373780 +ок -10.438563 +ря -10.515372 +уь -10.516926 +бе -10.524407 +му -10.529730 +им -10.532242 +жа -10.540752 +лл -10.548358 +ул -10.552495 +ӏе -10.571099 +тт -10.573360 +ив -10.594531 +кв -10.603087 +иж -10.605069 +ду -10.608709 +ит -10.613356 +иц -10.617017 +ун -10.625373 +че -10.639182 +жн -10.669274 +ип -10.673427 +ец -10.680723 +ту -10.686307 +ди -10.687356 +_ч -10.688756 +ца -10.706022 +ви -10.708150 +ое -10.732855 +уг -10.737195 +_ц -10.749198 +рм -10.772765 +ш_ -10.774622 +я_ -10.781329 +ес -10.785817 +ук -10.805811 +рж -10.860390 +ье -10.875460 +бр -10.875858 +см -10.879851 +иг -10.902825 +рг -10.908524 +_з -10.938612 +ге -10.947384 +ез -10.954104 +аш -10.995528 +ак -11.031943 +ек -11.052077 +ая -11.056138 +рд -11.063840 +вг -11.072039 +яз -11.090430 +ше -11.099248 +би -11.129354 +жи -11.132209 +ус -11.142726 +бу -11.159617 +чи -11.178674 +лг -11.208493 +дм -11.237916 +еж -11.263249 +уб -11.308279 +ьа -11.328328 +хи -11.328874 +уш -11.332153 +уд -11.332700 +ча -11.334343 +ыи -11.339283 +цк -11.340933 +г_ -11.346448 +из -11.347554 +шк -11.390773 +ши -11.423039 +юь -11.472230 +пр -11.494711 +нк -11.508246 +м_ -11.508864 +кс -11.510102 +уо -11.528790 +ем -11.556005 +пу -11.570130 +от -11.619352 +вы -11.634114 +ьш -11.643588 +аь -11.643588 +рь -11.674467 +св -11.679331 +рн -11.686307 +цу -11.689107 +ох -11.707440 +ег -11.718840 +чу -11.741912 +ля -11.744822 +тӏ -11.747008 +сп -11.770539 +зе -11.815721 +рк -11.820318 +нц -11.838854 +р_ -11.852914 +мс -11.868698 +ру -11.885460 +оп -11.887067 +ги -11.892704 +нд -11.899173 +ья -11.906486 +т_ -11.925345 +аг -12.039516 +су -12.054783 +оз -12.062024 +еп -12.068389 +ны -12.079367 +яб -12.083045 +лт -12.103444 +лх -12.108120 +ня -12.109995 +хе -12.117519 +аб -12.148976 +тю -12.150905 +оч -12.170336 +уз -12.175234 +лк -12.191023 +ху -12.194003 +еч -12.203980 +ач -12.203980 +_е -12.203980 +их -12.227193 +ие -12.241508 +мб -12.258043 +ть -12.301301 +же -12.315292 +ыш -12.319624 +аз -12.326148 +ее -12.326148 +дл -12.337085 +зо -12.341484 +ае -12.367039 +шо -12.374907 +др -12.374907 +ух -12.393054 +ры -12.394196 +вн -12.395339 +хӏ -12.430048 +пи -12.445352 +ъ_ -12.454852 +нз -12.481302 +ож -12.484947 +эл -12.493487 +ьм -12.499618 +нь -12.500848 +цх -12.508246 +нб -12.509483 +ут -12.529417 +хн -12.534444 +уч -12.579192 +еш -12.580491 +ьл -12.588310 +ап -12.614686 +нт -12.616017 +ке -12.618684 +лы -12.626714 +_ж -12.652441 +кл -12.659288 +зн -12.703897 +иш -12.731051 +ид -12.747008 +гл -12.758726 +еб -12.777972 +ум -12.798989 +юр -12.805051 +хч -12.809614 +зи -12.812664 +юм -12.818784 +ьи -12.828012 +х_ -12.829556 +кк -12.838854 +еи -12.841967 +рр -12.863945 +це -12.886263 +в_ -12.902419 +дн -12.902419 +тл -12.915475 +вр -12.926997 +нн -12.965516 +лм -12.996395 +тя -12.998130 +мц -12.999867 +кт -13.024410 +дс -13.069301 +аф -13.089505 +мы -13.108120 +зу -13.117519 +рл -13.138414 +оц -13.148013 +ще -13.165453 +лю -13.193009 +чк -13.198983 +эр -13.213019 +цо -13.215035 +ды -13.219077 +цӏ -13.225160 +фи -13.241508 +як -13.276875 +дв -13.283207 +пл -13.283207 +лб -13.304517 +к_ -13.304517 +ыс -13.308818 +ыб -13.319624 +уи -13.328328 +рв -13.339283 +ьх -13.345896 +мн -13.365918 +сь -13.383952 +шн -13.390773 +рш -13.404513 +зд -13.411433 +ъи -13.420711 +тс -13.432392 +яя -13.434740 +ты -13.470423 +иб -13.514441 +_ӏ -13.534444 +ын -13.544550 +д_ -13.552177 +рц -13.557284 +иа -13.570130 +мм -13.570130 +ьк -13.572714 +шу -13.575301 +с_ -13.585699 +дж -13.588310 +фо -13.596173 +л_ -13.598803 +ы_ -13.604078 +вя -13.617350 +оу -13.655176 +тк -13.671695 +ыт -13.688406 +ц_ -13.710993 +жь -13.731051 +жо -13.760197 +чх -13.795968 +ые -13.845086 +ъе -13.873468 +ищ -13.886263 +ув -13.889480 +жс -13.892704 +фа -13.915475 +гс -13.945291 +лж -13.958743 +оя -13.975736 +ым -13.992931 +ӏи -14.010334 +аж -14.017355 +щи -14.017355 +ык -14.017355 +хб -14.060210 +рб -14.060210 +жу -14.074783 +лн -14.104378 +кн -14.153803 +тн -14.153803 +чн -14.177199 +зь -14.200980 +мт -14.229229 +ср -14.229229 +лч -14.237403 +ят -14.270570 +сы -14.283207 +п_ -14.300230 +вт -14.317457 +шм -14.330513 +зл -14.330513 +оа -14.352539 +iе -14.365918 +_щ -14.379422 +мк -14.388496 +бы -14.420711 +юг -14.425372 +ну -14.425372 +мр -14.444169 +уж -14.444169 +юб -14.458430 +уе -14.468016 +дӏ -14.477666 +ял -14.482516 +ыл -14.487382 +еа -14.497163 +iа -14.507010 +хв -14.511960 +пы -14.526910 +ео -14.542017 +нч -14.577894 +тi -14.577894 +ям -14.593547 +чь -14.598803 +лд -14.598803 +вь -14.598803 +уп -14.620019 +жд -14.625373 +бс -14.630746 +ау -14.636139 +чч -14.646986 +рз -14.646986 +ьт -14.646986 +юж -14.657916 +ща -14.680027 +яж -14.680027 +мл -14.685608 +юс -14.702481 +чм -14.736833 +ьч -14.748468 +ещ -14.760197 +дк -14.777972 +ч_ -14.795968 +чӏ -14.808092 +iо -14.808092 +шл -14.832649 +ьг -14.832649 +цш -14.857631 +дз -14.876656 +вд -14.876656 +мя -14.889480 +зъ -14.895935 +юх -14.902419 +яс -14.935284 +эз -14.941947 +ьц -14.968914 +бц -15.010334 +хм -15.010334 +рч -15.045783 +ыр -15.052979 +гг -15.052979 +бн -15.082125 +тч -15.119406 +хл -15.126979 +ощ -15.149940 +лц -15.173273 +рп -15.181135 +юч -15.196989 +яч -15.221101 +хт -15.229229 +цн -15.278982 +ъя -15.295955 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ckb b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ckb new file mode 100644 index 00000000000..c978893fd88 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ckb @@ -0,0 +1,321 @@ +#TOTAL_BIGRAMS 195435 +#UNIQUE_BIGRAMS 317 +#UNSEEN_LOG_PROB -14.254401 +#EXPECTED_SCORE -6.954047 +e_ -4.481592 +n_ -4.677350 +in -5.269983 +a_ -5.279413 +di -5.531594 +i_ -5.575273 +er -5.606943 +_d -5.672824 +an -5.679997 +_b -5.737519 +we -5.797841 +_h -5.902579 +_x -6.030882 +_k -6.037655 +r_ -6.103638 +xw -6.144832 +ir -6.168530 +en -6.317175 +he -6.386505 +ki -6.525120 +bi -6.570705 +be -6.611267 +iy -6.657466 +ri -6.666436 +ne -6.740279 +ke -6.755353 +ti -6.765758 +de -6.768170 +re -6.791694 +zi -6.819773 +_w -6.870697 +ye -6.880231 +_m -6.884586 +_s -6.903904 +ek -6.910993 +na -6.934278 +ya -6.950620 +ma -6.968999 +_e -6.977346 +_g -7.064577 +ar -7.092514 +_c -7.098571 +ez -7.107705 +st -7.126149 +ik -7.144832 +se -7.168000 +_n -7.227601 +wa -7.238707 +te -7.251024 +ur -7.264581 +mi -7.333155 +rd -7.348713 +is -7.366876 +ku -7.395177 +al -7.407657 +av -7.434222 +id -7.436778 +va -7.515633 +_a -7.523761 +at -7.555349 +ha -7.581976 +ib -7.583391 +k_ -7.611988 +ro -7.661944 +em -7.702885 +ni -7.705965 +ba -7.707507 +el -7.719904 +es -7.729272 +_t -7.729272 +t_ -7.737126 +_r -7.745022 +da -7.746607 +_z -7.781913 +ka -7.790060 +ra -7.796610 +le -7.836549 +ci -7.867246 +la -7.872426 +_p -7.928871 +ge -7.987615 +im -7.997013 +li -8.000790 +sa -8.006474 +ev -8.010275 +me -8.029435 +ta -8.066554 +ed -8.086481 +v_ -8.088489 +ji -8.088489 +ve -8.092514 +ew -8.160588 +as -8.214386 +et -8.288617 +nd -8.290927 +_y -8.352328 +iv -8.359583 +un -8.371758 +m_ -8.473042 +ay -8.518338 +d_ -8.537410 +za -8.567901 +bu -8.570705 +si -8.610545 +ca -8.660450 +pi -8.669439 +go -8.684546 +vi -8.740279 +_j -8.743439 +ak -8.778668 +wi -8.798252 +gi -8.841620 +il -8.841620 +eb -8.851815 +ih -8.862084 +hi -8.918118 +ng -8.921693 +ot -8.983872 +s_ -9.068535 +ey -9.100596 +l_ -9.150065 +o_ -9.184012 +am -9.188312 +_v -9.227601 +pe -9.258917 +u_ -9.263446 +ov -9.295559 +ce -9.314234 +xe -9.328402 +aw -9.347511 +ok -9.371758 +az -9.371758 +ej -9.396420 +or -9.467805 +_l -9.488866 +yi -9.581976 +rt -9.627962 +ga -9.633815 +ze -9.657466 +gu -9.737126 +ah -9.788427 +oj -9.855230 +bo -9.918118 +ex -9.968999 +z_ -9.998901 +rk -10.006474 +rm -10.029435 +pa -10.068535 +fe -10.068535 +eh -10.076483 +eg -10.175450 +ij -10.201290 +sh -10.236479 +ja -10.245412 +_i -10.254401 +xa -10.272549 +je -10.290927 +ix -10.309543 +g_ -10.318941 +oz -10.347511 +_f -10.366876 +ol -10.406404 +rs -10.457388 +hu -10.467805 +du -10.488866 +_o -10.576329 +ws -10.576329 +h_ -10.633815 +do -10.645592 +lm -10.669439 +mu -10.743439 +ko -10.743439 +qe -10.768974 +rb -10.875890 +ax -10.889829 +j_ -10.903904 +nc -10.903904 +af -10.918118 +ab -10.932473 +mb -10.976416 +ny -10.991367 +su -10.991367 +it -10.991367 +ig -11.006474 +ad -11.021740 +ho -11.021740 +qa -11.052767 +om -11.084476 +xi -11.100596 +ic -11.150065 +aq -11.166938 +ul -11.218777 +rw -11.236479 +so -11.236479 +aj -11.254401 +yb -11.254401 +lk -11.272549 +_q -11.290927 +vb -11.309543 +ec -11.328402 +nb -11.366876 +um -11.426582 +wm -11.510240 +wl -11.510240 +iq -11.553962 +lb -11.553962 +iz -11.576329 +wr -11.599049 +lo -11.622133 +zd -11.645592 +vl -11.669439 +on -11.669439 +os -11.693686 +fa -11.693686 +tu -11.718348 +iw -11.718348 +ef -11.743439 +hw -11.848409 +fk -11.875890 +zo -11.932473 +uy -11.961619 +ht -11.961619 +ft -11.991367 +y_ -11.991367 +if -11.991367 +hl -12.116898 +lu -12.116898 +op -12.116898 +x_ -12.184012 +jn -12.184012 +jm -12.184012 +us -12.218777 +ag -12.218777 +uh -12.218777 +kg -12.328402 +ry -12.328402 +cu -12.406404 +ud -12.406404 +uk -12.406404 +w_ -12.447046 +rv -12.447046 +sk -12.447046 +ei -12.488866 +lp -12.488866 +nj -12.576329 +ru -12.576329 +up -12.576329 +dh -12.576329 +tp -12.576329 +sl -12.576329 +hm -12.622133 +zg -12.718348 +sm -12.821442 +xo -12.875890 +rg -12.875890 +hs -12.932473 +gr -12.991367 +fn -12.991367 +nw -12.991367 +mo -12.991367 +rf -12.991367 +xt -12.991367 +vn -13.052767 +vd -13.116898 +qi -13.116898 +hr -13.116898 +br -13.184012 +rh -13.184012 +ip -13.184012 +ck -13.254401 +tg -13.254401 +xs -13.328402 +hd -13.328402 +ns -13.328402 +ls -13.406404 +xn -13.406404 +ai -13.488866 +c_ -13.488866 +ea -13.488866 +sg -13.488866 +km -13.488866 +q_ -13.576329 +po -13.576329 +my -13.576329 +jo -13.669439 +rz -13.669439 +vg -13.768974 +oh -13.768974 +no -13.768974 +wq -13.768974 +ie -13.768974 +cb -13.875890 +hb -13.875890 +f_ -13.875890 +rj -13.875890 +rl -13.875890 +ll -13.991367 +nr -13.991367 +oc -13.991367 +ds -13.991367 +b_ -13.991367 +md -13.991367 +sp -13.991367 +jh -13.991367 +ac -14.116898 +ub -14.116898 +wd -14.116898 +sn -14.116898 +rx -14.116898 +zk -14.116898 +mg -14.116898 +rp -14.254401 +uj -14.254401 +ap -14.254401 +ju -14.254401 +_u -14.254401 +xu -14.254401 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/cmn b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/cmn new file mode 100644 index 00000000000..774265fc754 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/cmn @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 23567433 +#UNIQUE_BIGRAMS 35190 +#UNSEEN_LOG_PROB -16.016627 +#EXPECTED_SCORE -12.939902 +的_ -6.281547 +_的 -6.572575 +_年 -6.873297 +年_ -7.201137 +_中 -7.487564 +_一 -7.800171 +一_ -7.860949 +中_ -7.901884 +_是 -7.929048 +在_ -7.958148 +_日 -8.026160 +_大 -8.118345 +_在 -8.143587 +_月 -8.204943 +国_ -8.219941 +月_ -8.262563 +人_ -8.272050 +國_ -8.324501 +大_ -8.371289 +日_ -8.489169 +面_ -8.491106 +_人 -8.499013 +_分 -8.524394 +地_ -8.530357 +是_ -8.577075 +_国 -8.597630 +于_ -8.645806 +_國 -8.682836 +_面 -8.739140 +於_ -8.759237 +_頁 -8.812957 +頁面 -8.836019 +_生 -8.838847 +_地 -8.875208 +_于 -9.004934 +_主 -9.014716 +_以 -9.018553 +生_ -9.071813 +_建 -9.098544 +分_ -9.114471 +_西 -9.147835 +_南 -9.152530 +中国 -9.157381 +有_ -9.167061 +為_ -9.169914 +南_ -9.177798 +頁_ -9.191906 +市_ -9.246411 +以_ -9.252932 +_北 -9.278593 +立_ -9.294150 +_成 -9.298114 +代_ -9.315249 +_目 -9.334287 +_於 -9.335355 +_為 -9.344001 +和_ -9.349741 +_出 -9.352420 +物_ -9.354622 +西_ -9.374191 +_文 -9.375573 +作_ -9.381767 +_有 -9.411557 +_不 -9.413559 +_台 -9.419038 +_政 -9.419080 +_作 -9.420506 +及_ -9.434797 +海_ -9.434966 +_公 -9.444148 +_本 -9.450430 +部_ -9.454289 +_海 -9.466320 +本_ -9.474963 +成_ -9.476053 +文_ -9.478846 +北_ -9.487344 +前_ -9.496557 +_前 -9.508546 +目_ -9.522885 +_存 -9.530878 +個_ -9.537550 +家_ -9.560709 +存_ -9.561912 +主_ -9.563671 +_立 -9.573992 +区_ -9.575766 +_等 -9.576093 +存在 -9.585045 +_民 -9.594385 +斯_ -9.608843 +學_ -9.626008 +名_ -9.629398 +等_ -9.639811 +民_ -9.647106 +学_ -9.658242 +州_ -9.693795 +_山 -9.694100 +政_ -9.696993 +行_ -9.709495 +_之 -9.728532 +_第 -9.745090 +_小 -9.753995 +灣_ -9.765991 +港_ -9.766470 +下_ -9.779700 +出_ -9.784929 +不_ -9.810701 +布_ -9.838175 +類_ -9.869327 +建_ -9.874087 +_上 -9.877423 +一個 -9.879785 +世_ -9.898418 +_相 -9.900465 +上_ -9.905916 +與_ -9.907679 +_香 -9.915520 +_基 -9.918953 +山_ -9.930735 +_布 -9.933905 +_學 -9.935163 +用_ -9.940386 +香港 -9.945267 +_行 -9.947743 +_代 -9.951314 +_部 -9.953376 +_美 -9.959397 +_位 -9.973606 +會_ -9.985845 +省_ -9.997562 +_和 -10.000443 +_斯 -10.002263 +_其 -10.006035 +_学 -10.010764 +_個 -10.011079 +_新 -10.017727 +品_ -10.035478 +_或 -10.039531 +任_ -10.043143 +_由 -10.045147 +路_ -10.065141 +_世 -10.065469 +了_ -10.073032 +_下 -10.073296 +_可 -10.084751 +區_ -10.094422 +台_ -10.099115 +時_ -10.108410 +_工 -10.115999 +的一 -10.121035 +史_ -10.124242 +_同 -10.133358 +个_ -10.133977 +道_ -10.135216 +士_ -10.137076 +至_ -10.137903 +为_ -10.138593 +理_ -10.144886 +分布 -10.150024 +基_ -10.158535 +日本 -10.163019 +_市 -10.170549 +_所 -10.173727 +合_ -10.181526 +分類 -10.183518 +_十 -10.188509 +_關 -10.194020 +由_ -10.195742 +_標 -10.203445 +或_ -10.211914 +是一 -10.220434 +_與 -10.223578 +標_ -10.227903 +_加 -10.229592 +公_ -10.236887 +_科 -10.242290 +_合 -10.252260 +第_ -10.254427 +作品 -10.262675 +_時 -10.264481 +台灣 -10.267496 +面分 -10.286255 +子_ -10.286255 +之_ -10.290159 +位_ -10.294842 +_名 -10.305726 +_三 -10.306268 +_物 -10.310538 +_參 -10.316536 +_港 -10.321776 +相_ -10.326170 +_米 -10.338640 +科_ -10.345872 +长_ -10.361975 +人物 -10.363587 +關_ -10.365685 +_二 -10.371675 +建立 -10.386758 +_法 -10.391683 +所_ -10.397039 +院_ -10.398856 +_東 -10.408225 +期_ -10.419080 +要_ -10.419835 +利_ -10.426138 +洲_ -10.428920 +_電 -10.433146 +府_ -10.433230 +治_ -10.434839 +一个 -10.438742 +_用 -10.440613 +後_ -10.442997 +教_ -10.447862 +_天 -10.449402 +_为 -10.455406 +站_ -10.455836 +_路 -10.463941 +_长 -10.467403 +加_ -10.467837 +_州 -10.473135 +_教 -10.478016 +_任 -10.478889 +_他 -10.493465 +小作 -10.494877 +十_ -10.514622 +_各 -10.520725 +_家 -10.528299 +法_ -10.531920 +_英 -10.535277 +入_ -10.535641 +員_ -10.535641 +_正 -10.543020 +定_ -10.544025 +等地 -10.548143 +朝_ -10.548602 +工_ -10.549060 +_江 -10.561773 +_多 -10.562698 +以及 -10.571335 +_共 -10.571801 +_高 -10.581805 +_及 -10.586974 +_四 -10.588387 +_列 -10.590179 +出生 -10.591029 +陆_ -10.594811 +_德 -10.598318 +可_ -10.608508 +其_ -10.611666 +事_ -10.616367 +中國 -10.616943 +林_ -10.617616 +_提 -10.622723 +美_ -10.627654 +目標 -10.629204 +到_ -10.630174 +方_ -10.633671 +大陆 -10.638640 +二_ -10.640788 +国大 -10.644213 +不存 -10.644801 +相關 -10.645095 +_东 -10.645880 +度_ -10.650595 +_會 -10.651875 +球_ -10.654044 +面不 -10.659974 +縣_ -10.660469 +在頁 -10.661163 +標頁 -10.661163 +_自 -10.665531 +_灣 -10.681729 +_个 -10.683037 +_發 -10.688280 +_最 -10.692326 +平_ -10.701267 +化_ -10.702797 +_理 -10.706065 +表_ -10.723141 +_定 -10.736343 +_河 -10.741050 +东_ -10.752093 +關的 -10.758124 +軍_ -10.758548 +列_ -10.760458 +_事 -10.767484 +_接 -10.774544 +次_ -10.778517 +_後 -10.779377 +克_ -10.780777 +江_ -10.789960 +_球 -10.793976 +共_ -10.794302 +_方 -10.810481 +正_ -10.811691 +接_ -10.817203 +亞_ -10.819967 +可以 -10.820078 +_治 -10.821960 +同_ -10.822958 +_利 -10.823845 +_直 -10.827845 +德_ -10.828736 +_参 -10.829181 +_使 -10.831857 +地区 -10.833866 +_巴 -10.833978 +尔_ -10.845759 +里_ -10.848465 +种_ -10.862188 +三_ -10.865496 +動_ -10.874202 +_安 -10.874662 +東_ -10.875351 +_平 -10.876272 +_原 -10.879266 +县_ -10.879843 +界_ -10.884465 +_林 -10.886433 +会_ -10.888056 +_特 -10.897250 +_維 -10.900172 +種_ -10.902162 +_被 -10.909326 +_广 -10.909562 +拉_ -10.913453 +京_ -10.915107 +_陆 -10.928884 +_明 -10.939905 +_逝 -10.942071 +后_ -10.944120 +議_ -10.949195 +目前 -10.958544 +逝世 -10.958910 +_請 -10.959885 +_引 -10.964526 +指_ -10.967219 +生长 -10.969795 +_指 -10.970532 +外_ -10.971392 +線_ -10.971638 +_克 -10.971884 +_區 -10.972007 +布于 -10.982373 +_了 -10.984976 +他_ -10.986466 +_亞 -10.987459 +_朝 -10.993063 +小_ -10.995061 +多_ -10.995061 +_太 -10.996561 +美國 -11.003331 +_要 -11.003331 +業_ -11.008492 +_此 -11.009249 +_全 -11.011016 +_先 -11.012407 +_尼 -11.013672 +陆的 -11.017093 +_這 -11.017093 +政府 -11.018616 +特_ -11.018743 +时_ -11.026256 +_取 -11.037535 +_華 -11.037921 +_化 -11.039338 +被_ -11.039724 +式_ -11.045665 +司_ -11.051240 +村_ -11.058271 +新_ -11.059577 +_更 -11.067044 +數_ -11.069805 +的地 -11.069936 +_來 -11.073362 +尼_ -11.073362 +_數 -11.077986 +_道 -11.079442 +使用 -11.079972 +香_ -11.082492 +_軍 -11.082625 +長_ -11.088478 +类_ -11.088745 +_区 -11.088878 +_交 -11.089679 +_城 -11.097571 +_議 -11.098913 +_拉 -11.099854 +亚_ -11.102678 +自_ -11.104833 +_歷 -11.111048 +_開 -11.117290 +來_ -11.131915 +_考 -11.136731 +華_ -11.140595 +天_ -11.143778 +_也 -11.152809 +_尚 -11.155459 +_並 -11.156437 +_表 -11.156717 +_种 -11.157835 +請_ -11.162037 +_未 -11.163721 +_而 -11.164564 +者_ -11.167800 +_首 -11.167941 +始_ -11.171607 +_次 -11.172596 +_期 -11.181241 +高_ -11.181952 +_条 -11.182094 +_戰 -11.182521 +系_ -11.182521 +军_ -11.182806 +条_ -11.185940 +_通 -11.187938 +_包 -11.189939 +視_ -11.191942 +_金 -11.198401 +_福 -11.198833 +得_ -11.202579 +_清 -11.206769 +引_ -11.211987 +_車 -11.215622 +考_ -11.220872 +爾_ -11.223505 +間_ -11.225409 +主要 -11.225555 +_水 -11.225849 +未_ -11.227756 +示_ -11.230989 +場_ -11.237035 +位於 -11.237478 +_时 -11.249649 +_重 -11.250842 +_湖 -11.257571 +直_ -11.267496 +使_ -11.268402 +_入 -11.272334 +取_ -11.280990 +尚_ -11.281752 +_印 -11.281752 +_動 -11.286332 +年代 -11.289546 +發_ -11.295534 +電_ -11.298153 +行政 -11.299386 +_改 -11.302939 +人工 -11.303558 +_社 -11.306346 +条目 -11.308518 +先_ -11.313962 +族_ -11.316458 +培_ -11.317551 +歷史 -11.322716 +_栽 -11.327271 +栽培 -11.327271 +_委 -11.328057 +由人 -11.338799 +維基 -11.338799 +引种 -11.341179 +元_ -11.341338 +拔_ -11.341815 +車_ -11.342450 +_因 -11.342609 +种栽 -11.344996 +栽_ -11.344996 +工引 -11.345155 +第一 -11.351380 +_曾 -11.354903 +_到 -11.355063 +戰_ -11.355223 +主条 -11.361975 +_總 -11.363426 +_县 -11.369082 +校_ -11.371512 +_常 -11.372161 +_元 -11.374922 +海拔 -11.375736 +公司 -11.377852 +更_ -11.378830 +_省 -11.380297 +长于 -11.380624 +通_ -11.382257 +安_ -11.383074 +大學 -11.390779 +體_ -11.390779 +_系 -11.390779 +_类 -11.393576 +城_ -11.404819 +尚未 -11.404985 +人民 -11.405815 +王_ -11.406812 +原_ -11.412975 +e_ -11.413476 +_進 -11.414144 +_現 -11.421009 +_士 -11.426728 +_種 -11.429258 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/cym b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/cym new file mode 100644 index 00000000000..5a59f07e404 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/cym @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 6513014 +#UNIQUE_BIGRAMS 525 +#UNSEEN_LOG_PROB -18.865122 +#EXPECTED_SCORE -7.599537 +d_ -5.070268 +dd -5.280326 +n_ -5.526890 +_g -5.708227 +_c -6.106638 +yd -6.116671 +l_ -6.128710 +an -6.198394 +ed -6.290789 +r_ -6.316157 +_d -6.407466 +yn -6.440406 +wy -6.457942 +ae -6.603667 +th -6.636634 +_a -6.671072 +en -6.733131 +ll -6.748983 +di -6.750723 +ol -6.786051 +_m -6.796823 +u_ -6.798720 +od -6.876176 +er -6.893401 +s_ -6.896617 +_f -6.957229 +ar -7.041590 +ia -7.051870 +ad -7.070983 +ai -7.105219 +g_ -7.115933 +_b -7.116548 +ga -7.134706 +h_ -7.159224 +da -7.168562 +_s -7.168945 +el -7.186294 +ch -7.215493 +_r -7.267718 +on -7.281540 +ne -7.286615 +_h -7.289593 +_l -7.311769 +ra -7.337154 +oe -7.347073 +_e -7.363030 +la -7.383781 +_p -7.414175 +ei -7.439752 +ri -7.443411 +in -7.482807 +_y -7.521966 +cy -7.526492 +ro -7.544492 +ma -7.554326 +fe -7.562594 +ha -7.579739 +ym -7.594604 +gy -7.606254 +we -7.615086 +re -7.617085 +i_ -7.621833 +au -7.635246 +f_ -7.638994 +le -7.664788 +_t -7.678427 +do -7.680925 +e_ -7.687988 +_n -7.698440 +gw -7.702725 +io -7.705219 +ni -7.733461 +id -7.754689 +no -7.771966 +_o -7.772111 +et -7.797907 +or -7.808842 +de -7.813469 +hy -7.822666 +_w -7.854764 +ys -7.862218 +go -7.885234 +yf -7.893532 +ir -7.902408 +es -7.912033 +na -7.916200 +he -7.919629 +af -7.927912 +ff -7.964791 +ef -7.983338 +ng -8.013357 +rh -8.018058 +a_ -8.046531 +li -8.083065 +al -8.084508 +wr -8.094826 +il -8.124192 +o_ -8.128772 +ca -8.133803 +w_ -8.150637 +nn -8.151897 +dy -8.154041 +wn -8.159287 +fo -8.179824 +nw -8.191497 +eg -8.193895 +nt -8.201634 +aw -8.212501 +ly -8.254027 +wa -8.255177 +ho -8.262369 +st -8.272608 +if -8.274870 +me -8.347903 +y_ -8.365986 +fa -8.368620 +fr -8.401799 +ry -8.407428 +as -8.409159 +ew -8.409536 +t_ -8.425060 +rd -8.430094 +it -8.445455 +dw -8.449709 +dr -8.459110 +tr -8.464421 +fy -8.473683 +lo -8.483005 +ig -8.505450 +yr -8.515791 +rw -8.517088 +lw -8.528413 +gr -8.537685 +un -8.570151 +ny -8.610101 +nd -8.624628 +is -8.643106 +si -8.645588 +sa -8.655111 +eu -8.700742 +am -8.716869 +hw -8.720789 +ge -8.739413 +og -8.752251 +ta -8.798548 +se -8.819811 +rt -8.825528 +by -8.858152 +os -8.858358 +_i -8.871474 +sg -8.873758 +yw -8.903469 +be -8.929369 +hi -8.939231 +wi -8.978469 +pe -8.994084 +bl -9.044307 +te -9.048992 +iw -9.058646 +m_ -9.066106 +mr -9.115626 +br -9.139539 +ur -9.141414 +ce -9.191432 +fi -9.206795 +at -9.216382 +gl -9.217305 +co -9.222059 +sy -9.239494 +bo -9.323004 +_u -9.336688 +rf -9.394997 +ac -9.397983 +pa -9.413911 +ba -9.419058 +ti -9.437984 +so -9.460436 +lu -9.488803 +yl -9.496462 +ru -9.561422 +wl -9.563934 +b_ -9.593406 +ag -9.593919 +ob -9.610966 +du -9.616694 +mo -9.660659 +to -9.697704 +fn -9.698992 +mu -9.734216 +ud -9.752251 +pr -9.756460 +rc -9.796281 +hr -9.800028 +eb -9.824925 +hu -9.825126 +dl -9.832579 +fl -9.906761 +tu -9.976906 +mi -9.981153 +rs -9.991489 +eo -10.019034 +of -10.022717 +ic -10.026177 +gh -10.051341 +yg -10.052517 +ie -10.054400 +rn -10.063378 +gi -10.066463 +yc -10.103269 +ab -10.139039 +ty -10.141789 +sn -10.195323 +_j -10.203657 +nh -10.216251 +cr -10.270212 +rl -10.282850 +mw -10.288657 +ws -10.309026 +ec -10.314093 +wo -10.322862 +nr -10.347470 +ul -10.380456 +rb -10.397385 +em -10.399179 +om -10.419968 +po -10.436142 +fu -10.484512 +us -10.488007 +fw -10.489599 +yt -10.500147 +uo -10.533903 +p_ -10.538179 +ea -10.555409 +nu -10.557077 +ml -10.602848 +yh -10.611833 +c_ -10.620525 +md -10.625415 +pl -10.644435 +rg -10.656899 +my -10.665867 +op -10.677792 +oc -10.703048 +nc -10.711566 +wd -10.722005 +sw -10.732142 +df -10.751869 +ua -10.774583 +bw -10.788228 +ya -10.919361 +lf -10.928830 +ns -10.932721 +wm -10.944459 +lt -10.945333 +ph -10.950145 +jo -10.974898 +oh -10.976236 +rr -11.008272 +im -11.008729 +lm -11.115749 +ap -11.160681 +hn -11.197662 +ld -11.211253 +mp -11.248492 +mh -11.254974 +uc -11.256599 +nf -11.262573 +ow -11.275144 +cl -11.293931 +gu -11.307341 +ib -11.329972 +wp -11.342572 +ct -11.358769 +ot -11.455607 +sb -11.465595 +pw -11.508835 +pi -11.546106 +dn -11.567460 +ug -11.571499 +su -11.595975 +ms -11.602848 +cw -11.610447 +lc -11.610447 +mg -11.612526 +rm -11.619479 +uw -11.627166 +ey -11.633486 +ah -11.658330 +vi -11.680698 +hl -11.683609 +ci -11.690182 +bi -11.696049 +lb -11.701941 +wc -11.707116 +yb -11.730259 +ss -11.785489 +wg -11.833186 +sh -11.838043 +nl -11.885863 +ja -11.906123 +bu -11.919790 +av -12.024792 +uf -12.052752 +oi -12.118209 +tw -12.139039 +pu -12.168308 +ep -12.204441 +ou -12.226564 +wb -12.263117 +gf -12.358769 +k_ -12.364599 +eh -12.383411 +hd -12.425441 +ut -12.494064 +pt -12.510773 +ft -12.598720 +ts -12.602848 +ee -12.632079 +mb -12.641956 +ve -12.663350 +ue -12.686527 +mn -12.690914 +um -12.695315 +_v -12.717522 +va -12.720509 +tb -12.790973 +ip -12.795690 +tl -12.808345 +nb -12.821113 +oy -12.853534 +ev -12.900184 +mc -12.989235 +_k -13.000083 +hg -13.066938 +ao -13.091862 +ki -13.151078 +ke -13.159160 +ay -13.169327 +oo -13.181623 +tt -13.187811 +rp -13.210728 +ck -13.225503 +lg -13.231882 +fd -13.306219 +sl -13.322011 +dg -13.365767 +ds -13.365767 +sc -13.370451 +py -13.389341 +ii -13.396489 +cs -13.418148 +dh -13.427880 +ub -13.482609 +cu -13.505611 +oa -13.528985 +bb -13.550086 +wf -13.587770 +rk -13.601471 +sf -13.615303 +iy -13.618086 +lh -13.666227 +hf -13.680698 +ls -13.710081 +je -13.798844 +yo -13.808345 +tg -13.817910 +x_ -13.840478 +sm -13.976682 +tn -14.001899 +ok -14.042437 +ex -14.046179 +bc -14.049931 +iu -14.127099 +ks -14.143041 +ps -14.221266 +iv -14.238289 +wt -14.295044 +sr -14.299504 +cn -14.308464 +ui -14.331113 +nk -14.354123 +gn -14.372799 +fb -14.440137 +up -14.568805 +mm -14.606988 +ov -14.680698 +sp -14.692379 +nm -14.728003 +lp -14.795690 +ih -14.802004 +yp -14.808345 +ju -14.860107 +ak -14.906973 +uy -14.955414 +ht -14.955414 +za -14.962469 +gb -15.049931 +wh -15.057465 +hb -15.080305 +mf -15.087999 +az -15.135048 +hs -15.183683 +mt -15.191950 +sd -15.208629 +ik -15.225503 +hm -15.242576 +xa -15.251190 +vo -15.268572 +ka -15.268572 +iz -15.286166 +ye -15.349492 +gd -15.368107 +ln -15.454985 +yi -15.568805 +v_ -15.579611 +lk -15.579611 +cc -15.590500 +ox -15.634894 +qu -15.692379 +ky -15.716031 +lr -15.728003 +tc -15.814715 +wu -15.827539 +tm -15.853534 +ix -15.893427 +bn -15.934454 +rv -15.948393 +ji -15.962469 +nj -16.005537 +z_ -16.127099 +tv -16.159160 +ze -16.295044 +dm -16.331113 +aa -16.368107 +tf -16.386966 +lv -16.386966 +db -16.505611 +ax -16.547431 +gm -16.547431 +ek -16.657614 +gs -16.657614 +gt -16.728003 +fg -16.752251 +sk -16.776913 +xi -16.776913 +_q -16.827539 +hp -16.853534 +zo -16.853534 +td -16.880006 +dv -16.880006 +cb -16.906973 +tk -16.934454 +aj -16.934454 +pp -16.934454 +_z -16.991038 +bs -17.020184 +kh -17.020184 +bd -17.049931 +np -17.080305 +xe -17.111332 +dt -17.143041 +kl -17.242576 +rx -17.312966 +yx -17.312966 +uv -17.349492 +cq -17.349492 +pb -17.425441 +zz -17.464969 +_x -17.464969 +uk -17.590500 +kk -17.634894 +xf -17.680698 +wk -17.728003 +yu -17.776913 +uu -17.776913 +ij -17.776913 +ez -17.776913 +dc -17.827539 +tz -17.827539 +vn -17.880006 +zh -17.880006 +nz -17.934454 +ko -17.934454 +uh -17.991038 +zu -17.991038 +gg -17.991038 +yy -18.175462 +xp -18.175462 +kn -18.242576 +zi -18.242576 +pk -18.242576 +ku -18.242576 +cf -18.242576 +rj -18.312966 +vu -18.312966 +xy -18.312966 +tp -18.464969 +pc -18.547431 +vl -18.547431 +vd -18.547431 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/dan b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/dan new file mode 100644 index 00000000000..ba1f8155422 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/dan @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 414283098 +#UNIQUE_BIGRAMS 623 +#UNSEEN_LOG_PROB -18.143652 +#EXPECTED_SCORE -7.377445 +e_ -4.895735 +er -4.959078 +r_ -5.136972 +de -5.228220 +en -5.463488 +t_ -5.542233 +n_ -5.727639 +_s -5.780050 +_d -6.050326 +et -6.115516 +re -6.121880 +or -6.187413 +_f -6.209077 +an -6.343810 +_m -6.353751 +ge -6.420655 +te -6.424933 +_h -6.437481 +nd -6.514141 +le -6.643714 +ti -6.644376 +ar -6.689983 +d_ -6.695072 +st -6.720943 +ed -6.723688 +me -6.726930 +_t -6.755169 +in -6.829745 +_b -6.862908 +g_ -6.915502 +fo -6.919657 +ae -6.919661 +el -6.926968 +il -6.939560 +ne -6.943159 +ve -6.963094 +_k -6.986950 +ig -7.023864 +l_ -7.041359 +_v -7.055956 +s_ -7.070684 +li -7.113372 +ke -7.115969 +se -7.241705 +_a -7.245818 +sk -7.284944 +es -7.299802 +ng -7.342230 +ha -7.406072 +al -7.425241 +ra -7.477707 +om -7.623619 +_o -7.646510 +_l -7.661426 +_i -7.688438 +la -7.710315 +be -7.726947 +ma -7.743431 +va -7.745754 +ri -7.766828 +_p -7.792152 +ll -7.802721 +m_ -7.835363 +_g -7.864410 +ka -7.888717 +_e -7.898314 +ta -7.903924 +_n -7.915039 +is -7.927669 +sa -7.932330 +vi -7.950783 +so -7.973378 +si -7.980157 +on -8.018374 +ns -8.076386 +ag -8.093088 +ik -8.098666 +un -8.133685 +ko -8.149044 +a_ -8.154876 +eg -8.170985 +ol -8.207766 +em -8.255767 +id -8.282459 +_r -8.298535 +_u -8.334840 +bl -8.364572 +he -8.375336 +da -8.391176 +at -8.397554 +og -8.417475 +kk -8.427054 +rs -8.434432 +nt -8.457481 +k_ -8.461397 +od -8.493076 +je -8.535901 +rt -8.542662 +ld -8.570763 +ro -8.598963 +to -8.599059 +ad -8.603660 +dt -8.637346 +fr -8.642103 +ni -8.652273 +ds -8.668546 +ev -8.677181 +tr -8.701076 +gt -8.721113 +am -8.721122 +mm -8.723764 +iv -8.746674 +na -8.768683 +di -8.775789 +ud -8.779376 +rn -8.820471 +mi -8.843267 +_j -8.870807 +tt -8.883969 +ho -8.912878 +ls -8.920089 +ov -8.951782 +hv -8.952351 +fa -8.953948 +v_ -8.956091 +ga -8.959046 +lo -8.967614 +av -8.979852 +pe -9.031486 +ru -9.055363 +rd -9.055710 +it -9.086082 +ek -9.090113 +ft -9.110608 +pr -9.142435 +rk -9.152346 +lt -9.193428 +dr -9.197440 +nn -9.226547 +no -9.227286 +ba -9.230920 +op -9.239708 +io -9.258360 +mo -9.272148 +kr -9.290010 +ku -9.315216 +sp -9.331549 +ej -9.332934 +kt -9.375140 +ie -9.382154 +as -9.400679 +vo -9.404162 +ce -9.425588 +gs -9.461707 +ef -9.466737 +bo -9.500432 +br -9.543162 +us -9.555999 +pa -9.575239 +fi -9.579730 +ul -9.580647 +ss -9.675341 +ur -9.714515 +os -9.720064 +af -9.735828 +gr -9.756996 +_c -9.758371 +go -9.817841 +gi -9.826725 +kl -9.860344 +po -9.869293 +ab -9.877587 +ob -9.891421 +ir -9.911867 +sl -9.914066 +ok -9.992511 +ks -10.002057 +hu -10.009428 +ak -10.017509 +rg -10.018906 +do -10.030981 +rb -10.033238 +dl -10.051707 +pl -10.066627 +mu -10.087319 +fe -10.111509 +i_ -10.132737 +ts -10.137900 +rm -10.140857 +fl -10.172008 +lg -10.173692 +rl -10.189749 +bi -10.194777 +ug -10.209306 +ty -10.214630 +pi -10.219911 +nk -10.253865 +gg -10.259766 +lu -10.307327 +tu -10.369358 +ic -10.378694 +lv -10.400806 +ki -10.431352 +ny -10.435291 +yd -10.441791 +ot -10.448462 +ly -10.456014 +gl -10.480975 +eb -10.485856 +mt -10.504609 +gn -10.505217 +ut -10.519239 +by -10.558235 +pu -10.565209 +rr -10.572089 +im -10.580997 +ia -10.583287 +sm -10.591280 +mp -10.594211 +sv -10.599335 +ja -10.609027 +sy -10.646354 +oj -10.662322 +ub -10.687146 +o_ -10.698189 +ys -10.706079 +jo -10.711808 +rf -10.726372 +yn -10.747308 +yg -10.761215 +rv -10.774421 +p_ -10.787490 +dv -10.801847 +ch -10.836132 +jd -10.841394 +um -10.850708 +dd -10.851518 +su -10.852040 +rh -10.854410 +gh -10.866674 +lk -10.868430 +nu -10.889164 +vn -10.893874 +of -10.934571 +b_ -10.947744 +pp -10.971315 +ye -10.973603 +bu -11.001168 +hj -11.006280 +yr -11.028222 +vd -11.063456 +mb -11.070052 +lb -11.105837 +ci -11.107181 +ap -11.110966 +if -11.148900 +th -11.154438 +tn -11.170185 +ea -11.216013 +ry -11.237688 +fu -11.247848 +uk -11.255371 +hi -11.269289 +co -11.281598 +sn -11.288681 +yk -11.297595 +db -11.298998 +dn -11.311343 +tl -11.328069 +y_ -11.343659 +ec -11.355591 +au -11.358021 +ep -11.362067 +nl -11.364608 +yt -11.365600 +dg -11.367577 +j_ -11.370727 +ff -11.380600 +lm -11.403077 +ml -11.443929 +ue -11.447416 +nh -11.461439 +nc -11.463336 +eh -11.471145 +kn -11.480697 +ua -11.494958 +up -11.501941 +nm -11.547912 +ms -11.557421 +ac -11.630672 +kv -11.638434 +ju -11.649948 +pt -11.682497 +du -11.695732 +gu -11.702726 +yl -11.773732 +ee -11.795551 +_w -11.818800 +oc -11.830953 +oe -11.869498 +ib -11.871206 +ca -11.886274 +u_ -11.904248 +h_ -11.945133 +sh -11.946672 +lf -12.030027 +mr -12.039466 +df -12.046549 +nv -12.091845 +ou -12.097527 +dy -12.122930 +sd -12.135614 +nf -12.180865 +ky -12.199204 +sb -12.208865 +gd -12.211423 +aa -12.258371 +f_ -12.259207 +lp -12.266567 +tj -12.269280 +jl -12.274740 +gy -12.306211 +sc -12.309760 +ai -12.321084 +mf -12.361379 +mk -12.406457 +ps -12.439676 +jr -12.444327 +fy -12.471184 +eu -12.489531 +fs -12.504893 +_y -12.508703 +ct -12.567373 +ip -12.567754 +mh -12.617350 +oo -12.620834 +bt -12.626218 +dk -12.627584 +eo -12.719738 +ck -12.749285 +tv -12.752237 +gj -12.757943 +sj -12.763404 +vs -12.765124 +uc -12.805190 +we -12.833455 +js -12.841254 +uf -12.864179 +cl -12.888398 +bn -12.937928 +gm -12.956797 +dh -12.967998 +sf -12.981650 +bb -12.983990 +vt -12.984272 +hr -13.030959 +pg -13.040463 +pf -13.070284 +nb -13.104503 +mg -13.133062 +vr -13.137135 +vl -13.157290 +yv -13.157895 +fg -13.167251 +mn -13.174927 +gv -13.204404 +ph -13.210564 +dm -13.225763 +aj -13.228100 +ei -13.243249 +lj -13.276380 +wi -13.280636 +dj -13.292572 +yh -13.319375 +sr -13.320087 +vu -13.336959 +fj -13.363433 +x_ -13.394933 +bs -13.397148 +cc -13.462589 +yp -13.463965 +wa -13.465264 +jy -13.465895 +ym -13.468655 +oh -13.492418 +ay -13.512666 +nr -13.524764 +fh -13.524928 +my -13.528709 +rp -13.557054 +pd -13.563068 +oi -13.577257 +rc -13.578705 +bj -13.579387 +c_ -13.580410 +ow -13.591587 +ex -13.670392 +w_ -13.741060 +tf -13.773561 +yb -13.778935 +hy -13.784723 +uv -13.850534 +lh -13.853315 +gb -13.856307 +ew -13.928097 +sg -13.990324 +wo -14.049026 +pc -14.059571 +ui -14.067561 +kj -14.083796 +vf -14.129500 +fv -14.138516 +fd -14.149549 +pm -14.204108 +cy -14.210234 +iu -14.256581 +yo -14.316921 +ah -14.317987 +pb -14.339195 +ey -14.350645 +oa -14.354360 +dp -14.379672 +tk -14.390701 +jt -14.431747 +jn -14.458545 +cr -14.475104 +tb -14.504104 +pn -14.509210 +nj -14.565261 +cu -14.575683 +z_ -14.630893 +mv -14.643850 +hn -14.645902 +lr -14.715024 +uh -14.750388 +ih -14.760502 +tm -14.816977 +za -14.819492 +gf -14.848170 +tw -14.970511 +zi -14.974990 +hl -15.002846 +ax -15.020794 +oy -15.035104 +nz -15.052276 +az -15.060821 +ht -15.103215 +_z -15.122216 +ii -15.129812 +tc -15.180768 +tg -15.183876 +hm -15.194675 +gp -15.205160 +pk -15.211356 +tz -15.223962 +fb -15.270691 +kf -15.292607 +ws -15.315287 +ya -15.352829 +ix -15.368654 +np -15.409902 +wn -15.476453 +xe -15.478996 +ln -15.507425 +tp -15.516538 +ze -15.517354 +md -15.565515 +lc -15.604194 +bh -15.615688 +iz -15.627804 +pv -15.729331 +kh -15.748758 +ao -15.772147 +fk -15.792954 +ox -15.840794 +bm -15.918251 +td -15.928857 +zo -15.943047 +gk -15.949203 +qu -15.958043 +rj -16.012482 +vj -16.052395 +hw -16.066904 +py -16.070973 +vg -16.088581 +aw -16.096123 +dc -16.177151 +xi -16.199253 +uo -16.251546 +ez -16.257535 +jk -16.321976 +wh -16.328839 +xa -16.330847 +_q -16.357792 +uu -16.379598 +kd -16.418722 +mc -16.423000 +ux -16.439618 +vm -16.448934 +fc -16.453927 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/deu b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/deu new file mode 100644 index 00000000000..ae0e8a163ba --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/deu @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 506403942 +#UNIQUE_BIGRAMS 645 +#UNSEEN_LOG_PROB -17.787092 +#EXPECTED_SCORE -7.352522 +n_ -4.930113 +er -4.983595 +en -5.011295 +_d -5.228213 +e_ -5.281875 +r_ -5.504528 +de -5.517938 +ch -5.595458 +t_ -5.791414 +ei -5.848573 +ie -5.976529 +te -6.013532 +_s -6.212067 +in -6.272401 +s_ -6.308563 +ge -6.427589 +nd -6.449521 +_a -6.450739 +un -6.454324 +_e -6.600180 +st -6.647652 +be -6.710450 +_w -6.788992 +ne -6.789423 +di -6.837478 +an -6.859647 +d_ -6.911354 +re -6.927799 +_b -6.977174 +es -6.991805 +_u -7.058507 +he -7.064614 +au -7.078479 +se -7.098597 +ic -7.113888 +it -7.117890 +h_ -7.133447 +_m -7.148136 +_v -7.180764 +_g -7.257396 +is -7.259852 +ur -7.291194 +sc -7.303332 +on -7.334926 +ng -7.466262 +le -7.476478 +al -7.487188 +_n -7.562680 +_f -7.584714 +li -7.597524 +el -7.599979 +ar -7.620290 +m_ -7.643043 +si -7.691175 +as -7.708747 +da -7.728094 +nt -7.732547 +g_ -7.774138 +ss -7.778264 +_k -7.792280 +we -7.802132 +_h -7.842964 +ha -7.854849 +_i -7.862159 +at -7.906401 +or -7.926322 +hr -7.951363 +me -7.956486 +ra -7.963771 +em -7.967528 +rt -7.969228 +vo -7.974927 +us -8.002429 +rd -8.009026 +ht -8.036544 +_z -8.041552 +et -8.041979 +mi -8.049967 +ll -8.107269 +ri -8.109652 +ti -8.124880 +na -8.182710 +la -8.206242 +ta -8.235053 +wi -8.262421 +l_ -8.287943 +ve -8.292673 +ig -8.353646 +ni -8.374030 +ma -8.408841 +_p -8.440693 +nn -8.446947 +_l -8.451044 +wa -8.456021 +ah -8.502670 +uf -8.510825 +eh -8.515059 +ac -8.519946 +ru -8.554020 +rs -8.555839 +_t -8.560577 +lt -8.613938 +zu -8.642726 +_r -8.648011 +ro -8.649255 +eg -8.661565 +ns -8.662862 +ze -8.678194 +fu -8.716416 +ol -8.736044 +f_ -8.750977 +il -8.752593 +_j -8.770162 +ut -8.796922 +i_ -8.815290 +eu -8.843738 +ab -8.859418 +uc -8.881637 +tr -8.908570 +am -8.959967 +so -8.972095 +ag -8.997593 +ls -8.997998 +eb -9.016564 +fe -9.021775 +ts -9.042260 +ke -9.064053 +ko -9.077250 +tt -9.120203 +hi -9.130202 +_o -9.131302 +sa -9.147809 +ed -9.153487 +tz -9.160169 +ir -9.230128 +hl -9.242879 +rn -9.246754 +ka -9.256965 +sp -9.277592 +pr -9.280927 +wu -9.289026 +os -9.291471 +fa -9.308018 +tu -9.324582 +um -9.331243 +gr -9.339502 +bi -9.377651 +om -9.429693 +io -9.430222 +ba -9.471716 +ft -9.493339 +kt -9.501210 +mm -9.502034 +ub -9.502939 +ho -9.531885 +hn -9.562123 +oc -9.578351 +ga -9.601670 +gt -9.606577 +ja -9.623715 +nu -9.652946 +rg -9.663792 +to -9.678104 +du -9.687266 +a_ -9.692037 +no -9.726905 +ck -9.750011 +pa -9.751111 +rk -9.760863 +fr -9.798230 +af -9.832811 +lu -9.889603 +ih -9.897331 +nz -9.898443 +k_ -9.923419 +wo -9.925194 +fo -9.934057 +rl -9.940459 +ec -9.950803 +gi -9.957951 +rb -9.966699 +tl -9.968390 +ef -10.003845 +lo -10.012302 +ue -10.049222 +z_ -10.049394 +do -10.058182 +im -10.060574 +nk -10.085177 +gl -10.117871 +ad -10.120784 +fi -10.129929 +rc -10.136882 +o_ -10.141782 +ff -10.147390 +br -10.156049 +zw -10.157145 +pe -10.175178 +mu -10.181874 +_c -10.195295 +rh -10.199694 +mo -10.200606 +zi -10.203630 +oh -10.208452 +bu -10.212421 +od -10.226299 +vi -10.233988 +ot -10.242204 +ew -10.254141 +pi -10.286631 +ku -10.297902 +rf -10.310849 +id -10.318094 +ug -10.319047 +of -10.342898 +hm -10.370466 +uh -10.381992 +dr -10.430088 +zt -10.433725 +ik -10.452668 +ek -10.458529 +ld -10.461101 +hu -10.463999 +kl -10.467488 +nf -10.471730 +rm -10.476175 +po -10.481122 +rz -10.482442 +tw -10.551587 +og -10.558557 +bl -10.568200 +kr -10.587354 +ul -10.588481 +je -10.590478 +gu -10.591234 +rr -10.594151 +su -10.607294 +th -10.624903 +ai -10.693791 +ob -10.702543 +b_ -10.716377 +hs -10.773237 +iv -10.848915 +fl -10.853767 +u_ -10.859213 +rw -10.872927 +gs -10.879599 +lb -10.900081 +ud -10.912108 +op -10.948949 +ak -10.953512 +ib -10.970167 +lg -10.973096 +ia -10.976100 +mt -10.996821 +pl -11.014240 +ea -11.059641 +if -11.108516 +ez -11.109951 +bt -11.113973 +ki -11.138298 +hw -11.149573 +bo -11.166996 +ap -11.174404 +mp -11.227087 +bs -11.229158 +sg -11.231828 +ju -11.310777 +sl -11.319582 +mb -11.369331 +iz -11.370651 +pf -11.383342 +nl -11.404717 +pp -11.427960 +y_ -11.443491 +ep -11.454658 +up -11.485457 +dt -11.490301 +ow -11.495334 +ua -11.639744 +za -11.641317 +pt -11.656163 +ee -11.658445 +nh -11.670866 +dl -11.674903 +pu -11.734687 +sb -11.736485 +lf -11.740573 +ex -11.780345 +p_ -11.824381 +sh -11.828344 +nb -11.837354 +co -11.840600 +ok -11.861883 +ln -11.882473 +oz -11.892782 +va -11.901475 +ms -11.940492 +tg -11.943059 +nc -11.949422 +go -11.985034 +ou -11.989047 +aa -12.015377 +fg -12.032760 +eo -12.044181 +ce -12.064160 +ds -12.122428 +lm -12.135430 +ph -12.135609 +sw -12.141158 +zo -12.201481 +nw -12.259330 +sk -12.303118 +ev -12.334233 +ks -12.355321 +uk -12.360555 +w_ -12.364756 +jo -12.395294 +ip -12.416412 +qu -12.419733 +sy -12.426979 +ov -12.438161 +ca -12.484428 +az -12.484444 +gn -12.492844 +lc -12.495984 +tm -12.500665 +lk -12.530357 +bg -12.604089 +sm -12.613824 +rp -12.626073 +oo -12.627911 +av -12.642847 +nm -12.653747 +ay -12.662752 +ys -12.790724 +v_ -12.815667 +fs -12.857468 +sz -12.876537 +_q -12.897361 +sr -12.907504 +tf -12.930716 +x_ -12.934369 +rv -12.957863 +fn -13.059677 +nr -13.109818 +zl -13.110549 +ty -13.141544 +dw -13.147477 +gk -13.168228 +iu -13.191866 +tn -13.200288 +dn -13.231348 +xi -13.245057 +lz -13.254186 +c_ -13.307459 +ly -13.337046 +uz -13.383449 +ye -13.389306 +sf -13.429444 +ps -13.434418 +nv -13.448490 +ci -13.500368 +bn -13.517838 +ml -13.521988 +mg -13.537148 +hk -13.574195 +tb -13.598477 +tk -13.634906 +tp -13.658473 +kn -13.661201 +mf -13.664231 +lr -13.667230 +ym -13.670049 +td -13.672093 +ax -13.693749 +gh -13.706374 +uv -13.716808 +ui -13.746493 +sd -13.771214 +cl -13.804415 +bz -13.804904 +hd -13.829577 +oa -13.855566 +lv -13.861830 +bw -13.870338 +hb -13.884400 +ry -13.888974 +sv -13.908336 +sn -13.959110 +ey -13.961517 +bh -13.964064 +_y -13.978937 +ws -13.982253 +xp -13.988352 +ii -13.991550 +xt -13.996478 +dg -14.001048 +yp -14.013527 +ae -14.019334 +hf -14.037279 +yo -14.069488 +cr -14.085248 +hz -14.091799 +fb -14.113601 +cu -14.176196 +lp -14.218746 +kf -14.257223 +oi -14.281696 +dp -14.298648 +dh -14.318068 +pd -14.319931 +oj -14.347817 +yr -14.349481 +hg -14.356755 +zb -14.359567 +mn -14.393193 +lh -14.406062 +tc -14.431142 +lw -14.440487 +xe -14.444800 +ct -14.449318 +bb -14.492597 +gg -14.495753 +aw -14.509243 +fh -14.536403 +dd -14.541897 +dm -14.594490 +ny -14.673954 +km -14.687195 +kk -14.722342 +ix -14.725271 +dy -14.750414 +kg -14.776882 +gm -14.792724 +yl -14.797340 +yn -14.806291 +kz -14.811360 +uw -14.837646 +df -14.868419 +hy -14.898818 +gb -14.905535 +tv -14.933165 +ox -14.937360 +fm -14.937539 +zd -14.995547 +fz -15.004696 +xa -15.020706 +by -15.070027 +oe -15.091252 +cd -15.102133 +np -15.102634 +dk -15.139795 +db -15.145359 +gy -15.166577 +kw -15.167101 +md -15.172983 +gz -15.176988 +kh -15.198358 +rj -15.233487 +pz -15.235464 +fw -15.249711 +mw -15.272985 +oy -15.286243 +ya -15.350730 +hh -15.359567 +iw -15.421858 +ao -15.427622 +mk -15.489056 +bj -15.530660 +gf -15.549528 +bf -15.556513 +zz -15.565879 +eq -15.620231 +my -15.623679 +cs -15.636683 +zm -15.683742 +yc -15.701698 +nj -15.733164 +vp -15.745475 +dc -15.751435 +_x -15.758366 +wn -15.760105 +gd -15.813902 +fd -15.829743 +uo -15.831403 +wj -15.835729 +fp -15.854005 +yt -15.909913 +ux -15.916594 +bk -15.994616 +fk -16.020138 +bm -16.020517 +ky -16.023360 +sj -16.034408 +bc -16.048049 +bd -16.063574 +kb -16.079071 +zg -16.096932 +xu -16.105543 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/div b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/div new file mode 100644 index 00000000000..e0e84f76a6e --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/div @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 65217988 +#UNIQUE_BIGRAMS 1037 +#UNSEEN_LOG_PROB -16.374920 +#EXPECTED_SCORE -7.676352 +ނ_ -4.643617 +_އ -4.805471 +އ_ -5.285282 +_ކ -5.506370 +_މ -5.519111 +ގ_ -5.564743 +ށ_ -5.708178 +ވ_ -5.813861 +_ހ -6.226351 +_ބ -6.392610 +ގއ -6.415538 +އނ -6.425399 +އވ -6.445723 +ކރ -6.502250 +ނނ -6.517589 +_ފ -6.537437 +ރ_ -6.542509 +_ދ -6.613067 +_ވ -6.693559 +ކމ -6.693946 +މނ -6.859414 +ރއ -6.861480 +ނގ -6.865622 +_ނ -6.876820 +ރނ -6.950267 +ސ_ -6.960363 +_ގ -6.973845 +ހނ -6.996791 +މށ -6.999127 +މއ -7.052907 +ފއ -7.084593 +އކ -7.130116 +_ސ -7.177746 +ގނ -7.179063 +_ރ -7.219639 +މ_ -7.237444 +އއ -7.248942 +ކށ -7.275700 +ކނ -7.276283 +ނމ -7.341801 +އމ -7.396983 +ކ_ -7.433146 +ހރ -7.444611 +އސ -7.475155 +ތ_ -7.500540 +ނވ -7.502412 +އގ -7.516625 +ތކ -7.584856 +ވނ -7.611323 +މހ -7.614692 +ލ_ -7.642357 +މގ -7.658289 +އދ -7.679927 +ދ_ -7.695552 +ތރ -7.697397 +ވރ -7.732350 +_ތ -7.736118 +ބނ -7.791608 +އހ -7.818137 +ތއ -7.834060 +ގތ -7.837638 +ދނ -7.849165 +ރކ -7.871656 +ރގ -7.896763 +ތނ -7.936632 +ވސ -7.983994 +ފރ -8.046047 +ދއ -8.071082 +ރމ -8.076936 +ހއ -8.117860 +އތ -8.126074 +އރ -8.130506 +ދވ -8.149576 +ނޑ -8.159251 +ނކ -8.177580 +ނއ -8.189736 +_ލ -8.195742 +ކއ -8.200062 +ބއ -8.222371 +އލ -8.234726 +ބރ -8.293091 +ލއ -8.294535 +ރވ -8.305754 +ހމ -8.306965 +ވއ -8.333579 +ރތ -8.369262 +ނސ -8.375662 +ސއ -8.376941 +ވހ -8.387303 +ސލ -8.400914 +ޅވ -8.403488 +ނތ -8.475699 +ވދ -8.486464 +ނދ -8.500512 +ފނ -8.505182 +ށވ -8.534576 +ސޓ -8.536989 +ތގ -8.550395 +ލނ -8.551515 +ޓ_ -8.561593 +އށ -8.571220 +_ޖ -8.588215 +މކ -8.590627 +ލމ -8.633909 +ޓރ -8.641928 +ކތ -8.651077 +ހ_ -8.652260 +ރށ -8.654282 +އފ -8.655717 +މސ -8.661089 +މވ -8.707562 +މތ -8.711989 +_ޕ -8.735991 +މދ -8.744134 +ދޅ -8.763269 +ބލ -8.799487 +އޖ -8.828468 +އޅ -8.888310 +ފހ -8.896604 +ހގ -8.901505 +ހކ -8.905403 +މރ -8.919312 +އޔ -8.931033 +ވމ -8.943188 +ނޓ -8.945226 +ލގ -8.966359 +ޅނ -8.994855 +މލ -8.996944 +ހދ -9.000282 +ޑ_ -9.000792 +ސރ -9.000894 +ޑއ -9.010936 +ފ_ -9.017387 +ކޅ -9.066390 +ދމ -9.074477 +ނށ -9.088390 +ލބ -9.139685 +އބ -9.141908 +ޖހ -9.146677 +ރސ -9.151814 +ރހ -9.182681 +ވލ -9.258964 +ލކ -9.277075 +އޓ -9.303557 +ވފ -9.312363 +ދކ -9.316123 +ޔ_ -9.318790 +ނފ -9.319949 +ކގ -9.323913 +ފލ -9.327931 +ކލ -9.340309 +ސނ -9.341859 +ށފ -9.357663 +ތތ -9.369232 +ސކ -9.374450 +ގޅ -9.409644 +_ޓ -9.422853 +ދރ -9.428726 +ލތ -9.466006 +ކޓ -9.469311 +ޅގ -9.482622 +ޅ_ -9.483651 +ޝނ -9.493982 +ޕޓ -9.562613 +ލވ -9.563952 +ލހ -9.570430 +ހތ -9.585256 +ޓގ -9.588522 +ސތ -9.634938 +ބދ -9.639183 +ކވ -9.661456 +ވގ -9.666733 +ބ_ -9.678920 +ދލ -9.691724 +_ޑ -9.694526 +ޓނ -9.695442 +މބ -9.695699 +ދގ -9.712954 +ތމ -9.717621 +ސވ -9.773059 +ނލ -9.794744 +_ޝ -9.795805 +ސބ -9.804166 +ޕރ -9.811641 +ގރ -9.850814 +ހށ -9.853799 +މޖ -9.865019 +ރފ -9.886048 +ޓމ -9.904014 +ޔތ -9.908982 +ށގ -9.917942 +ތށ -9.935294 +ދޔ -9.974215 +ހޅ -9.986886 +ފށ -9.991991 +ހލ -9.996074 +ފތ -10.015886 +ޖ_ -10.023555 +ސފ -10.055473 +ނބ -10.065204 +ބޑ -10.078777 +ޔނ -10.082585 +ހފ -10.087752 +ޖއ -10.088522 +ށއ -10.089727 +ފޅ -10.090764 +ސމ -10.093154 +ޖގ -10.107457 +ޓއ -10.120942 +ހސ -10.122864 +މޑ -10.127237 +އޗ -10.137167 +ޕއ -10.157993 +ދތ -10.190350 +ވތ -10.197527 +ލސ -10.204036 +ހވ -10.205889 +_ޔ -10.211360 +ލފ -10.230102 +ސގ -10.243260 +ތވ -10.298826 +މމ -10.300555 +ޔރ -10.314967 +ނހ -10.319709 +ޔއ -10.335057 +_ޒ -10.335428 +ވޑ -10.347194 +ނޖ -10.347569 +ޖލ -10.354648 +މޗ -10.385682 +ސޕ -10.386037 +ދފ -10.391048 +މޝ -10.407118 +ޅއ -10.419063 +ބފ -10.431901 +ނރ -10.455779 +ޓކ -10.459824 +ޑގ -10.481198 +ފސ -10.499463 +ޔގ -10.513719 +ޖނ -10.514658 +ގމ -10.514981 +ޑވ -10.517833 +ރޕ -10.542431 +ޅމ -10.542728 +އޑ -10.545006 +ރލ -10.563701 +ނޔ -10.576987 +ޑކ -10.584576 +ތހ -10.587500 +ކސ -10.625751 +ގގ -10.626485 +ބޔ -10.632267 +މފ -10.635606 +ފދ -10.649184 +ޑމ -10.649717 +ރޔ -10.660597 +ދށ -10.667380 +ސސ -10.694214 +ބބ -10.746270 +ޖމ -10.746992 +ށދ -10.747373 +ނޝ -10.751295 +މޔ -10.775248 +ޔސ -10.782827 +ނޏ -10.791505 +ޒމ -10.798107 +ޒރ -10.806997 +ށހ -10.812277 +ލޑ -10.819975 +ބކ -10.821775 +އޒ -10.822977 +ބހ -10.838488 +ގވ -10.838853 +ޗއ -10.841570 +ޒ_ -10.843886 +_ޗ -10.844374 +އޕ -10.844943 +ވކ -10.845757 +ބސ -10.850691 +ޕލ -10.868158 +ޔކ -10.870392 +ޔވ -10.893561 +ޝރ -10.910365 +ޑނ -10.918348 +ލށ -10.920105 +_ޤ -10.925129 +ޓލ -10.943134 +ޗގ -10.957051 +ކދ -10.963529 +ފޔ -10.974571 +ބގ -11.012911 +ރޝ -11.028260 +ޑށ -11.038228 +ނޒ -11.039112 +ރދ -11.040789 +ޔބ -11.048077 +_ޚ -11.057428 +ޔމ -11.070118 +ބތ -11.073404 +ގލ -11.081243 +ކޑ -11.097050 +ގސ -11.097777 +ޅށ -11.098213 +ޓވ -11.108043 +ށނ -11.112444 +ޑޕ -11.119563 +ޑރ -11.126717 +މޓ -11.135250 +ކޔ -11.135399 +ޗށ -11.139585 +ޕނ -11.168775 +ޝދ -11.190893 +ޝއ -11.208688 +ޕ_ -11.209945 +މޕ -11.226440 +ޕގ -11.254377 +ފކ -11.287281 +ތލ -11.290823 +ال -11.301783 +لل -11.304914 +له -11.304914 +މޒ -11.326680 +ޏވ -11.347396 +ރޓ -11.375566 +ه_ -11.377566 +ތފ -11.381161 +ރޖ -11.385119 +ޒނ -11.386895 +ކކ -11.391108 +ގކ -11.392415 +ކފ -11.404957 +ލޔ -11.442820 +ލޓ -11.486901 +ވޒ -11.494093 +ޒފ -11.494221 +ޓސ -11.495051 +ވޔ -11.498375 +ގފ -11.507362 +ޓތ -11.512782 +ޖރ -11.515823 +ޅފ -11.535979 +ތޖ -11.543157 +ރބ -11.562831 +ދސ -11.575805 +ކޝ -11.578441 +އޝ -11.592103 +ދދ -11.596138 +ރޒ -11.604242 +ހބ -11.605689 +ބވ -11.616414 +ފގ -11.623167 +ކޕ -11.632548 +ތބ -11.639941 +ޢލ -11.646238 +ޅކ -11.647657 +ށލ -11.662496 +ޒވ -11.672209 +ކހ -11.673220 +ސދ -11.675751 +ސޔ -11.695131 +ފޓ -11.734690 +ވޓ -11.789076 +ގބ -11.795589 +ޑލ -11.810052 +_ޢ -11.815144 +ލރ -11.838124 +ބމ -11.838853 +މޅ -11.866835 +ރޑ -11.872713 +ޓށ -11.875537 +ގދ -11.889988 +ޓބ -11.907643 +ޝހ -11.928186 +ޑތ -11.929911 +ޝލ -11.931725 +ޔފ -11.956567 +ފވ -11.966979 +ތޅ -11.969106 +ސހ -11.973014 +ދا -12.008120 +_ޙ -12.021025 +މޢ -12.037391 +ޕސ -12.073547 +ޑސ -12.098262 +ޤއ -12.110339 +ލލ -12.144885 +ޕކ -12.145988 +ޒހ -12.147593 +ދބ -12.163640 +ޔލ -12.184700 +ޖކ -12.221414 +ޑޔ -12.235106 +ލޖ -12.240876 +ގޑ -12.246347 +ޗނ -12.250545 +ޕޕ -12.258760 +ޤނ -12.317618 +ޚބ -12.326339 +ޒކ -12.330207 +ޒލ -12.343367 +ބޖ -12.349242 +ޗކ -12.349242 +ޝވ -12.382873 +ހޔ -12.392832 +ޖސ -12.395570 +ކބ -12.418154 +ޔށ -12.422641 +ޒޔ -12.444422 +ބޅ -12.453079 +ތދ -12.460790 +ވޅ -12.464411 +އޤ -12.466538 +ޓޑ -12.466788 +ފޝ -12.492435 +ލދ -12.532764 +ފޒ -12.569135 +ރރ -12.573983 +ޅތ -12.579388 +ޝކ -12.589169 +ވބ -12.595042 +ށޓ -12.601215 +ޑޓ -12.603966 +ޓފ -12.619473 +ވވ -12.654558 +ޖޓ -12.683224 +ޙއ -12.704033 +ޅދ -12.715890 +ޗމ -12.719317 +ނޗ -12.726346 +ބޓ -12.734765 +ތސ -12.761550 +ށށ -12.797162 +ޝމ -12.799684 +ޖތ -12.845838 +ށމ -12.933454 +ޒގ -12.937613 +ލޝ -12.940219 +ސޒ -12.943352 +ވށ -12.948238 +ޗ_ -12.964767 +ޖވ -12.973458 +މޤ -12.982023 +ތޚ -13.003117 +ޗލ -13.012043 +ޑފ -13.027290 +ޚދ -13.028953 +ފމ -13.037298 +ޒއ -13.053756 +ދހ -13.057711 +ޝ_ -13.071165 +ޔޒ -13.089173 +ކޒ -13.101952 +ތޔ -13.102536 +ޚލ -13.165366 +ގޒ -13.183362 +ފބ -13.256377 +ތޒ -13.258977 +ޓޓ -13.275114 +_ا -13.280166 +ސޑ -13.281487 +ހޒ -13.294319 +ޖބ -13.310409 +ށތ -13.311983 +ޚއ -13.333743 +ކޗ -13.343367 +ގށ -13.348895 +ޤމ -13.353750 +ސޖ -13.364209 +ޗރ -13.371927 +ބށ -13.393426 +ޝޔ -13.407779 +ޅލ -13.469420 +އޙ -13.491161 +ޤވ -13.543289 +ޢމ -13.577224 +ޝޕ -13.582913 +ޖޕ -13.606171 +ޑޑ -13.612530 +ގހ -13.614193 +ޑބ -13.637120 +މޙ -13.660131 +ޚރ -13.673942 +ޤ_ -13.698141 +ލޕ -13.712917 +ޤބ -13.714998 +ފޑ -13.739598 +ޕޝ -13.768325 +ޙމ -13.782905 +ފފ -13.785089 +ށކ -13.800157 +ޓޗ -13.820175 +ޒޓ -13.831740 +ރޚ -13.832707 +ޝފ -13.887639 +ށރ -13.894361 +ސޝ -13.922593 +އޚ -13.927065 +ތޢ -13.940914 +ޓޝ -13.943352 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ekk b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ekk new file mode 100644 index 00000000000..4d01c843a18 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ekk @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 19492458 +#UNIQUE_BIGRAMS 680 +#UNSEEN_LOG_PROB -18.956931 +#EXPECTED_SCORE -7.439680 +s_ -5.611747 +e_ -5.644368 +a_ -5.687932 +i_ -5.705587 +_k -5.854276 +st -5.915368 +se -5.935578 +as -5.997164 +is -6.131480 +ta -6.135408 +al -6.155822 +d_ -6.221860 +li -6.243506 +_s -6.264075 +_a -6.369398 +us -6.450518 +es -6.476046 +_t -6.501128 +va -6.518739 +aa -6.547050 +ma -6.576726 +te -6.589256 +el -6.622606 +_v -6.626498 +in -6.644823 +t_ -6.678269 +_p -6.700259 +_m -6.731299 +l_ -6.764904 +an -6.910378 +ni -6.922797 +ar -6.931390 +ri -6.934238 +le -6.942353 +oo -6.990565 +ol -7.042960 +tu -7.052272 +ku -7.070670 +ja -7.073099 +at -7.075902 +mi -7.078801 +ko -7.096175 +_l -7.097918 +_o -7.124285 +ti -7.130090 +si -7.140505 +ne -7.144134 +sa -7.144961 +ee -7.202970 +la -7.208947 +_e -7.223098 +ik -7.257588 +id -7.281294 +na -7.316265 +su -7.326469 +ka -7.333280 +ud -7.366116 +nd -7.390324 +_n -7.416410 +it -7.456759 +da -7.474235 +il -7.493578 +ad -7.504746 +en -7.538211 +ii -7.546503 +ul -7.564184 +me -7.569137 +u_ -7.583645 +ks -7.608562 +ra -7.644216 +uu -7.700097 +de -7.710570 +ai -7.726612 +em -7.729311 +er -7.762933 +on -7.768458 +im -7.793346 +_j -7.835546 +ll -7.847224 +et -7.881930 +un -7.954979 +ak -7.957681 +ts -7.996582 +_r -8.009417 +_u -8.012224 +ed -8.032370 +di -8.055222 +am -8.064722 +re -8.065575 +ga -8.074585 +ut -8.076861 +ng -8.117106 +or -8.121737 +oi -8.154894 +pa -8.159754 +ke -8.181764 +gu -8.185293 +nn -8.221396 +to -8.237568 +he -8.303477 +ur -8.303617 +du -8.307387 +b_ -8.349206 +ah -8.370261 +nu -8.370384 +lu -8.371977 +lt -8.380954 +_h -8.384438 +ek -8.397032 +gi -8.401680 +ei -8.453745 +ki -8.463509 +os -8.494833 +vo -8.522381 +av -8.531911 +po -8.535119 +ju -8.554328 +lo -8.579533 +_i -8.603084 +aj -8.603487 +ia -8.604466 +ab -8.677163 +ge -8.692698 +pe -8.726502 +mu -8.731370 +ig -8.753687 +ha -8.755573 +om -8.770783 +ea -8.777134 +eg -8.800044 +ev -8.814567 +pi -8.816300 +ot -8.850193 +vi -8.856629 +um -8.888264 +uh -8.942981 +ro -8.963747 +so -8.972123 +ir -9.002891 +r_ -9.003726 +k_ -9.009932 +oh -9.013409 +ag -9.029486 +ve -9.038098 +g_ -9.051388 +nt -9.103932 +hi -9.105481 +mo -9.115791 +uk -9.152428 +n_ -9.165969 +m_ -9.190970 +rg -9.237255 +ld -9.242895 +op -9.314744 +sk -9.323348 +ht -9.364908 +ui -9.371119 +lm -9.384512 +ae -9.416575 +og -9.475051 +au -9.494246 +tr -9.498950 +pr -9.499486 +od -9.500451 +ru -9.510672 +rt -9.515756 +ss -9.531883 +lj -9.547971 +kk -9.563680 +lg -9.637922 +pu -9.667771 +ug -9.702315 +io -9.730395 +mb -9.794348 +no -9.795663 +eh -9.803975 +ap -9.813534 +ub -9.820812 +jo -9.821481 +be -9.833654 +rd -9.867546 +bi -9.887878 +rj -9.913917 +kt -9.922654 +_b -9.941525 +rv -9.946190 +ba -9.967299 +hu -9.969079 +br -9.971083 +v_ -10.030918 +ou -10.049544 +iv -10.052527 +_f -10.062623 +tt -10.066110 +_d -10.070958 +eb -10.136178 +vu -10.161808 +tl -10.170227 +ok -10.216149 +o_ -10.238507 +oe -10.264217 +oj -10.294200 +ho -10.325009 +ns -10.354357 +eo -10.358432 +kr -10.392051 +uv -10.404336 +je -10.426268 +_g -10.439363 +ie -10.440288 +rm -10.489024 +fi -10.522817 +rr -10.642529 +ib -10.659907 +kl -10.667711 +ep -10.679923 +ob -10.703179 +ua -10.749954 +sm -10.776063 +ao -10.802917 +sv -10.816868 +gr -10.817268 +hk -10.850364 +ov -10.881162 +rk -10.884936 +rn -10.896882 +pp -10.899141 +bu -10.901122 +iu -10.919640 +tm -10.929422 +hv -10.987293 +rs -11.050407 +hj -11.059856 +sl -11.112469 +dr -11.151165 +ls -11.169630 +_c -11.193525 +lk -11.208159 +gl -11.227196 +ej -11.284936 +up -11.289376 +ch -11.300533 +do -11.322163 +dm -11.335299 +uj -11.346819 +ln -11.407650 +eu -11.425250 +ih -11.425250 +ip -11.462151 +sp -11.469898 +ue -11.552410 +ic -11.583645 +mm -11.612786 +rl -11.620223 +pl -11.668554 +mp -11.674590 +pt -11.705401 +dl -11.735118 +nk -11.774247 +y_ -11.808879 +th -11.809942 +fr -11.828665 +go -11.848725 +lv -11.873505 +sn -11.894203 +_w -11.976814 +fo -12.008178 +p_ -12.009093 +af -12.111486 +rb -12.129281 +km -12.148642 +vs -12.154367 +tk -12.176466 +gn -12.184367 +h_ -12.195780 +co -12.197865 +bl -12.212543 +tn -12.242357 +ca -12.247386 +ps -12.276833 +bo -12.278303 +lb -12.323490 +ds -12.337979 +fe -12.382337 +kv -12.427287 +hn -12.437104 +ij -12.445336 +of -12.467801 +wi -12.495314 +gs -12.663744 +dn -12.674348 +sj -12.684544 +fu -12.767780 +ms -12.856114 +mn -12.862714 +kn -12.893358 +sr -12.895048 +fa -12.900131 +ac -12.932746 +ef -12.946702 +sc -13.039616 +rh -13.043360 +ce -13.102020 +lh -13.127624 +ck -13.169289 +hh -13.192658 +gm -13.229149 +wa -13.267316 +nc -13.269506 +oa -13.296060 +tj -13.349906 +tv -13.381942 +ci -13.385897 +nf -13.400229 +gp -13.401029 +gk -13.455693 +lf -13.482550 +if -13.484245 +w_ -13.564464 +nr -13.575264 +hm -13.587056 +zi -13.627698 +f_ -13.637097 +oc -13.655125 +sh -13.660865 +ew -13.674348 +ph -13.698743 +ry -13.713581 +_x -13.714576 +dt -13.770363 +bj -13.816601 +z_ -13.846815 +xi -13.877676 +rp -13.889983 +nl -13.900131 +c_ -13.912632 +ec -13.929855 +we -13.975621 +ow -13.996034 +lp -13.997244 +yo -13.998455 +ex -14.071754 +hr -14.113125 +_y -14.113125 +x_ -14.122335 +ay -14.150323 +kp -14.200998 +rc -14.203788 +vk -14.231994 +ty -14.282722 +vl -14.311026 +ey -14.324629 +ze -14.344507 +nb -14.344507 +ez -14.350679 +bs -14.380362 +za -14.381942 +fl -14.386690 +pk -14.388276 +nz -14.391454 +uo -14.405841 +dv -14.417131 +_z -14.485094 +dk -14.503886 +vn -14.535174 +dj -14.545756 +dw -14.563568 +sb -14.581602 +kh -14.610933 +nh -14.642766 +mk -14.663744 +gh -14.683083 +ct -14.686982 +pm -14.754933 +cl -14.756981 +dz -14.765202 +mt -14.914916 +sf -14.933324 +ml -14.959025 +ny -14.968485 +xa -14.990000 +wo -15.021656 +kj -15.069208 +hl -15.102671 +cu -15.166564 +jn -15.182990 +mj -15.185746 +ix -15.196822 +ly -15.199604 +nv -15.250628 +pn -15.262216 +ys -15.268045 +az -15.291600 +iz -15.336829 +ff -15.361544 +uc -15.393045 +vd -15.471579 +cr -15.474946 +xv -15.478320 +zo -15.478320 +ft -15.495314 +aw -15.509054 +ax -15.522926 +j_ -15.572557 +tp -15.572557 +gt -15.587056 +ox -15.685031 +oy -15.736632 +qu -15.744737 +sy -15.748807 +ws -15.794348 +ya -15.802785 +hz -15.832708 +dg -15.841373 +vr -15.850090 +gd -15.903530 +dd -15.926394 +bn -15.949626 +rz -15.954318 +cc -16.036504 +yl -16.041487 +sd -16.051506 +tz -16.056541 +nj -16.092291 +rw -16.118381 +my -16.118381 +dh -16.144950 +bb -16.150323 +zu -16.150323 +vg -16.161130 +bk -16.194045 +mh -16.210788 +yd -16.244869 +rf -16.339896 +ky -16.339896 +mf -16.346048 +uf -16.346048 +bh -16.428510 +sw -16.428510 +tg -16.435053 +fg -16.448228 +tf -16.448228 +oz -16.468220 +bm -16.481703 +ji -16.481703 +dy -16.587056 +_q -16.594361 +mv -16.631450 +vp -16.631450 +jm -16.646557 +jp -16.677254 +ye -16.716567 +wn -16.807022 +wh -16.832708 +uz -16.832708 +cd -16.832708 +hs -16.858861 +wr -16.885496 +pc -16.912632 +mr -16.921792 +lz -16.921792 +hw -16.931010 +xf -16.987594 +kf -17.016740 +hy -17.026588 +fj -17.046488 +zy -17.046488 +pz -17.056541 +cs -17.056541 +wu -17.087130 +zz -17.107888 +tw -17.107888 +pg -17.118381 +df -17.128950 +gy -17.150323 +wt -17.161130 +lc -17.205185 +sz -17.216413 +sg -17.227728 +np -17.227728 +bc -17.239133 +cy -17.239133 +md -17.273898 +vy -17.273898 +vh -17.297549 +nm -17.309522 +lr -17.333770 +xe -17.409058 +xx -17.409058 +zb -17.461525 +vj -17.461525 +cq -17.515973 +gb -17.515973 +mg -17.515973 +yn -17.515973 +pj -17.543987 +yw -17.543987 +pf -17.558201 +cz -17.587056 +vm -17.587056 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ell b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ell new file mode 100644 index 00000000000..d877277c13f --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ell @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 313979087 +#UNIQUE_BIGRAMS 1019 +#UNSEEN_LOG_PROB -17.320770 +#EXPECTED_SCORE -7.407205 +σ_ -5.019884 +α_ -5.392886 +_τ -5.495573 +ου -5.512366 +ι_ -5.569544 +ν_ -5.652723 +το -5.874899 +_ε -5.897721 +_π -5.970404 +ει -5.974903 +_α -5.989194 +τη -6.063648 +ο_ -6.064298 +αι -6.081462 +_κ -6.086315 +κα -6.102793 +ια -6.157389 +_σ -6.175065 +πο -6.203021 +η_ -6.286603 +υ_ -6.320898 +στ -6.366765 +τα -6.405426 +ησ -6.460595 +ικ -6.712197 +αν -6.725189 +ασ -6.802877 +ισ -6.806402 +τι -6.925917 +_μ -6.930622 +ρο -6.941979 +εσ -6.968839 +ρι -7.012791 +ερ -7.024635 +οσ -7.060505 +ων -7.090182 +να -7.099706 +ρα -7.103715 +με -7.111580 +_δ -7.121980 +μα -7.138222 +τε -7.185153 +ον -7.199998 +ατ -7.229322 +εν -7.281651 +_ο -7.325648 +ιο -7.341973 +ην -7.347196 +ε_ -7.372978 +απ -7.415879 +κο -7.418869 +ση -7.487093 +υσ -7.519412 +αρ -7.561687 +ιν -7.586178 +πα -7.612964 +υν -7.618440 +δι -7.646706 +νο -7.653669 +οι -7.671670 +ετ -7.683292 +_γ -7.695853 +αλ -7.731867 +πρ -7.766807 +νε -7.823944 +λο -7.871036 +ολ -7.881400 +γι -7.883779 +ντ -7.901118 +σε -7.921011 +μο -7.944228 +οτ -7.975672 +πε -7.980467 +ωσ -7.988352 +λε -8.009223 +ρε -8.040806 +ορ -8.043254 +ημ -8.070972 +κε -8.078339 +επ -8.101771 +ελ -8.156824 +συ -8.184739 +τω -8.186040 +ομ -8.188330 +πι -8.200541 +λα -8.206432 +θε -8.240821 +νι -8.250068 +λη -8.273682 +κη -8.287141 +λι -8.295952 +οπ -8.376743 +σι -8.386035 +υτ -8.390601 +δε -8.430788 +λλ -8.444625 +_χ -8.497472 +ευ -8.510790 +ητ -8.520689 +μι -8.523092 +χε -8.581299 +ιτ -8.632058 +τρ -8.641644 +νη -8.651679 +ρω -8.685524 +αγ -8.724326 +ακ -8.731736 +εκ -8.760214 +εχ -8.792974 +υρ -8.806750 +υμ -8.829098 +υπ -8.837485 +αυ -8.851546 +_β -8.860249 +αθ -8.923845 +ιδ -8.944874 +_υ -8.957585 +γο -8.981755 +νω -8.986248 +αμ -9.000236 +αφ -9.021147 +ργ -9.072482 +ρη -9.082551 +ογ -9.086200 +γε -9.093563 +σο -9.100988 +ιε -9.125595 +_λ -9.137271 +_φ -9.138571 +σα -9.139871 +γα -9.167401 +ηρ -9.182265 +αδ -9.184894 +δο -9.197202 +_ι -9.208140 +δη -9.218740 +ιω -9.244821 +_ν -9.263965 +θη -9.264336 +ωρ -9.264706 +φο -9.279632 +χο -9.288823 +βα -9.292071 +σμ -9.331349 +μη -9.341291 +χρ -9.350491 +κρ -9.416950 +ιμ -9.425260 +ω_ -9.436146 +_θ -9.436499 +_η -9.441768 +εγ -9.456010 +μπ -9.458740 +κτ -9.472341 +ηκ -9.569923 +οδ -9.598855 +σκ -9.618882 +ωτ -9.625502 +ιρ -9.627881 +πλ -9.641024 +φα -9.656542 +εμ -9.679308 +οκ -9.679647 +φε -9.700507 +ρχ -9.770533 +ιχ -9.790598 +οχ -9.796306 +ιλ -9.815501 +δα -9.835395 +γρ -9.836521 +χα -9.840408 +ξε -9.850757 +λυ -9.852047 +υλ -9.862549 +εδ -9.868052 +εξ -9.877767 +δυ -9.906426 +κι -9.907612 +χω -9.915973 +ηλ -9.967385 +δρ -9.977832 +κλ -10.021637 +σχ -10.036264 +ιπ -10.087301 +χι -10.118866 +πω -10.139501 +χη -10.145323 +ηθ -10.153484 +τυ -10.156874 +υγ -10.168070 +βο -10.177255 +ζε -10.181118 +γκ -10.194790 +εω -10.200775 +κυ -10.215702 +πτ -10.217183 +γη -10.237647 +ιζ -10.243551 +πη -10.264757 +οφ -10.280569 +λω -10.282800 +ηγ -10.297858 +γω -10.304996 +βρ -10.316054 +σω -10.335401 +μμ -10.354254 +αζ -10.374064 +αξ -10.389494 +κω -10.409128 +φι -10.425756 +βε -10.482291 +ξη -10.485838 +υθ -10.498492 +εα -10.504687 +ωπ -10.515226 +νδ -10.520501 +εφ -10.533362 +ωμ -10.571401 +μβ -10.596068 +θο -10.616294 +ζο -10.647183 +ιγ -10.705950 +εο -10.725454 +υχ -10.749877 +εθ -10.750882 +υο -10.791946 +φω -10.797414 +φη -10.812135 +γμ -10.823056 +_ζ -10.847282 +σσ -10.858329 +σπ -10.869504 +_ρ -10.872541 +αβ -10.888384 +ρυ -10.901060 +οβ -10.904641 +ψη -10.944163 +ιθ -10.947263 +ρκ -10.963613 +μω -10.965238 +ρμ -10.975906 +θα -10.997866 +χν -11.006688 +ιβ -11.021656 +αχ -11.034099 +ρτ -11.043185 +θυ -11.050115 +ξα -11.076029 +_ξ -11.099107 +βλ -11.100972 +βι -11.105329 +δω -11.116742 +σφ -11.184188 +ξι -11.223696 +ρν -11.260377 +υκ -11.261155 +φυ -11.270341 +μφ -11.277794 +γν -11.288938 +οθ -11.312608 +νθ -11.333278 +ιη -11.339789 +γγ -11.341958 +εβ -11.343570 +θρ -11.363359 +θω -11.413051 +_ω -11.428280 +οε -11.445592 +τσ -11.522501 +θμ -11.559204 +υα -11.597277 +ζω -11.620744 +ιξ -11.633023 +αε -11.641909 +ζη -11.661378 +υξ -11.663135 +ωθ -11.691410 +ψε -11.705459 +γυ -11.798371 +οη -11.804965 +νν -11.852735 +χθ -11.917150 +τ_ -11.917505 +θν -11.924472 +λ_ -11.957661 +ωγ -11.965468 +ζι -11.982436 +σθ -11.999154 +ρ_ -12.060421 +υδ -12.087621 +κδ -12.090504 +ηφ -12.090845 +_ψ -12.122580 +ξυ -12.127465 +υψ -12.130284 +υε -12.161098 +ξο -12.182681 +χυ -12.199094 +υβ -12.203747 +κπ -12.211375 +μυ -12.213316 +νσ -12.214145 +ρθ -12.214713 +e_ -12.221631 +φρ -12.222795 +ηπ -12.246601 +γχ -12.251298 +ρφ -12.338397 +θλ -12.345003 +φτ -12.352649 +εψ -12.389994 +ζα -12.393895 +οψ -12.395677 +κ_ -12.409160 +θι -12.409535 +υφ -12.437885 +ωη -12.484310 +ιφ -12.493793 +αο -12.530974 +εζ -12.535277 +ηχ -12.561757 +νυ -12.625338 +λμ -12.654222 +υη -12.656356 +γλ -12.679832 +φθ -12.686631 +_s -12.695626 +χτ -12.736026 +ρδ -12.745051 +s_ -12.745682 +μν -12.772855 +ρρ -12.795379 +υζ -12.811738 +τμ -12.825647 +πυ -12.898295 +βη -12.905716 +εε -12.936326 +ρξ -12.940980 +κκ -12.949169 +μ_ -13.051285 +λτ -13.075037 +οξ -13.095684 +er -13.095965 +ξω -13.138630 +ηδ -13.223059 +σβ -13.225697 +_a -13.228120 +t_ -13.251007 +in -13.261346 +ωλ -13.276631 +τζ -13.296095 +πν -13.297621 +τλ -13.327870 +_m -13.332602 +ψο -13.354332 +te -13.380407 +_t -13.382221 +l_ -13.383252 +αψ -13.385561 +ωδ -13.397907 +n_ -13.413465 +οο -13.414869 +_e -13.466985 +an -13.515179 +_c -13.529017 +λθ -13.559259 +ηξ -13.578185 +ηψ -13.582237 +ρβ -13.608568 +r_ -13.626413 +λπ -13.627459 +ωα -13.644952 +d_ -13.692520 +on -13.707624 +π_ -13.746629 +es -13.749094 +ar -13.755117 +ιψ -13.770894 +or -13.775011 +ρσ -13.780820 +ψυ -13.781078 +_b -13.820287 +ττ -13.829890 +ψα -13.847053 +ne -13.869504 +a_ -13.885756 +_l -13.887984 +βγ -13.909529 +_p -13.913777 +_f -13.935138 +ro -13.936867 +al -13.950696 +βω -13.964439 +st -13.969811 +κθ -13.974611 +οα -13.976017 +me -13.985898 +βδ -13.989631 +λκ -14.010788 +_h -14.043699 +et -14.053197 +λφ -14.064017 +re -14.099711 +he -14.101003 +_d -14.122560 +th -14.147692 +ωι -14.151952 +ωβ -14.152956 +ti -14.153961 +le -14.194650 +ma -14.231651 +οζ -14.254280 +na -14.280649 +_n -14.290376 +_w -14.291757 +_i -14.314701 +οω -14.344023 +se -14.368209 +βυ -14.383841 +y_ -14.425901 +it -14.429749 +ic -14.431474 +si -14.431677 +κν -14.432185 +en -14.439211 +nt -14.456255 +k_ -14.474340 +κφ -14.475177 +nd -14.487157 +o_ -14.502006 +_r -14.502112 +λγ -14.533587 +ce -14.543318 +_g -14.549365 +υω -14.553668 +σδ -14.555326 +li -14.563314 +de -14.564538 +λβ -14.595371 +σλ -14.603355 +ββ -14.618069 +ea -14.620382 +io -14.638316 +at -14.643128 +ri -14.652446 +υι -14.659326 +_o -14.673664 +ζυ -14.683061 +αη -14.697029 +ll -14.720158 +ng -14.743285 +la -14.763847 +ππ -14.804423 +ch -14.809428 +ou -14.810219 +ta -14.828820 +ζ_ -14.833910 +di -14.850782 +χλ -14.856632 +is -14.857314 +po -14.859361 +co -14.871017 +κμ -14.875983 +ho -14.877642 +em -14.892519 +ra -14.933915 +i_ -14.943294 +il -14.958721 +ρπ -14.978611 +el -14.987539 +g_ -15.021675 +ve -15.023051 +rt -15.023816 +ac -15.028877 +so -15.032876 +ωφ -15.032876 +as -15.033492 +ur -15.033800 +ωζ -15.038741 +φλ -15.070010 +am -15.087341 +p_ -15.088302 +ba -15.101487 +ai -15.121822 +m_ -15.138133 +lo -15.146608 +vi -15.163204 +_k -15.165060 +ha -15.168440 +h_ -15.190435 +fo -15.203725 +_v -15.225213 +to -15.236699 +ol -15.239185 +im -15.258867 +un -15.279735 +rn -15.321270 +κσ -15.356692 +tr -15.361133 +νγ -15.368307 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/eng b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/eng new file mode 100644 index 00000000000..ad14d4203b2 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/eng @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 547203287 +#UNIQUE_BIGRAMS 698 +#UNSEEN_LOG_PROB -18.808975 +#EXPECTED_SCORE -7.564846 +e_ -4.877769 +_t -5.386961 +th -5.424394 +s_ -5.507954 +d_ -5.619903 +he -5.672584 +_a -6.012393 +er -6.066699 +an -6.170573 +in -6.179163 +t_ -6.192668 +_s -6.218022 +re -6.274997 +r_ -6.459287 +n_ -6.492111 +_w -6.548986 +_c -6.604534 +on -6.608872 +y_ -6.666759 +ed -6.690613 +nd -6.706498 +en -6.724824 +es -6.786682 +or -6.803307 +ar -6.835956 +_f -6.840772 +te -6.862742 +at -6.896845 +_p -6.938959 +al -6.988471 +ng -7.005513 +st -7.007354 +ti -7.032467 +_h -7.066590 +_m -7.175842 +_b -7.182290 +nt -7.200186 +g_ -7.261115 +l_ -7.285473 +ha -7.321229 +se -7.322342 +it -7.338900 +le -7.371559 +ve -7.374875 +_r -7.395790 +ou -7.401205 +as -7.427908 +ea -7.436661 +is -7.461083 +de -7.467441 +co -7.476625 +me -7.477866 +ro -7.498135 +_d -7.511473 +ri -7.542576 +ne -7.561132 +hi -7.581331 +h_ -7.645572 +io -7.711586 +_l -7.719093 +ra -7.719261 +ic -7.722389 +ce -7.742566 +li -7.747855 +ll -7.763993 +_o -7.805042 +_e -7.816716 +ma -7.879476 +om -7.926360 +la -7.928097 +ch -7.933254 +_i -7.940828 +ca -7.946793 +si -7.956729 +fo -7.989361 +il -8.028863 +ho -8.068822 +_n -8.075036 +el -8.092771 +ur -8.098834 +ta -8.120638 +wa -8.172728 +rs -8.175613 +pe -8.212983 +ec -8.237414 +_g -8.239459 +un -8.262148 +di -8.263498 +ns -8.278063 +be -8.282522 +ac -8.300810 +ai -8.319490 +us -8.344365 +to -8.372798 +ge -8.375894 +wi -8.377757 +pr -8.378015 +m_ -8.408225 +ut -8.437893 +ee -8.454489 +id -8.464841 +tr -8.467974 +ly -8.477323 +lo -8.478381 +ol -8.505968 +nc -8.508798 +so -8.511199 +ie -8.516446 +na -8.517057 +ad -8.520168 +ot -8.524595 +am -8.525483 +et -8.526353 +rt -8.536811 +o_ -8.582765 +ni -8.617289 +po -8.618406 +ts -8.618664 +ct -8.629605 +ow -8.638737 +sh -8.649092 +ss -8.651891 +mi -8.675566 +mo -8.676845 +ir -8.680091 +em -8.688542 +no -8.690858 +pa -8.708804 +wh -8.721797 +fi -8.730485 +ay -8.770625 +we -8.773025 +ia -8.801891 +sa -8.815059 +vi -8.849464 +im -8.941421 +pl -8.961520 +ig -8.968753 +os -8.976222 +k_ -8.988188 +ke -8.989571 +iv -8.995324 +ul -9.019063 +a_ -9.034877 +su -9.050145 +ci -9.067133 +ld -9.070008 +op -9.120626 +ry -9.128652 +ev -9.130636 +fr -9.153161 +wo -9.175365 +fe -9.179284 +_u -9.181099 +w_ -9.189707 +oo -9.237330 +gh -9.243789 +da -9.251025 +ag -9.274220 +mp -9.303816 +ba -9.322399 +ov -9.325053 +tu -9.337528 +bo -9.338724 +bu -9.339682 +_v -9.371492 +rd -9.374546 +av -9.395715 +_j -9.424015 +_y -9.433381 +ga -9.483575 +sp -9.484171 +ty -9.497172 +bl -9.505673 +ls -9.508318 +gr -9.508971 +rn -9.518297 +ap -9.535011 +ck -9.565125 +ab -9.565880 +cl -9.638244 +ei -9.657491 +ep -9.672869 +ex -9.689967 +cr -9.755163 +tt -9.769458 +fa -9.779669 +oc -9.781139 +ew -9.785095 +rm -9.802127 +do -9.830398 +ff -9.832305 +p_ -9.838150 +od -9.841136 +ye -9.851000 +sc -9.863506 +uc -9.872414 +ef -9.877861 +ue -9.896064 +_k -9.899228 +gi -9.902313 +c_ -9.916040 +ey -9.919488 +cu -9.925780 +du -9.945220 +ru -9.957562 +pp -9.973223 +rc -10.007319 +au -10.012419 +pi -10.013040 +go -10.022242 +br -10.048601 +ki -10.055717 +ug -10.070181 +um -10.074843 +rr -10.078242 +eg -10.090231 +rk -10.109582 +ua -10.116285 +ds -10.125668 +ak -10.136350 +ht -10.169533 +lu -10.211143 +if -10.227768 +bi -10.229903 +mb -10.236555 +yo -10.264750 +of -10.265639 +va -10.285748 +af -10.299196 +pu -10.304074 +nn -10.325184 +rg -10.330609 +ft -10.338172 +rl -10.367847 +lt -10.371933 +mm -10.376587 +qu -10.378877 +up -10.395177 +ny -10.402239 +mu -10.432962 +ud -10.463136 +wn -10.463821 +ui -10.477390 +dr -10.521899 +ys -10.529237 +gu -10.543233 +oi -10.570181 +tw -10.582220 +nu -10.587427 +ip -10.590380 +ub -10.607517 +hr -10.627556 +ju -10.672116 +tl -10.677966 +eo -10.679530 +f_ -10.696134 +fu -10.700905 +og -10.702662 +ph -10.708822 +ob -10.717555 +cc -10.745816 +oa -10.770901 +ms -10.778411 +pt -10.788826 +ok -10.792249 +jo -10.813866 +rv -10.882266 +hu -10.913566 +ib -10.920520 +aw -10.927959 +ks -10.934786 +gn -10.975740 +fl -11.053070 +vo -11.055728 +sl -11.149571 +u_ -11.188255 +nk -11.225181 +nl -11.245252 +xp -11.254974 +nf -11.261264 +i_ -11.268478 +gl -11.301270 +nv -11.324578 +ik -11.324707 +gs -11.325694 +ps -11.343657 +eb -11.345117 +iz -11.388599 +sm -11.419962 +ws -11.432281 +ja -11.500788 +ze -11.531022 +sy -11.539544 +kn -11.548069 +dd -11.563498 +_q -11.618368 +x_ -11.653591 +sk -11.721351 +je -11.733770 +oy -11.771330 +lf -11.789750 +dy -11.793395 +cy -11.835392 +xt -11.840991 +tc -11.862349 +yi -11.868182 +b_ -11.915042 +bs -11.975210 +eu -11.979036 +ek -11.983108 +nm -12.058542 +eq -12.062890 +lm -12.078963 +rp -12.095345 +dl -12.161285 +rf -12.173997 +hy -12.185458 +wr -12.217445 +hn -12.258335 +oe -12.272666 +dg -12.309437 +lv -12.317583 +lp -12.337736 +tm -12.395557 +rb -12.400281 +my -12.409489 +gg -12.421805 +eh -12.435600 +xi -12.451135 +ya -12.455986 +ix -12.512141 +sw -12.521458 +lk -12.546002 +gy -12.571292 +sd -12.602089 +ka -12.615627 +oh -12.616206 +cs -12.657785 +ah -12.716250 +xa -12.769414 +dv -12.828577 +az -12.865582 +za -12.883122 +xe -12.923456 +ym -12.929387 +lb -13.019598 +hs -13.033900 +yp -13.071035 +ae -13.073828 +aj -13.076012 +uf -13.103269 +ax -13.103478 +by -13.139472 +xc -13.145503 +dm -13.152424 +ox -13.166657 +gt -13.202220 +zi -13.202518 +bb -13.227588 +hl -13.301603 +nj -13.304908 +rw -13.312693 +kl -13.313041 +yl -13.329177 +yr -13.380156 +sf -13.385055 +dn -13.391840 +_z -13.430875 +z_ -13.483775 +tn -13.504491 +tb -13.516737 +ko -13.531928 +wl -13.543780 +pm -13.546901 +dw -13.555953 +iu -13.556620 +lr -13.637938 +yt -13.649817 +lw -13.653923 +yn -13.657564 +nr -13.703814 +sb -13.714477 +oj -13.745898 +rh -13.776870 +hm -13.798157 +zo -13.842975 +sq -13.870668 +py -13.876406 +sn -13.877516 +nh -13.899467 +yc -13.911361 +lc -13.911970 +ky -13.945769 +sr -13.970229 +uy -14.021526 +ji -14.081783 +bc -14.127437 +iq -14.153481 +bt -14.173607 +ez -14.179515 +nw -14.193179 +v_ -14.204981 +bj -14.220700 +vy -14.234406 +fy -14.264704 +uk -14.274442 +hb -14.281932 +mc -14.296289 +hw -14.313578 +nb -14.322950 +tf -14.343191 +aq -14.361777 +yb -14.368175 +ii -14.386071 +lg -14.428693 +kr -14.430788 +hd -14.462340 +mn -14.556652 +dc -14.576484 +uo -14.603926 +wt -14.629092 +fs -14.645148 +yd -14.661111 +ao -14.734749 +ku -14.735827 +gm -14.770556 +pd -14.859456 +xu -14.866291 +np -14.884836 +yw -14.893878 +wd -14.899588 +nz -14.933837 +oz -14.941946 +cn -14.942693 +cq -14.947850 +dh -14.995800 +cb -15.025214 +kh -15.037486 +aa -15.042015 +wy -15.046916 +bm -15.049059 +tp -15.053266 +ln -15.077037 +df -15.104267 +zz -15.139995 +q_ -15.216328 +ij -15.282563 +gd -15.291840 +uz -15.361389 +xh -15.371189 +ej -15.375105 +uv -15.455868 +mf -15.509955 +ux -15.527905 +bd -15.533521 +ml -15.542930 +dj -15.570506 +sg -15.597180 +tz -15.619437 +kg -15.663641 +db -15.668301 +wk -15.669125 +bn -15.702899 +yu -15.722295 +td -15.758375 +fg -15.784328 +pg -15.840304 +_x -15.869523 +nq -15.876168 +cd -15.927332 +gb -15.935405 +dt -15.953862 +dp -15.982937 +wm -15.988754 +mt -16.015576 +kt -16.061898 +mr -16.121363 +tg -16.145814 +kf -16.157908 +fp -16.158101 +bv -16.173802 +vu -16.176338 +xo -16.178879 +ih -16.192438 +pc -16.253950 +kp -16.263874 +wf -16.283299 +zy -16.287510 +dq -16.296607 +qi -16.304054 +pb -16.310468 +xy -16.325329 +zu -16.325979 +iw -16.338159 +qa -16.404077 +cm -16.438787 +kw -16.482779 +zh -16.489798 +vd -16.511802 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/epo b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/epo new file mode 100644 index 00000000000..6e2d54ef193 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/epo @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 162799666 +#UNIQUE_BIGRAMS 618 +#UNSEEN_LOG_PROB -19.015353 +#EXPECTED_SCORE -7.381372 +o_ -5.080250 +j_ -5.342836 +s_ -5.414359 +_k -5.483713 +n_ -5.784016 +_p -5.874099 +aj -5.991549 +on -6.039775 +a_ -6.057688 +ta -6.131997 +an -6.134530 +is -6.200030 +st -6.217461 +as -6.268302 +oj -6.312629 +ka -6.341434 +_s -6.345894 +_e -6.388056 +ti -6.468565 +es -6.559624 +er -6.565477 +ro -6.590822 +to -6.640439 +e_ -6.645299 +ra -6.674242 +_t -6.691230 +en -6.694215 +ri -6.762158 +i_ -6.783823 +in -6.795495 +nt -6.800571 +_a -6.806114 +li -6.817620 +_m -6.913754 +re -6.966209 +or -6.966410 +ia -7.002890 +ar -7.012013 +te -7.019398 +ko -7.028734 +po -7.122099 +io -7.246759 +la -7.259464 +u_ -7.265433 +pr -7.298654 +no -7.309925 +al -7.316447 +_l -7.383573 +_d -7.417001 +ni -7.435081 +ki -7.447746 +ma -7.472363 +ci -7.485692 +_i -7.502753 +un -7.506342 +lo -7.531236 +_f -7.550803 +r_ -7.560337 +tr -7.620864 +_r -7.624289 +pe -7.657254 +na -7.698803 +mo -7.702140 +_n -7.704360 +_v -7.708700 +do -7.716679 +vi -7.734983 +iu -7.762209 +si -7.801713 +el -7.819978 +ig -7.867305 +di -7.868489 +au -7.872297 +jn -7.872968 +gi -7.879762 +va -7.881761 +om -7.900268 +am -7.904485 +le -7.904648 +at -7.929755 +ur -7.934573 +ek -7.938362 +nd -7.945450 +_c -7.947238 +il -7.955928 +me -7.982422 +it -8.025388 +de -8.034884 +_g -8.062163 +se -8.068037 +pl -8.204635 +pa -8.204878 +mi -8.227834 +_o -8.238198 +ie -8.242816 +da -8.244255 +ve -8.303055 +ga -8.316143 +ku -8.329636 +nu -8.333236 +_h -8.396145 +rt -8.416969 +m_ -8.430765 +ol -8.431698 +vo -8.434994 +ce -8.456773 +_b -8.492629 +ne -8.515352 +em -8.520625 +ov -8.543373 +_j -8.562353 +sp -8.578899 +sa -8.583750 +nk -8.590938 +tu -8.596221 +_u -8.611414 +ul -8.615833 +ng -8.664697 +su -8.686427 +ir -8.688297 +kt -8.690020 +os -8.696494 +ed -8.697131 +ik -8.709780 +so -8.715237 +go -8.778229 +et -8.844860 +fa -8.845353 +id -8.894017 +bo -8.903980 +ad -8.905891 +gr -8.942465 +l_ -8.974470 +ns -9.024011 +fo -9.038299 +co -9.042691 +zi -9.046950 +nc -9.047691 +ok -9.064099 +jo -9.073350 +ja -9.073565 +uj -9.082526 +mp -9.110193 +iv -9.160058 +av -9.165772 +fi -9.170044 +im -9.184465 +du -9.195812 +eg -9.225737 +ak -9.227478 +ru -9.233392 +ag -9.249408 +ap -9.267465 +lt -9.271835 +us -9.279211 +ec -9.328770 +bl -9.334388 +ut -9.348669 +kr -9.352574 +mu -9.360515 +um -9.376077 +d_ -9.392153 +ha -9.393213 +ge -9.398538 +ac -9.434540 +lu -9.436473 +ev -9.456786 +fe -9.478722 +ej -9.512181 +rm -9.532540 +ua -9.547966 +ez -9.567925 +ca -9.580455 +op -9.607597 +sk -9.659499 +br -9.666136 +ic -9.673332 +ba -9.744851 +zo -9.775893 +rd -9.784253 +ab -9.804769 +ho -9.815389 +be -9.817443 +kv -9.818091 +pi -9.868510 +ks -9.875877 +az -9.895991 +za -9.909027 +rn -9.962038 +rs -10.040388 +ib -10.056841 +ud -10.134004 +eb -10.140861 +ep -10.158680 +rk -10.170784 +ze -10.187428 +og -10.204496 +ub -10.209083 +hi -10.231526 +kl -10.237067 +uz -10.245379 +uk -10.285905 +je -10.293893 +fr -10.297368 +iz -10.322702 +of -10.327796 +ke -10.336959 +rb -10.340160 +gv -10.366896 +t_ -10.368102 +ef -10.408760 +mb -10.427628 +he -10.461039 +bi -10.467148 +ju -10.504289 +up -10.555582 +pu -10.600899 +kc -10.622992 +rg -10.629351 +od -10.631289 +eo -10.678581 +nf -10.721403 +sc -10.723649 +gu -10.746243 +ea -10.746608 +lp -10.762268 +ip -10.805261 +oc -10.807101 +af -10.858431 +ue -10.880196 +ot -10.895443 +fl -10.902533 +ld -10.914080 +uo -10.918533 +kz -10.939125 +if -10.987406 +lk -11.003507 +ui -11.036893 +ob -11.041836 +ch -11.069545 +fu -11.136196 +rc -11.244325 +ii -11.246714 +pt -11.254032 +vu -11.339862 +ug -11.349056 +bu -11.360684 +rv -11.367365 +gn -11.374711 +k_ -11.447562 +oz -11.498521 +lf -11.522018 +ls -11.590245 +lv -11.607728 +gl -11.633652 +nv -11.666720 +rl -11.681721 +eu -11.681750 +lm -11.707688 +ll -11.722855 +sm -11.803169 +gh -11.854291 +lg -11.883524 +dr -11.928965 +ft -11.948936 +hu -11.961923 +xi -11.997860 +uf -12.008045 +rp -12.022240 +ei -12.048277 +_z -12.091867 +g_ -12.232123 +sv -12.232976 +nj -12.260192 +nh -12.296420 +sl -12.354313 +zu -12.394257 +cx -12.394447 +ee -12.459741 +b_ -12.524366 +nl -12.593938 +uc -12.640030 +kd -12.657157 +kn -12.662433 +jt -12.673332 +dz -12.715207 +sh -12.752146 +bs -12.758149 +y_ -12.776685 +x_ -12.856129 +lb -12.871786 +ai -12.872716 +ae -12.893739 +rf -12.903551 +ux -12.918910 +ao -12.976598 +gx -13.017456 +p_ -13.063445 +ln -13.133944 +dm -13.134980 +oe -13.151495 +uv -13.189983 +jm -13.229525 +z_ -13.249235 +bt -13.272547 +h_ -13.273688 +vr -13.286115 +_w -13.287266 +f_ -13.296420 +_y -13.324690 +nn -13.367318 +tl -13.466145 +tp -13.469056 +th -13.484411 +oh -13.494501 +ou -13.545189 +rr -13.581555 +tm -13.628256 +lj -13.638391 +mt -13.643144 +c_ -13.736942 +ah -13.783542 +bj -13.787296 +np -13.793574 +oo -13.795589 +v_ -13.797732 +lh -13.807482 +tn -13.809517 +lr -13.812191 +oa -13.881249 +ji -13.894953 +tt -13.902669 +sx -13.972034 +eh -13.995000 +xa -14.093647 +gm -14.110164 +sr -14.121333 +cu -14.122597 +sd -14.145220 +nm -14.179504 +ss -14.206222 +oi -14.217489 +md -14.222223 +nr -14.222393 +wa -14.224597 +mn -14.243380 +sf -14.254075 +kk -14.272372 +ry -14.281519 +yi -14.310036 +nz -14.310216 +ya -14.392064 +jr -14.393780 +mf -14.412789 +vj -14.449791 +jp -14.489396 +ay -14.497573 +zh -14.522175 +dp -14.538320 +kf -14.554648 +ij -14.579818 +wi -14.653042 +gd -14.714135 +xe -14.802789 +ps -14.837134 +ms -14.856195 +jb -14.897520 +dl -14.904570 +cd -15.014960 +ck -15.021135 +xu -15.034159 +cl -15.174251 +_q -15.182137 +lc -15.185105 +_x -15.231058 +xo -15.249926 +jl -15.255115 +we -15.275707 +tv -15.291969 +js -15.334909 +qi -15.358542 +tk -15.370505 +uu -15.391683 +tj -15.397026 +fm -15.409314 +sn -15.438138 +tf -15.483701 +dv -15.483701 +yo -15.509098 +mk -15.524470 +ph -15.532848 +ml -15.532848 +w_ -15.545083 +hn -15.566425 +nb -15.580686 +rz -15.636019 +ts -15.659220 +ds -15.712944 +gt -15.716280 +zl -15.740334 +aa -15.779175 +dh -15.781170 +ey -15.851210 +qu -15.872317 +hr -15.904570 +jz -15.925376 +jk -15.972461 +kp -15.997172 +kh -16.038924 +rj -16.061777 +ew -16.071508 +jh -16.112359 +cp -16.175891 +ow -16.175891 +sb -16.183125 +tb -16.191723 +km -16.197705 +mm -16.202375 +gk -16.207731 +ny -16.211761 +cj -16.241662 +ty -16.249926 +ht -16.263107 +bm -16.291969 +sz -16.317797 +dk -16.334543 +yu -16.344094 +tc -16.359659 +uy -16.365633 +bp -16.476814 +jf -16.482483 +jd -16.493888 +ww -16.497163 +gy -16.554862 +ly -16.603448 +rh -16.622992 +fg -16.640087 +dn -16.656471 +cs -16.661974 +pp -16.681401 +dj -16.694500 +wo -16.698264 +ff -16.706770 +tw -16.751045 +iw -16.752023 +bv -16.755941 +mv -16.756922 +mr -16.839731 +vt -16.843894 +hm -16.855407 +zr -16.857510 +vl -16.860670 +cm -16.875510 +yt -16.904570 +vs -16.934227 +cr -16.958850 +kb -16.978170 +pc -17.008227 +tz -17.043705 +oy -17.064203 +uh -17.067851 +ct -17.102349 +aw -17.106095 +ih -17.121176 +gb -17.122439 +pd -17.130046 +sg -17.131318 +cz -17.149239 +xr -17.184445 +zb -17.188410 +dy -17.199038 +zn -17.200372 +zy -17.201707 +gs -17.263107 +bb -17.279932 +dd -17.292681 +df -17.305543 +vm -17.314182 +ye -17.358170 +dw -17.382190 +bn -17.385221 +bd -17.402006 +gc -17.423654 +vd -17.425213 +gz -17.425213 +cc -17.463139 +ex -17.471168 +yd -17.515310 +kj -17.545507 +gg -17.560846 +fv -17.600803 +jg -17.616744 +cf -17.634666 +mc -17.649166 +bf -17.660137 +ax -17.661974 +dt -17.680470 +jv -17.712468 +mj -17.754961 +yn -17.764795 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/est b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/est new file mode 100644 index 00000000000..c69056d8fce --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/est @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 89940701 +#UNIQUE_BIGRAMS 627 +#UNSEEN_LOG_PROB -18.979854 +#EXPECTED_SCORE -7.350134 +a_ -5.304256 +e_ -5.509810 +_k -5.579500 +s_ -5.721599 +i_ -5.805275 +st -5.920180 +se -5.978047 +_s -5.997894 +ta -6.102121 +d_ -6.110061 +_t -6.178206 +is -6.201843 +t_ -6.299751 +al -6.353710 +as -6.385592 +va -6.395673 +_v -6.431262 +le -6.436418 +ma -6.441433 +te -6.524820 +us -6.524922 +_p -6.542873 +es -6.547021 +el -6.580486 +_m -6.583879 +li -6.682338 +_a -6.744393 +aa -6.753755 +in -6.802353 +ku -6.802376 +tu -6.853778 +ko -6.861451 +ar -6.929540 +da -6.996972 +_o -7.002679 +ee -7.010058 +si -7.013101 +l_ -7.024055 +_e -7.042534 +at -7.046863 +ol -7.078281 +mi -7.090867 +id -7.127976 +ud -7.173753 +ja -7.210673 +an -7.223757 +u_ -7.246147 +na -7.283627 +_n -7.288177 +_l -7.301130 +ul -7.319063 +me -7.320516 +ka -7.324480 +ne -7.348489 +la -7.349786 +ks -7.350173 +ik -7.350833 +oo -7.355031 +ni -7.361305 +ad -7.364191 +en -7.383268 +sa -7.397899 +ri -7.411787 +ti -7.467780 +ii -7.493579 +ll -7.567672 +it -7.595578 +ra -7.647693 +ga -7.652397 +im -7.656722 +nu -7.679083 +_j -7.706294 +uu -7.721364 +pa -7.752233 +b_ -7.785662 +nd -7.810056 +ai -7.819388 +oi -7.831533 +em -7.846592 +ak -7.861589 +su -7.868066 +il -7.958667 +am -7.985523 +ea -7.988303 +_r -7.997890 +_u -8.013320 +gi -8.016245 +ah -8.062044 +ed -8.086762 +on -8.092823 +et -8.105239 +ke -8.108042 +ei -8.128757 +ag -8.131201 +to -8.158569 +ju -8.184761 +de -8.185924 +ut -8.186751 +vo -8.201743 +re -8.213846 +om -8.240780 +lt -8.258300 +nn -8.260094 +_i -8.268731 +ts -8.296492 +_h -8.316082 +er -8.317258 +ha -8.322774 +ek -8.325097 +ab -8.340233 +eg -8.344398 +he -8.368614 +lu -8.380368 +gu -8.385160 +av -8.403586 +or -8.420859 +ki -8.422080 +ui -8.439906 +po -8.500310 +ot -8.525790 +so -8.528233 +ur -8.548648 +ig -8.553335 +mu -8.574545 +du -8.587327 +un -8.615720 +pe -8.692970 +di -8.700984 +aj -8.720853 +ng -8.735095 +ve -8.736832 +oh -8.739319 +ev -8.749735 +ge -8.753634 +vi -8.754597 +ht -8.837406 +nt -8.841123 +ae -8.884183 +uh -8.906632 +k_ -8.926093 +n_ -8.989350 +m_ -9.088582 +ro -9.094944 +um -9.146974 +pi -9.173783 +os -9.189291 +uk -9.197480 +tt -9.215991 +mm -9.217374 +r_ -9.273279 +kk -9.276061 +lj -9.303094 +bi -9.309889 +mo -9.325282 +sk -9.333879 +lo -9.341456 +ir -9.401632 +ob -9.409257 +lm -9.435918 +ld -9.444040 +eh -9.449435 +hi -9.485131 +ub -9.495098 +ru -9.497044 +pr -9.592897 +od -9.649293 +tl -9.671790 +rv -9.682664 +op -9.691961 +pu -9.703028 +au -9.712364 +lg -9.720203 +eb -9.732459 +g_ -9.744311 +ba -9.778051 +ss -9.800376 +hu -9.801607 +ap -9.829766 +rt -9.838081 +ia -9.846902 +ug -9.886239 +rd -9.904386 +og -9.933234 +no -9.994340 +ok -10.008132 +oe -10.015552 +o_ -10.042686 +v_ -10.080865 +jo -10.094953 +ou -10.103575 +ib -10.197734 +ie -10.213760 +rg -10.215800 +iv -10.319531 +rj -10.321295 +kr -10.378183 +uv -10.380384 +tr -10.475633 +ho -10.536226 +ua -10.556182 +rr -10.570917 +ep -10.583242 +be -10.602092 +kt -10.612025 +hk -10.669489 +ln -10.677190 +kl -10.741480 +mb -10.771139 +rm -10.806640 +io -10.858351 +vu -10.860079 +hv -10.928834 +br -10.975065 +ov -10.995911 +sm -11.028276 +ao -11.053794 +rn -11.063305 +je -11.123084 +rk -11.136202 +hj -11.175248 +ih -11.197263 +tm -11.220615 +_f -11.241862 +_d -11.270305 +pp -11.287403 +sj -11.324562 +eu -11.329053 +oj -11.332938 +sv -11.369224 +up -11.377564 +sp -11.391761 +tk -11.402012 +ns -11.405575 +rs -11.485005 +ip -11.495202 +dl -11.496360 +ps -11.568966 +nk -11.658273 +ue -11.683640 +lv -11.689985 +eo -11.695667 +tn -11.703777 +ds -11.810178 +dm -11.819772 +pl -11.830773 +_b -11.841859 +_g -11.852437 +lk -11.872626 +fo -11.902159 +dr -11.973258 +bl -11.996141 +iu -12.052669 +sl -12.061007 +fi -12.062790 +ij -12.122261 +ms -12.151373 +ls -12.171210 +bu -12.206938 +p_ -12.266230 +uj -12.287242 +ej -12.359244 +gl -12.444654 +gr -12.462650 +dn -12.467457 +km -12.606687 +sn -12.659051 +mn -12.673963 +h_ -12.709407 +pt -12.784827 +ef -12.829547 +rl -12.843273 +kv -12.844570 +tj -12.851311 +oa -12.892064 +do -12.948892 +mp -12.959457 +gm -12.961119 +tv -13.192751 +_c -13.238215 +rb -13.350841 +fa -13.408799 +dk -13.514829 +nl -13.595922 +vs -13.725938 +go -13.729202 +gn -13.729637 +uo -13.731382 +nf -13.741891 +fe -13.753807 +co -13.764259 +y_ -13.803626 +hn -13.853090 +hm -13.871243 +gh -14.052873 +af -14.074019 +nv -14.079007 +bo -14.091274 +ic -14.123263 +kn -14.212104 +dj -14.249106 +sr -14.285480 +_о -14.403923 +от -14.403923 +gs -14.410894 +ml -14.422823 +lb -14.479223 +yr -14.509956 +jn -14.517836 +py -14.582873 +ch -14.589185 +е_ -14.597512 +_w -14.601095 +ca -14.609491 +_q -14.639063 +ит -14.650156 +qu -14.657185 +lf -14.727243 +sh -14.817918 +th -14.850244 +ме -14.876541 +ен -14.876541 +vn -14.880406 +fr -14.894017 +ly -14.903818 +of -14.929116 +vk -14.941176 +hh -14.956904 +bk -15.005146 +ck -15.006201 +kp -15.028544 +dv -15.100543 +dg -15.113563 +mt -15.118119 +lp -15.123835 +rp -15.130725 +gp -15.153929 +lh -15.157442 +ет -15.193652 +те -15.197263 +ть -15.197263 +hr -15.214236 +vj -15.245674 +rh -15.279726 +gk -15.281641 +fg -15.308078 +ac -15.338327 +ce -15.340987 +f_ -15.341653 +wa -15.453084 +fl -15.468274 +w_ -15.513328 +oc -15.536774 +та -15.541357 +mk -15.573848 +vl -15.579335 +_y -15.594334 +sc -15.595922 +bs -15.619147 +тв -15.645215 +ве -15.645215 +т_ -15.645215 +_н -15.650156 +на -15.650156 +аж -15.650156 +жм -15.650156 +ми -15.650156 +тм -15.650156 +ни -15.650156 +ь_ -15.650156 +_ч -15.650156 +чт -15.650156 +то -15.650156 +об -15.650156 +бы -15.650156 +ы_ -15.650156 +dt -15.653459 +yo -15.747396 +fu -15.787660 +bj -15.802251 +pm -15.836569 +tp -15.867882 +c_ -15.876541 +x_ -15.960991 +_с -15.963039 +vr -15.965090 +ry -16.019459 +_к -16.022659 +we -16.035531 +_д -16.079285 +др -16.079285 +ру -16.079285 +уг -16.079285 +hl -16.103928 +nj -16.116409 +if -16.126702 +ко -16.145183 +ом -16.145183 +мм -16.145183 +нт -16.145183 +ар -16.145183 +ри -16.145183 +ии -16.145183 +и_ -16.145183 +vp -16.147510 +wi -16.147510 +ед -16.153344 +ду -16.153344 +щи -16.153344 +ие -16.153344 +rc -16.185261 +ct -16.224026 +пр -16.298349 +vd -16.321808 +kg -16.353693 +ax -16.369903 +ay -16.382181 +jj -16.393184 +ew -16.416846 +mg -16.475565 +ec -16.481423 +az -16.508086 +jm -16.555192 +tz -16.641111 +db -16.641111 +zi -16.665914 +kh -16.734220 +gb -16.764259 +nh -16.767835 +kj -16.818844 +nc -16.828146 +sw -16.837508 +ny -16.839388 +сл -16.846932 +ле -16.846932 +ую -16.846932 +ющ -16.846932 +_z -16.862138 +gt -16.864050 +cl -16.879439 +yl -16.950796 +nr -16.952829 +ff -16.989929 +га -17.075957 +а_ -17.075957 +_в -17.082621 +ва -17.082621 +аш -17.082621 +ше -17.082621 +ег -17.082621 +го -17.082621 +о_ -17.082621 +гу -17.082621 +у_ -17.082621 +тп -17.089315 +ра -17.089315 +ав -17.089315 +вл -17.089315 +ля -17.089315 +яе -17.089315 +ст -17.089315 +ат -17.089315 +ью -17.089315 +ю_ -17.089315 +_x -17.089315 +ky -17.091554 +zu -17.093796 +ey -17.118690 +ft -17.120975 +ci -17.125555 +bm -17.167442 +nb -17.193652 +z_ -17.200884 +cu -17.208152 +oy -17.222798 +sf -17.275266 +bb -17.277813 +ex -17.303530 +cr -17.348329 +xx -17.359076 +ez -17.369903 +bc -17.372622 +pk -17.378077 +vm -17.391804 +xi -17.400103 +cc -17.414042 +pn -17.430949 +yn -17.445191 +dz -17.448056 +sd -17.456687 +vg -17.494693 +wo -17.515580 +_п -17.542888 +ре -17.542888 +ды -17.542888 +ыд -17.542888 +ущ -17.542888 +zw -17.545954 +mf -17.573848 +rf -17.592748 +rw -17.602292 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/eus b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/eus new file mode 100644 index 00000000000..147cfa02e8c --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/eus @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 72171549 +#UNIQUE_BIGRAMS 535 +#UNSEEN_LOG_PROB -20.812145 +#EXPECTED_SCORE -7.111608 +a_ -4.873155 +n_ -4.903274 +en -5.232097 +_e -5.377391 +ar -5.703813 +er -5.800279 +_b -5.836592 +o_ -5.849796 +ta -5.851361 +an -5.928284 +ko -5.995410 +ra -6.026953 +re -6.139677 +te -6.147473 +ak -6.177830 +at -6.281508 +et -6.336815 +_a -6.399286 +k_ -6.413666 +tz -6.466910 +_d -6.472331 +rr -6.539940 +ze -6.594885 +_h -6.627293 +ri -6.629035 +it -6.661467 +e_ -6.667906 +in -6.680923 +tu -6.686617 +ba -6.705426 +ia -6.785286 +_z -6.862181 +al -6.865844 +ai -6.893725 +za -6.910961 +be -6.928208 +na -6.936059 +i_ -6.951961 +_g -6.985436 +u_ -6.986026 +ek -6.989919 +_i -7.016017 +la -7.028482 +or -7.043724 +de -7.068198 +ik -7.089964 +ea -7.096504 +di -7.099228 +ur -7.128601 +rt -7.201442 +iz -7.272976 +ka -7.296783 +zi -7.297173 +ha -7.375057 +da -7.409134 +nt -7.411535 +ir -7.542509 +es -7.569253 +du -7.575306 +eg -7.600042 +on -7.637445 +gi -7.681474 +go -7.686959 +ne -7.716262 +ga -7.724352 +oa -7.737875 +le -7.751018 +bi -7.759361 +st -7.763321 +nd -7.796983 +un -7.843695 +ez -7.867619 +ut -7.869834 +gu -7.886936 +_l -7.922736 +_o -7.953336 +az -7.958080 +au -7.963593 +ma -7.967480 +ro -7.985155 +ua -8.046037 +ue -8.053559 +ki -8.063375 +ag -8.064193 +as -8.074809 +_p -8.079901 +ti -8.120837 +ld -8.127708 +zu -8.129202 +_k -8.135754 +il -8.164441 +z_ -8.175755 +el -8.182453 +io -8.203960 +_m -8.207749 +sa -8.258996 +_n -8.268550 +ho -8.274635 +_s -8.361986 +ku -8.384215 +ab -8.404027 +ie -8.421548 +sk -8.422378 +hi -8.425914 +li -8.428405 +eh -8.460542 +uz -8.465706 +ke -8.484449 +zt -8.486943 +t_ -8.494176 +me -8.527564 +ge -8.528767 +r_ -8.542938 +_u -8.547396 +ru -8.580699 +_j -8.615981 +us -8.618163 +ni -8.648252 +zk -8.649688 +si -8.785775 +he -8.798251 +id -8.885976 +_t -8.894874 +pa -8.915652 +uk -8.917130 +so -8.934239 +em -8.950059 +ok -8.955011 +do -8.981836 +ol -9.063075 +os -9.114878 +pe -9.120041 +ts -9.123906 +is -9.127826 +no -9.147689 +ad -9.161096 +ot -9.203754 +ng -9.204461 +to -9.252422 +rd -9.264444 +ja -9.268827 +su -9.294996 +_f -9.314426 +ud -9.346118 +ed -9.358570 +am -9.363750 +eu -9.418303 +tx -9.463270 +pr -9.528516 +rk -9.553880 +lo -9.567845 +oi -9.600434 +se -9.616836 +oe -9.642425 +bu -9.654263 +l_ -9.657085 +tr -9.680515 +lt -9.708339 +ei -9.716926 +ah -9.779692 +mi -9.791300 +mo -9.828203 +ul -9.863372 +po -9.874756 +ug -9.879060 +ib -9.891462 +lu -10.029176 +mu -10.045561 +nb -10.099478 +ee -10.103211 +oz -10.161267 +bo -10.222905 +fi -10.247578 +om -10.269271 +og -10.275402 +ap -10.348318 +im -10.361907 +ig -10.375997 +s_ -10.428667 +zo -10.429466 +xi -10.467057 +rg -10.473125 +ob -10.479104 +sp -10.568619 +pi -10.572541 +np -10.587196 +fa -10.588088 +pu -10.598464 +af -10.610853 +op -10.665226 +jo -10.668215 +nu -10.674311 +ip -10.694873 +rm -10.744697 +xe -10.800896 +kt -10.846913 +ep -10.858223 +rn -10.858520 +hu -10.871045 +lk -10.877311 +rb -10.917343 +gr -10.930469 +eb -10.990330 +nl -11.091342 +fo -11.134551 +fr -11.137926 +um -11.170545 +rl -11.173543 +od -11.202787 +nk -11.237214 +ub -11.289994 +pl -11.333334 +lb -11.467793 +xa -11.571232 +sl -11.712475 +fe -11.744560 +zp -11.784267 +_c -11.891671 +bl -11.900050 +ae -11.906098 +je -11.907557 +kr -11.940648 +oh -11.972427 +ih -11.984608 +eo -12.010189 +sm -12.147099 +dr -12.183086 +br -12.239677 +_r -12.253666 +ui -12.327774 +xo -12.402971 +rp -12.458818 +fu -12.495171 +_x -12.522432 +ef -12.596142 +nf -12.599736 +ns -12.612949 +ix -12.618595 +ju -12.644983 +kl -12.698988 +co -12.813037 +of -12.887726 +xu -12.941120 +ex -12.943323 +lm -12.999182 +vi -13.005579 +if -13.046428 +dm -13.088815 +kn -13.094574 +th -13.100004 +ca -13.130512 +d_ -13.141850 +ll -13.142392 +_v -13.172712 +ce -13.227451 +nc -13.240547 +ch -13.306455 +ic -13.321929 +kz -13.368948 +uh -13.369794 +zl -13.396059 +lp -13.452754 +sg -13.480588 +sn -13.498753 +ax -13.513405 +lg -13.519025 +zb -13.521844 +va -13.617338 +sb -13.628434 +ci -13.633506 +up -13.650371 +ii -13.704047 +aj -13.708857 +c_ -13.815484 +ac -13.927507 +ps -13.934063 +m_ -13.999673 +pt -14.037493 +y_ -14.095448 +iu -14.142392 +ve -14.146737 +ji -14.196159 +_w -14.223048 +b_ -14.226876 +av -14.229561 +aa -14.251617 +ux -14.387250 +x_ -14.398431 +sc -14.454773 +oo -14.459719 +oc -14.461071 +qu -14.468302 +xk -14.483333 +tb -14.550338 +ou -14.583326 +ox -14.628687 +ij -14.839312 +ec -14.840484 +lf -14.854036 +ao -14.854628 +xv -14.952642 +ay -15.010849 +bj -15.016801 +nz -15.070128 +cu -15.092302 +gn -15.112697 +yo -15.142031 +gl -15.159483 +pp -15.161679 +sf -15.166082 +rz -15.179372 +rh -15.202552 +g_ -15.217706 +_y -15.241515 +ph -15.243840 +we -15.254740 +ck -15.268088 +wa -15.297572 +rc -15.348371 +bs -15.360934 +h_ -15.402754 +_q -15.404487 +ev -15.407959 +ew -15.411440 +tt -15.429852 +w_ -15.434271 +tm -15.532227 +rs -15.562862 +iv -15.609072 +ct -15.616083 +nn -15.627169 +fl -15.680761 +uc -15.710464 +tl -15.722303 +mp -15.733150 +sh -15.741887 +jk -15.771772 +p_ -15.777374 +mt -15.795451 +uv -15.808011 +kh -15.824156 +ry -15.833464 +wi -15.880925 +zg -15.936255 +cr -15.942536 +nr -16.000328 +ya -16.088119 +xt -16.095098 +vo -16.099302 +uo -16.114823 +ej -16.144925 +ss -16.163879 +pk -16.165348 +sr -16.165348 +mm -16.169762 +nh -16.174190 +lv -16.183086 +ty -16.193535 +bk -16.201045 +j_ -16.222284 +hn -16.265723 +pn -16.272037 +uf -16.289544 +nj -16.350039 +jr -16.411440 +v_ -16.436042 +f_ -16.470116 +cl -16.503156 +ys -16.529388 +ow -16.556105 +oj -16.571597 +hr -16.587257 +tw -16.591199 +ny -16.611071 +gm -16.725549 +ft -16.749576 +hl -16.769537 +ov -16.769537 +sz -16.782999 +rj -16.854628 +uj -16.868913 +pz -16.893039 +bg -16.905255 +mb -16.942536 +wo -16.967936 +ht -17.006895 +ey -17.041532 +tn -17.066008 +rf -17.071504 +rv -17.162412 +gh -17.198036 +oy -17.207081 +yp -17.210109 +yl -17.363460 +hy -17.377006 +xx -17.401023 +ly -17.421932 +zq -17.508737 +ms -17.512470 +lh -17.527498 +sq -17.558032 +bb -17.561895 +mn -17.605081 +my -17.637321 +nv -17.712609 +gb -17.734239 +vr -17.747375 +gt -17.751780 +dj -17.801146 +lc -17.828802 +gg -17.852261 +tc -17.856999 +dn -17.880925 +cc -17.885758 +dd -17.905255 +bh -17.910170 +yi -17.915102 +pv -17.920052 +ye -17.950109 +zn -17.955180 +bn -17.980806 +yr -18.077021 +bp -18.088119 +pg -18.088119 +dw -18.093700 +yc -18.116242 +uy -18.139143 +db -18.156560 +nm -18.174190 +by -18.198036 +zz -18.240741 +sy -18.291146 +ws -18.377006 +iq -18.411440 +fg -18.461071 +ls -18.461071 +ff -18.468302 +zy -18.475570 +vk -18.519964 +dy -18.527498 +cq -18.573545 +cy -18.695536 +hk -18.704047 +wn -18.704047 +aw -18.729887 +mg -18.756199 +cn -18.782999 +sd -18.801146 +tp -18.810306 +ln -18.819525 +yd -18.856999 +dg -18.856999 +lz -18.866522 +rx -18.885758 +yw -18.905255 +ym -18.935002 +bc -18.955180 +gd -18.955180 +pd -18.965376 +tg -19.060533 +ds -19.082559 +xf -19.104927 +ww -19.150731 +wh -19.162412 +aq -19.174190 +dl -19.174190 +yk -19.174190 +zc -19.174190 +ks -19.186064 +gk -19.246946 +lj -19.259437 +yn -19.259437 +gy -19.272037 +zm -19.310511 +dv -19.377006 +vd -19.377006 +hb -19.390681 +gw -19.461071 +yv -19.475570 +df -19.475570 +kd -19.519964 +vy -19.535071 +eq -19.597132 +zj -19.645495 +bt -19.661983 +mc -19.678662 +vl -19.729887 +zd -19.729887 +cd -19.747375 +tk -19.765077 +wt -19.765077 +py -19.819525 +pf -19.856999 +iy -19.876108 +yt -19.935002 +wr -19.935002 +lw -19.996402 +ml -19.996402 +uu -20.174190 +bd -20.198036 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fao b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fao new file mode 100644 index 00000000000..7af25604dd7 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fao @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 52538674 +#UNIQUE_BIGRAMS 552 +#UNSEEN_LOG_PROB -20.525542 +#EXPECTED_SCORE -7.432895 +r_ -5.423722 +a_ -5.432320 +in -5.453232 +_s -5.571315 +i_ -5.695199 +ar -5.856243 +n_ -6.034256 +d_ -6.034746 +_v -6.067369 +an -6.116003 +_f -6.169836 +_h -6.174898 +um -6.185765 +_t -6.220655 +id -6.280509 +m_ -6.280571 +er -6.325155 +ei -6.367556 +ur -6.397663 +ti -6.407026 +t_ -6.436343 +st -6.511738 +ta -6.519582 +ri -6.595094 +_e -6.645815 +di -6.676169 +vi -6.682953 +ra -6.687452 +nd -6.815384 +sk -6.907491 +na -6.919667 +ni -6.951560 +nn -6.983854 +la -7.058158 +ka -7.064415 +da -7.104702 +al -7.120683 +il -7.123694 +ng -7.131057 +ir -7.167250 +ad -7.178026 +_a -7.193284 +_l -7.235697 +or -7.268620 +du -7.285751 +_m -7.295717 +u_ -7.304876 +ki -7.334290 +l_ -7.335252 +nu -7.341514 +ve -7.358236 +tt -7.378860 +rd -7.383782 +he -7.387740 +li -7.423893 +va -7.441212 +un -7.459394 +_k -7.505453 +ja -7.535720 +ga -7.546512 +yr -7.550120 +ma -7.581124 +ae -7.624899 +is -7.645853 +en -7.661474 +_b -7.676360 +_u -7.699439 +su -7.713439 +sa -7.735332 +_g -7.828357 +ha -7.834724 +ik -7.844808 +ku -7.869711 +et -7.910336 +ru -7.920718 +am -7.935525 +si -7.947148 +it -7.953859 +gi -7.987542 +se -8.016609 +_n -8.029674 +ig -8.037431 +av -8.049406 +fy -8.079893 +ev -8.098611 +el -8.113813 +ro -8.114604 +_i -8.122404 +g_ -8.135317 +as -8.150701 +tu -8.216734 +lo -8.217502 +to -8.229667 +og -8.248250 +kk -8.282306 +le -8.293893 +ok -8.320219 +_o -8.338351 +fr -8.364421 +ld -8.367103 +lu -8.386416 +fo -8.393666 +on -8.393952 +ll -8.413631 +vu -8.450303 +oy -8.451350 +gu -8.469953 +me -8.498628 +gj -8.507885 +mi -8.516336 +re -8.524514 +ol -8.570499 +at -8.582344 +ag -8.598102 +ko -8.611487 +s_ -8.618053 +_r -8.630688 +od -8.636632 +te -8.688732 +ul -8.689015 +rt -8.691851 +es -8.710250 +k_ -8.759227 +eg -8.794908 +ft -8.825563 +ns -8.830455 +rn -8.873776 +ds -8.873982 +pp -8.886221 +jo -8.904435 +vn -8.905949 +rs -8.920179 +up -8.925096 +gg -8.944420 +ge -8.964293 +fe -8.986978 +pa -8.996933 +mu -9.032612 +fa -9.045033 +be -9.076265 +ut -9.098145 +om -9.112102 +ey -9.118056 +rk -9.125705 +ss -9.134599 +tr -9.137643 +_p -9.141359 +ot -9.148806 +ak -9.165455 +_d -9.168091 +ug -9.316450 +im -9.345291 +ba -9.357956 +ne -9.382782 +ty -9.388918 +vo -9.405042 +so -9.418302 +kr -9.442458 +yt -9.455856 +bo -9.505009 +sl -9.514417 +gr -9.541213 +rl -9.545578 +nt -9.548454 +ov -9.549050 +gt -9.567329 +hv -9.603742 +us -9.607379 +iv -9.625680 +br -9.625745 +yg -9.637201 +fi -9.637223 +hj -9.650072 +ho -9.711159 +ef -9.716139 +ys -9.748558 +mb -9.768610 +tk -9.850330 +go -9.878821 +ms -9.895202 +lt -9.920100 +dr -9.955733 +no -9.966462 +ju -10.012406 +mo -10.028176 +_j -10.030299 +em -10.031362 +af -10.048417 +kt -10.085738 +rv -10.090730 +ke -10.092168 +sj -10.095679 +fl -10.097603 +gv -10.123069 +ek -10.142678 +by -10.149773 +kj -10.151209 +ip -10.172125 +lk -10.205488 +ny -10.207208 +v_ -10.208344 +kn -10.210751 +gd -10.212932 +ap -10.218320 +mm -10.246164 +rb -10.254559 +ls -10.263847 +gl -10.274283 +yn -10.289050 +bu -10.308941 +gn -10.317466 +gs -10.352435 +tl -10.383644 +ed -10.385736 +p_ -10.404000 +y_ -10.411087 +sm -10.439061 +os -10.448393 +pi -10.526477 +rr -10.534478 +aa -10.549626 +ks -10.561487 +hu -10.568892 +yd -10.601330 +pe -10.616856 +lv -10.622776 +ud -10.632420 +nl -10.635387 +rg -10.671476 +_y -10.704729 +yv -10.713462 +ps -10.721554 +sp -10.759608 +pu -10.770072 +pr -10.780515 +sv -10.825800 +vt -10.886520 +mt -10.886572 +dg -10.888341 +uk -10.932416 +ya -10.932577 +dn -10.940758 +ly -10.945516 +rj -10.983430 +lm -11.017577 +vs -11.093127 +fu -11.093847 +nk -11.097212 +ry -11.113364 +lj -11.125828 +yk -11.126626 +de -11.168865 +if -11.196567 +rm -11.244131 +do -11.267363 +ml -11.271498 +vd -11.318131 +tv -11.320587 +ab -11.344737 +pl -11.347668 +hi -11.351538 +mf -11.385883 +yl -11.400730 +vg -11.419261 +ts -11.426271 +bi -11.475152 +tj -11.479615 +kl -11.511327 +dd -11.516708 +sf -11.520979 +dv -11.576001 +po -11.713094 +tn -11.737733 +ah -11.743371 +dk -11.797765 +io -11.806885 +kv -11.818839 +o_ -11.826598 +rf -11.895437 +my -11.906147 +ym -11.910369 +yu -11.915982 +sb -11.949474 +tb -12.006971 +dl -12.051036 +nf -12.104812 +ky -12.114642 +bl -12.135742 +sn -12.152521 +ia -12.199535 +df -12.209514 +tg -12.239874 +ep -12.272109 +lf -12.298840 +tm -12.301471 +vl -12.315120 +ua -12.396282 +mr -12.414755 +ai -12.428767 +ib -12.440931 +pg -12.473512 +lg -12.497129 +sh -12.510686 +lp -12.521140 +sy -12.532158 +yp -12.565726 +op -12.589054 +ji -12.592781 +dy -12.613453 +rh -12.620353 +e_ -12.641427 +dm -12.648462 +of -12.667809 +mj -12.693407 +bj -12.714293 +fj -12.737171 +oa -12.807476 +pt -12.812801 +au -12.854493 +gm -12.856324 +je -12.861220 +mh -12.868389 +gf -12.927702 +nm -12.944704 +sg -12.961471 +md -12.980874 +mv -13.008214 +ob -13.019343 +dj -13.032167 +rp -13.052785 +nv -13.054887 +lb -13.107475 +uf -13.116226 +f_ -13.147778 +oh -13.157781 +ii -13.161550 +sd -13.208344 +ih -13.270480 +ph -13.275917 +nj -13.443834 +yi -13.444140 +sr -13.448431 +uv -13.453967 +tf -13.456126 +ln -13.476638 +eb -13.484800 +kf -13.527935 +vm -13.543917 +vr -13.564727 +mg -13.622429 +th -13.639849 +db -13.771511 +hy -13.839522 +j_ -13.894496 +vj -13.924069 +nr -13.951648 +dt -13.967836 +vb -14.087021 +ie -14.118911 +ic -14.128715 +oe -14.141561 +bs -14.182841 +dh -14.183863 +nb -14.192063 +nh -14.207565 +ub -14.222710 +aj -14.321571 +b_ -14.372498 +ac -14.409069 +vv -14.429525 +co -14.467589 +mn -14.542933 +py -14.554119 +_c -14.600435 +ea -14.671745 +ch -14.690500 +ou -14.724292 +c_ -14.759656 +vh -14.764233 +ue -14.773432 +iu -14.779598 +pk -14.802171 +_w -14.868799 +vf -14.871266 +lh -14.942973 +gb -14.960376 +ce -15.065676 +h_ -15.074176 +fs -15.078920 +mk -15.084634 +ao -15.211206 +ff -15.270751 +bb -15.292627 +oj -15.333993 +oi -15.402513 +eu -15.465724 +mp -15.517593 +uu -15.533134 +uh -15.537046 +pn -15.681092 +gp -15.708767 +pm -15.728013 +ec -15.729504 +yf -15.793567 +lr -15.823509 +cl -15.890320 +ui -15.962128 +oo -15.963882 +gh -15.965638 +z_ -15.983318 +eo -16.034008 +hr -16.063794 +dz -16.123314 +oc -16.179271 +ca -16.181310 +ay -16.208085 +wa -16.256707 +np -16.271837 +pb -16.276189 +km -16.300363 +jg -16.345380 +pf -16.373081 +x_ -16.396578 +gy -16.422875 +kh -16.427708 +we -16.575414 +wi -16.575414 +ex -16.599753 +ci -16.627286 +hl -16.678210 +jb -16.689774 +cc -16.695592 +nc -16.713186 +bf -16.752059 +za -16.788895 +ee -16.826697 +pj -16.858974 +w_ -16.956878 +rc -17.043250 +vk -17.046964 +ck -17.050687 +ze -17.103845 +hn -17.111601 +kg -17.191549 +tp -17.229024 +uo -17.241735 +kb -17.356858 +sc -17.366106 +tz -17.375413 +_z -17.408472 +yo -17.413257 +td -17.427708 +ow -17.462001 +jv -17.554119 +ct -17.564727 +dp -17.580787 +uj -17.586181 +wt -17.624509 +ew -17.630068 +bt -17.635649 +hs -17.652523 +jy -17.663883 +pv -17.669597 +fc -17.885325 +cu -17.912167 +bm -17.988665 +cd -18.046964 +rw -18.061914 +wo -18.131177 +fn -18.171143 +fb -18.195665 +fh -18.229024 +ax -18.237486 +tc -18.289324 +cr -18.380090 +yh -18.427708 +jd -18.476951 +ux -18.507325 +pd -18.570061 +xe -18.602482 +wh -18.704362 +hd -18.739986 +bc -18.752059 +ej -18.752059 +ht -18.752059 +dw -18.776512 +wn -18.776512 +eh -18.826697 +fm -18.839522 +yb -18.839522 +nz -18.839522 +hf -19.003020 +iz -19.077021 +bn -19.092288 +ix -19.123314 +fd -19.155023 +aw -19.203933 +fg -19.220612 +xc -19.237486 +xa -19.380090 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fas b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fas new file mode 100644 index 00000000000..adb12b95cbc --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fas @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 116189490 +#UNIQUE_BIGRAMS 4501 +#UNSEEN_LOG_PROB -17.242737 +#EXPECTED_SCORE -7.997919 +_ا -4.946177 +ي_ -5.047795 +ن_ -5.422815 +ه_ -5.498425 +د_ -5.552802 +ان -5.598322 +اي -5.698424 +ت_ -5.792047 +_م -5.842599 +_ب -5.993096 +ار -6.112303 +ر_ -6.115115 +را -6.436182 +_د -6.504184 +ست -6.524668 +ين -6.595432 +_ك -6.693250 +_ت -6.737207 +اس -6.749944 +_ن -6.785222 +دا -6.799482 +م_ -6.847541 +ام -6.852935 +ند -6.939050 +ري -7.015383 +ل_ -7.089170 +ما -7.109602 +رد -7.114563 +_س -7.144000 +_ش -7.157012 +ال -7.247213 +با -7.267611 +ني -7.280557 +اد -7.338143 +ور -7.343358 +ا_ -7.370647 +ده -7.373479 +ود -7.384674 +_خ -7.418840 +وا -7.450803 +ها -7.454901 +_ر -7.482044 +بر -7.486159 +يا -7.491632 +_ه -7.497575 +رو -7.608778 +دي -7.655617 +لي -7.687372 +ير -7.687848 +سا -7.692942 +مي -7.693973 +لا -7.801787 +يد -7.804951 +اه -7.840825 +_پ -7.858888 +_گ -7.864777 +نا -7.868351 +_ج -7.892465 +تا -7.915768 +ات -7.970526 +_ف -7.985100 +ش_ -8.081919 +سي -8.087589 +از -8.098492 +_ح -8.145159 +اب -8.178029 +تي -8.186063 +_ع -8.218157 +خو -8.248212 +كر -8.260965 +ز_ -8.268323 +يم -8.269840 +_و -8.273885 +_ق -8.275469 +كا -8.276154 +ته -8.293160 +فت -8.344975 +بي -8.368120 +اش -8.370148 +مر -8.386530 +ون -8.390708 +تر -8.394238 +مو -8.426915 +وي -8.427517 +گر -8.428651 +در -8.461009 +زي -8.481008 +شت -8.485055 +ره -8.496485 +ول -8.526482 +نه -8.540237 +بو -8.550875 +كن -8.559380 +ب_ -8.591245 +هم -8.619501 +زا -8.621510 +تو -8.623644 +يل -8.642118 +يي -8.649747 +س_ -8.659304 +اف -8.676266 +شد -8.677862 +نو -8.684284 +فر -8.731768 +مه -8.760273 +يت -8.767187 +رس -8.769499 +شو -8.789743 +دو -8.795227 +دن -8.821540 +گا -8.836737 +_ز -8.843126 +جا -8.855456 +وز -8.862842 +او -8.886165 +قا -8.903774 +شا -8.929866 +هر -8.939296 +من -8.959812 +يك -8.963489 +يس -8.977084 +يش -8.982142 +ع_ -9.127860 +اع -9.150177 +نگ -9.150989 +مت -9.176792 +_چ -9.183065 +مل -9.186222 +رت -9.186389 +خا -9.190518 +يز -9.240842 +اق -9.243466 +مد -9.244505 +رف -9.271117 +گي -9.275950 +_ص -9.288343 +كي -9.292214 +نت -9.322517 +فا -9.354843 +رم -9.359942 +يه -9.366754 +وس -9.368181 +ك_ -9.369306 +ف_ -9.374184 +اخ -9.381213 +مع -9.398606 +تن -9.403702 +ق_ -9.423995 +هد -9.427282 +مس -9.437397 +_ي -9.443643 +كش -9.445633 +رك -9.447574 +حا -9.454152 +رن -9.457622 +خت -9.465554 +عا -9.469483 +يو -9.487482 +هي -9.504336 +تم -9.508400 +پي -9.524816 +وم -9.543680 +سر -9.551038 +پا -9.561115 +رب -9.573918 +شه -9.584299 +زم -9.589340 +نش -9.594112 +به -9.628284 +رگ -9.630231 +اك -9.643616 +رش -9.647634 +شي -9.661796 +جه -9.662078 +هن -9.668157 +عل -9.676851 +_ط -9.707397 +يگ -9.757342 +شن -9.770013 +نم -9.779258 +عي -9.788233 +اح -9.792906 +دم -9.797474 +له -9.810549 +سو -9.819171 +قي -9.824982 +كت -9.832876 +جم -9.833476 +شر -9.834303 +گف -9.859113 +مح -9.863756 +قر -9.867973 +بت -9.877496 +في -9.892961 +دس -9.901806 +جو -9.902032 +ط_ -9.903482 +_ل -9.903958 +شك -9.936707 +نس -9.948118 +وج -9.954708 +بل -9.958953 +پر -9.959175 +صو -9.963607 +اج -9.967194 +و_ -9.967393 +ح_ -9.992977 +نظ -9.996321 +لم -10.025790 +طر -10.026048 +نج -10.039106 +وش -10.053034 +مش -10.055965 +لو -10.088285 +تب -10.092942 +نن -10.109733 +شم -10.123033 +وب -10.140979 +اص -10.153157 +كو -10.164356 +زن -10.166467 +وه -10.172859 +چن -10.180779 +تق -10.191628 +گ_ -10.191657 +لت -10.199769 +سل -10.212367 +گو -10.217813 +نب -10.218331 +اط -10.219545 +قد -10.240767 +دگ -10.244407 +رز -10.246412 +جر -10.276097 +كل -10.281526 +گز -10.309254 +عد -10.328252 +عت -10.375206 +نك -10.380244 +تص -10.383541 +خي -10.386846 +حم -10.389342 +صا -10.389492 +عم -10.399721 +زه -10.439395 +بد -10.441794 +عه -10.449083 +عر -10.451898 +تح -10.453168 +ضا -10.453795 +ظر -10.470469 +كم -10.473520 +وت -10.482552 +هو -10.482694 +رخ -10.488017 +مق -10.489783 +تل -10.491713 +بس -10.502426 +مج -10.522303 +وع -10.534203 +يب -10.552436 +گذ -10.555778 +لب -10.563461 +كس -10.581214 +قل -10.583079 +عن -10.586989 +صل -10.607087 +نف -10.607474 +تع -10.609452 +ج_ -10.615809 +هس -10.616998 +تف -10.621060 +اگ -10.640789 +قت -10.642594 +قه -10.645356 +كه -10.646410 +نق -10.652733 +يق -10.654513 +بن -10.660650 +فز -10.664273 +ضو -10.666148 +سب -10.675885 +تش -10.680667 +_غ -10.689006 +بع -10.691920 +مك -10.696590 +زو -10.708487 +وق -10.715318 +صد -10.721133 +شگ -10.723336 +سن -10.723462 +تخ -10.731166 +خب -10.763005 +دل -10.773726 +زد -10.775575 +بخ -10.778909 +خر -10.812033 +يج -10.825804 +حد -10.825917 +حت -10.827947 +خص -10.837617 +رح -10.856187 +خش -10.857338 +رج -10.864474 +صر -10.866049 +اث -10.866999 +ص_ -10.890330 +جد -10.924095 +زش -10.928541 +لس -10.935406 +جل -10.941400 +سه -10.942353 +سم -10.943379 +چه -10.947444 +ذا -10.952135 +جن -10.952651 +رق -10.967443 +حر -10.969931 +غا -10.975821 +يع -10.994699 +خ_ -11.003364 +مب -11.012595 +فو -11.036495 +سط -11.068003 +حي -11.075567 +طل -11.085624 +ظا -11.091275 +اض -11.098635 +زر -11.105431 +قو -11.115810 +نر -11.123213 +يف -11.138359 +بز -11.162548 +حس -11.199360 +پو -11.208293 +فق -11.225493 +قب -11.230378 +يخ -11.231781 +صي -11.241037 +مخ -11.266842 +وض -11.271715 +هز -11.282655 +فه -11.284698 +لف -11.298111 +سپ -11.302966 +طو -11.303405 +غي -11.303437 +خل -11.309758 +تگ -11.310546 +رض -11.328220 +حو -11.335518 +هش -11.336513 +مط -11.338473 +بق -11.340564 +سف -11.355421 +مص -11.364199 +اغ -11.365771 +مچ -11.366688 +قع -11.368132 +قط -11.394096 +فع -11.403786 +دت -11.406210 +حق -11.418937 +سخ -11.431332 +لك -11.439861 +حل -11.443660 +رص -11.451707 +دف -11.493591 +حض -11.501345 +ث_ -11.517126 +جي -11.560496 +طا -11.576144 +نز -11.592999 +ذش -11.601115 +ضر -11.609471 +ثر -11.622841 +كز -11.629592 +اظ -11.647525 +طب -11.652713 +دش -11.659967 +اپ -11.668551 +دك -11.675885 +چي -11.676739 +سك -11.685137 +تد -11.687387 +_ث -11.695766 +وص -11.700717 +پن -11.705934 +_ض -11.715506 +ثا -11.724218 +طق -11.735055 +يح -11.774226 +چا -11.776054 +خد -11.787420 +وف -11.792521 +هت -11.818431 +سع -11.844952 +صن -11.845409 +وط -11.847421 +ظه -11.848062 +تج -11.852325 +زب -11.853749 +يچ -11.867741 +تك -11.874532 +عض -11.886752 +نخ -11.892689 +عب -11.907161 +فن -11.908641 +طي -11.909118 +گل -11.929170 +شخ -11.934798 +ثب -11.944994 +هب -11.950340 +هف -11.957977 +نط -11.987118 +국_ -11.998911 +لن -12.002269 +حك -12.015831 +يط -12.046913 +ضي -12.055027 +عو -12.071393 +ژه -12.073532 +مز -12.081421 +بك -12.097438 +پس -12.102616 +تغ -12.115286 +رع -12.121967 +نع -12.126013 +بط -12.134921 +لح -12.137772 +چ_ -12.143041 +ژي -12.161922 +ژا -12.185036 +بگ -12.185036 +چو -12.189495 +وك -12.228589 +دب -12.230258 +غر -12.230616 +ض_ -12.244166 +ضع -12.244528 +شب -12.247845 +بح -12.248027 +وپ -12.251958 +دد -12.257297 +پل -12.260279 +خس -12.268097 +رر -12.290751 +_한 -12.302621 +دق -12.318516 +صف -12.323280 +طه -12.327103 +هه -12.334267 +مث -12.335807 +وح -12.338698 +وگ -12.346307 +فل -12.350127 +بش -12.378409 +لل -12.401265 +حص -12.403282 +غ_ -12.405638 +جب -12.412391 +حب -12.434559 +ذي -12.434627 +فك -12.444491 +قش -12.445044 +_ذ -12.458190 +رژ -12.491266 +چر -12.503327 +لق -12.505274 +_t -12.512221 +سد -12.533854 +طع -12.533927 +جت -12.568355 +_국 -12.577282 +خن -12.577282 +تس -12.586646 +_ژ -12.605248 +مم -12.612928 +한국 -12.617369 +جش -12.628648 +شع -12.633295 +ذر -12.635031 +تظ -12.652913 +رچ -12.662219 +لع -12.670695 +لد -12.682074 +يژ -12.691981 +گس -12.695519 +وو -12.696507 +گش -12.701213 +دع -12.701213 +صح -12.712670 +مذ -12.718265 +پ_ -12.719018 +رل -12.726236 +_ظ -12.729605 +جز -12.738742 +هل -12.756590 +خط -12.781638 +اذ -12.786718 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fin b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fin new file mode 100644 index 00000000000..ede9d6d4d46 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fin @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 327845794 +#UNIQUE_BIGRAMS 551 +#UNSEEN_LOG_PROB -19.996888 +#EXPECTED_SCORE -7.194269 +a_ -4.531234 +n_ -4.632826 +ta -5.274231 +an -5.607696 +en -5.632937 +aa -5.933328 +in -5.951820 +i_ -5.965692 +_k -6.004355 +is -6.161869 +sa -6.212405 +ka -6.266430 +si -6.301156 +st -6.340252 +_t -6.365308 +tt -6.390245 +va -6.407309 +_s -6.414963 +ai -6.470772 +la -6.539087 +al -6.565483 +_v -6.575968 +_m -6.591567 +it -6.642468 +ll -6.669082 +se -6.735747 +li -6.742217 +te -6.780672 +ma -6.816304 +t_ -6.846543 +_p -6.914070 +ti -6.924548 +_o -6.965513 +el -6.983825 +oi -7.015671 +at -7.041302 +et -7.082942 +as -7.115346 +na -7.143175 +_j -7.152611 +ss -7.160651 +ol -7.164287 +_a -7.186266 +tu -7.201172 +_e -7.209176 +ku -7.244517 +_h -7.297978 +uo -7.318736 +ja -7.327508 +ne -7.344726 +es -7.364463 +ii -7.375007 +to -7.383156 +on -7.386705 +mi -7.387763 +_l -7.388885 +ki -7.400215 +ut -7.404856 +os -7.412880 +ko -7.453756 +le -7.477858 +jo -7.489000 +uu -7.521964 +ee -7.571217 +ke -7.606890 +ar -7.621313 +pa -7.627954 +ks -7.628340 +il -7.676095 +nn -7.681166 +ik -7.689434 +us -7.724866 +un -7.733416 +e_ -7.797165 +ri -7.815835 +de -7.818956 +ha -7.821919 +er -7.827213 +s_ -7.874372 +lu -7.884491 +ia -7.907512 +_n -7.927170 +me -8.007611 +nt -8.022811 +ak -8.029467 +ei -8.084449 +ra -8.089120 +im -8.108193 +mu -8.114925 +ot -8.155881 +ie -8.191979 +ty -8.196227 +ni -8.200546 +uk -8.205782 +vi -8.214741 +su -8.232380 +ul -8.255868 +am -8.277293 +vu -8.314532 +lo -8.367267 +_y -8.377078 +ns -8.395618 +ok -8.397884 +ht -8.399151 +he -8.473186 +av -8.483538 +o_ -8.502752 +kk -8.520515 +pi -8.560577 +au -8.660386 +om -8.664046 +_r -8.686991 +yt -8.711912 +ui -8.712118 +ah -8.770071 +iv -8.788184 +pe -8.844578 +yo -8.846900 +sk -8.858470 +no -8.866914 +pu -8.879198 +nk -8.890103 +ur -8.913278 +ek -8.919688 +aj -8.927004 +mm -8.946389 +_i -8.950589 +em -8.971776 +vo -8.978036 +lm -8.980430 +lt -9.002895 +u_ -9.032655 +lk -9.055328 +oh -9.126516 +op -9.162514 +or -9.175018 +hd -9.214068 +my -9.217823 +id -9.219842 +yh -9.265668 +rk -9.282670 +_u -9.300762 +ou -9.340323 +ir -9.346111 +nu -9.354481 +so -9.358317 +ap -9.414121 +ro -9.415344 +io -9.420349 +eh -9.506612 +rj -9.513414 +ay -9.522810 +ve -9.526472 +eu -9.547219 +uv -9.550865 +ud -9.552080 +da -9.579459 +oo -9.595064 +ys -9.606144 +ts -9.608697 +tk -9.621819 +re -9.634019 +od -9.638151 +ih -9.642210 +hi -9.666816 +po -9.709005 +ov -9.739854 +je -9.744319 +rt -9.755044 +yl -9.796276 +yk -9.802004 +ea -9.832328 +ev -9.846774 +y_ -9.855199 +ru -9.950537 +ny -10.006046 +sy -10.008120 +ij -10.029011 +do -10.064021 +yy -10.073118 +mo -10.085558 +ky -10.093257 +ua -10.123084 +ue -10.137713 +ed -10.176023 +oj -10.176792 +oa -10.193962 +ju -10.194225 +pp -10.205649 +ng -10.222279 +yi -10.247451 +mp -10.280900 +yn -10.309369 +yv -10.376898 +rv -10.381587 +uh -10.389776 +um -10.413311 +ho -10.448310 +lj -10.458862 +up -10.485964 +hy -10.488351 +di -10.578618 +hu -10.584958 +ym -10.666426 +rr -10.736576 +gi -10.861440 +hj -10.992458 +_b -11.020613 +ad -11.054738 +rs -11.086711 +lv -11.091091 +hm -11.110322 +sv -11.156947 +tr -11.193993 +ly -11.303312 +ae -11.364268 +du -11.387210 +pr -11.389003 +_f -11.402603 +rh -11.410607 +yr -11.427986 +py -11.484778 +nh -11.488781 +rm -11.513771 +_d -11.522586 +lp -11.550785 +l_ -11.579089 +r_ -11.584376 +ip -11.616500 +yd -11.621039 +eo -11.684410 +dy -11.759042 +ls -11.847865 +_c -11.864374 +ji -11.873724 +ya -11.891954 +ge -11.897635 +ga -11.936675 +oy -11.944250 +nd -11.944493 +uj -11.944701 +d_ -11.946402 +ey -11.947479 +ye -12.021656 +ry -12.043930 +th -12.157067 +kr -12.206584 +_g -12.219223 +bu -12.279051 +nj -12.286418 +k_ -12.287320 +_w -12.334564 +rn -12.458769 +oe -12.464403 +fi -12.470934 +m_ -12.473334 +ps -12.485320 +sp -12.507851 +be -12.524167 +nv -12.546870 +lb -12.552331 +vy -12.603474 +ch -12.621719 +hk -12.632017 +ic -12.648169 +nl -12.662305 +ao -12.725276 +rd -12.752954 +ej -12.776689 +br -12.809357 +np -12.821059 +ig -12.828338 +rl -12.870326 +ba -12.877660 +gl -12.890567 +kt -12.945222 +ep -12.956021 +rg -12.984626 +co -13.059811 +ca -13.082993 +ac -13.125736 +h_ -13.167313 +hn -13.176369 +go -13.214677 +yp -13.225595 +hv -13.254116 +sl -13.293028 +rp -13.304870 +ce -13.338342 +of -13.340714 +bo -13.349690 +fo -13.353646 +sm -13.361683 +iu -13.393388 +fe -13.398083 +ck -13.408955 +fr -13.415142 +kl -13.421453 +g_ -13.463483 +nm -13.477419 +wi -13.484715 +bi -13.519224 +gr -13.571301 +kn -13.583944 +wa -13.670229 +p_ -13.716275 +og -13.764390 +nc -13.810052 +fa -13.826643 +ob -13.835172 +w_ -13.838907 +nr -13.840971 +jy -13.848936 +ag -13.870590 +sh -13.904535 +hl -13.957595 +pl -13.965951 +hr -13.969617 +eb -13.981028 +eg -13.987875 +cu -14.013627 +af -14.026641 +ew -14.041553 +ms -14.098232 +dr -14.164644 +sc -14.269286 +ci -14.269981 +ab -14.287826 +ow -14.318426 +oc -14.335701 +yj -14.345653 +ld -14.355582 +pt -14.389841 +tl -14.400079 +c_ -14.405417 +bl -14.500030 +sj -14.528970 +f_ -14.570230 +gu -14.596699 +ug -14.663190 +rc -14.680996 +x_ -14.683773 +ph -14.692369 +sr -14.719180 +we -14.819436 +if -14.862046 +ec -14.919254 +wo -14.929516 +ds -15.016833 +gh -15.022681 +by -15.040960 +ib -15.128571 +ex -15.193045 +b_ -15.215975 +cl -15.225384 +kh -15.287386 +ub -15.317258 +nf -15.317797 +fy -15.327535 +lg -15.335701 +lh -15.337522 +lf -15.347761 +z_ -15.375553 +sn -15.396279 +mb -15.523985 +ct -15.533137 +cr -15.583458 +tn -15.616680 +ws -15.666848 +dv -15.727154 +v_ -15.759744 +az -15.761210 +fl -15.842651 +aw -15.847573 +gy -15.921847 +gn -15.931715 +ww -15.933917 +j_ -15.937779 +rb -15.937779 +sb -16.007381 +uc -16.022534 +ef -16.048844 +ff -16.076250 +iy -16.094301 +km -16.095841 +jn -16.118830 +tm -16.152733 +ax -16.171124 +tv -16.218992 +wh -16.294796 +bb -16.354751 +fu -16.398938 +dw -16.405035 +ix -16.409625 +wn -16.471858 +kp -16.506673 +dn -16.520258 +xa -16.550772 +ze -16.559246 +ft -16.575915 +za -16.601504 +dd -16.636045 +dj -16.707242 +ox -16.708184 +qu -16.720961 +tw -16.728109 +wr -16.757548 +zz -16.758036 +tj -16.758036 +nz -16.759500 +mc -16.785113 +xi -16.824407 +tp -16.880643 +ez -16.890233 +_z -16.909064 +dg -16.922120 +gt -16.924307 +nb -17.047055 +gs -17.068669 +vd -17.072303 +_x -17.080818 +fg -17.081428 +sd -17.086318 +bn -17.112269 +uf -17.121024 +yw -17.133624 +_q -17.136157 +fk -17.159803 +zi -17.161093 +bc -17.414998 +bs -17.430461 +df -17.457927 +dm -17.461099 +sw -17.509544 +vr -17.568198 +vl -17.597571 +dp -17.633806 +mk -17.677417 +bm -17.747346 +tc -17.747346 +cy -17.752195 +ux -17.857990 +wt -17.871645 +pd -17.896125 +ln -17.898273 +xe -17.901502 +jk -17.918845 +oz -17.934193 +cs -17.986946 +gg -17.993821 +cc -18.042889 +xt -18.062030 +dh -18.099853 +iz -18.160448 +pm -18.164321 +tg -18.206293 +sf -18.244048 +uy -18.255019 +uz -18.312594 +gm -18.321216 +zu -18.329889 +zy -18.335701 +rw -18.362146 +mt -18.372563 +cq -18.384560 +xp -18.386067 +fs -18.389085 +mw -18.399699 +dl -18.411925 +gp -18.492403 +vg -18.500540 +qv -18.535225 +kf -18.540249 +ml -18.557123 +rx -18.639186 +hp -18.682963 +pk -18.682963 +wl -18.703480 +zo -18.751224 +dt -18.816767 +xb -18.826963 +td -18.843427 +hs -18.866377 +lc -18.872700 +gd -18.904738 +wf -18.915577 +tb -18.928693 +hc -19.007671 +xf -19.012318 +bj -19.123535 +wc -19.123535 +gf -19.193045 +lz -19.257775 +mn -19.260536 +dz -19.260536 +dq -19.314028 +gb -19.334246 +gk -19.340075 +yu -19.351804 +mh -19.360664 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fra b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fra new file mode 100644 index 00000000000..d15e6ea3a75 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fra @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 257729107 +#UNIQUE_BIGRAMS 626 +#UNSEEN_LOG_PROB -19.450127 +#EXPECTED_SCORE -7.286241 +e_ -4.560184 +s_ -4.615084 +es -5.228431 +re -5.624622 +t_ -5.705716 +_p -5.729926 +on -5.920039 +nt -5.999353 +en -6.077784 +le -6.094247 +_d -6.142093 +te -6.198364 +_c -6.216871 +er -6.219572 +r_ -6.311148 +_s -6.312408 +an -6.409342 +de -6.457321 +ur -6.545248 +_a -6.564717 +ou -6.617063 +_e -6.666457 +me -6.730573 +ai -6.800138 +is -6.809557 +ti -6.821085 +n_ -6.835299 +_l -6.855856 +_m -6.905006 +ie -6.912968 +ne -6.918282 +qu -6.922372 +se -6.923771 +it -6.932406 +in -6.991996 +co -7.015348 +ue -7.064145 +ns -7.087123 +ar -7.124408 +ra -7.178061 +ce -7.197708 +_r -7.232866 +eu -7.290264 +em -7.296691 +tr -7.299266 +pa -7.299387 +et -7.345973 +_t -7.380073 +st -7.408774 +_f -7.419472 +io -7.463141 +po -7.469453 +pr -7.485334 +at -7.490068 +ve -7.526115 +ri -7.556420 +ro -7.584634 +el -7.662934 +al -7.669459 +so -7.671820 +ma -7.684158 +or -7.710254 +ec -7.730085 +li -7.742109 +pe -7.773889 +ta -7.778591 +us -7.821506 +ui -7.821696 +i_ -7.823986 +_q -7.837356 +si -7.856796 +oi -7.865362 +ll -7.887532 +ir -7.890053 +au -7.894826 +nc -7.971038 +ch -7.989260 +rs -7.993572 +nd -8.015830 +ut -8.017045 +il -8.047367 +_v -8.049194 +ss -8.058273 +om -8.072940 +di -8.089521 +rt -8.094693 +la -8.191293 +su -8.210167 +un -8.223431 +_b -8.233106 +_n -8.234157 +da -8.280202 +mi -8.307849 +ge -8.365852 +na -8.411796 +as -8.421730 +to -8.424653 +no -8.430956 +ca -8.433801 +pl -8.441648 +ni -8.475668 +_i -8.513591 +vi -8.540299 +ic -8.547720 +x_ -8.556151 +av -8.559547 +_g -8.563826 +lo -8.566413 +_u -8.568018 +sa -8.569525 +lu -8.582496 +ci -8.592502 +l_ -8.611576 +nn -8.647290 +ee -8.659509 +ac -8.672254 +mo -8.677317 +he -8.688617 +ux -8.704238 +ts -8.794376 +ct -8.806210 +fi -8.835599 +_o -8.845641 +_j -8.883238 +mp -8.897740 +ol -8.925160 +iq -8.936429 +va -8.953578 +fa -8.966597 +mm -9.004136 +ha -9.033905 +a_ -9.041696 +vo -9.059016 +ep -9.069628 +bl -9.095695 +ev -9.147071 +u_ -9.150289 +uv -9.151731 +fo -9.154022 +ag -9.155668 +am -9.165403 +ul -9.172624 +tt -9.184972 +ap -9.194444 +do -9.228223 +c_ -9.255755 +ea -9.270731 +os -9.273621 +tu -9.315892 +iv -9.326718 +im -9.336702 +gr -9.362336 +d_ -9.371380 +ot -9.374208 +rm -9.436969 +_h -9.448625 +fe -9.461567 +rn -9.600409 +eg -9.601852 +ia -9.607168 +ba -9.617267 +oc -9.631165 +id -9.631298 +ga -9.650006 +rr -9.656592 +cu -9.676178 +fr -9.687640 +br -9.695528 +ed -9.701136 +cr -9.701957 +op -9.705191 +rd -9.750419 +rc -9.755945 +jo -9.792665 +ab -9.795657 +ig -9.796617 +ff -9.838438 +pu -9.851028 +mb -9.881022 +if -9.885902 +ng -9.897452 +be -9.901949 +ef -9.920805 +up -9.932956 +du -9.938498 +sp -9.941082 +dr -9.953583 +cl -9.966793 +pp -9.986471 +ex -9.993426 +bi -9.994837 +ho -10.043748 +ad -10.056013 +gn -10.076931 +uc -10.106902 +ei -10.109732 +pi -10.117677 +je -10.148941 +hi -10.150372 +gi -10.154379 +bo -10.172934 +nu -10.177316 +ua -10.183090 +o_ -10.307682 +ru -10.400622 +ib -10.410882 +vr -10.452852 +gu -10.478586 +ls -10.479969 +sc -10.545085 +mu -10.558420 +od -10.559619 +ip -10.573482 +pt -10.599570 +og -10.602257 +ob -10.623205 +ud -10.635753 +ub -10.653424 +nf -10.656772 +bu -10.686519 +rg -10.708691 +um -10.717429 +ph -10.782976 +th -10.829971 +rv -10.857365 +cc -10.859651 +ju -10.941765 +ja -10.985052 +m_ -10.993255 +ay -11.005839 +eb -11.043506 +nv -11.051729 +ys -11.076151 +lt -11.106908 +af -11.110121 +y_ -11.126272 +go -11.165001 +fu -11.174881 +f_ -11.177160 +oy -11.180508 +ix -11.203650 +p_ -11.274474 +of -11.302604 +ps -11.305760 +z_ -11.339060 +ye -11.352317 +sq -11.372315 +xi -11.524235 +eq -11.532171 +ez -11.533448 +ya -11.592967 +nq -11.639034 +xe -11.674567 +g_ -11.734609 +ds -11.737359 +ug -11.745889 +gl -11.746139 +h_ -11.761974 +xp -11.773627 +sy -11.776746 +ov -11.825810 +eo -11.917179 +aq -11.927064 +_w -11.930162 +rl -11.947236 +fl -11.987925 +k_ -12.018231 +oo -12.107896 +hu -12.114632 +oe -12.122124 +ej -12.168966 +rq -12.200788 +uj -12.206147 +lq -12.343024 +xt -12.374186 +uo -12.392338 +tc -12.394989 +_k -12.401606 +aj -12.413437 +uf -12.437113 +_y -12.444458 +ly -12.444832 +rf -12.449928 +rp -12.459418 +sm -12.510632 +ck -12.568143 +lm -12.582217 +rb -12.582766 +hn -12.621396 +vu -12.625848 +lg -12.690575 +fs -12.693575 +oq -12.798296 +oj -12.808620 +ki -12.818169 +ah -12.830756 +bs -12.839099 +b_ -12.855683 +yo -12.963374 +ke -12.969333 +ty -13.002482 +ym -13.018138 +rk -13.026662 +_z -13.048121 +ry -13.056871 +nr -13.058398 +wa -13.061984 +yn -13.078788 +ae -13.106217 +hr -13.128902 +sh -13.144480 +cs -13.151696 +yp -13.161612 +wi -13.309876 +ao -13.331755 +gt -13.340380 +ms -13.346023 +zo -13.377488 +ld -13.392639 +ax -13.461311 +az -13.461626 +q_ -13.523625 +xc -13.526793 +ka -13.533880 +w_ -13.536604 +ze -13.540868 +sf -13.555418 +xa -13.565562 +gm -13.597539 +we -13.599067 +za -13.616326 +yc -13.628256 +sl -13.663049 +bt -13.667849 +bj -13.672592 +ey -13.680949 +ew -13.681684 +ko -13.690982 +dm -13.701085 +ak -13.702055 +ii -13.705042 +hy -13.708784 +ny -13.770729 +cq -13.796383 +eh -13.821447 +oh -13.838730 +lb -13.918046 +_x -13.941721 +ox -13.965522 +iz -14.009250 +ow -14.022510 +cy -14.022790 +gh -14.088556 +mn -14.163819 +lc -14.176409 +uh -14.211978 +dg -14.268082 +nz -14.298439 +zi -14.340787 +dy -14.361611 +lp -14.413192 +oa -14.448175 +nj -14.467701 +ok -14.473802 +ik -14.500541 +my -14.512004 +tb -14.527520 +sr -14.564205 +mt -14.605192 +yt -14.649102 +ek -14.654867 +oz -14.655445 +hm -14.666902 +nl -14.675079 +lh -14.682861 +yr -14.686547 +dv -14.728936 +iu -14.729240 +wo -14.743756 +zy -14.748063 +by -14.801569 +nk -14.817644 +sk -14.825749 +lf -14.827375 +uy -14.848358 +yl -14.866637 +gs -14.867473 +lv -14.874686 +tl -14.888373 +sb -14.892283 +nh -14.941280 +aa -14.976939 +nm -15.061888 +ft -15.063229 +hs -15.102273 +aw -15.110963 +uz -15.125097 +ks -15.201078 +dh -15.248229 +bb -15.294722 +cf -15.305562 +sn -15.317856 +v_ -15.334644 +ws -15.355848 +ht -15.362672 +xu -15.383817 +dj -15.403576 +dd -15.461247 +rh -15.491358 +yd -15.502488 +uq -15.526067 +gy -15.532950 +tp -15.535341 +kh -15.598372 +gg -15.607566 +fp -15.629532 +xv -15.659061 +bd -15.705864 +hl -15.749296 +pc -15.812964 +ky -15.850175 +pd -15.857801 +yv -15.896203 +ji -15.923776 +fg -16.021672 +lk -16.045326 +ln -16.085244 +tm -16.094223 +gb -16.094615 +zz -16.101289 +tz -16.111161 +bc -16.171029 +zu -16.173096 +py -16.194346 +tn -16.211235 +tw -16.222747 +mc -16.228323 +wn -16.242142 +uk -16.380947 +ij -16.402121 +nb -16.445425 +kr -16.484412 +df -16.488524 +cd -16.493164 +ku -16.512920 +kl -16.553801 +dl -16.557576 +xo -16.587582 +sw -16.591446 +sg -16.591999 +qa -16.610924 +cg -16.708260 +dn -16.713063 +bv -16.717279 +bn -16.729392 +cn -16.777631 +vd -16.804289 +gd -16.814576 +xq -16.817159 +rz -16.847203 +pm -16.851830 +sd -16.855144 +lr -16.862462 +wh -16.907169 +pn -16.911993 +np -16.944807 +fm -16.999499 +xy -17.001701 +mw -17.047979 +xx -17.062463 +yg -17.089531 +dw -17.123497 +rw -17.135536 +uu -17.147677 +dc -17.188900 +bm -17.203188 +tv -17.214211 +tg -17.276833 +xb -17.311013 +yu -17.368580 +vy -17.376178 +wr -17.474694 +dt -17.481849 +ml -17.538268 +gp -17.642072 +yb -17.676838 +ih -17.682714 +hz -17.706463 +md -17.707661 +sj -17.748987 +kt -17.750221 +cp -17.792804 +sv -17.795348 +mr -17.797897 +zb -17.808138 +kk -17.817159 +vl -17.831450 +fd -17.845883 +wt -17.849845 +yw -17.875191 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fry b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fry new file mode 100644 index 00000000000..371fb322fd1 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/fry @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 12722630 +#UNIQUE_BIGRAMS 625 +#UNSEEN_LOG_PROB -17.235341 +#EXPECTED_SCORE -7.510770 +n_ -4.770255 +e_ -5.011539 +er -5.352999 +an -5.502029 +en -5.502589 +_f -5.525623 +t_ -5.643695 +ar -5.934309 +r_ -5.958637 +te -6.208886 +de -6.213608 +_s -6.213650 +in -6.378410 +_w -6.434907 +oa -6.481679 +ne -6.524328 +_d -6.535004 +ie -6.542257 +st -6.563282 +_m -6.586753 +fa -6.644790 +_b -6.671057 +ei -6.725360 +_h -6.832774 +ke -6.890074 +s_ -6.916446 +me -6.935655 +_o -6.946509 +ch -7.026433 +ea -7.053517 +ge -7.058013 +_t -7.060841 +re -7.103650 +at -7.113399 +_n -7.127632 +le -7.207118 +_k -7.214610 +d_ -7.224666 +ri -7.350669 +el -7.357682 +aa -7.380912 +je -7.382255 +rd -7.397029 +et -7.418131 +ro -7.438719 +al -7.488902 +sk -7.493963 +i_ -7.506012 +_g -7.512561 +be -7.528468 +oe -7.545039 +we -7.672121 +_l -7.700310 +li -7.702245 +fo -7.731517 +ng -7.765115 +_p -7.770131 +la -7.782386 +ha -7.785986 +k_ -7.795679 +es -7.813221 +nt -7.830565 +un -7.832322 +fe -7.833821 +rs -7.861614 +sj -7.874409 +da -7.885119 +ns -7.901999 +wa -7.931981 +h_ -7.953351 +oc -7.981591 +l_ -7.982279 +ed -7.987047 +ma -8.003830 +tr -8.015168 +_a -8.022993 +ti -8.025177 +ur -8.032106 +nd -8.034037 +ts -8.036446 +se -8.046605 +it -8.080674 +nn -8.106788 +wi -8.132842 +_r -8.229083 +om -8.251923 +ra -8.253618 +_e -8.263237 +ko -8.276080 +ji -8.315419 +ni -8.330087 +is -8.342585 +rt -8.344685 +am -8.363422 +_j -8.371925 +di -8.373804 +ik -8.387372 +jo -8.404945 +wu -8.405560 +ut -8.419741 +ol -8.436065 +m_ -8.445482 +yn -8.453053 +on -8.477096 +ht -8.480656 +as -8.507889 +fr -8.519119 +ld -8.520326 +ll -8.526794 +g_ -8.540789 +ak -8.584390 +tt -8.594436 +a_ -8.599705 +si -8.618212 +ee -8.637591 +_u -8.653622 +ys -8.663980 +ek -8.716532 +he -8.738450 +to -8.767213 +ka -8.792231 +id -8.830643 +ta -8.834727 +ou -8.834831 +hi -8.839290 +ru -8.851600 +rr -8.875527 +il -8.885736 +pe -8.897963 +mi -8.901050 +op -8.903980 +ij -8.921359 +sl -8.952929 +na -9.019752 +no -9.020635 +gr -9.034126 +ki -9.056774 +us -9.075067 +ry -9.075250 +rk -9.088585 +bi -9.107476 +uw -9.110858 +mm -9.112865 +sy -9.115190 +bo -9.139158 +mo -9.139286 +_i -9.146273 +ig -9.151100 +pr -9.164572 +eg -9.166200 +or -9.177712 +ad -9.186671 +em -9.192896 +kr -9.243204 +p_ -9.262227 +pl -9.268717 +so -9.318819 +yk -9.361743 +rn -9.365777 +ac -9.367124 +im -9.378401 +ly -9.386726 +ze -9.408061 +fi -9.418266 +br -9.422384 +ic -9.439053 +kk -9.471047 +do -9.477823 +wo -9.479684 +kt -9.514591 +o_ -9.515255 +lj -9.549515 +ef -9.554281 +rl -9.568245 +pa -9.596761 +sp -9.611322 +ii -9.627734 +ls -9.631237 +ot -9.631777 +ks -9.652070 +lo -9.679146 +of -9.724473 +tu -9.732264 +_v -9.773650 +dt -9.790121 +tw -9.798175 +nk -9.802827 +_y -9.817077 +ve -9.831572 +sa -9.848199 +dr -9.897639 +su -9.955235 +au -10.026655 +um -10.041995 +ja -10.065983 +ju -10.069512 +y_ -10.087906 +gi -10.088523 +ho -10.104165 +eu -10.132651 +u_ -10.138391 +ba -10.175678 +go -10.195087 +iz -10.199481 +po -10.205761 +hu -10.207637 +rg -10.230070 +va -10.253964 +kl -10.271097 +ud -10.280516 +iu -10.284046 +bl -10.298112 +ep -10.318095 +rj -10.352373 +rm -10.358166 +bu -10.365179 +ty -10.378401 +ff -10.432692 +ul -10.435358 +ds -10.439132 +ss -10.445601 +oo -10.447658 +lt -10.461023 +og -10.493840 +os -10.546289 +ap -10.554452 +ew -10.560090 +eb -10.563690 +gj -10.563861 +f_ -10.567299 +ok -10.567987 +du -10.583563 +uk -10.589666 +ga -10.590890 +ag -10.592115 +dd -10.598254 +ip -10.605480 +lu -10.610967 +ec -10.613097 +ft -10.647788 +pp -10.653622 +md -10.732650 +yl -10.735934 +vi -10.758543 +rf -10.760116 +by -10.780316 +aw -10.785310 +sn -10.786311 +j_ -10.795150 +ku -10.807087 +uo -10.833951 +rb -10.837474 +rw -10.841838 +pi -10.874675 +zz -10.877019 +_c -10.877233 +fu -10.881719 +mt -10.887722 +gs -10.917899 +ia -10.926701 +tj -10.945363 +ms -10.947601 +ai -10.970626 +ez -10.982279 +nl -10.996109 +ir -11.002376 +fl -11.022993 +yt -11.031275 +nf -11.044627 +th -11.047264 +mp -11.081257 +pu -11.090377 +nj -11.112300 +rp -11.118842 +lk -11.121871 +hj -11.128964 +ww -11.130997 +io -11.138647 +sd -11.149682 +nw -11.190442 +sm -11.191503 +ov -11.244717 +w_ -11.262436 +mr -11.264945 +gu -11.296257 +iv -11.301113 +mu -11.313758 +mb -11.325933 +ev -11.340562 +wy -11.344390 +if -11.366076 +lg -11.380818 +dw -11.383845 +fy -11.403984 +af -11.404599 +rh -11.454643 +eo -11.465826 +sw -11.470645 +eh -11.471289 +zj -11.485200 +ue -11.498918 +fj -11.512768 +ib -11.521409 +kw -11.531444 +tl -11.566095 +ps -11.582693 +nb -11.605480 +ob -11.605833 +pt -11.683522 +ui -11.685388 +yd -11.687630 +dy -11.714816 +od -11.727065 +nh -11.735547 +kj -11.761690 +uz -11.804043 +ug -11.819534 +hs -11.828992 +my -11.876806 +dl -11.877659 +ny -11.929353 +gg -11.952087 +gl -11.958842 +nu -11.974272 +zi -11.987565 +bb -11.988947 +ab -11.997731 +co -12.007036 +fs -12.053035 +wr -12.068538 +_z -12.114059 +vo -12.116574 +ub -12.138903 +yf -12.155879 +av -12.192033 +ca -12.222057 +lm -12.255488 +sc -12.267738 +sh -12.341739 +az -12.348228 +sb -12.392050 +tb -12.476772 +rc -12.484550 +np -12.516750 +lp -12.567471 +lv -12.591765 +ky -12.664992 +ce -12.678309 +ym -12.693252 +py -12.731300 +zo -12.779917 +lf -12.798377 +nr -12.808917 +up -12.826930 +kn -12.832709 +gy -12.862801 +dk -12.878940 +tf -12.880649 +ej -12.915269 +tk -12.969717 +mk -13.019693 +oh -13.040561 +nz -13.070487 +kb -13.115064 +vr -13.121113 +b_ -13.130235 +dn -13.174629 +yp -13.181987 +pf -13.245543 +kh -13.253272 +lb -13.271097 +mj -13.272219 +sf -13.288011 +dz -13.370873 +c_ -13.379306 +tm -13.404906 +tn -13.454961 +wt -13.460064 +gt -13.479360 +pj -13.483250 +nc -13.483250 +pk -13.484550 +js -13.508136 +ck -13.516085 +yg -13.522743 +ci -13.524078 +fd -13.526752 +rz -13.548326 +pn -13.551045 +td -13.584085 +ph -13.591065 +bs -13.599485 +vd -13.637998 +lw -13.640892 +uc -13.646697 +rv -13.659846 +jn -13.686508 +jk -13.691001 +hy -13.703048 +db -13.752271 +hn -13.761690 +oj -13.763266 +uu -13.791929 +nm -13.829404 +dj -13.837681 +ae -13.839342 +hr -13.846006 +ws -13.851024 +jt -13.886648 +nv -13.891810 +oi -13.928468 +df -13.946258 +iw -13.949842 +uv -13.969717 +yz -13.980674 +v_ -14.004704 +fk -14.034840 +jw -14.040561 +pv -14.044388 +fb -14.057862 +ow -14.059797 +vv -14.063675 +jd -14.067564 +ay -14.109041 +gd -14.115064 +fh -14.129218 +ye -14.141462 +za -14.151745 +ey -14.153810 +ah -14.200014 +oy -14.208576 +gh -14.223683 +lc -14.225854 +fn -14.228029 +dv -14.234571 +tg -14.234571 +zy -14.238950 +sg -14.243342 +ln -14.256598 +vy -14.324769 +pd -14.341150 +ua -14.343506 +lh -14.376892 +hh -14.376892 +dh -14.391440 +mh -14.423474 +pb -14.448609 +ml -14.448609 +sr -14.451146 +bj -14.458787 +cl -14.487151 +yo -14.489758 +cr -14.500231 +z_ -14.510781 +pm -14.518745 +cd -14.521409 +km -14.586873 +zg -14.586873 +tp -14.598079 +ex -14.612209 +x_ -14.635109 +zu -14.676081 +ct -14.706076 +gw -14.724377 +mf -14.746025 +jy -14.749145 +cu -14.761690 +fg -14.761690 +hd -14.774345 +aj -14.774345 +kf -14.793539 +lr -14.822816 +tz -14.836022 +dm -14.879794 +mc -14.910896 +qu -14.910896 +dg -14.921413 +zw -14.942682 +gn -14.982508 +mn -15.042473 +hg -15.077332 +jl -15.101048 +hl -15.145566 +fw -15.149682 +dp -15.174629 +wj -15.204289 +yb -15.221515 +uf -15.261044 +gb -15.352966 +ya -15.357720 +mw -15.376892 +sv -15.396322 +hb -15.396322 +hf -15.396322 +jb -15.401221 +pg -15.401221 +kp -15.508136 +gm -15.508136 +yi -15.529431 +kg -15.551045 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ful b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ful new file mode 100644 index 00000000000..7da6fa9fba1 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ful @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 5846839 +#UNIQUE_BIGRAMS 548 +#UNSEEN_LOG_PROB -18.708837 +#EXPECTED_SCORE -7.201513 +e_ -4.590845 +aa -5.076305 +i_ -5.136309 +de -5.217204 +a_ -5.444276 +_n -5.606611 +nd -5.731491 +o_ -5.882337 +al -5.975322 +_m -6.077346 +_h -6.226486 +_w -6.251515 +an -6.261438 +ee -6.266026 +oo -6.304534 +di -6.312101 +n_ -6.334269 +on -6.349681 +_d -6.355367 +l_ -6.502952 +ii -6.569754 +la -6.572288 +na -6.577251 +_k -6.665945 +ng -6.691654 +ma -6.702638 +be -6.758153 +en -6.758874 +ta -6.828875 +in -6.875918 +ad -6.931819 +ga -7.044076 +u_ -7.061999 +_a -7.075048 +nn -7.085567 +_b -7.085734 +am -7.096365 +ar -7.131604 +m_ -7.134791 +_t -7.146245 +ka -7.146769 +_f -7.152972 +go -7.156452 +wo -7.174446 +ha -7.192343 +_s -7.205321 +da -7.251309 +_j -7.259527 +_l -7.283623 +wa -7.287434 +do -7.295435 +no -7.318054 +ol -7.325515 +du -7.352602 +aw -7.356802 +le -7.361501 +_g -7.414693 +re -7.451665 +ba -7.480547 +uu -7.485270 +ni -7.494584 +um -7.519269 +ko -7.557710 +ab -7.571396 +ll -7.571584 +_y -7.612671 +er -7.634765 +ti -7.794422 +mu -7.838133 +wi -7.956583 +kk -7.975772 +sa -7.994780 +dd -7.995599 +ay -8.008058 +ak -8.064606 +ri -8.068840 +ra -8.082821 +ud -8.119201 +te -8.134791 +ed -8.138263 +or -8.145721 +ji -8.151462 +r_ -8.211268 +to -8.228853 +mb -8.274349 +ey -8.277178 +bi -8.297530 +it -8.300249 +ya -8.310397 +ja -8.321484 +ul -8.322036 +li -8.328526 +jo -8.332736 +_i -8.340394 +_p -8.353247 +iy -8.372826 +yi -8.421911 +yd -8.438251 +ur -8.440221 +ne -8.460938 +at -8.496945 +ew -8.500783 +he -8.504990 +un -8.511459 +el -8.527576 +bu -8.578170 +ir -8.579113 +_c -8.618139 +id -8.626306 +me -8.677921 +mi -8.716740 +gu -8.732711 +ho -8.737758 +ye -8.760265 +_e -8.783997 +se -8.783997 +fo -8.810784 +fa -8.826605 +ge -8.829071 +ot -8.842148 +od -8.844188 +ag -8.858204 +ku -8.914481 +im -8.918654 +je -9.015190 +bb -9.016084 +hi -9.019154 +ke -9.046292 +si -9.049034 +tu -9.063088 +ah -9.082487 +mo -9.096331 +rd -9.096736 +gi -9.141604 +nj -9.190071 +et -9.232336 +bo -9.245156 +ki -9.258998 +of -9.267641 +ok -9.280167 +ow -9.295745 +ro -9.305236 +_r -9.305236 +rn -9.310710 +il -9.334089 +ib -9.343516 +aj -9.352198 +mm -9.353489 +y_ -9.357045 +fu -9.366786 +eb -9.381522 +lo -9.402410 +tt -9.439792 +yo -9.443395 +ub -9.452183 +_o -9.462765 +fe -9.489831 +nt -9.492317 +ob -9.548673 +es -9.557570 +wd -9.610017 +em -9.628062 +pu -9.639038 +ru -9.654068 +ca -9.665044 +f_ -9.693160 +nk -9.701765 +om -9.713525 +ik -9.728100 +ju -9.736705 +nu -9.769711 +as -9.777702 +fi -9.783345 +gg -9.831767 +is -9.834693 +og -9.873514 +hu -9.876295 +ld -9.926316 +rt -9.932090 +su -9.967720 +oy -9.987372 +wr -10.039654 +ej -10.092824 +wu -10.126079 +ce -10.146350 +wn -10.164076 +k_ -10.169181 +ln -10.169749 +so -10.196137 +af -10.209807 +pi -10.218600 +ut -10.252211 +ug -10.258847 +h_ -10.264299 +uj -10.280780 +we -10.291564 +po -10.320300 +_u -10.333612 +w_ -10.387790 +lu -10.534879 +md -10.549967 +pa -10.568583 +oj -10.574215 +pe -10.596582 +ij -10.598494 +lb -10.663042 +wt -10.677921 +pp -10.704850 +mn -10.767989 +s_ -10.839433 +eg -10.842601 +yn -10.879777 +yy -10.901797 +lt -10.906999 +jj -10.956644 +iw -10.968461 +us -10.992390 +uk -10.997931 +t_ -11.039914 +ek -11.056635 +co -11.066126 +yr -11.132158 +ft -11.140489 +d_ -11.162944 +dt -11.221838 +ap -11.224197 +wb -11.264906 +ig -11.300561 +yu -11.307423 +ci -11.308049 +bd -11.330749 +cc -11.348012 +mt -11.372662 +op -11.395081 +ef -11.417854 +uw -11.425978 +rk -11.428016 +if -11.438936 +fr -11.447180 +td -11.448558 +os -11.448558 +rg -11.449938 +ac -11.543324 +cu -11.573838 +dn -11.658249 +lm -11.684810 +lf -11.688877 +ih -11.739445 +ys -11.748755 +wl -11.968461 +db -12.041473 +sn -12.063484 +mp -12.098764 +rb -12.099847 +sd -12.100931 +sk -12.175445 +br -12.299316 +kn -12.321878 +yt -12.346083 +kt -12.382510 +ae -12.386468 +tn -12.415830 +hd -12.448558 +ec -12.461025 +eh -12.487704 +st -12.514885 +yl -12.529399 +p_ -12.549967 +wm -12.555898 +hr -12.630602 +sl -12.635304 +uy -12.641598 +b_ -12.711041 +rm -12.724338 +bt -12.737758 +oh -12.764980 +nc -12.764980 +rs -12.777053 +ep -12.926556 +ip -12.930404 +rl -12.957625 +uh -12.967473 +yb -12.979380 +ny -13.028014 +bn -13.082621 +bl -13.214783 +lk -13.294350 +lh -13.309300 +wg -13.334567 +_z -13.378563 +ze -13.407763 +uf -13.410447 +ve -13.415830 +az -13.421234 +nb -13.429377 +ea -13.456858 +io -13.462417 +lj -13.484872 +sm -13.578359 +ue -13.590482 +hm -13.608861 +yk -13.652677 +kl -13.675094 +sr -13.711041 +mr -13.727681 +sg -13.842601 +av -13.857174 +c_ -13.916983 +zi -13.916983 +ns -13.963526 +qu -13.971431 +ie -13.971431 +fd -13.971431 +gn -14.015701 +gh -14.040434 +wk -14.108538 +hn -14.152796 +zu -14.161813 +gd -14.198455 +oc -14.264906 +fn -14.269772 +ou -14.299316 +_q -14.314319 +pr -14.355104 +x_ -14.375938 +uc -14.429377 +ym -14.456858 +yg -14.467998 +aq -14.484872 +ox -14.513441 +oe -14.542587 +g_ -14.602708 +pt -14.615039 +ss -14.633735 +sc -14.640022 +ml -14.671871 +rw -14.678326 +tg -14.697866 +sp -14.704438 +ks -14.751305 +tr -14.879313 +ه_ -14.916983 +th -14.916983 +ia -14.924637 +ls -14.963526 +zo -14.979380 +lp -15.003492 +_v -15.044597 +gr -15.052961 +ic -15.061373 +sh -15.095521 +ال -15.112903 +lg -15.130497 +fp -15.130497 +dm -15.139375 +jk -15.193823 +za -15.193823 +ai -15.212439 +uz -15.212439 +ds -15.250407 +yh -15.269772 +iz -15.269772 +qe -15.289401 +va -15.360284 +ez -15.360284 +gl -15.370701 +rr -15.370701 +nl -15.370701 +km -15.391763 +dl -15.402410 +له -15.413136 +ly -15.445802 +dh -15.445802 +_ا -15.456858 +dr -15.479225 +ws -15.513441 +cr -15.513441 +لي -15.513441 +dg -15.513441 +au -15.525029 +ht -15.536711 +oz -15.572335 +oi -15.584408 +sb -15.584408 +lw -15.596582 +up -15.608861 +zz -15.621244 +ov -15.646335 +rh -15.659046 +mc -15.671871 +jd -15.684810 +vr -15.697866 +bh -15.751305 +fl -15.778786 +rj -15.792725 +_و -15.835369 +rf -15.849869 +iv -15.894263 +iq -15.894263 +vi -15.924637 +nf -15.940067 +wh -15.987372 +gb -16.003492 +ax -16.052961 +xe -16.052961 +ui -16.086908 +kr -16.086908 +لل -16.104186 +eu -16.139375 +tp -16.193823 +nr -16.231298 +hb -16.231298 +zd -16.250407 +bg -16.269772 +ao -16.289401 +qm -16.309300 +mj -16.309300 +dy -16.329478 +ي_ -16.349942 +pd -16.370701 +tb -16.370701 +ex -16.370701 +fs -16.413136 +ua -16.479225 +nm -16.501946 +kh -16.548488 +ts -16.621244 +lc -16.621244 +hc -16.646335 +mv -16.671871 +hy -16.671871 +_ع -16.697866 +عل -16.697866 +يه -16.697866 +_ص -16.724338 +صل -16.724338 +vo -16.724338 +lv -16.751305 +mg -16.778786 +rp -16.778786 +وا -16.806800 +وس -16.864516 +سل -16.864516 +لم -16.864516 +م_ -16.864516 +gt -16.864516 +qa -16.864516 +ct -16.955663 +ms -17.019794 +cf -17.052961 +dk -17.086908 +pn -17.086908 +fg -17.121673 +pl -17.157297 +jn -17.193823 +pw -17.193823 +z_ -17.193823 +hg -17.231298 +jt -17.269772 +tw -17.309300 +wz -17.309300 +ry -17.349942 +fb -17.349942 +bp -17.349942 +pm -17.349942 +mf -17.349942 +vu -17.349942 +vv -17.349942 +mw -17.434831 +oa -17.434831 +hl -17.434831 +pc -17.479225 +لا -17.525029 +ph -17.621244 +zy -17.621244 +rz -17.621244 +zh -17.671871 +eo -17.778786 +cl -17.778786 +ch -17.778786 +ev -17.778786 +mk -17.835369 +_x -17.835369 +kj -17.955663 +jp -17.955663 +pf -17.955663 +lz -17.955663 +kd -17.955663 +cp -18.019794 +qd -18.019794 +xx -18.019794 +xi -18.019794 +mz -18.086908 +zt -18.086908 +ei -18.157297 +tl -18.231298 +j_ -18.231298 +xp -18.231298 +ام -18.231298 +مر -18.231298 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gla b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gla new file mode 100644 index 00000000000..d5d05a4c56f --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gla @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 79099662 +#UNIQUE_BIGRAMS 643 +#UNSEEN_LOG_PROB -20.379013 +#EXPECTED_SCORE -6.913654 +ai -4.763718 +h_ -5.030362 +ch -5.084435 +n_ -5.117781 +ha -5.231816 +ea -5.249792 +_a -5.265558 +an -5.436591 +dh -5.552578 +ir -5.750101 +_c -5.773369 +r_ -5.899648 +ac -5.922489 +th -5.932018 +_t -6.094367 +nn -6.118298 +s_ -6.142159 +_s -6.201659 +a_ -6.225871 +ad -6.237087 +bh -6.255303 +in -6.326891 +_b -6.332356 +id -6.386519 +he -6.450742 +ar -6.572365 +_d -6.600188 +il -6.662416 +e_ -6.708445 +ei -6.741782 +d_ -6.743188 +na -6.769742 +ra -6.828045 +ag -6.979430 +_f -7.036226 +le -7.039152 +ui -7.143498 +oi -7.175341 +l_ -7.183468 +_g -7.185256 +_m -7.222102 +la -7.242124 +gu -7.247657 +as -7.248289 +is -7.255127 +ig -7.280985 +gh -7.287829 +ta -7.307988 +hi -7.365489 +_l -7.392000 +mh -7.414831 +ic -7.438393 +al -7.441084 +us -7.442717 +io -7.446298 +ma -7.510591 +hd -7.542838 +se -7.556960 +on -7.574267 +ga -7.578625 +da -7.651223 +ho -7.678213 +_r -7.700940 +te -7.733913 +sa -7.742563 +g_ -7.753821 +re -7.760327 +ri -7.791490 +hu -7.793217 +_i -7.879264 +de -7.923798 +t_ -7.926054 +it -7.932688 +_n -7.941260 +_e -8.049719 +at -8.054958 +am -8.070745 +ia -8.071461 +om -8.103148 +st -8.158157 +or -8.164842 +ne -8.170659 +rt -8.249116 +co -8.262602 +o_ -8.284933 +ba -8.321896 +ao -8.364397 +_p -8.366412 +sg -8.372716 +si -8.403481 +li -8.408691 +m_ -8.428726 +eo -8.445369 +ua -8.451505 +cu -8.494208 +ll -8.501758 +ro -8.548016 +hl -8.637626 +ob -8.665430 +rr -8.669955 +fa -8.708874 +ca -8.714433 +fh -8.728437 +ur -8.731411 +rs -8.763447 +so -8.840630 +_u -8.851660 +_o -8.892985 +hr -8.905595 +os -8.920683 +lt -8.981881 +un -9.032855 +nt -9.087778 +ge -9.103815 +um -9.145867 +ns -9.244927 +ti -9.249216 +im -9.262507 +ol -9.280826 +ru -9.317653 +lb -9.321546 +og -9.322409 +to -9.336797 +lu -9.355433 +hn -9.371254 +ni -9.387287 +cl -9.391801 +pa -9.398666 +eu -9.436761 +ab -9.473787 +ib -9.474332 +ce -9.526160 +oc -9.583483 +di -9.650284 +iu -9.651868 +tr -9.659267 +du -9.663802 +fe -9.682504 +lo -9.702197 +do -9.737494 +ht -9.753903 +bi -9.763478 +c_ -9.785603 +ut -9.813708 +be -9.814430 +fi -9.815662 +mo -9.850177 +me -9.876115 +ot -9.881387 +od -9.915451 +gr -9.928296 +gi -9.953845 +ng -10.001210 +tu -10.036461 +cr -10.059145 +ph -10.064038 +rd -10.074187 +uc -10.074600 +sh -10.099997 +ug -10.103544 +ci -10.114745 +br -10.185131 +u_ -10.202412 +bu -10.222363 +rb -10.382227 +mi -10.393370 +mu -10.406826 +dr -10.412110 +su -10.454528 +en -10.455552 +ud -10.457449 +go -10.522306 +pr -10.523943 +gl -10.529027 +bl -10.548836 +rg -10.594581 +ul -10.605536 +rl -10.610432 +er -10.764128 +ub -10.897040 +po -10.909089 +nd -10.941866 +mp -10.978897 +es -10.999958 +sn -11.009289 +fo -11.012111 +_h -11.020422 +nu -11.045184 +no -11.062828 +sr -11.096338 +i_ -11.177908 +rn -11.177950 +fu -11.209781 +rm -11.272557 +nc -11.279250 +bo -11.291953 +sl -11.318445 +nm -11.331264 +et -11.366803 +sp -11.384590 +nb -11.389964 +ie -11.488766 +sb -11.501824 +if -11.514414 +oo -11.542538 +y_ -11.576559 +gn -11.604855 +el -11.644594 +pe -11.725045 +ds -11.815235 +ed -11.859551 +ct -11.879066 +rc -11.907022 +pl -11.909335 +fr -11.932460 +ap -11.939894 +_w -12.015883 +sm -12.023229 +ki -12.032291 +rp -12.038953 +nr -12.041412 +hc -12.089725 +sc -12.090997 +ok -12.197478 +sd -12.210991 +pi -12.213934 +lm -12.264547 +ls -12.332628 +of -12.357968 +p_ -12.368249 +gs -12.476344 +tt -12.565517 +bc -12.570389 +hb -12.580967 +ve -12.583204 +ms -12.658088 +ou -12.666127 +_j -12.718884 +k_ -12.735953 +hs -12.785699 +dm -12.884297 +ip -12.902476 +b_ -12.918484 +wi -13.078559 +ss -13.103705 +cn -13.138478 +pu -13.155352 +mg -13.203917 +mr -13.209608 +lg -13.210472 +ck -13.216015 +ke -13.234705 +wa -13.251860 +mb -13.275176 +vi -13.316443 +iv -13.342919 +we -13.358351 +mc -13.364686 +cs -13.374337 +ow -13.385224 +_v -13.423587 +tl -13.445598 +op -13.458064 +bb -13.471468 +pt -13.472919 +ay -13.489185 +ts -13.539332 +af -13.543899 +em -13.547170 +lc -13.572054 +ae -13.572277 +ee -13.611916 +ld -13.621998 +_k -13.626374 +nl -13.638650 +ry -13.655497 +tc -13.701650 +ly -13.815103 +eb -13.830963 +ew -13.831495 +jo -13.861314 +ec -13.871393 +up -13.957267 +w_ -13.993994 +mm -13.995483 +dl -14.085517 +ue -14.104347 +fl -14.131260 +ey -14.164366 +va -14.250615 +_y -14.282972 +bs -14.293554 +cc -14.314955 +ex -14.335924 +ja -14.366033 +eg -14.375306 +gm -14.375694 +au -14.421785 +f_ -14.449266 +rk -14.474371 +yo -14.476448 +ik -14.507547 +wo -14.539332 +cd -14.547607 +av -14.555930 +sk -14.558128 +nk -14.566069 +x_ -14.636326 +ml -14.640978 +oa -14.658796 +ty -14.701893 +ef -14.703838 +ev -14.712626 +mn -14.825657 +oh -14.836822 +ff -14.886229 +ep -15.009552 +tm -15.018605 +lf -15.040566 +ov -15.084250 +ye -15.181208 +gt -15.192091 +tw -15.208571 +ak -15.208571 +dt -15.237873 +pp -15.248484 +hg -15.272105 +ka -15.281518 +ft -15.286612 +az -15.292456 +ks -15.301267 +nh -15.333286 +ky -15.352235 +wn -15.363724 +ys -15.368346 +qu -15.374531 +wh -15.376081 +ih -15.381521 +ws -15.395604 +je -15.409825 +oy -15.439507 +ny -15.450082 +hm -15.459913 +lr -15.465679 +dn -15.474786 +ps -15.499921 +aw -15.506698 +yl -15.555051 +np -15.581637 +zo -15.583427 +rv -15.608723 +cg -15.630763 +yp -15.644711 +dd -15.650328 +ii -15.673972 +aa -15.680662 +lp -15.682579 +gy -15.721468 +ox -15.733342 +ya -15.782869 +hp -15.809855 +cf -15.814052 +dy -15.816155 +ah -15.823540 +vo -15.833091 +yr -15.842705 +hy -15.844851 +xi -15.873034 +ju -15.875224 +sw -15.895094 +yt -15.904013 +iz -15.916368 +cm -15.934529 +zi -16.028934 +nf -16.074777 +_q -16.087421 +z_ -16.147056 +gg -16.152360 +tn -16.159017 +cy -16.161689 +sy -16.172425 +oe -16.199621 +yn -16.209262 +my -16.216188 +ek -16.223148 +xa -16.230141 +ze -16.247064 +ji -16.251326 +dg -16.252750 +ko -16.282972 +yd -16.290262 +rf -16.330278 +_z -16.379187 +td -16.385419 +v_ -16.390111 +nv -16.391678 +bp -16.401118 +tb -16.412209 +lw -16.478945 +kl -16.543681 +za -16.557688 +lk -16.577172 +by -16.587912 +dc -16.596923 +hw -16.698009 +rh -16.698009 +bt -16.711647 +gb -16.731357 +ax -16.739316 +cp -16.761435 +uh -16.813002 +ix -16.823540 +df -16.829900 +ym -16.829900 +py -16.834156 +ez -16.857790 +dp -16.862129 +dw -16.904013 +wr -16.944847 +mt -16.989241 +ln -16.993994 +kh -17.044875 +pd -17.047344 +wt -17.052293 +gd -17.057259 +rw -17.059749 +cb -17.062243 +nz -17.087421 +fg -17.097617 +_x -17.115635 +uk -17.123426 +xt -17.141771 +ux -17.217577 +sq -17.280066 +fs -17.303478 +dv -17.306431 +tv -17.333286 +lv -17.333286 +xp -17.360651 +yb -17.372982 +db -17.416989 +kr -17.416989 +wl -17.442752 +pb -17.522923 +tz -17.547170 +yw -17.557688 +yc -17.596923 +xy -17.607812 +nj -17.633542 +kn -17.678747 +uf -17.686421 +kc -17.701893 +hf -17.733342 +zh -17.749328 +uo -17.781841 +iq -17.794225 +tf -17.798376 +zu -17.853464 +kf -17.892872 +oz -17.937960 +eh -17.965705 +aj -17.970382 +kj -17.979780 +pm -17.989241 +ku -18.008350 +xc -18.067243 +pc -18.102742 +sx -18.181886 +hh -18.181886 +lh -18.181886 +tp -18.231544 +xe -18.242815 +q_ -18.312356 +wy -18.312356 +vd -18.348425 +kp -18.348425 +tg -18.385419 +qa -18.436268 +ql -18.488975 +ij -18.495701 +oj -18.502459 +xf -18.509248 +kt -18.516069 +zz -18.522923 +uy -18.543681 +qi -18.607812 +vl -18.607812 +bw -18.615116 +fy -18.615116 +yf -18.629838 +fc -18.637255 +pn -18.652206 +jl -18.674926 +km -18.690274 +oq -18.721468 +sv -18.737322 +eq -18.794225 +ej -18.870846 +gw -18.879616 +wu -18.915240 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gle b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gle new file mode 100644 index 00000000000..fcd0d2f1114 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gle @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 27898112 +#UNIQUE_BIGRAMS 556 +#UNSEEN_LOG_PROB -20.703391 +#EXPECTED_SCORE -7.159526 +ai -5.046509 +ea -5.435660 +ch -5.463707 +h_ -5.576828 +_a -5.695327 +n_ -5.707797 +_c -5.838672 +a_ -5.890970 +s_ -5.971418 +in -6.012972 +r_ -6.034902 +an -6.054658 +ir -6.074916 +ha -6.096116 +_s -6.150398 +ei -6.166172 +e_ -6.187278 +_b -6.300205 +ac -6.328473 +ta -6.348944 +th -6.357276 +ar -6.377345 +bh -6.469167 +il -6.520038 +_d -6.597855 +nn -6.608326 +_m -6.650320 +oi -6.664508 +_t -6.669664 +l_ -6.725761 +le -6.746300 +na -6.747850 +_f -6.768201 +is -6.771542 +he -6.791258 +ag -6.795176 +_g -6.847837 +ui -6.848725 +i_ -6.880403 +ra -6.924970 +_l -6.926146 +dh -6.956165 +io -6.967931 +t_ -6.972438 +mh -6.994470 +la -7.062082 +ad -7.139048 +it -7.182985 +gu -7.190874 +id -7.193354 +us -7.206829 +on -7.231982 +as -7.341263 +hu -7.457794 +d_ -7.458604 +ri -7.462384 +re -7.464017 +te -7.485208 +se -7.488288 +nt -7.497856 +eo -7.505653 +al -7.506330 +ma -7.536832 +at -7.556868 +ga -7.572178 +_n -7.579044 +ht -7.604542 +ia -7.642838 +_r -7.676974 +_i -7.729521 +hi -7.739499 +ne -7.751785 +si -7.804787 +ao -7.807901 +sa -7.809118 +am -7.814801 +gh -7.817050 +ua -7.837545 +li -7.893083 +co -7.965066 +ho -7.976860 +rt -7.995150 +or -7.998293 +ca -8.009910 +ig -8.027195 +st -8.028666 +fa -8.131589 +_p -8.150067 +ti -8.157682 +de -8.166436 +om -8.176036 +sc -8.179345 +im -8.184301 +da -8.208312 +o_ -8.213168 +os -8.237840 +_e -8.289927 +ce -8.290785 +u_ -8.311057 +oc -8.316224 +ur -8.345210 +ol -8.349302 +iu -8.388450 +cu -8.422537 +ba -8.441900 +ni -8.529513 +fe -8.563485 +hr -8.602712 +ic -8.619108 +un -8.622814 +di -8.712033 +_o -8.744758 +m_ -8.745423 +ll -8.747267 +ge -8.809826 +hf -8.950180 +fo -8.973360 +ab -8.973828 +lt -9.011684 +ae -9.028815 +ng -9.048012 +gc -9.056660 +c_ -9.077155 +g_ -9.084970 +lo -9.089921 +nu -9.093109 +ib -9.093589 +_u -9.101919 +du -9.129256 +be -9.132532 +en -9.136543 +tu -9.165233 +ro -9.168800 +ru -9.194384 +do -9.211267 +mi -9.218149 +me -9.221634 +hl -9.240591 +ob -9.260307 +_h -9.282904 +ph -9.292406 +tr -9.299329 +fu -9.309859 +lu -9.405796 +su -9.411208 +fh -9.425930 +gl -9.454815 +ci -9.460561 +cl -9.543068 +ul -9.552007 +mb -9.567423 +sh -9.581260 +mo -9.634851 +fi -9.667575 +og -9.694188 +ud -9.721127 +pa -9.755132 +bl -9.761717 +lg -9.762480 +rr -9.785023 +po -9.793029 +cr -9.804729 +rs -9.873547 +br -9.883477 +no -9.918631 +dt -9.924499 +ot -9.930441 +to -9.932714 +ns -9.936813 +hn -10.036208 +nd -10.041321 +so -10.042411 +mu -10.142025 +rd -10.143486 +bi -10.333986 +pr -10.397088 +sp -10.412652 +ec -10.433598 +sl -10.442637 +gr -10.450793 +ts -10.458194 +rl -10.465780 +od -10.467756 +gt -10.504996 +bo -10.522232 +rb -10.522688 +y_ -10.527337 +bu -10.582965 +ug -10.600843 +rm -10.629229 +el -10.660108 +pe -10.746933 +ap -10.750314 +_テ -10.789684 +rg -10.829312 +pl -10.831854 +gi -10.848635 +ct -10.864359 +fr -10.869478 +sn -10.905230 +if -10.932663 +rc -10.956203 +uc -10.987045 +um -10.991144 +et -11.010217 +dr -11.045523 +es -11.075118 +rn -11.078693 +gm -11.219813 +ry -11.221788 +mp -11.263005 +ss -11.319243 +mt -11.372817 +av -11.405129 +nc -11.439761 +gn -11.457685 +v_ -11.466292 +ms -11.546467 +ls -11.574739 +er -11.580429 +rf -11.581696 +sm -11.740372 +hm -11.767159 +nm -11.807368 +hs -11.843210 +sr -11.919883 +nf -11.922089 +ut -11.940061 +pi -11.947599 +up -11.948007 +nr -11.973360 +ip -11.986940 +hd -12.010643 +bp -12.107043 +of -12.107955 +af -12.179555 +b_ -12.198632 +hc -12.269629 +dl -12.300600 +p_ -12.312125 +_w -12.322683 +ue -12.326662 +gs -12.439618 +go -12.465415 +_j -12.471863 +ub -12.475393 +テ_ -12.485737 +ed -12.511473 +mr -12.538292 +rp -12.613426 +hb -12.646533 +cn -12.703342 +mn -12.730146 +ml -12.745335 +テュ -12.844160 +ュ_ -12.844160 +lm -12.854464 +lb -12.860220 +we -12.888174 +cf -12.921086 +op -13.093871 +_v -13.120796 +df -13.138873 +テゥ -13.341347 +ゥ_ -13.341347 +lc -13.348880 +jo -13.402747 +テコ -13.407235 +コ_ -13.407235 +k_ -13.437321 +cm -13.447684 +nl -13.461618 +cs -13.463369 +ie -13.499445 +ou -13.523602 +tl -13.546931 +_k -13.730849 +f_ -13.737897 +vi -13.789684 +ke -13.794820 +mc -13.831289 +nb -13.835063 +tn -13.867931 +ey -13.912687 +vo -13.927115 +テウ -14.041921 +ウ_ -14.041921 +oh -14.043666 +dm -14.046288 +oo -14.132822 +ve -14.134681 +lf -14.177158 +pu -14.189666 +ck -14.214028 +lr -14.261989 +ay -14.268098 +ds -14.274233 +ow -14.295912 +wa -14.335990 +ee -14.340274 +gf -14.362977 +wi -14.383830 +pt -14.400509 +tt -14.402747 +hp -14.419647 +ly -14.445952 +ja -14.495259 +fl -14.509662 +ny -14.561237 +hg -14.594113 +au -14.608251 +ld -14.681096 +em -14.697491 +sf -14.762121 +w_ -14.783837 +mm -14.797026 +iv -14.869478 +eu -14.874129 +_y -14.919883 +yo -14.927920 +ev -14.960525 +dd -14.990513 +np -15.024580 +va -15.028032 +tm -15.095228 +dc -15.102487 +ki -15.109783 +ew -15.192568 +dy -15.212064 +rk -15.259958 +tc -15.290721 +ok -15.322153 +ty -15.343495 +oe -15.345647 +eb -15.373915 +iz -15.378313 +ah -15.466878 +dn -15.512077 +oy -15.512077 +gb -15.566246 +ft -15.568757 +qu -15.589006 +_q -15.591557 +bb -15.604381 +gg -15.700241 +je -15.711296 +tf -15.730849 +ps -15.733664 +yl -15.826774 +ka -15.826774 +ep -15.838846 +ze -15.841880 +ya -15.847968 +eg -15.860220 +hy -15.932764 +pf -16.005744 +lp -16.026305 +my -16.026305 +cc -16.036697 +pd -16.054184 +tw -16.068328 +nk -16.104308 +fc -16.130038 +ji -16.152464 +wo -16.160017 +sg -16.163809 +ww -16.179075 +ju -16.190632 +x_ -16.278337 +wn -16.299036 +ex -16.303212 +sd -16.311599 +ff -16.345647 +ye -16.367342 +ik -16.376112 +ak -16.411736 +by -16.429883 +db -16.462201 +pc -16.480999 +cd -16.529093 +ws -16.533992 +fs -16.563739 +ov -16.589006 +oa -16.594113 +yn -16.614723 +bc -16.625140 +sy -16.625140 +bt -16.705758 +z_ -16.756384 +mf -16.851021 +fg -16.869478 +az -16.913485 +yd -16.913485 +cp -16.945762 +aw -16.945762 +za -16.978777 +yr -16.998955 +sb -16.998955 +cg -17.012565 +bs -17.054184 +rv -17.082612 +cy -17.118954 +dv -17.217964 +pp -17.241811 +tz -17.282453 +uk -17.307399 +ky -17.324273 +vd -17.332785 +lv -17.358625 +ks -17.376112 +wh -17.466878 +uu -17.485737 +dw -17.533992 +aa -17.583917 +ys -17.594113 +ef -17.594113 +uv -17.604381 +dg -17.604381 +lw -17.614723 +sk -17.667575 +ek -17.689270 +テ永 -17.700241 +永_ -17.700241 +pn -17.722437 +ox -17.756384 +ko -17.767880 +nh -17.802927 +uf -17.802927 +cb -17.900774 +kr -17.926309 +kl -17.939248 +lk -17.939248 +yc -17.978777 +wr -18.005744 +ih -18.061239 +xt -18.089808 +vf -18.118954 +dp -18.148702 +_z -18.179075 +zg -18.194505 +qa -18.194505 +zi -18.274233 +ez -18.274233 +wl -18.290721 +bn -18.307399 +eh -18.324273 +gy -18.376112 +ii -18.376112 +zo -18.411736 +uo -18.429883 +xa -18.448262 +yp -18.485737 +ln -18.504845 +ix -18.504845 +nv -18.583917 +ax -18.667575 +md -18.667575 +ym -18.689270 +テ單 -18.756384 +單_ -18.756384 +tb -18.756384 +td -18.756384 +vy -18.756384 +py -18.779468 +aj -18.826774 +gp -18.851021 +yw -18.900774 +ij -18.926309 +kk -18.978777 +yt -18.978777 +oz -19.118954 +zy -19.118954 +zz -19.148702 +kh -19.179075 +pm -19.210102 +jm -19.241811 +sw -19.274233 +fy -19.307399 +fn -19.307399 +fd -19.307399 +tv -19.341347 +rw -19.376112 +yv -19.376112 +rh -19.376112 +xo -19.448262 +eq -19.524211 +oj -19.563739 +pb -19.604381 +vl -19.604381 +sq -19.604381 +fb -19.646201 +_x -19.733664 +xp -19.733664 +ku -19.779468 +wy -19.779468 +tp -19.779468 +xi -19.826774 +mw -19.826774 +nw -19.826774 +fp -19.826774 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/glg b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/glg new file mode 100644 index 00000000000..9b93c769ece --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/glg @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 26984206 +#UNIQUE_BIGRAMS 615 +#UNSEEN_LOG_PROB -19.693474 +#EXPECTED_SCORE -7.311502 +s_ -4.982991 +a_ -5.137027 +o_ -5.400485 +e_ -5.646334 +_c -5.855066 +on -5.914338 +_p -5.951845 +es -5.965201 +n_ -6.068658 +en -6.146396 +ra -6.247766 +os -6.266532 +te -6.313010 +as -6.364560 +an -6.366579 +co -6.403026 +nt -6.426949 +er -6.454780 +re -6.484373 +ar -6.498526 +_a -6.543989 +ta -6.557613 +_d -6.578773 +ci -6.591864 +de -6.615543 +_s -6.630690 +or -6.665028 +do -6.674417 +_e -6.684050 +io -6.697870 +se -6.826320 +ca -6.847744 +ro -6.862861 +_m -6.867073 +st -6.874030 +to -6.874482 +ad -6.894360 +me -6.934597 +in -6.953219 +po -6.966885 +ri -6.984908 +da -7.010875 +al -7.013746 +r_ -7.041685 +_t -7.126855 +ma -7.142573 +ia -7.154931 +is -7.196658 +ue -7.253959 +un -7.285333 +qu -7.298949 +_f -7.301570 +no -7.318001 +ic -7.336296 +ti -7.351662 +pa -7.388552 +tr -7.393066 +na -7.397458 +ac -7.511593 +_n -7.518517 +la -7.550845 +nd -7.594259 +_q -7.647550 +id -7.648258 +_r -7.687132 +ec -7.719591 +pr -7.724490 +mo -7.731291 +om -7.760336 +li -7.787955 +it -7.794244 +lo -7.812973 +ns -7.857537 +di -7.865296 +ai -7.884661 +_i -7.896282 +le -7.920429 +si -7.989500 +pe -7.990384 +ha -7.999139 +ce -8.035907 +ol -8.036314 +l_ -8.048916 +am -8.067857 +so -8.096209 +ir -8.132613 +el -8.150245 +at -8.150747 +em -8.171699 +_l -8.210132 +u_ -8.222535 +ou -8.227301 +nc -8.243932 +_v -8.273075 +_u -8.305980 +ve -8.379604 +ni -8.390076 +_o -8.397575 +fo -8.412800 +rt -8.422875 +sa -8.435591 +ei -8.441899 +mi -8.506889 +ga -8.529746 +im -8.537235 +su -8.551426 +ua -8.553012 +ne -8.569512 +nh -8.572114 +ba -8.603546 +xe -8.617778 +ab -8.621690 +_b -8.629038 +_g -8.632895 +mp -8.633936 +ur -8.653523 +i_ -8.658440 +us -8.661121 +oi -8.680031 +fi -8.689403 +ul -8.696306 +vi -8.737655 +ll -8.744014 +il -8.751368 +cu -8.757857 +od -8.763306 +tu -8.772956 +gu -8.821377 +va -8.844637 +du -8.866930 +ex -8.877779 +iv -8.888356 +_h -8.891501 +eu -8.896206 +br -8.915929 +ct -8.955301 +eg -8.961418 +ed -8.986094 +et -9.019471 +gr -9.020692 +xi -9.064934 +rm -9.068431 +ob -9.068690 +ut -9.074674 +go -9.079293 +_x -9.093009 +ui -9.122296 +vo -9.150215 +za -9.182314 +sp -9.219950 +oc -9.250495 +rr -9.293898 +bi -9.325691 +fe -9.326514 +ea -9.468601 +sc -9.492202 +be -9.545621 +ap -9.553916 +lu -9.589226 +uc -9.597983 +cr -9.625254 +pi -9.640364 +ix -9.654255 +bo -9.657965 +op -9.659823 +ch -9.671504 +nu -9.726380 +ot -9.737655 +mu -9.740397 +fa -9.747962 +oa -9.751046 +ig -9.760336 +ie -9.800631 +rs -9.803017 +ib -9.803542 +ru -9.826029 +ip -9.827679 +lt -9.877704 +ep -9.887697 +pl -9.893127 +rc -9.965421 +um -9.967454 +ov -9.979764 +xo -9.998018 +ng -10.019554 +bl -10.042263 +if -10.045141 +rd -10.077242 +pu -10.078571 +iz -10.102647 +cl -10.121641 +ub -10.158135 +ax -10.161744 +mb -10.195827 +ev -10.207095 +og -10.209815 +au -10.218197 +zo -10.228744 +ez -10.245002 +rn -10.271786 +fr -10.274035 +he -10.291149 +eo -10.316969 +ho -10.337437 +up -10.343607 +xa -10.356236 +lm -10.389628 +av -10.536897 +ox -10.551747 +ud -10.562703 +ag -10.607044 +sm -10.641901 +eb -10.644894 +hi -10.652361 +iu -10.689492 +xu -10.697815 +z_ -10.719647 +nv -10.729780 +ef -10.730598 +cc -10.769826 +nf -10.833277 +bu -10.842182 +fu -10.851734 +az -10.864037 +dr -10.899138 +ug -10.942461 +rg -10.994523 +af -11.061159 +lg -11.093856 +eq -11.253331 +rv -11.263942 +of -11.301233 +nz -11.316424 +t_ -11.405131 +ao -11.431026 +d_ -11.455591 +lv -11.460706 +xp -11.660992 +_j -11.664632 +nx -11.670371 +y_ -11.739437 +pt -11.753582 +xt -11.786066 +fl -11.844637 +rb -11.888964 +oe -11.909796 +ux -11.956416 +bs -11.971581 +rp -12.013850 +th -12.060360 +rq -12.075279 +gl -12.085467 +hu -12.138235 +ld -12.198777 +gn -12.207348 +aq -12.249161 +m_ -12.254375 +sd -12.256990 +rl -12.305421 +ae -12.363684 +lc -12.393578 +ii -12.410360 +_w -12.414441 +_z -12.430879 +rz -12.441248 +rx -12.520705 +ge -12.523221 +rf -12.612475 +k_ -12.620196 +sf -12.740168 +lb -12.751184 +g_ -12.763027 +jo -12.774969 +h_ -12.826077 +oo -12.936581 +c_ -12.991254 +oz -13.005692 +_k -13.021165 +ck -13.071362 +wa -13.126713 +sq -13.137272 +uz -13.146453 +uo -13.164993 +b_ -13.187261 +bt -13.235432 +oq -13.255159 +ls -13.263021 +nn -13.284199 +nm -13.284732 +bx -13.290613 +sl -13.291685 +x_ -13.309487 +cn -13.325862 +oh -13.360306 +ke -13.369896 +ss -13.467654 +ja -13.476159 +iq -13.489625 +xc -13.495170 +pc -13.500117 +uf -13.555041 +nq -13.558907 +wi -13.567320 +p_ -13.571219 +nl -13.663244 +sh -13.682094 +ay -13.712632 +nr -13.730689 +lf -13.735056 +dm -13.746767 +ps -13.762284 +ee -13.767494 +tt -13.802969 +ey -13.884712 +lq -13.900159 +ry -13.947520 +ki -13.952597 +rk -14.017613 +zu -14.031871 +dn -14.032767 +yo -14.048987 +ts -14.167943 +ow -14.200789 +we -14.204822 +ju -14.209878 +gi -14.221066 +ka -14.261446 +uv -14.264599 +xv -14.273042 +_y -14.277282 +tl -14.299750 +mn -14.306234 +ms -14.314924 +tc -14.350222 +tm -14.431764 +ph -14.465234 +je -14.483488 +gm -14.488395 +sb -14.499498 +lp -14.532060 +hn -14.547340 +w_ -14.610133 +ew -14.659088 +tb -14.682797 +ak -14.753397 +ff -14.802969 +ds -14.833863 +oy -14.854305 +ko -14.876648 +ly -14.884712 +ok -14.887950 +eh -14.902614 +uq -14.932395 +gh -14.940778 +ya -14.944145 +ah -14.995614 +f_ -15.016727 +ty -15.018500 +v_ -15.022054 +nk -15.029187 +ik -15.102529 +dq -15.106296 +wo -15.152282 +aw -15.156181 +pp -15.189757 +ny -15.218006 +sn -15.246820 +tn -15.301908 +zi -15.304069 +hr -15.334673 +ze -15.345762 +ks -15.381831 +mm -15.428224 +bd -15.461610 +sk -15.464025 +vu -15.490855 +dv -15.533327 +lx -15.603463 +nb -15.606127 +ft -15.649438 +by -15.702618 +dy -15.746033 +bn -15.833863 +sg -15.840122 +aa -15.840122 +dh -15.924061 +ji -15.934068 +dw -15.937419 +ht -15.961098 +lk -15.992125 +wn -16.002617 +pn -16.020276 +tw -16.059903 +my -16.063560 +fs -16.067226 +sr -16.067226 +wh -16.085699 +bm -16.111965 +ku -16.115756 +lz -16.123369 +bb -16.134865 +hl -16.146453 +dd -16.165976 +gt -16.173859 +yl -16.181786 +ys -16.213937 +yn -16.222088 +sv -16.226180 +tp -16.234401 +mc -16.255159 +zz -16.284732 +bc -16.297595 +ye -16.332465 +dx -16.354695 +kh -16.505703 +gb -16.520705 +gs -16.525741 +hy -16.556329 +uk -16.630329 +ky -16.679987 +sy -16.679987 +kl -16.691258 +rw -16.731416 +nj -16.790794 +yc -16.790794 +cd -16.809095 +ws -16.865433 +ln -16.865433 +yu -16.865433 +dt -16.891196 +zq -16.897709 +dg -16.910825 +ih -16.957691 +yr -16.985172 +tz -16.999111 +ek -17.013187 +tv -17.027400 +cs -17.056255 +hm -17.138717 +gf -17.138717 +lh -17.146453 +sw -17.146453 +yd -17.162050 +aj -17.162050 +xh -17.169912 +yw -17.226180 +gg -17.259347 +dl -17.293294 +xx -17.328060 +pd -17.336884 +rh -17.372729 +pm -17.400210 +vd -17.418825 +pg -17.437684 +wr -17.447207 +cq -17.456793 +vy -17.476159 +bh -17.485940 +kn -17.495787 +kt -17.515687 +mw -17.566671 +kr -17.587580 +kk -17.619523 +uj -17.696927 +ej -17.743097 +mt -17.778721 +bv -17.778721 +hc -17.790794 +oj -17.815247 +ij -17.827631 +gd -17.840122 +ml -17.878257 +cy -17.878257 +dc -17.944145 +tg -17.944145 +jp -17.985172 +bf -18.013187 +dp -18.013187 +wy -18.027400 +zh -18.041756 +ym -18.056255 +iy -18.193759 +hw -18.209878 +cb -18.226180 +xf -18.259347 +wt -18.276221 +iw -18.276221 +yt -18.293294 +mh -18.328060 +td -18.345762 +vl -18.363684 +cp -18.363684 +wk -18.381831 +uh -18.400210 +kd -18.495787 +gp -18.495787 +hs -18.495787 +pb -18.515687 +vg -18.535865 +py -18.535865 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gom b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gom new file mode 100644 index 00000000000..8fe2afd379e --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gom @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 5352517 +#UNIQUE_BIGRAMS 568 +#UNSEEN_LOG_PROB -17.772137 +#EXPECTED_SCORE -7.494440 +an -5.280973 +a_ -5.520466 +m_ -5.534290 +i_ -5.658340 +ch -5.694286 +_a -5.734118 +n_ -6.013467 +or -6.033173 +ea -6.047613 +_k -6.061911 +ta -6.117268 +he -6.158472 +ho -6.299239 +_t -6.354386 +t_ -6.388100 +le -6.411289 +_m -6.487020 +r_ -6.509706 +ar -6.520900 +em -6.537003 +k_ -6.570452 +o_ -6.586655 +on -6.592600 +am -6.597473 +nn -6.602912 +un -6.605272 +_p -6.620944 +_s -6.643672 +ka -6.719700 +ni -6.731251 +ha -6.749493 +at -6.881572 +_d -6.929360 +_v -7.001468 +_b -7.032643 +dd -7.095430 +sa -7.097865 +_z -7.128689 +za -7.129482 +ko -7.136632 +nt -7.144047 +lo -7.173004 +e_ -7.182252 +ac -7.257502 +_g -7.266065 +nk -7.313682 +so -7.361062 +ra -7.410968 +as -7.431294 +ll -7.438756 +_h -7.438803 +na -7.441331 +al -7.448469 +po -7.477381 +ol -7.487503 +it -7.492057 +to -7.540411 +im -7.556000 +od -7.616177 +ai -7.627166 +va -7.643510 +la -7.649289 +te -7.685007 +ri -7.697262 +bh -7.748102 +ak -7.793605 +kh -7.805917 +ti -7.811053 +ma -7.822661 +hi -7.822661 +vo -7.823881 +di -7.839046 +el -7.863632 +ro -7.889284 +tt -7.949907 +nc -7.958865 +in -7.993272 +pa -8.005272 +mh -8.012702 +ad -8.045012 +de -8.048219 +li -8.071233 +go -8.073772 +h_ -8.114800 +av -8.115622 +_n -8.161035 +ik -8.184682 +oi -8.224598 +is -8.250549 +st -8.297351 +_l -8.311668 +ot -8.320429 +re -8.328465 +_f -8.341345 +_o -8.343007 +ok -8.355931 +nv -8.372808 +gh -8.396590 +mo -8.420125 +ru -8.440207 +il -8.459718 +bo -8.465708 +ap -8.467329 +vi -8.469907 +_u -8.473927 +do -8.479881 +l_ -8.480940 +ng -8.491087 +s_ -8.503554 +en -8.522955 +ne -8.527921 +ag -8.534203 +_c -8.540712 +er -8.594500 +da -8.605797 +hu -8.619619 +os -8.638077 +xe -8.657646 +rt -8.671976 +_r -8.705565 +ur -8.709170 +ve -8.737306 +et -8.774475 +_j -8.778849 +du -8.798037 +ir -8.799717 +ek -8.816268 +ox -8.849824 +mi -8.855931 +th -8.869987 +je -8.871374 +d_ -8.876053 +sl -8.876306 +gl -8.876813 +ke -8.893892 +_x -8.901735 +om -8.901735 +pu -8.930247 +ba -8.935252 +oy -8.937894 +v_ -8.948508 +xi -8.951440 +ov -8.995197 +nd -9.055157 +ga -9.106976 +dh -9.132920 +pl -9.159956 +tr -9.165672 +og -9.191599 +ev -9.202356 +tu -9.223792 +pr -9.240977 +ya -9.297860 +ud -9.356196 +ji -9.359556 +tl -9.382579 +me -9.423268 +_i -9.424934 +zo -9.450165 +es -9.491863 +su -9.505120 +dl -9.523451 +mc -9.544431 +rs -9.558386 +ye -9.567765 +ic -9.570221 +eo -9.582155 +ec -9.587744 +ki -9.593979 +no -9.596064 +au -9.596481 +mn -9.601707 +pi -9.646154 +y_ -9.653515 +um -9.654819 +up -9.663099 +iv -9.673186 +fa -9.740531 +us -9.741453 +ed -9.755363 +ge -9.790021 +ia -9.843744 +_e -9.846471 +z_ -9.854932 +ob -9.861186 +ut -9.863193 +p_ -9.866460 +u_ -9.931563 +rg -9.941070 +uk -9.972678 +id -9.983825 +eu -9.993135 +oc -10.001399 +_y -10.003335 +gi -10.032114 +rk -10.083829 +ul -10.091160 +gu -10.137771 +oz -10.143552 +op -10.149050 +ku -10.156721 +fu -10.187508 +ix -10.203309 +lu -10.207765 +dr -10.215756 +ab -10.223469 +mb -10.270636 +ay -10.275973 +g_ -10.306368 +ez -10.312182 +mu -10.408905 +az -10.411472 +kr -10.411840 +rn -10.418464 +xa -10.418464 +se -10.421049 +dn -10.424749 +dt -10.427716 +vh -10.456968 +lt -10.476805 +aj -10.514947 +rl -10.521667 +rv -10.524046 +sr -10.546445 +zu -10.587329 +rp -10.601079 +vn -10.627699 +rm -10.636253 +ou -10.637970 +gn -10.657864 +sk -10.660043 +kx -10.671427 +fo -10.674507 +sp -10.696255 +gt -10.699837 +eg -10.714255 +vs -10.721519 +uc -10.724252 +ax -10.738457 +ig -10.759329 +pe -10.759796 +mk -10.796718 +rc -10.812627 +b_ -10.816997 +xo -10.823332 +vr -10.880111 +bi -10.884690 +nh -10.894405 +vt -10.941865 +xt -10.941865 +ub -10.973491 +gd -10.985464 +kn -11.017513 +oj -11.029858 +iu -11.046864 +kt -11.049718 +si -11.050862 +iz -11.056017 +rj -11.083244 +yc -11.087930 +be -11.090867 +vu -11.133223 +ui -11.147826 +vp -11.195703 +br -11.235442 +ry -11.240650 +io -11.260351 +uz -11.267642 +ib -11.273635 +mp -11.310127 +nl -11.332890 +ug -11.369505 +nu -11.378089 +vl -11.427716 +x_ -11.538005 +rd -11.547655 +tk -11.547655 +gr -11.566334 +iy -11.577823 +ip -11.594396 +ex -11.608635 +ie -11.651346 +fe -11.752803 +fi -11.797197 +ns -11.826265 +ep -11.881127 +co -11.892354 +lk -11.905737 +yk -11.918201 +ux -11.924473 +nj -11.932879 +sv -11.955181 +xr -11.993135 +kl -12.009711 +ah -12.013050 +rb -12.035505 +uj -12.081491 +dk -12.110995 +ca -12.143552 +sh -12.159493 +kd -12.160727 +tn -12.160727 +dv -12.160727 +vc -12.180609 +rz -12.183114 +pn -12.203309 +lh -12.207128 +ja -12.283008 +lm -12.284352 +tv -12.371646 +vd -12.391784 +ci -12.396136 +zh -12.418095 +oa -12.431433 +fr -12.447904 +yo -12.507865 +ju -12.514158 +jo -12.560623 +lc -12.567151 +uv -12.573709 +ei -12.593563 +dp -12.635824 +ts -12.637541 +zv -12.722429 +zi -12.724252 +ss -12.733401 +eb -12.755596 +lg -12.781930 +tm -12.781930 +kp -12.808754 +kv -12.822355 +lp -12.843991 +ht -12.894405 +mz -12.894405 +lz -12.900575 +lv -12.906771 +ce -12.955181 +jk -12.957323 +xn -12.965924 +hn -12.972408 +sc -13.009711 +hl -13.020869 +md -13.032114 +ao -13.043447 +bu -13.073337 +hp -13.077990 +ue -13.080323 +oh -13.087343 +vz -13.157029 +oe -13.169392 +ze -13.212235 +dg -13.253754 +ty -13.259029 +pk -13.296504 +rx -13.296504 +ua -13.318363 +hr -13.318363 +bd -13.321119 +sn -13.351786 +tb -13.412207 +vk -13.415148 +xv -13.426974 +_w -13.438897 +j_ -13.493805 +af -13.506296 +dc -13.567151 +c_ -13.640980 +ej -13.644427 +ms -13.658299 +mt -13.711541 +if -13.733401 +ny -13.755596 +mv -13.759329 +pt -13.816511 +zt -13.843991 +jy -13.884180 +ih -13.900575 +ph -13.900575 +ld -13.904703 +hv -13.942395 +rr -13.985464 +wa -13.985464 +sf -14.011936 +dx -14.011936 +sm -14.016396 +ct -14.048005 +ks -14.066384 +dy -14.089691 +of -14.122967 +qu -14.137467 +we -14.137467 +nb -14.191915 +hy -14.227665 +ly -14.274970 +xu -14.285697 +ae -14.291090 +py -14.301937 +oo -14.301937 +yt -14.318363 +ml -14.351786 +w_ -14.351786 +bs -14.397590 +ds -14.403419 +nz -14.409272 +mm -14.409272 +ee -14.415148 +ck -14.469143 +px -14.475269 +ew -14.512582 +tc -14.531607 +gs -14.544431 +tp -14.544431 +sg -14.576999 +bl -14.576999 +f_ -14.590235 +jp -14.596899 +cr -14.596899 +cl -14.617076 +cu -14.679361 +aw -14.774357 +uy -14.812627 +lj -14.828224 +ey -14.828224 +hd -14.836086 +np -14.884180 +pd -14.892354 +xm -14.908843 +nx -14.925521 +ff -14.968082 +zl -14.985464 +td -15.011936 +gc -15.011936 +bt -15.029858 +km -15.029858 +cy -15.048005 +rf -15.075662 +cc -15.132618 +bj -15.152114 +wo -15.181861 +lf -15.232845 +wi -15.264323 +ow -15.318363 +_q -15.340559 +ls -15.386002 +dm -15.409272 +vg -15.421049 +ky -15.421049 +gv -15.432923 +gx -15.432923 +hm -15.531607 +gp -15.544431 +sy -15.610319 +lb -15.623866 +sw -15.737076 +vm -15.751873 +ef -15.751873 +zb -15.766824 +eq -15.766824 +zm -15.766824 +ps -15.781930 +sd -15.797197 +yu -15.797197 +my -15.812627 +kc -15.828224 +uf -15.828224 +ys -15.828224 +vx -15.859933 +mg -15.876053 +tx -15.994234 +yd -16.048005 +zr -16.066384 +gm -16.142333 +yb -16.142333 +ij -16.142333 +pm -16.161961 +hh -16.243262 +nw -16.264323 +cp -16.264323 +jb -16.285697 +eh -16.307392 +wr -16.307392 +ln -16.307392 +aq -16.329418 +xh -16.351786 +cd -16.397590 +zk -16.397590 +by -16.421049 +mj -16.421049 +kk -16.421049 +zn -16.444895 +yl -16.469143 +mr -16.469143 +gy -16.518896 +yn -16.518896 +fl -16.544431 +nr -16.596899 +ym -16.596899 +bg -16.737076 +jh -16.766824 +jd -16.925521 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gsw b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gsw new file mode 100644 index 00000000000..7ca150cd6c4 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gsw @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 5900045 +#UNIQUE_BIGRAMS 575 +#UNSEEN_LOG_PROB -17.268872 +#EXPECTED_SCORE -7.534759 +ch -4.907750 +e_ -4.938650 +t_ -5.182179 +er -5.294310 +sc -5.638181 +r_ -5.907244 +is -6.077890 +_s -6.125665 +nd -6.145435 +h_ -6.210075 +_g -6.285013 +de -6.363676 +d_ -6.395209 +an -6.513115 +_d -6.516939 +he -6.527751 +_i -6.549757 +_h -6.583433 +_w -6.593835 +un -6.623978 +i_ -6.642596 +_u -6.717636 +_b -6.732926 +_a -6.808395 +et -6.814657 +te -6.821777 +_v -6.880809 +m_ -6.909506 +si -6.929545 +re -6.931812 +_m -6.985553 +it -6.997501 +s_ -7.020556 +or -7.045793 +be -7.147479 +n_ -7.148796 +in -7.170366 +al -7.270821 +st -7.324680 +ie -7.339257 +en -7.344056 +_e -7.363454 +ge -7.365953 +li -7.367608 +ha -7.401107 +g_ -7.421545 +ig -7.423978 +ht -7.437054 +_z -7.448071 +ne -7.465814 +ri -7.478972 +_f -7.620053 +on -7.645433 +ar -7.647834 +me -7.680719 +ei -7.706638 +l_ -7.708324 +le -7.718537 +_n -7.719207 +ic -7.765704 +gs -7.772371 +ur -7.772745 +se -7.799897 +ss -7.805685 +vo -7.872247 +di -7.919180 +ra -7.919298 +mi -7.934292 +um -7.934472 +wi -7.945701 +rt -7.958174 +ni -7.969039 +nt -7.980110 +ii -7.985119 +ti -7.990644 +el -8.002509 +rd -8.011063 +la -8.015865 +ai -8.033311 +wo -8.042952 +om -8.059883 +nn -8.063869 +wa -8.112036 +ma -8.180333 +hi -8.184453 +es -8.208121 +at -8.209785 +il -8.213482 +as -8.226899 +ll -8.229099 +ro -8.229906 +_o -8.238669 +_k -8.243552 +_l -8.251578 +a_ -8.259874 +hr -8.320258 +us -8.358671 +_r -8.374408 +rs -8.398134 +ta -8.398712 +_c -8.399042 +_j -8.411893 +oo -8.414561 +bi -8.434726 +da -8.450379 +ng -8.460507 +ue -8.460938 +zu -8.486407 +ho -8.493792 +aa -8.496616 +ol -8.517253 +na -8.520302 +ve -8.527412 +em -8.538189 +ru -8.565906 +gr -8.606789 +ba -8.608792 +we -8.661581 +id -8.679816 +ab -8.685342 +am -8.688063 +no -8.716993 +ir -8.741797 +lt -8.778156 +au -8.814025 +o_ -8.875286 +os -8.876320 +eg -8.942630 +ts -8.952288 +_p -8.963352 +hu -9.006591 +oc -9.058448 +ac -9.084230 +dr -9.089016 +ag -9.095288 +sa -9.097564 +ut -9.123515 +oh -9.127749 +ze -9.150637 +ga -9.150776 +gi -9.173046 +rg -9.195092 +f_ -9.215007 +mm -9.217334 +ka -9.248823 +mo -9.268142 +jo -9.268444 +tt -9.277672 +du -9.280710 +br -9.299848 +ot -9.310832 +gl -9.312541 +af -9.312852 +fu -9.326131 +uf -9.328330 +ed -9.348433 +fa -9.353383 +vu -9.357708 +od -9.371895 +hl -9.373840 +fr -9.396074 +uu -9.414144 +do -9.414811 +lo -9.417651 +ee -9.425196 +ns -9.427720 +z_ -9.437351 +to -9.441596 +_t -9.452005 +ad -9.472530 +im -9.482641 +tz -9.504320 +so -9.508232 +bu -9.513763 +zi -9.524168 +fi -9.526330 +ec -9.528134 +ft -9.552348 +sp -9.572500 +rr -9.580528 +fe -9.608888 +io -9.619427 +gh -9.620774 +kt -9.644258 +ff -9.646412 +ls -9.646804 +tu -9.691597 +zw -9.699301 +nz -9.705616 +ko -9.719774 +gm -9.747040 +pr -9.761824 +eb -9.767354 +ke -9.805356 +hn -9.807546 +tr -9.820090 +k_ -9.851146 +ia -9.857937 +eh -9.918647 +mu -9.930768 +rn -9.946365 +fl -9.948296 +rh -9.960426 +ob -9.960913 +ub -9.963840 +ik -9.978074 +pe -10.001444 +su -10.008731 +ah -10.013019 +ld -10.021127 +og -10.039538 +lu -10.063411 +ib -10.096225 +th -10.098100 +rm -10.117255 +rb -10.122425 +nu -10.123788 +za -10.152445 +pa -10.154951 +rc -10.200261 +gn -10.216752 +hw -10.231375 +rk -10.249418 +u_ -10.261674 +vi -10.275852 +tl -10.303088 +rl -10.351784 +fo -10.360116 +dt -10.409148 +po -10.416147 +rz -10.427889 +ud -10.449609 +of -10.478623 +rf -10.492295 +gt -10.507164 +wu -10.524708 +ug -10.525068 +bs -10.570082 +nk -10.588413 +uc -10.606979 +bl -10.627335 +ih -10.630432 +gf -10.631983 +sl -10.648373 +ck -10.665746 +if -10.678113 +dl -10.710115 +ul -10.715450 +rw -10.739078 +gg -10.775904 +go -10.781058 +zt -10.794893 +ek -10.797066 +je -10.797501 +ak -10.811935 +ku -10.822081 +op -10.835870 +zo -10.843488 +ju -10.862027 +ef -10.899837 +ez -10.900772 +mb -10.908742 +eo -10.930529 +iz -10.934831 +pi -10.944436 +eu -10.986979 +lb -10.997439 +ja -11.003952 +bb -11.005962 +yt -11.009487 +gw -11.045211 +sy -11.050906 +ap -11.085027 +bo -11.086090 +gu -11.091415 +ki -11.133094 +pp -11.138045 +oi -11.169803 +kl -11.183387 +ea -11.278583 +kr -11.290783 +b_ -11.298153 +up -11.318617 +mt -11.320492 +pt -11.331793 +lg -11.336844 +va -11.341278 +ua -11.346362 +ev -11.372057 +hd -11.418822 +ds -11.428225 +iv -11.435657 +ep -11.512869 +sg -11.515730 +p_ -11.525068 +y_ -11.531569 +sb -11.543197 +dd -11.601270 +hs -11.691395 +tw -11.693013 +ou -11.711755 +pf -11.719155 +ok -11.759279 +lf -11.766928 +lm -11.773762 +tg -11.804919 +wy -11.821638 +pu -11.851146 +bt -11.856576 +co -11.856576 +ew -11.887741 +mp -11.913922 +yy -11.955076 +lk -11.978567 +ys -11.984500 +fg -11.996440 +oe -11.997439 +ph -12.010495 +ly -12.015548 +ex -12.069179 +mf -12.111833 +iw -12.122697 +ry -12.134743 +dw -12.208049 +sf -12.210365 +_y -12.229025 +ov -12.269500 +hm -12.281623 +ui -12.326131 +nc -12.347636 +ow -12.378552 +oz -12.400859 +sd -12.431599 +bg -12.450635 +hb -12.461627 +pl -12.490886 +sr -12.503610 +ml -12.513584 +sw -12.516447 +lv -12.539553 +yc -12.545388 +sm -12.557129 +iu -12.576415 +ca -12.597477 +ip -12.656244 +v_ -12.668927 +yn -12.737407 +w_ -12.757585 +sk -12.759279 +by -12.784935 +yz -12.844836 +ej -12.848438 +dn -12.866586 +hy -12.892382 +yl -12.903580 +fd -12.912979 +bn -12.916755 +ln -12.933874 +nh -12.951198 +nl -12.953136 +zr -12.980542 +ks -13.008479 +xi -13.014536 +rv -13.016561 +ym -13.030815 +ms -13.059753 +yb -13.063934 +ce -13.063934 +dy -13.078667 +ae -13.093551 +az -13.115084 +dp -13.117255 +yr -13.154673 +vp -13.223168 +zy -13.223168 +qu -13.232551 +av -13.234907 +ty -13.270707 +nr -13.282841 +bz -13.295078 +ay -13.295078 +uk -13.307419 +x_ -13.329903 +td -13.337476 +sh -13.360438 +hp -13.386386 +bh -13.396898 +lc -13.412810 +dg -13.469927 +fs -13.515015 +lz -13.532293 +lp -13.579405 +rp -13.612711 +uz -13.618850 +tc -13.646804 +aw -13.665746 +gk -13.707660 +nf -13.750828 +zl -13.778049 +ey -13.805794 +yp -13.805794 +lh -13.809300 +tm -13.809300 +_q -13.826959 +ij -13.837658 +lr -13.873909 +tb -13.873909 +sz -13.877585 +dh -13.884964 +gy -13.926240 +ye -13.945400 +mg -13.945400 +c_ -13.972658 +fn -14.041083 +uh -14.082904 +ax -14.091415 +yg -14.095690 +ci -14.099977 +tn -14.104277 +sn -14.112916 +fb -14.139148 +ps -14.234907 +cv -14.258675 +bd -14.263476 +ji -14.263476 +tf -14.357868 +nb -14.368173 +cl -14.415479 +tk -14.415479 +hz -14.437012 +yo -14.481067 +ny -14.486670 +sv -14.509301 +mn -14.520751 +yf -14.561557 +oa -14.573431 +kz -14.591428 +yd -14.628108 +dm -14.653091 +xa -14.665746 +j_ -14.678513 +xt -14.684940 +yw -14.684940 +uw -14.684940 +kh -14.724110 +xe -14.730743 +oj -14.791855 +km -14.907332 +yv -14.914866 +bw -14.914866 +ix -14.953136 +lw -15.024689 +my -15.041083 +kg -15.049351 +zb -15.066030 +oy -15.074442 +ao -15.074442 +fm -15.091415 +cd -15.125972 +bj -15.125972 +zs -15.143566 +gd -15.143566 +ya -15.161378 +kw -15.179412 +hf -15.179412 +hg -15.188514 +zm -15.216170 +pd -15.225508 +vy -15.273126 +zz -15.292622 +tj -15.352743 +zd -15.373353 +db -15.373353 +ox -15.383770 +kf -15.426205 +xp -15.426205 +ux -15.492295 +df -15.573431 +ct -15.585404 +wj -15.621930 +kn -15.621930 +gz -15.646804 +bf -15.672116 +nw -15.697879 +zg -15.710935 +nj -15.778049 +mc -15.834083 +dk -15.862938 +tp -15.862938 +nv -15.892382 +kd -15.892382 +hk -15.907332 +dc -15.937706 +dv -16.000441 +_x -16.000441 +cs -16.016561 +md -16.016561 +gc -16.016561 +vg -16.049351 +hh -16.066030 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gug b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gug new file mode 100644 index 00000000000..ab344a57656 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/gug @@ -0,0 +1,425 @@ +#TOTAL_BIGRAMS 3165243 +#UNIQUE_BIGRAMS 421 +#UNSEEN_LOG_PROB -18.271957 +#EXPECTED_SCORE -7.081161 +a_ -4.473303 +e_ -4.711863 +i_ -5.426682 +_o -5.469137 +pe -5.526661 +ra -5.530974 +gu -5.951636 +te -5.970546 +ar -5.976073 +ua -6.030838 +o_ -6.034209 +_a -6.106893 +ha -6.141869 +re -6.201567 +he -6.295035 +va -6.332267 +an -6.369318 +_p -6.386489 +ta -6.424586 +er -6.492648 +ve -6.519618 +et -6.525484 +ku -6.533822 +_h -6.575163 +ap -6.601256 +up -6.646613 +ei -6.661855 +_t -6.663009 +_r -6.665922 +ch -6.672927 +av -6.675487 +_n -6.775652 +nd -6.822189 +mb -6.836078 +ue -6.861346 +ko -6.907165 +_u -6.907220 +ai -6.920300 +y_ -7.000786 +na -7.034269 +po -7.037020 +u_ -7.045725 +ik -7.062930 +oi -7.067263 +_m -7.140871 +ka -7.155223 +ne -7.239567 +py -7.304370 +at -7.322568 +je -7.355555 +_i -7.361839 +_g -7.380705 +ja -7.434723 +ek -7.475025 +pa -7.477866 +_j -7.485932 +or -7.492402 +em -7.502697 +_k -7.542251 +ic -7.580388 +ru -7.587997 +ev -7.612674 +on -7.613924 +oh -7.621443 +mi -7.622341 +mo -7.626659 +en -7.647710 +tu -7.688686 +ag -7.703051 +am -7.707617 +oj -7.714780 +ep -7.737654 +as -7.743112 +_c -7.760995 +ri -7.767634 +iv -7.894204 +ni -7.953640 +ma -7.965553 +hu -8.029140 +ur -8.081638 +om -8.086338 +eh -8.094662 +ak -8.098779 +_y -8.113473 +ia -8.124879 +yv -8.148353 +op -8.155223 +de -8.163694 +pi -8.170244 +ov -8.184494 +ip -8.194874 +ro -8.207079 +no -8.227153 +me -8.227973 +it -8.243636 +um -8.257239 +ah -8.261708 +ui -8.297112 +ty -8.327684 +sa -8.366269 +in -8.396668 +og -8.452416 +yp -8.492730 +yr -8.493058 +es -8.583357 +bo -8.595647 +ir -8.633702 +vo -8.649356 +ry -8.663702 +ok -8.665737 +eg -8.684367 +yt -8.686243 +ot -8.690568 +da -8.716792 +su -8.793389 +bi -8.830465 +hi -8.836912 +vy -8.868093 +ay -8.909575 +aa -8.929438 +ng -8.962480 +pu -8.995600 +uk -9.002362 +ec -9.036182 +ga -9.083862 +du -9.124497 +im -9.144220 +aj -9.174451 +os -9.197280 +sy -9.218845 +ho -9.224287 +se -9.229203 +uv -9.235234 +_v -9.250421 +ju -9.332084 +ti -9.368979 +yh -9.381692 +by -9.397898 +ba -9.402826 +un -9.453055 +nt -9.456894 +di -9.476891 +bu -9.484381 +ej -9.490925 +s_ -9.510406 +_s -9.514067 +vi -9.561839 +al -9.599178 +_e -9.614460 +ya -9.672044 +jo -9.736681 +do -9.754287 +n_ -9.774903 +_l -9.805982 +io -9.813755 +yk -9.815397 +le -9.832334 +l_ -9.852418 +ym -9.858329 +ca -9.871504 +go -9.899527 +to -9.970003 +li -9.973207 +hy -9.981477 +la -10.006576 +is -10.007514 +ke -10.008453 +el -10.028307 +ij -10.095036 +uh -10.112590 +oa -10.124243 +st -10.230298 +us -10.277038 +yg -10.284408 +ci -10.303290 +ih -10.310218 +lo -10.371694 +_f -10.400359 +il -10.425213 +ae -10.438434 +_d -10.449864 +co -10.451778 +au -10.463958 +my -10.486668 +si -10.503772 +so -10.611604 +_b -10.641872 +ol -10.674277 +ge -10.686994 +ug -10.696796 +yo -10.742917 +z_ -10.755469 +ig -10.804351 +rr -10.814165 +ac -10.826528 +eo -10.868519 +ie -10.924114 +ad -10.929438 +uy -10.935673 +ez -10.959074 +ee -10.962708 +ul -10.973665 +ys -11.049886 +r_ -11.050853 +dy -11.054726 +ll -11.089066 +ey -11.091053 +rt -11.138558 +oy -11.204791 +id -11.222108 +be -11.296969 +yi -11.347144 +cu -11.361464 +ye -11.361464 +ut -11.363864 +nc -11.391761 +yj -11.482749 +d_ -11.503772 +tr -11.557711 +ou -11.577077 +ky -11.586858 +lt -11.603781 +ea -11.630989 +ed -11.635332 +jy -11.673532 +sp -11.700583 +ce -11.814165 +lu -11.893445 +rg -11.933889 +fe -11.950029 +rn -11.990258 +nu -12.012684 +ao -12.014569 +br -12.144736 +pr -12.153016 +za -12.159257 +fr -12.167620 +vu -12.178143 +mu -12.238534 +sc -12.317760 +uj -12.423960 +fa -12.423960 +ab -12.444138 +qu -12.449227 +gi -12.459459 +uc -12.469764 +oo -12.487976 +gr -12.493222 +mp -12.514400 +bl -12.552226 +dr -12.557711 +eu -12.565979 +fi -12.574294 +cr -12.585456 +ki -12.669072 +oc -12.690003 +pl -12.748395 +nz -12.751534 +rc -12.760995 +t_ -12.876208 +ns -12.953640 +lv -12.957260 +oe -13.001428 +c_ -13.016456 +rl -13.050853 +gy -13.066408 +rd -13.098030 +ud -13.126279 +od -13.138558 +ib -13.142674 +ix -13.142674 +ub -13.303866 +az -13.317760 +x_ -13.322422 +ob -13.327098 +rm -13.360265 +fo -13.365066 +nh -13.404060 +zo -13.454333 +ny -13.454333 +eb -13.527796 +iz -13.538602 +gw -13.582658 +k_ -13.639689 +wa -13.723520 +gl -13.748395 +if -13.767336 +oz -13.819098 +af -13.832334 +rs -13.852418 +ef -13.859175 +rk -13.865964 +b_ -13.872786 +xi -13.950029 +wi -13.964528 +_x -13.964528 +lm -13.971833 +ld -13.979175 +fl -14.193005 +ii -14.193005 +_z -14.245157 +ct -14.245157 +iq -14.254035 +yn -14.374716 +of -14.413976 +_w -14.423960 +th -14.434013 +cc -14.434013 +fu -14.434013 +rp -14.485360 +cl -14.485360 +pn -14.538602 +kw -14.593885 +sl -14.628101 +vr -14.651370 +kr -14.711242 +lc -14.723520 +ss -14.773706 +we -14.852418 +lg -14.852418 +tl -14.935673 +oq -14.979175 +h_ -14.993972 +sk -15.024029 +ox -15.054726 +hn -15.070323 +yc -15.070323 +g_ -15.070323 +xu -15.086090 +sr -15.102032 +ex -15.167620 +m_ -15.184494 +zi -15.271957 +xv -15.308483 +p_ -15.308483 +pc -15.384431 +aq -15.384431 +tt -15.384431 +mn -15.423960 +nj -15.423960 +gn -15.423960 +zu -15.423960 +rb -15.423960 +sh -15.444138 +_q -15.444138 +sn -15.485360 +lk -15.549491 +iu -15.571517 +zt -15.639689 +uq -15.711242 +gt -15.760995 +rz -15.812525 +lb -15.979175 +rv -16.008922 +cn -16.008922 +lf -16.039296 +nb -16.134453 +ew -16.167620 +ls -16.201567 +xe -16.236333 +sm -16.271957 +rq -16.271957 +cy -16.345957 +bs -16.384431 +ji -16.384431 +yu -16.423960 +uz -16.464602 +f_ -16.506422 +lh -16.506422 +v_ -16.549491 +nn -16.593885 +tk -16.593885 +kj -16.593885 +j_ -16.593885 +nr -16.639689 +xa -16.686994 +tn -16.686994 +ze -16.686994 +mc -16.735904 +xt -16.735904 +nx -16.893445 +tm -16.893445 +pt -16.893445 +sq -16.893445 +ps -16.950029 +dg -16.950029 +ms -16.950029 +mm -17.008922 +lp -17.008922 +tb -17.070323 +kl -17.134453 +np -17.134453 +zq -17.201567 +wo -17.201567 +ht -17.201567 +nm -17.271957 +yl -17.271957 +uu -17.423960 +yd -17.506422 +dm -17.506422 +mr -17.506422 +sf -17.593885 +gd -17.686994 +eq -17.686994 +zm -17.686994 +sv -17.686994 +bc -17.786530 +ux -17.786530 +hk -17.893445 +pp -17.893445 +kt -17.893445 +w_ -17.893445 +ff -17.893445 +bd -18.008922 +zz -18.008922 +rf -18.008922 +zc -18.008922 +ax -18.134453 +vl -18.134453 +nv -18.271957 +nf -18.271957 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/guj b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/guj new file mode 100644 index 00000000000..bc75ed022e1 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/guj @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 13064951 +#UNIQUE_BIGRAMS 1645 +#UNSEEN_LOG_PROB -14.637457 +#EXPECTED_SCORE -7.969582 +ન_ -4.260065 +ત_ -5.086199 +ર_ -5.382589 +_ક -5.462694 +મ_ -5.599878 +_મ -5.779567 +_હ -5.794486 +_સ -5.832838 +_અ -5.850238 +વ_ -5.919649 +_પ -5.982927 +_ત -6.066757 +કર -6.097088 +_આ -6.220827 +લ_ -6.265237 +_જ -6.275038 +ય_ -6.436548 +_વ -6.436969 +_બ -6.516502 +હત -6.544987 +_ન -6.652068 +અન -6.665826 +ક_ -6.689474 +_ર -6.707434 +થ_ -6.757093 +રત -7.081226 +ણ_ -7.107665 +_દ -7.233508 +_લ -7.277974 +તન -7.290972 +_ગ -7.299453 +ટ_ -7.320991 +_ભ -7.353886 +મન -7.430678 +વન -7.443903 +_શ -7.475509 +રવ -7.482325 +આવ -7.487310 +પર -7.514370 +તમ -7.520683 +જ_ -7.639374 +ડ_ -7.672873 +સ_ -7.748412 +_એ -7.787547 +ધ_ -7.793267 +મટ -7.793708 +_થ -7.796283 +શ_ -7.870032 +હ_ -7.915857 +રન -7.963131 +ગ_ -7.967685 +ળ_ -7.989437 +સમ -8.012862 +દ_ -8.013005 +_ખ -8.105443 +_ચ -8.148348 +રક -8.171529 +લક -8.211329 +વમ -8.237952 +_ઉ -8.311435 +વર -8.332389 +જવ -8.343932 +રમ -8.355423 +સર -8.407602 +દર -8.427614 +મર -8.466341 +હવ -8.474449 +સથ -8.475391 +કમ -8.490881 +તર -8.511446 +જર -8.535829 +રણ -8.553394 +થય -8.566605 +વત -8.570336 +જન -8.576646 +કન -8.604442 +ટલ -8.611508 +વલ -8.625004 +ધર -8.631076 +બન -8.641093 +ભર -8.643034 +_ફ -8.645552 +વધ -8.660667 +રહ -8.668868 +આપ -8.689463 +એ_ -8.690192 +હર -8.707768 +ઓ_ -8.717776 +નવ -8.722992 +દવ -8.776803 +રજ -8.799995 +કલ -8.801226 +મળ -8.842348 +નથ -8.863228 +પડ -8.871635 +હય -8.872307 +નન -8.913672 +_ટ -8.914578 +ગમ -8.919435 +_ઘ -8.939626 +ગર -8.941471 +શક -8.949528 +ઈ_ -8.952753 +ભગ -8.956204 +નમ -8.993427 +પત -8.999349 +વવ -9.007623 +_ધ -9.062950 +હલ -9.065492 +પ_ -9.072728 +સન -9.080240 +છ_ -9.085869 +ચ_ -9.128805 +_છ -9.136180 +બ_ -9.182780 +બજ -9.197227 +પસ -9.225967 +ઉપ -9.243597 +જમ -9.253471 +પછ -9.255562 +મજ -9.258196 +લવ -9.273696 +ખ_ -9.277255 +નક -9.288190 +અમ -9.310594 +મત -9.333422 +વસ -9.341637 +શર -9.374682 +_ય -9.379455 +મય -9.419274 +મહ -9.439909 +યન -9.447060 +સધ -9.448216 +દશ -9.464585 +ટક -9.479169 +બદ -9.488023 +ગય -9.489848 +ખત -9.494381 +બધ -9.503569 +શન -9.507181 +જય -9.517665 +કટ -9.531981 +મક -9.536238 +નહ -9.542319 +પન -9.557465 +લગ -9.562383 +પલ -9.591734 +પણ -9.600108 +એવ -9.609652 +રખ -9.615271 +મણ -9.617524 +_ઓ -9.619087 +નર -9.626836 +મદ -9.637175 +દન -9.638054 +ઇ_ -9.652023 +ચર -9.675128 +ફર -9.686002 +લન -9.693115 +સવ -9.702284 +જત -9.703389 +કય -9.710958 +રપ -9.726028 +મલ -9.736729 +હન -9.747130 +ળવ -9.793610 +યલ -9.829028 +પહ -9.850480 +વદ -9.858761 +ધન -9.868947 +તઓ -9.874741 +ભવ -9.877128 +પય -9.880455 +તલ -9.889643 +જણ -9.897310 +વશ -9.898364 +તવ -9.909259 +રય -9.916177 +તથ -9.934214 +પવ -9.937459 +બર -9.946801 +_ડ -9.957191 +કહ -9.961259 +ચલ -9.973751 +સદ -9.979202 +લમ -9.989381 +જક -9.989605 +ગત -9.996921 +રર -10.009045 +કઈ -10.009842 +ગજ -10.011551 +કપ -10.022650 +ગન -10.030251 +કવ -10.035109 +ટન -10.037079 +મમ -10.046157 +ડન -10.047092 +યર -10.074096 +લસ -10.089895 +કસ -10.090015 +સહ -10.105625 +અગ -10.128681 +લખ -10.140100 +વય -10.148849 +વક -10.150103 +આજ -10.158030 +કત -10.158282 +નગ -10.175163 +યદ -10.185028 +બલ -10.209661 +થવ -10.210576 +ખવ -10.212017 +બહ -10.214114 +ચન -10.228217 +રસ -10.230338 +ગળ -10.234190 +મગ -10.234190 +મથ -10.257791 +પક -10.263616 +ખર -10.264566 +શહ -10.280273 +ગણ -10.286603 +સત -10.287293 +એન -10.289917 +રડ -10.300741 +હજ -10.307582 +વડ -10.315581 +એટ -10.336845 +નત -10.351342 +કડ -10.364092 +રલ -10.373876 +ડવ -10.394835 +યગ -10.395876 +તય -10.395876 +ડય -10.403932 +વચ -10.411582 +મડ -10.428223 +રફ -10.428984 +ઘણ -10.440140 +ફ_ -10.448911 +ગલ -10.453084 +ભ_ -10.464741 +ણન -10.475391 +રશ -10.484538 +નય -10.507341 +દલ -10.508950 +ઓન -10.521880 +થત -10.532472 +પચ -10.535911 +કળ -10.549417 +સબ -10.575466 +ઘટ -10.622564 +બત -10.623783 +પટ -10.629370 +વગ -10.630420 +એમ -10.639022 +કઇ -10.650869 +ળન -10.656918 +ખલ -10.672693 +લત -10.693023 +હમ -10.694303 +યમ -10.698151 +_ઝ -10.698701 +ણમ -10.701089 +ડત -10.703113 +રગ -10.708646 +આગ -10.710495 +ઈન -10.711050 +એક -10.712346 +કશ -10.718846 +ખન -10.730243 +બળ -10.734376 +રબ -10.736823 +નસ -10.750265 +જબ -10.773658 +વથ -10.786084 +યજ -10.796848 +સક -10.797634 +ઇન -10.798617 +સલ -10.800191 +છત -10.804925 +કદ -10.818022 +ગવ -10.822415 +ણવ -10.824215 +ટમ -10.827623 +ધક -10.836884 +મવ -10.842348 +ળત -10.842550 +ધમ -10.843767 +દખ -10.849054 +ષ_ -10.856405 +વખ -10.858044 +નજ -10.858864 +નલ -10.870394 +વળ -10.873912 +લધ -10.890796 +તક -10.899207 +ભજ -10.906819 +રથ -10.908941 +સચ -10.923022 +જહ -10.925168 +દમ -10.925168 +ચક -10.932055 +આન -10.946583 +ઠ_ -10.947673 +જઈ -10.952041 +હથ -10.966994 +ઘર -10.968763 +જગ -10.970092 +અહ -10.975863 +વજ -10.979202 +_ઊ -10.988147 +વહ -11.003480 +ટર -11.005522 +ડલ -11.010981 +ચવ -11.011664 +બબ -11.018520 +જટ -11.027482 +ઘ_ -11.028866 +બક -11.037428 +તપ -11.041844 +હસ -11.045807 +આત -11.054001 +સપ -11.054706 +મસ -11.076956 +સફ -11.084849 +મબ -11.091340 +સભ -11.094958 +લબ -11.100524 +નદ -11.102465 +મચ -11.102708 +ખડ -11.104895 +ચત -11.106112 +સગ -11.129918 +અથ -11.135373 +હશ -11.137610 +લડ -11.141845 +અધ -11.149601 +_ઇ -11.154879 +વપ -11.154879 +તણ -11.176696 +ખબ -11.188760 +સસ -11.212934 +ડક -11.230073 +પગ -11.239920 +રઓ -11.249029 +રદ -11.256845 +ણત -11.259279 +લલ -11.263887 +શમ -11.273696 +દધ -11.289917 +સજ -11.297402 +યત -11.313050 +ળક -11.314455 +જદ -11.314455 +અત -11.330291 +બસ -11.331998 +અપ -11.343143 +વણ -11.347452 +આખ -11.349756 +જડ -11.369779 +ખસ -11.384465 +ઈએ -11.398705 +યવ -11.410981 +ઓછ -11.412184 +હદ -11.413388 +જપ -11.413690 +તજ -11.419727 +લય -11.422453 +હક -11.425487 +વટ -11.458046 +પમ -11.472408 +અદ -11.474606 +ફળ -11.481536 +યક -11.487548 +ધવ -11.495178 +આધ -11.497730 +હચ -11.505414 +રચ -11.515400 +થન -11.520906 +અસ -11.531654 +લર -11.536238 +થડ -11.546441 +ફટ -11.555387 +બઈ -11.565057 +નશ -11.569413 +છળ -11.575129 +આર -11.578502 +સડ -11.581545 +ગભ -11.581884 +ડમ -11.590712 +જથ -11.593780 +વબ -11.608531 +ચમ -11.610602 +જઇ -11.611292 +કબ -11.625876 +લજ -11.627272 +અવ -11.631470 +_ઠ -11.636735 +ઢ_ -11.637790 +બઠ -11.644138 +આદ -11.653001 +ઠક -11.657631 +રધ -11.671252 +નઓ -11.685366 +નચ -11.686821 +શધ -11.693023 +નટ -11.696684 +નખ -11.698885 +લઈ -11.704770 +સય -11.705139 +મશ -11.714015 +દય -11.745897 +ચટ -11.748554 +નધ -11.751216 +અલ -11.762298 +ળમ -11.763833 +આઇ -11.769605 +ગધ -11.772692 +ળખ -11.777336 +ફન -11.777724 +ઉમ -11.783162 +ઝડ -11.786279 +ણક -11.794100 +નભ -11.802359 +ટણ -11.804727 +ઝ_ -11.812650 +છડ -11.820616 +વઘ -11.821415 +દક -11.837894 +ટડ -11.839512 +અભ -11.843970 +ફલ -11.848443 +ભળ -11.848850 +ટવ -11.850480 +દદ -11.862354 +ડપ -11.872255 +ખમ -11.875986 +શળ -11.885146 +ઇએ -11.887654 +લટ -11.891845 +લઇ -11.898153 +કચ -11.900262 +ગડ -11.909153 +બમ -11.919382 +શય -11.922379 +ધત -11.925812 +ઝન -11.927531 +રટ -11.937892 +બગ -11.938325 +ણસ -11.940494 +ભષ -11.947455 +થક -11.951823 +ભમ -11.964565 +ઓળ -11.965889 +ખક -11.972974 +જલ -11.978757 +વભ -11.989044 +ફસ -11.992190 +ઉ_ -11.996695 +નબ -11.998953 +કએ -12.008021 +આય -12.011209 +અર -12.044407 +ભન -12.044874 +દગ -12.051421 +ધધ -12.059882 +ભલ -12.060354 +s_ -12.068868 +જશ -12.078388 +થશ -12.078866 +એસ -12.079343 +ગઉ -12.090858 +કઓ -12.113188 +હઠ -12.114167 +તહ -12.114167 +પદ -12.118089 +આક -12.119071 +રએ -12.125964 +રઆ -12.138854 +_ઈ -12.140349 +કણ -12.146344 +ઠવ -12.149852 +તત -12.152363 +બટ -12.175163 +શખ -12.176696 +કથ -12.206135 +ઓએ -12.210838 +આશ -12.215557 +દડ -12.219765 +ડશ -12.245272 +આસ -12.251720 +ફડ -12.255494 +રષ -12.256034 +ભઈ -12.258737 +ઠળ -12.259820 +ણય -12.260362 +છટ -12.261988 +ડઓ -12.266877 +સટ -12.270692 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hat b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hat new file mode 100644 index 00000000000..349f7ad9d6d --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hat @@ -0,0 +1,490 @@ +#TOTAL_BIGRAMS 3722955 +#UNIQUE_BIGRAMS 486 +#UNSEEN_LOG_PROB -18.506089 +#EXPECTED_SCORE -7.115707 +n_ -4.009135 +an -4.527253 +e_ -4.961317 +on -5.348679 +en -5.485005 +_n -5.593592 +na -5.613091 +ou -5.644052 +_p -5.645157 +yo -5.914520 +_m -6.076185 +_a -6.276489 +re -6.306999 +_k -6.310101 +te -6.462720 +i_ -6.494617 +ye -6.546521 +_s -6.606618 +sy -6.634530 +u_ -6.720105 +ti -6.747574 +ko -6.786229 +po -6.810730 +is -6.848592 +ma -6.867970 +_d -6.924088 +_e -6.959050 +_l -6.965331 +_y -7.001667 +ra -7.023230 +as -7.061591 +t_ -7.068753 +it -7.075845 +l_ -7.100575 +la -7.115651 +s_ -7.126331 +_g -7.218319 +_t -7.227291 +se -7.240532 +ay -7.337793 +mo -7.356786 +_r -7.380869 +a_ -7.400049 +al -7.405886 +de -7.445325 +le -7.447691 +ek -7.450806 +pr -7.462309 +et -7.482889 +ri -7.488793 +ns -7.495561 +nt -7.525164 +in -7.562475 +li -7.562548 +el -7.594023 +pa -7.608697 +un -7.615292 +ot -7.656605 +es -7.670828 +ne -7.692869 +me -7.697124 +so -7.736995 +k_ -7.738401 +ta -7.739974 +ge -7.741549 +_b -7.747366 +at -7.776128 +mi -7.847163 +yi -7.863676 +_f -7.927433 +ik -7.933104 +to -7.935189 +m_ -8.006741 +_j -8.007239 +ni -8.008636 +wa -8.058385 +si -8.058489 +il -8.058592 +y_ -8.078147 +wo -8.086128 +pi -8.106277 +pe -8.180896 +tr -8.240591 +ke -8.249644 +ch -8.250352 +om -8.261844 +ka -8.267684 +di -8.276068 +_v -8.299929 +da -8.300540 +em -8.305558 +we -8.324439 +ny -8.340428 +ve -8.342187 +nm -8.348742 +st -8.361303 +_o -8.362578 +o_ -8.366665 +nd -8.385592 +er -8.391305 +ap -8.392216 +ep -8.425937 +no -8.457193 +ar -8.508204 +zi -8.537998 +vi -8.585885 +ez -8.591106 +ey -8.605071 +id -8.606732 +je -8.619020 +lo -8.627731 +sa -8.647641 +ol -8.648263 +iz -8.668934 +ba -8.740057 +_c -8.792358 +pl -8.829603 +op -8.830132 +nn -8.859890 +av -8.876550 +va -8.913819 +np -8.913819 +ak -8.925265 +pw -8.940796 +fe -8.951308 +ev -8.998492 +ya -9.078147 +iv -9.134312 +ks -9.143268 +d_ -9.156033 +im -9.162459 +os -9.165348 +kr -9.197750 +ad -9.209631 +ga -9.219993 +ki -9.233459 +or -9.240942 +am -9.244229 +ut -9.302251 +fi -9.335412 +bl -9.338169 +aj -9.362705 +sp -9.370636 +fo -9.375518 +he -9.399395 +ib -9.454880 +uv -9.461695 +kt -9.484276 +mw -9.496540 +ly -9.525949 +ja -9.537998 +ze -9.537998 +p_ -9.560060 +be -9.576830 +nk -9.585736 +ab -9.608546 +dw -9.627118 +ef -9.628038 +_i -9.655589 +za -9.676683 +ji -9.678905 +jo -9.683040 +ha -9.684634 +ir -9.696803 +ed -9.728012 +b_ -9.730972 +do -9.769348 +if -9.773752 +ip -9.844667 +ok -9.845380 +bi -9.935095 +gr -9.938513 +fa -9.960738 +vo -9.965379 +rt -9.970813 +ll -10.007040 +nb -10.054053 +iy -10.067713 +tw -10.081922 +od -10.111197 +eg -10.135401 +bo -10.154266 +ag -10.194567 +go -10.198660 +fr -10.217915 +nv -10.317747 +sk -10.323198 +j_ -10.331163 +dy -10.335162 +nj -10.355834 +my -10.392867 +oz -10.394953 +ob -10.397042 +ej -10.398087 +v_ -10.398087 +f_ -10.415447 +zy -10.425538 +og -10.441615 +lm -10.467170 +zo -10.507780 +az -10.536270 +ur -10.550148 +h_ -10.555970 +sw -10.571218 +_z -10.579496 +ac -10.597996 +jw -10.622223 +vw -10.622223 +kl -10.629572 +z_ -10.645002 +gw -10.653091 +ij -10.702603 +_w -10.720800 +us -10.735260 +ic -10.745202 +af -10.770025 +lw -10.770025 +ea -10.819588 +by -10.831543 +vl -10.878190 +nf -10.909154 +of -10.915876 +ie -10.940796 +wi -10.949966 +ng -10.989601 +ry -11.071460 +hi -11.088236 +lt -11.157360 +ty -11.178761 +up -11.185965 +r_ -11.217915 +vr -11.240239 +vy -11.242115 +g_ -11.409110 +ul -11.450806 +th -11.466073 +ro -11.467170 +br -11.470465 +ec -11.480395 +ew -11.600401 +ov -11.619782 +ai -11.680812 +ig -11.722108 +sm -11.729985 +tm -11.752538 +kw -11.755214 +ca -11.760582 +nc -11.770025 +ce -11.774091 +iw -11.808426 +au -11.809817 +_h -11.885502 +ho -11.897279 +rr -11.912137 +jy -11.931684 +gz -11.937752 +co -11.949966 +oj -11.957652 +dr -11.959194 +fw -11.974707 +uj -12.001468 +gl -12.019053 +eb -12.020662 +zw -12.078147 +fy -12.088236 +gi -12.096698 +py -12.125844 +pt -12.162681 +oc -12.182358 +jm -12.229964 +ia -12.246816 +dm -12.271511 +um -12.336164 +ui -12.344201 +fl -12.425005 +oi -12.433554 +ue -12.479289 +ud -12.481503 +ss -12.488167 +dl -12.547246 +ps -12.547246 +nz -12.570629 +ld -12.575351 +rd -12.580089 +rl -12.589612 +uy -12.618563 +oy -12.623446 +io -12.714275 +du -12.724729 +uc -12.748532 +dj -12.759238 +nw -12.772734 +ph -12.811208 +_u -12.811208 +cl -12.811208 +pm -12.845023 +hw -12.868015 +ub -12.870915 +bs -12.894326 +rk -12.900239 +rn -12.927150 +sf -12.951500 +uk -13.059832 +bj -13.113771 +ci -13.124113 +x_ -13.184161 +zn -13.228104 +nr -13.246816 +vm -13.254370 +ah -13.273428 +aw -13.281122 +eu -13.300540 +sl -13.348236 +nu -13.427137 +bw -13.427137 +c_ -13.427137 +w_ -13.431412 +gm -13.457329 +rs -13.461695 +qu -13.470465 +ow -13.510604 +rg -13.519678 +ky -13.561230 +uz -13.580089 +ex -13.613698 +dd -13.678270 +eo -13.724729 +xi -13.740554 +gu -13.745868 +ck -13.756554 +ju -13.778168 +lu -13.794594 +lg -13.839332 +mb -13.850737 +tu -13.862232 +kn -13.897279 +md -13.921126 +dt -13.933199 +vn -13.988813 +sh -14.007838 +tn -14.027117 +sc -14.046657 +gt -14.100096 +ht -14.100096 +ax -14.106918 +su -14.127577 +tc -14.141516 +oo -14.162681 +km -14.184161 +lv -14.184161 +rc -14.213307 +tl -14.250588 +lb -14.336164 +mm -14.352283 +ls -14.376806 +wr -14.393389 +ru -14.418626 +tp -14.461695 +ix -14.479289 +oe -14.488167 +uf -14.497100 +tt -14.506089 +tb -14.580089 +cq -14.618563 +nh -14.618563 +bu -14.618563 +ug -14.638192 +pp -14.658092 +sr -14.709076 +dh -14.729985 +ee -14.729985 +dn -14.761928 +kh -14.783623 +ct -14.783623 +oh -14.805649 +hn -14.805649 +nl -14.945374 +cs -14.995127 +pn -15.007838 +ym -15.020662 +ae -15.020662 +yc -15.033601 +oa -15.059832 +hu -15.059832 +kz -15.073129 +cu -15.086550 +mp -15.127577 +jt -15.155591 +lk -15.155591 +wy -15.184161 +pj -15.243054 +lf -15.243054 +cr -15.243054 +ts -15.288858 +ua -15.452977 +ff -15.524236 +sd -15.561230 +pu -15.580089 +ln -15.580089 +bc -15.599198 +hr -15.618563 +zm -15.658092 +rp -15.740554 +ms -15.783623 +yn -15.828017 +tf -15.921126 +tz -15.921126 +lp -15.921126 +xa -15.945374 +wn -15.945374 +yv -16.020662 +ys -16.020662 +_q -16.046657 +aa -16.046657 +ds -16.073129 +rv -16.073129 +hy -16.127577 +bv -16.155591 +ao -16.213307 +yl -16.213307 +wt -16.243054 +zr -16.401752 +cc -16.542615 +gn -16.542615 +fp -16.618563 +rh -16.658092 +uq -16.698734 +tg -16.740554 +cy -16.783623 +sn -16.828017 +iu -16.921126 +tv -16.921126 +iq -17.020662 +rb -17.020662 +rj -17.020662 +fn -17.073129 +uw -17.073129 +ux -17.073129 +dg -17.073129 +fh -17.127577 +hf -17.127577 +rq -17.127577 +rm -17.127577 +ws -17.243054 +mu -17.243054 +ku -17.243054 +mh -17.304455 +ei -17.304455 +sq -17.304455 +mn -17.368585 +fu -17.368585 +dc -17.435699 +cp -17.435699 +bn -17.506089 +vs -17.506089 +ox -17.580089 +yp -17.658092 +tk -17.828017 +yw -17.828017 +rw -17.921126 +xt -17.921126 +pk -17.921126 +_x -17.921126 +aq -17.921126 +nx -18.020662 +hp -18.020662 +q_ -18.020662 +zz -18.020662 +gp -18.020662 +ih -18.020662 +yk -18.127577 +fb -18.127577 +mt -18.127577 +zu -18.127577 +hm -18.127577 +lr -18.243054 +nq -18.243054 +eh -18.243054 +cn -18.243054 +bb -18.368585 +cm -18.368585 +gj -18.368585 +lc -18.368585 +gs -18.506089 +wh -18.506089 +gh -18.506089 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hau b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hau new file mode 100644 index 00000000000..bc6fb45b121 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hau @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 170215752 +#UNIQUE_BIGRAMS 700 +#UNSEEN_LOG_PROB -19.201883 +#EXPECTED_SCORE -6.951930 +a_ -4.383385 +n_ -4.588238 +an -4.766943 +ar -5.080918 +ka -5.365127 +in -5.482110 +i_ -5.537137 +_k -5.604576 +wa -5.620529 +ma -5.733161 +_s -5.755994 +ya -5.772843 +sa -5.833906 +da -5.978678 +r_ -6.107382 +ta -6.141437 +na -6.257115 +_m -6.261359 +_a -6.407731 +ba -6.413787 +e_ -6.444097 +_d -6.446402 +am -6.506864 +as -6.534269 +ak -6.549392 +ra -6.680627 +_b -6.712958 +ha -6.727794 +_w -6.790711 +ai -6.798889 +sh -6.801948 +u_ -6.868540 +ab -6.873149 +_t -6.876240 +un -6.879469 +ga -6.906135 +ki -6.983769 +su -7.003032 +_y -7.044264 +at -7.106898 +_g -7.181740 +_h -7.199620 +ay -7.202681 +ni -7.215300 +ku -7.232626 +iy -7.258857 +al -7.268892 +ri -7.290905 +uk -7.394127 +ci -7.436150 +la -7.464370 +ad -7.501266 +mu -7.540980 +_c -7.566684 +hi -7.626072 +_z -7.803742 +_i -7.827421 +ik -7.837022 +fa -7.877942 +um -7.906560 +za -7.941491 +o_ -7.978321 +on -7.999395 +bi -8.006244 +_r -8.022313 +_f -8.027337 +ce -8.092234 +_n -8.109385 +ir -8.115815 +ke -8.118244 +au -8.128641 +aw -8.145450 +en -8.146257 +mi -8.213838 +_j -8.252584 +fi -8.253693 +li -8.284745 +_l -8.298902 +ur -8.307708 +ts -8.311438 +af -8.325821 +nn -8.339864 +ko -8.383090 +nd -8.413713 +ag -8.422809 +ji -8.438837 +re -8.446404 +du -8.463852 +bu -8.468183 +uw -8.518278 +di -8.583272 +yi -8.583876 +tu -8.583977 +hu -8.598284 +he -8.692323 +aj -8.724350 +ac -8.728378 +ew -8.750916 +do -8.751983 +ne -8.805730 +gi -8.818175 +ja -8.839305 +nc -8.857813 +is -8.888233 +ut -8.931167 +ru -8.933995 +ah -8.939016 +ll -8.947594 +ok -8.947636 +ng -8.949118 +us -8.958967 +id -8.961466 +nt -9.070984 +it -9.113449 +ns -9.127086 +ud -9.140560 +rk -9.151783 +kw -9.168410 +je -9.175685 +er -9.220047 +ti -9.252806 +im -9.255145 +om -9.274368 +mm -9.366572 +il -9.399811 +ug -9.406180 +so -9.447658 +s_ -9.473232 +zu -9.482205 +yy -9.507171 +gu -9.532619 +si -9.539282 +nk -9.540563 +lo -9.546179 +ro -9.573139 +nu -9.582037 +ul -9.594649 +bo -9.608794 +to -9.667390 +m_ -9.683441 +bb -9.702968 +dd -9.745239 +ye -9.749624 +gw -9.793087 +ub -9.819074 +ig -9.842469 +ny -9.845319 +yu -9.865158 +or -10.007748 +ho -10.024732 +go -10.035642 +_u -10.074028 +ih -10.107645 +h_ -10.133107 +be -10.153959 +l_ -10.161802 +yo -10.168049 +ek -10.179002 +me -10.218052 +mn -10.218416 +d_ -10.237761 +ot -10.264753 +tt -10.278636 +mf -10.281576 +st -10.311002 +rs -10.318916 +fu -10.324317 +le -10.327211 +k_ -10.357637 +oy -10.365632 +uh -10.400103 +mo -10.443822 +ge -10.498831 +az -10.505322 +_p -10.513624 +em -10.514963 +de -10.535195 +no -10.541927 +wo -10.544988 +wu -10.552110 +mb -10.598836 +es -10.613394 +c_ -10.655906 +od -10.660275 +t_ -10.663089 +ib -10.663969 +rn -10.683867 +nz -10.715583 +ca -10.736717 +rg -10.759001 +te -10.801103 +ol -10.841730 +ic -10.842679 +lu -10.880031 +ow -10.928880 +oj -10.954368 +ry -10.970570 +sk -10.990608 +rt -11.030225 +ch -11.031768 +_o -11.049749 +ij -11.061855 +zi -11.065738 +iw -11.067429 +el -11.070797 +rw -11.089754 +kk -11.091769 +if -11.105542 +ia -11.145573 +se -11.173432 +os -11.215379 +oh -11.218365 +y_ -11.235388 +uy -11.247908 +ss -11.249970 +fe -11.250920 +ky -11.267812 +co -11.272438 +lh -11.305071 +lk -11.337055 +jo -11.337625 +uf -11.400985 +ed -11.401214 +ob -11.406796 +et -11.435101 +uc -11.473316 +of -11.494117 +p_ -11.530687 +rf -11.564327 +ap -11.594308 +rb -11.610675 +rd -11.613779 +bc -11.668790 +oc -11.673987 +rr -11.721911 +nm -11.726039 +og -11.756272 +_e -11.780994 +lm -11.803812 +rm -11.820330 +oo -11.850748 +uj -11.862820 +zo -11.871908 +nj -11.875184 +ou -11.877861 +th -11.908259 +cc -11.919082 +iz -12.005620 +io -12.019453 +ie -12.023435 +pa -12.043152 +g_ -12.096234 +ju -12.098425 +ea -12.099950 +rl -12.102445 +rc -12.162686 +br -12.224254 +eb -12.229820 +gh -12.258396 +cu -12.284459 +po -12.335061 +op -12.391778 +fr -12.401650 +pc -12.437073 +rz -12.442489 +fo -12.446977 +wi -12.452335 +ms -12.501667 +tr -12.513017 +dp -12.641754 +yr -12.660892 +pd -12.685080 +ve -12.690785 +ec -12.722283 +ef -12.739801 +vi -12.794028 +pr -12.834809 +ht -12.840393 +ej -12.854196 +hm -12.870733 +we -12.887591 +lb -12.912337 +bd -12.921119 +f_ -12.925662 +b_ -12.931676 +ei -12.931808 +lt -12.945248 +mp -12.976057 +ff -13.021777 +py -13.038295 +eg -13.038581 +ee -13.042437 +pe -13.063614 +ld -13.083046 +ck -13.117281 +_v -13.206038 +_q -13.214231 +eh -13.234347 +w_ -13.245992 +gy -13.257732 +rj -13.264805 +dr -13.271998 +kh -13.279479 +cr -13.376194 +nh -13.414919 +ey -13.430555 +mt -13.435617 +qa -13.455854 +iv -13.461675 +ty -13.523309 +pl -13.527106 +ly -13.533825 +ip -13.548069 +bl -13.560098 +kt -13.565842 +ft -13.628436 +ph -13.652464 +ev -13.652682 +ue -13.697243 +gg -13.699046 +eo -13.717080 +ua -13.730266 +lw -13.751968 +sc -13.771037 +sp -13.776973 +ls -13.783532 +pi -13.786283 +gr -13.798549 +sl -13.819227 +jj -13.824505 +gl -13.853569 +va -13.881566 +lc -13.926124 +nb -13.957196 +ep -13.964359 +nl -13.990746 +aq -14.002383 +pp -14.008935 +tw -14.013694 +oa -14.014114 +ct -14.024247 +aa -14.030899 +mh -14.034166 +sy -14.048169 +iu -14.071326 +sm -14.094862 +xa -14.102892 +fk -14.153582 +kr -14.159309 +av -14.176940 +qi -14.185758 +mc -14.191456 +md -14.239010 +pt -14.303356 +ui -14.309022 +pu -14.316785 +uz -14.332261 +ex -14.373943 +ae -14.385143 +rh -14.401008 +qu -14.427470 +gb -14.437590 +ov -14.441923 +nf -14.472810 +ze -14.490065 +yw -14.496711 +sd -14.500439 +oi -14.552644 +wh -14.553052 +eu -14.555907 +bt -14.588111 +z_ -14.601954 +ax -14.606810 +cl -14.610622 +ds -14.618062 +hr -14.620836 +sw -14.676121 +lf -14.695781 +ez -14.702092 +hn -14.751501 +fc -14.754075 +zz -14.824628 +up -14.836234 +ws -14.860486 +nw -14.865537 +by -14.899587 +fl -14.906859 +fs -14.909986 +mk -14.925464 +cs -14.927048 +_x -14.963682 +dl -14.978928 +x_ -15.024529 +rp -15.043009 +lj -15.061439 +vo -15.063759 +wn -15.118486 +xi -15.141891 +sr -15.164436 +np -15.184180 +fg -15.217699 +iq -15.289882 +hl -15.408361 +ps -15.425417 +lg -15.439661 +zh -15.439661 +ks -15.476283 +ys -15.488700 +ym -15.492212 +tc -15.501618 +mr -15.529810 +ao -15.557337 +gt -15.561840 +ww -15.623828 +bs -15.625541 +fy -15.635862 +cy -15.640617 +bn -15.644519 +tl -15.678787 +v_ -15.754543 +oz -15.761118 +لي -15.778163 +sb -15.788680 +rv -15.792043 +my -15.825120 +bw -15.917049 +yp -15.933929 +kz -15.959625 +gs -16.011312 +ii -16.014114 +yn -16.020298 +sn -16.035019 +kl -16.041293 +q_ -16.048743 +hy -16.159774 +ml -16.244757 +dy -16.256653 +ه_ -16.349143 +lv -16.385687 +dh -16.396614 +hd -16.444944 +kp -16.451765 +gn -16.502798 +dg -16.517035 +ox -16.608080 +fp -16.618275 +ال -16.627685 +لل -16.674791 +له -16.674791 +yt -16.687259 +nr -16.694432 +oe -16.706165 +yl -16.735459 +dm -16.739163 +م_ -16.752202 +عل -16.776735 +يه -16.776735 +وس -16.777687 +سل -16.777687 +لم -16.777687 +dc -16.777687 +_ص -16.779593 +صل -16.779593 +uq -16.796860 +sg -16.808486 +cd -16.811408 +ux -16.812383 +dw -16.829062 +cp -16.860990 +tn -16.863009 +sf -16.883358 +dj -16.894673 +bz -16.929161 +hs -16.942977 +kn -16.976467 +tm -16.980846 +uv -16.991850 +gm -17.015237 +lr -17.027640 +ix -17.048169 +qo -17.068994 +dk -17.072494 +xt -17.112769 +qw -17.119994 +cn -17.121202 +bk -17.123621 +kc -17.144344 +wl -17.151730 +_ا -17.201960 +td -17.229047 +j_ -17.247392 +vl -17.307990 +tz -17.323199 +tf -17.378448 +xe -17.417977 +_ع -17.429900 +_و -17.431397 +ي_ -17.434397 +tp -17.435899 +zb -17.457093 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/heb b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/heb new file mode 100644 index 00000000000..534c26780c6 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/heb @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 129114198 +#UNIQUE_BIGRAMS 852 +#UNSEEN_LOG_PROB -16.327638 +#EXPECTED_SCORE -7.907020 +_ה -4.650784 +ת_ -5.219348 +ה_ -5.242912 +ם_ -5.381841 +_ב -5.523695 +_מ -5.865058 +_ל -5.928934 +ים -5.984484 +ות -6.009313 +ר_ -6.114403 +י_ -6.176513 +_א -6.430802 +_ש -6.475215 +ן_ -6.519664 +ו_ -6.588947 +ני -6.831037 +ל_ -6.948344 +יו -6.969507 +בי -7.054297 +רי -7.103122 +ור -7.113958 +לי -7.179858 +רו -7.254071 +_ו -7.289613 +ית -7.290568 +הי -7.327577 +מו -7.327699 +_י -7.355773 +יי -7.359855 +נו -7.369998 +א_ -7.394812 +ול -7.407415 +המ -7.465933 +לה -7.492038 +_נ -7.497680 +_כ -7.516596 +ד_ -7.520238 +או -7.526681 +בר -7.600774 +די -7.605747 +שי -7.615639 +הו -7.636236 +יה -7.723779 +יר -7.725041 +מי -7.727807 +_ע -7.744407 +בו -7.785431 +לו -7.790263 +אי -7.797432 +_ח -7.839397 +מש -7.882638 +שו -7.896067 +וד -7.929602 +עו -7.945028 +ב_ -7.952559 +וב -7.979765 +ון -7.996832 +ונ -8.005242 +תו -8.019756 +חו -8.034789 +וא -8.047199 +נה -8.073774 +ך_ -8.077135 +ינ -8.110870 +יל -8.127170 +רא -8.130470 +של -8.167042 +דו -8.190141 +שר -8.216853 +יש -8.244480 +אל -8.259851 +קו -8.265304 +לא -8.292660 +_ר -8.307812 +שנ -8.315378 +_ת -8.316706 +תי -8.322997 +ע_ -8.326020 +עי -8.329262 +חי -8.340088 +כו -8.354008 +במ -8.373924 +_פ -8.381808 +ש_ -8.411089 +הא -8.415798 +הש -8.421541 +אח -8.421851 +_ק -8.435825 +יד -8.451042 +ומ -8.454130 +פר -8.454895 +וע -8.462990 +ק_ -8.463251 +בע -8.466381 +מה -8.483245 +ין -8.503856 +סי -8.523734 +רה -8.524503 +שה -8.533886 +ום -8.552288 +פו -8.557368 +חר -8.562078 +פי -8.598771 +הר -8.604549 +הת -8.622140 +מר -8.641126 +וי -8.693113 +בא -8.706489 +וש -8.708970 +_ס -8.709199 +הח -8.709564 +למ -8.712893 +יב -8.714352 +קי -8.734857 +רב -8.743816 +יא -8.748921 +ופ -8.775258 +קר -8.778719 +הע -8.793834 +כל -8.803023 +צי -8.826115 +אש -8.840129 +בה -8.843272 +מת -8.851609 +_ד -8.854012 +יק -8.876622 +ימ -8.894479 +וס -8.927444 +וו -8.936147 +על -8.939440 +הב -8.940813 +טי -8.944650 +מא -8.946132 +גי -8.948653 +מע -8.953503 +מד -8.955637 +אר -8.956875 +ח_ -8.963085 +רת -8.976520 +בש -8.996367 +סו -9.003317 +ף_ -9.008648 +בת -9.009005 +יע -9.013508 +תר -9.038227 +וח -9.040561 +בל -9.055478 +כי -9.068179 +ט_ -9.073473 +_ג -9.078659 +שב -9.081901 +צו -9.084435 +ספ -9.090848 +גו -9.093879 +וק -9.095804 +נת -9.101249 +וכ -9.120848 +בנ -9.128752 +הפ -9.133601 +טו -9.141367 +דר -9.143507 +ער -9.161580 +חל -9.168706 +וצ -9.174687 +מנ -9.178152 +הג -9.214950 +אמ -9.216857 +הק -9.229498 +מצ -9.232977 +מס -9.237570 +תה -9.259207 +מק -9.265135 +חד -9.275091 +לב -9.335096 +וה -9.344080 +יט -9.357233 +לת -9.363748 +הל -9.366835 +שמ -9.369619 +מח -9.373053 +הס -9.380794 +ס_ -9.385786 +הצ -9.397781 +רכ -9.406081 +_צ -9.419898 +את -9.446564 +עב -9.464111 +תח -9.468442 +יח -9.472000 +_ז -9.477144 +אנ -9.479822 +יכ -9.494964 +כנ -9.501825 +לח -9.519160 +קב -9.523249 +הכ -9.531660 +בד -9.531751 +הנ -9.535826 +שא -9.537718 +יצ -9.562580 +וג -9.564330 +חב -9.571428 +לכ -9.581743 +שת -9.588945 +לק -9.600305 +אה -9.604127 +לפ -9.621572 +לש -9.627526 +לל -9.635173 +יס -9.644730 +יפ -9.645043 +טר -9.663383 +בח -9.667058 +צר -9.680455 +פע -9.694487 +דה -9.702918 +מל -9.711286 +גד -9.718027 +ץ_ -9.721693 +כת -9.729677 +מכ -9.753245 +לע -9.772417 +אב -9.783502 +תק -9.801566 +ממ -9.833487 +הה -9.874266 +פת -9.886937 +הם -9.903076 +עה -9.903911 +זו -9.907019 +עת -9.912791 +_ט -9.918998 +מב -9.928124 +חק -9.957064 +הד -9.959587 +גר -9.976328 +כב -9.981222 +צא -9.981786 +רג -9.983799 +לד -9.994827 +נש -10.005791 +כמ -10.005860 +קה -10.009541 +רח -10.041568 +בכ -10.054877 +בס -10.057316 +שפ -10.066297 +עד -10.066393 +כה -10.076215 +עש -10.078871 +רש -10.081798 +פנ -10.086031 +חת -10.093544 +עצ -10.110058 +לם -10.118083 +זי -10.122784 +צע -10.144057 +ג_ -10.151981 +זר -10.153928 +רט -10.186800 +זה -10.190529 +קש -10.193679 +בק -10.218733 +חש -10.227762 +שע -10.229223 +דש -10.232123 +שח -10.238751 +פה -10.239696 +גל -10.246480 +תנ -10.247146 +וך -10.248695 +נס -10.248763 +רך -10.255412 +צב -10.262958 +מפ -10.268074 +לנ -10.303446 +פש -10.305962 +רד -10.307547 +וז -10.326605 +עמ -10.329808 +נג -10.340316 +תפ -10.347198 +רס -10.357291 +לט -10.359961 +אפ -10.373431 +חמ -10.390638 +יג -10.410332 +סר -10.421154 +לר -10.437966 +תמ -10.438137 +רמ -10.448545 +כר -10.467231 +גב -10.495795 +מן -10.499381 +הז -10.511547 +דב -10.513310 +שכ -10.514502 +סט -10.518021 +צמ -10.532065 +רצ -10.534267 +דע -10.538698 +כא -10.557503 +ענ -10.560874 +תב -10.562377 +כן -10.567794 +דת -10.577272 +רק -10.581272 +קל -10.581632 +מט -10.582420 +ז_ -10.597057 +חה -10.618855 +פל -10.622091 +פק -10.633779 +צה -10.645025 +קט -10.645151 +אז -10.648321 +נק -10.653154 +זכ -10.656883 +צל -10.664571 +יך -10.681078 +בב -10.682969 +כד -10.687458 +חז -10.696201 +שק -10.710546 +נח -10.714202 +אד -10.722353 +דמ -10.740744 +וף -10.752493 +וט -10.768192 +קד -10.771332 +בפ -10.779224 +פט -10.792778 +זמ -10.795874 +לג -10.830574 +נט -10.837530 +מג -10.843800 +נמ -10.868426 +בצ -10.890147 +נע -10.913987 +דם -10.915562 +סת -10.942708 +נפ -10.970936 +פח -10.971766 +חנ -10.990331 +חס -10.992742 +בט -10.995295 +נד -10.998286 +הט -11.002956 +תא -11.005573 +סד -11.012804 +סק -11.019886 +פס -11.028496 +תן -11.058710 +בג -11.073034 +קצ -11.074069 +כש -11.079765 +נא -11.086747 +לך -11.092323 +לצ -11.093959 +שט -11.106863 +כז -11.128139 +זא -11.156705 +נר -11.159489 +תם -11.163225 +לס -11.173667 +סב -11.213629 +אס -11.229720 +רץ -11.238872 +טה -11.247377 +מז -11.254893 +דל -11.256369 +סף -11.271260 +רפ -11.273167 +יז -11.278653 +וץ -11.279820 +עם -11.314004 +שם -11.325429 +פג -11.339230 +טח -11.361519 +טל -11.374751 +עק -11.380519 +קנ -11.387237 +טע -11.402431 +רנ -11.409010 +גה -11.459061 +נכ -11.462084 +גנ -11.467833 +כך -11.502652 +סג -11.505216 +צח -11.513816 +צד -11.535444 +תל -11.536008 +תע -11.539995 +סמ -11.546431 +הן -11.551721 +סה -11.579596 +צפ -11.598182 +גע -11.618591 +שך -11.625177 +סכ -11.650924 +עז -11.659610 +רע -11.664825 +אם -11.671260 +קת -11.674727 +נב -11.674800 +פן -11.686611 +גמ -11.702164 +תג -11.712289 +תכ -11.716795 +כס -11.723619 +סל -11.775949 +טב -11.807522 +תת -11.819022 +גש -11.848758 +אן -11.870349 +טנ -11.887308 +רם -11.912759 +כפ -11.947952 +צת -11.965585 +נם -12.005687 +כח -12.023022 +דד -12.029641 +קא -12.039720 +רר -12.047977 +יף -12.058996 +עט -12.075877 +כם -12.088863 +בן -12.089496 +רל -12.111776 +עס -12.112716 +שג -12.113904 +נצ -12.114944 +זק -12.121601 +גם -12.158518 +בז -12.174183 +תש -12.195617 +דק -12.196299 +טת -12.217162 +קמ -12.238764 +קפ -12.240547 +אג -12.255767 +סע -12.262834 +קס -12.271537 +חצ -12.285470 +פא -12.317964 +קע -12.323223 +שש -12.352959 +אצ -12.359051 +גן -12.399410 +שד -12.442733 +גת -12.467263 +דא -12.504696 +לז -12.509379 +נל -12.565100 +דנ -12.600609 +אכ -12.634099 +טן -12.636943 +ננ -12.637512 +צג -12.686095 +קן -12.692812 +פצ -12.720599 +ען -12.724072 +דג -12.729374 +צט -12.760437 +סח -12.775165 +גז -12.778851 +קח -12.788147 +אק -12.843410 +נן -12.852141 +פכ -12.874874 +נז -12.890656 +אט -12.944425 +רף -12.949807 +פך -12.960544 +תף -12.964469 +רן -12.967508 +סם -13.024278 +ככ -13.060570 +סנ -13.084926 +חם -13.095645 +אף -13.101329 +לף -13.110194 +רז -13.130091 +יץ -13.133400 +עכ -13.187412 +שז -13.202184 +כע -13.224684 +מם -13.228325 +זב -13.280959 +טא -13.316881 +לן -13.330283 +e_ -13.334317 +קם -13.334894 +טמ -13.419898 +טכ -13.474176 +קף -13.509705 +דן -13.545195 +סס -13.574884 +דח -13.595206 +חפ -13.619470 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hin b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hin new file mode 100644 index 00000000000..4ffa953679e --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hin @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 168311532 +#UNIQUE_BIGRAMS 1840 +#UNSEEN_LOG_PROB -15.021738 +#EXPECTED_SCORE -7.865151 +न_ -4.818290 +_म -5.004805 +र_ -5.066905 +त_ -5.257585 +_क -5.259918 +ह_ -5.371557 +म_ -5.374291 +_स -5.501503 +_ह -5.691670 +ल_ -5.779231 +_ब -5.877739 +य_ -5.922050 +क_ -6.011660 +_ज -6.174511 +_ल -6.185635 +कर -6.262687 +_प -6.328993 +स_ -6.353306 +_द -6.355536 +_र -6.377916 +_अ -6.448221 +_न -6.549292 +ग_ -6.662975 +_व -7.027106 +ए_ -7.049546 +द_ -7.070661 +_आ -7.108169 +_ग -7.143293 +रह -7.247035 +_उ -7.312692 +_त -7.446658 +_च -7.469076 +_इ -7.558433 +सक -7.578871 +_भ -7.692014 +नह -7.707795 +लग -7.757434 +हत -7.764797 +रत -7.787809 +पन -7.819903 +व_ -7.850970 +सम -7.872065 +ई_ -7.904437 +नक -7.907098 +जन -7.913679 +रन -7.931355 +ट_ -7.937938 +_श -7.980777 +ज_ -8.046805 +_ख -8.046889 +मन -8.068573 +सर -8.077765 +पर -8.091405 +मल -8.100450 +श_ -8.151056 +लए -8.159770 +कह -8.164704 +लक -8.170693 +इस -8.198202 +अप -8.242015 +मर -8.246949 +थ_ -8.321935 +गय -8.347360 +दन -8.348846 +हन -8.351290 +_य -8.391428 +तर -8.406901 +रक -8.435900 +हल -8.457284 +बर -8.470206 +कय -8.474682 +उस -8.513122 +च_ -8.522119 +बत -8.542403 +_फ -8.544568 +वल -8.547548 +ख_ -8.557809 +जत -8.588602 +कत -8.604315 +प_ -8.617377 +बद -8.624041 +हर -8.690448 +कन -8.694398 +दर -8.697316 +ड_ -8.717607 +सल -8.727274 +जर -8.738887 +कस -8.742723 +ब_ -8.753174 +पह -8.760553 +बन -8.778019 +सन -8.792495 +गर -8.798605 +वर -8.805541 +बह -8.878079 +नत -8.882045 +जस -8.892794 +भर -8.904346 +कल -8.924450 +कम -8.930217 +रप -8.949785 +भ_ -8.978440 +उन -8.983126 +मह -9.012369 +दय -9.015267 +ण_ -9.021887 +लन -9.053510 +हम -9.067441 +लत -9.090872 +छ_ -9.099328 +ध_ -9.116011 +चन -9.117348 +तन -9.119640 +हए -9.119916 +दख -9.132519 +सह -9.150580 +सथ -9.173422 +रज -9.178355 +नय -9.208469 +_ए -9.222390 +सब -9.227154 +मत -9.239370 +तय -9.272490 +दश -9.275860 +हग -9.342101 +लय -9.395804 +नम -9.400755 +रण -9.406252 +सस -9.452406 +नव -9.459009 +कछ -9.471155 +फ_ -9.508988 +_घ -9.510625 +गत -9.511751 +पल -9.514450 +वन -9.545218 +रव -9.568349 +चह -9.588222 +चल -9.588770 +कई -9.591340 +_ट -9.593173 +बल -9.616661 +खल -9.629700 +पस -9.635374 +गल -9.636199 +_ऐ -9.654064 +अन -9.661820 +चर -9.682822 +बक -9.697159 +दल -9.700436 +मज -9.702942 +रय -9.746713 +ऐस -9.755888 +लख -9.759843 +धक -9.765659 +धर -9.771970 +पक -9.783096 +शर -9.784547 +वह -9.791048 +_ड -9.795063 +नन -9.795855 +खन -9.799425 +अध -9.809997 +यर -9.817542 +झ_ -9.830899 +मझ -9.838083 +सभ -9.845974 +फर -9.847584 +रख -9.854963 +मय -9.866239 +पत -9.867896 +मक -9.875356 +सव -9.884474 +सद -9.891409 +बज -9.893381 +ओ_ -9.910594 +आप -9.925854 +वद -9.929026 +जल -9.933286 +जए -9.956774 +मद -9.958700 +सच -9.967976 +भग -9.978548 +रग -9.995047 +यह -10.004622 +लस -10.018762 +रब -10.035118 +तक -10.038882 +बच -10.043769 +रम -10.044267 +नस -10.074679 +चक -10.076233 +यद -10.079327 +हस -10.090657 +_छ -10.103349 +बस -10.106020 +ष_ -10.110230 +जव -10.114300 +पय -10.114547 +आ_ -10.119087 +शन -10.122924 +दस -10.139323 +दव -10.152618 +वक -10.164128 +यन -10.167979 +धन -10.198635 +हआ -10.199320 +_ध -10.231780 +यग -10.236643 +पछ -10.245419 +वस -10.256092 +लव -10.256731 +तल -10.279957 +वध -10.281119 +मम -10.285841 +आय -10.302804 +वज -10.312146 +खत -10.312658 +कप -10.316118 +मग -10.320529 +जय -10.339606 +बध -10.351999 +हई -10.362162 +दत -10.363719 +जह -10.365380 +एग -10.370149 +रर -10.371851 +सत -10.379801 +नर -10.385683 +हच -10.393202 +आत -10.394529 +अग -10.400321 +जद -10.427957 +आद -10.429139 +जग -10.432238 +फस -10.434585 +भव -10.454269 +कव -10.454281 +टन -10.467485 +चत -10.495128 +शक -10.506031 +तम -10.508713 +जब -10.509338 +अल -10.524787 +गव -10.529771 +अभ -10.541451 +इन -10.574217 +घट -10.584921 +नद -10.591704 +वत -10.599409 +रल -10.609846 +तथ -10.611656 +खर -10.616008 +कट -10.617866 +आर -10.627936 +रस -10.631127 +ठक -10.642481 +अम -10.644771 +ठ_ -10.646749 +भज -10.660529 +रश -10.661181 +दम -10.669757 +नज -10.670553 +हद -10.693407 +जम -10.697088 +डय -10.709063 +नग -10.715001 +कश -10.718448 +टक -10.722744 +इत -10.733372 +यज -10.734175 +गन -10.742110 +शह -10.742844 +डल -10.760505 +लब -10.796671 +गह -10.802905 +वश -10.804759 +मस -10.816660 +टर -10.819322 +सप -10.823789 +यत -10.830719 +हज -10.835771 +_थ -10.843404 +आई -10.857155 +फल -10.865367 +मच -10.865687 +मट -10.867992 +मड -10.873980 +सध -10.882903 +पच -10.899508 +लम -10.902507 +सफ -10.916919 +पट -10.927197 +पए -10.932012 +शम -10.946368 +जक -10.953558 +_ठ -10.958462 +रध -10.981032 +नल -10.987144 +रद -11.007804 +एव -11.009871 +सग -11.011374 +शय -11.017811 +तब -11.036774 +बठ -11.056428 +यक -11.057286 +टल -11.076464 +तज -11.078149 +हय -11.086755 +कभ -11.089087 +लट -11.095206 +जप -11.107579 +आन -11.107598 +तह -11.118191 +खब -11.119106 +पड -11.151243 +हक -11.154835 +एस -11.155030 +टम -11.156888 +अस -11.181482 +दग -11.196069 +उप -11.215586 +छल -11.216075 +आध -11.227807 +कद -11.235826 +आग -11.243495 +मश -11.243932 +नश -11.245055 +बट -11.268060 +भष -11.289870 +नट -11.296150 +वव -11.297617 +_झ -11.307837 +re -11.313869 +अत -11.320386 +te -11.329556 +इल -11.342986 +हथ -11.350487 +लफ -11.353400 +उठ -11.355891 +e_ -11.368074 +दक -11.377234 +पज -11.378077 +अव -11.389829 +रच -11.390312 +छट -11.396722 +ठन -11.399800 +अद -11.406159 +दब -11.421924 +वच -11.430226 +सज -11.431765 +पद -11.431859 +यम -11.447167 +खद -11.447406 +लह -11.449658 +कफ -11.450234 +अक -11.450977 +गज -11.467170 +यल -11.477496 +कए -11.480309 +नए -11.480652 +लल -11.482932 +यव -11.490706 +शल -11.506155 +er -11.519531 +भल -11.536363 +बड -11.563035 +लज -11.577475 +ar -11.580333 +गद -11.601139 +_ई -11.601272 +दद -11.623358 +शष -11.625090 +कब -11.627122 +बग -11.632745 +सय -11.639019 +गभ -11.645925 +वब -11.671364 +डक -11.675227 +वट -11.690614 +भत -11.691748 +t_ -11.693904 +इट -11.703792 +हव -11.705480 +बब -11.708001 +sh -11.708919 +रफ -11.719575 +सट -11.735972 +पश -11.739133 +आख -11.741802 +सख -11.762469 +दह -11.763094 +मब -11.771940 +खक -11.772120 +st -11.774610 +लर -11.778068 +लड -11.779423 +वभ -11.785613 +_ओ -11.788279 +चय -11.790494 +सझ -11.790949 +आव -11.794507 +गम -11.810613 +ha -11.825966 +रट -11.827710 +जच -11.839096 +वय -11.842271 +नप -11.862268 +जट -11.864216 +s_ -11.875412 +गठ -11.876056 +_ऊ -11.900491 +आस -11.904428 +बय -11.906829 +उत -11.924579 +शश -11.926480 +धय -11.927214 +एम -11.930088 +कड -11.931862 +तत -11.937129 +बव -11.938340 +लप -11.946673 +तव -11.949789 +हट -11.949925 +तओ -11.954816 +नब -11.963451 +अर -11.967049 +गध -11.968732 +इड -11.982020 +_ढ -11.984345 +षण -11.998340 +नभ -11.999322 +_p -12.013889 +डर -12.015733 +कग -12.022635 +भम -12.031830 +_t -12.034453 +in -12.040940 +वप -12.047746 +घर -12.066705 +रड -12.078149 +चव -12.111708 +_s -12.112012 +बम -12.116382 +शव -12.124665 +नच -12.126426 +दए -12.143234 +nt -12.150464 +शत -12.152062 +फन -12.160200 +रष -12.160200 +r_ -12.161888 +सइ -12.176574 +बई -12.208875 +कज -12.212735 +लइ -12.257235 +एक -12.269667 +भय -12.270302 +खस -12.276115 +गग -12.283745 +फट -12.288754 +वड -12.298955 +आक -12.300857 +लई -12.311100 +चप -12.316643 +दप -12.317955 +भई -12.337342 +धत -12.339162 +इक -12.340850 +ढ_ -12.342096 +लश -12.344367 +खड -12.349503 +शद -12.351696 +_ऑ -12.356678 +डन -12.400958 +पठ -12.411800 +अह -12.413810 +लद -12.418917 +थन -12.439481 +अज -12.445493 +कथ -12.460826 +तस -12.483030 +रआ -12.487109 +छप -12.488192 +नओ -12.490163 +सए -12.492483 +एन -12.493224 +रए -12.493422 +घ_ -12.495400 +वष -12.497629 +हब -12.498026 +es -12.501103 +डत -12.505482 +बढ -12.509925 +लभ -12.514030 +घष -12.519606 +आल -12.534785 +तभ -12.535396 +वई -12.539268 +_a -12.546019 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hin-rom b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hin-rom new file mode 100644 index 00000000000..b895091c78a --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hin-rom @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 314729412 +#UNIQUE_BIGRAMS 848 +#UNSEEN_LOG_PROB -17.970396 +#EXPECTED_SCORE -7.653680 +e_ -5.128350 +i_ -5.322242 +a_ -5.407719 +ha -5.608498 +ar -5.799251 +_k -5.964581 +an -6.010352 +_s -6.015218 +r_ -6.080165 +_a -6.112470 +_m -6.117152 +_h -6.162288 +_b -6.296168 +n_ -6.422070 +ka -6.501531 +in -6.505304 +ai -6.527415 +_p -6.633973 +hi -6.710866 +_t -6.738292 +ma -6.776632 +ra -6.822770 +_d -6.835081 +t_ -6.837704 +aa -6.907176 +ne -6.927706 +s_ -7.016860 +at -7.029039 +re -7.044159 +ta -7.057675 +al -7.067667 +d_ -7.090524 +ch -7.100861 +_l -7.111213 +na -7.128010 +h_ -7.141432 +ba -7.151803 +er -7.164073 +_c -7.274439 +y_ -7.287372 +l_ -7.289113 +ad -7.326617 +sa -7.352798 +sh -7.405961 +_r -7.409453 +nd -7.418603 +ri -7.428585 +ya -7.432141 +di -7.443877 +ah -7.455787 +he -7.493617 +ho -7.499843 +la -7.500366 +am -7.505668 +me -7.506566 +on -7.507959 +_n -7.537602 +th -7.547629 +pa -7.568171 +ur -7.582298 +_j -7.597946 +is -7.603207 +ay -7.609018 +li -7.657218 +_g -7.675782 +se -7.768647 +ng -7.807924 +_f -7.809132 +da -7.821958 +te -7.849632 +ti -7.862329 +ja -7.869890 +de -7.874192 +m_ -7.878114 +st -7.954247 +ga -7.982574 +ap -7.992240 +or -7.992565 +kh -7.994752 +es -8.018052 +as -8.032548 +il -8.059871 +hu -8.079854 +_u -8.087998 +bh -8.092663 +en -8.104969 +ni -8.117069 +le -8.155544 +ye -8.163587 +au -8.173066 +us -8.180286 +g_ -8.239364 +o_ -8.249785 +iy -8.271228 +vi -8.280602 +ki -8.281241 +ab -8.290936 +si -8.309585 +ee -8.320886 +ag -8.329690 +un -8.345638 +lo -8.360335 +it -8.423969 +_w -8.433237 +wa -8.433649 +ak -8.476588 +ot -8.579267 +mo -8.587266 +oo -8.633533 +to -8.637766 +um -8.644934 +ro -8.647156 +_i -8.656452 +ir -8.699550 +u_ -8.710918 +rn -8.737206 +_v -8.744714 +ke -8.746709 +ea -8.753178 +k_ -8.768801 +nt -8.773214 +be -8.797195 +do -8.831829 +ik -8.872487 +so -8.892721 +ow -8.907212 +dh -8.931215 +pr -8.963322 +ll -8.967015 +ac -8.970718 +ed -8.972969 +ul -8.973618 +ic -8.979118 +gh -8.985456 +mi -8.993649 +aj -9.012152 +el -9.012658 +mu -9.020617 +p_ -9.030651 +ut -9.072081 +_y -9.088368 +ie -9.133501 +od -9.153029 +ve -9.160444 +eh -9.169342 +et -9.188220 +ov -9.189429 +id -9.207895 +su -9.269614 +_o -9.281563 +ge -9.288208 +du -9.302297 +nl -9.306395 +pe -9.312380 +sk -9.328434 +ko -9.379826 +tu -9.405915 +lu -9.411398 +em -9.443618 +ol -9.453721 +ei -9.487227 +b_ -9.533724 +rt -9.540733 +os -9.542086 +ky -9.547338 +oa -9.554365 +ss -9.563805 +bo -9.588529 +ku -9.605441 +om -9.614252 +x_ -9.618447 +up -9.630554 +rd -9.658162 +wn -9.664859 +ou -9.669883 +ce -9.674056 +gi -9.704666 +fa -9.705271 +im -9.723190 +ex -9.725106 +mb -9.726713 +nu -9.742498 +ek -9.769057 +oh -9.771314 +ia -9.775543 +po -9.777513 +ud -9.778213 +ht -9.781695 +_e -9.783552 +uj -9.789617 +uc -9.797958 +tr -9.801700 +ry -9.811758 +pl -9.826625 +ip -9.826641 +jh -9.827124 +ct -9.840443 +og -9.848907 +io -9.855610 +co -9.858322 +pn -9.875442 +ly -9.882331 +oi -9.882954 +ca -9.888183 +bu -9.898007 +no -9.903555 +fi -9.905295 +bi -9.914166 +j_ -9.927139 +nk -9.941797 +go -9.957900 +fr -10.042349 +eg -10.044913 +ru -10.056621 +va -10.064286 +ji -10.066202 +fo -10.087180 +pu -10.103528 +ig -10.105789 +pi -10.117831 +gu -10.150948 +ts -10.171048 +av -10.174095 +ec -10.196038 +lt -10.244195 +ns -10.251139 +ps -10.256523 +yo -10.276454 +nc -10.283636 +we -10.313226 +rm -10.317606 +c_ -10.318929 +fu -10.326620 +tt -10.379399 +ep -10.399542 +kr -10.416720 +hr -10.417096 +ju -10.421742 +aw -10.501178 +ty -10.504197 +bl -10.550139 +rs -10.559212 +ua -10.567230 +gr -10.570907 +ey -10.571214 +za -10.571716 +ci -10.580997 +w_ -10.582711 +fe -10.584864 +pk -10.584920 +ph -10.605284 +ob -10.626128 +ck -10.633624 +az -10.639769 +je -10.653495 +uk -10.670324 +af -10.686958 +f_ -10.689469 +ev -10.694307 +iv -10.700930 +oc -10.702762 +wi -10.710208 +ok -10.711160 +eo -10.719174 +br -10.729411 +hn -10.733135 +wo -10.735994 +mp -10.741932 +op -10.760388 +v_ -10.780066 +_z -10.793118 +yu -10.815215 +ub -10.833183 +dr -10.868415 +kt -10.884929 +gs -10.896626 +tn -10.926559 +z_ -10.929041 +py -10.980924 +ug -11.004442 +nh -11.005950 +cr -11.010765 +pp -11.013037 +rc -11.014554 +rb -11.053052 +mm -11.055625 +uy -11.074738 +nn -11.082193 +ld -11.127582 +ks -11.130436 +ew -11.135459 +pt -11.140997 +hh -11.153932 +nj -11.199991 +gn -11.202408 +rk -11.207776 +eb -11.242850 +dk -11.277683 +yi -11.280006 +hl -11.284093 +jo -11.321754 +rv -11.364456 +sm -11.365544 +ny -11.376057 +ue -11.379228 +sl -11.392919 +ls -11.394202 +wh -11.425860 +dd -11.436645 +tk -11.441175 +bs -11.487701 +qu -11.511941 +if -11.526039 +lm -11.541341 +_q -11.560375 +rr -11.593337 +hy -11.598473 +rh -11.611352 +sw -11.675338 +cl -11.706189 +zi -11.719082 +lk -11.732808 +ds -11.743079 +sc -11.744431 +rl -11.755736 +ms -11.780098 +ao -11.797680 +gy -11.819781 +gl -11.825460 +uh -11.825693 +mh -11.886212 +yr -11.907151 +tm -11.919776 +iz -11.922194 +hk -11.938366 +hm -11.951632 +tc -11.965351 +ze -11.977426 +ui -11.980905 +sp -11.981646 +ij -12.007044 +ff -12.030977 +oj -12.068129 +bb -12.074165 +cy -12.086552 +ax -12.093186 +sn -12.103922 +of -12.155312 +rg -12.194480 +sr -12.235581 +oy -12.285374 +pd -12.296353 +mn -12.329236 +gt -12.344317 +kn -12.354507 +lz -12.369711 +hw -12.371653 +ib -12.377690 +cc -12.403932 +dy -12.419668 +ix -12.459906 +ln -12.474910 +rf -12.478254 +tl -12.519322 +mr -12.532406 +hs -12.549287 +lp -12.569485 +dn -12.573503 +iw -12.621051 +qa -12.626258 +ef -12.646219 +cu -12.675548 +cs -12.686928 +ys -12.688017 +my -12.694687 +hc -12.731248 +aq -12.770105 +xy -12.818953 +nf -12.824794 +ft -12.841721 +jy -12.844955 +ae -12.851987 +lh -12.879979 +q_ -12.880117 +tw -12.890940 +dw -12.918853 +ej -12.924972 +bt -12.940094 +rj -12.956251 +yw -12.973992 +jp -12.997866 +kk -13.002523 +sy -13.016812 +gg -13.040484 +zz -13.058595 +rw -13.066595 +wt -13.072190 +vo -13.124447 +mj -13.150052 +ws -13.153723 +iq -13.184545 +lb -13.198353 +kl -13.245118 +yd -13.245474 +df -13.251631 +md -13.279528 +ih -13.300741 +xx -13.302777 +ez -13.305931 +zo -13.336140 +dm -13.403684 +sg -13.407564 +dv -13.463318 +ii -13.487754 +lw -13.495251 +xt -13.496521 +oe -13.515721 +by -13.537357 +uv -13.546542 +wr -13.643166 +dt -13.655298 +mk -13.661224 +ml -13.687593 +lg -13.734556 +nm -13.737558 +ww -13.740441 +vy -13.742074 +hd -13.751526 +dl -13.755831 +_x -13.819350 +hp -13.834404 +lf -13.836817 +ox -13.844483 +ym -13.885241 +nv -13.897500 +bn -13.901844 +pc -13.935921 +hq -13.963336 +rp -13.989193 +np -14.001319 +uz -14.009763 +bd -14.014004 +jr -14.040948 +nw -14.044971 +fl -14.066359 +oz -14.093026 +qt -14.098001 +eq -14.138432 +xi -14.151052 +sb -14.179518 +zu -14.183607 +jn -14.206822 +rx -14.229713 +vd -14.241030 +vr -14.300833 +zy -14.346703 +tp -14.365351 +mt -14.386205 +xp -14.392500 +hb -14.396054 +xm -14.422987 +xe -14.434512 +yp -14.443267 +dg -14.475693 +uf -14.487438 +kd -14.576468 +bk -14.609775 +cn -14.625910 +gd -14.659681 +xa -14.675188 +kb -14.685538 +uw -14.725711 +gw -14.726208 +yy -14.753297 +lv -14.777296 +wl -14.785298 +rz -14.810630 +lc -14.811684 +wd -14.826525 +dp -14.842866 +lr -14.845563 +mz -14.852597 +yt -14.893588 +jl -14.915237 +nb -14.931188 +bj -14.997116 +jk -15.089026 +jj -15.116120 +jb -15.174593 +vg -15.189932 +nz -15.230241 +hv -15.237661 +sf -15.240852 +db -15.320394 +vn -15.368062 +vt -15.404578 +xu -15.442450 +eu -15.450228 +xc -15.451870 +iu -15.455573 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hrv b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hrv new file mode 100644 index 00000000000..520e5d5ed1c --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hrv @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 155052814 +#UNIQUE_BIGRAMS 572 +#UNSEEN_LOG_PROB -19.352364 +#EXPECTED_SCORE -7.566075 +a_ -5.042417 +e_ -5.474105 +je -5.544369 +i_ -5.561806 +o_ -5.677712 +_p -5.788373 +_s -5.928516 +st -6.155965 +ko -6.297842 +na -6.321233 +ra -6.341295 +_n -6.425356 +_k -6.465748 +u_ -6.530424 +ni -6.610747 +ij -6.663219 +po -6.780010 +ti -6.787372 +an -6.787658 +_o -6.843496 +pr -6.845993 +li -6.883823 +ta -6.885452 +no -6.888807 +_d -6.944734 +ka -6.989672 +va -7.017606 +os -7.074727 +m_ -7.084678 +re -7.093070 +ov -7.101247 +oj -7.108614 +la -7.166897 +ri -7.171414 +ja -7.177676 +ci -7.197396 +en -7.198315 +_t -7.203020 +nj -7.227461 +ma -7.246623 +od -7.248033 +_i -7.268262 +_m -7.314439 +vi -7.357256 +ne -7.374363 +vo -7.375634 +at -7.378070 +in -7.380622 +ak -7.389788 +_z -7.402000 +im -7.408421 +_v -7.412701 +_b -7.414134 +ed -7.421116 +ad -7.446352 +is -7.453697 +da -7.457229 +to -7.474227 +al -7.509410 +om -7.514131 +av -7.531831 +ro -7.552834 +di -7.571470 +sk -7.582952 +lj -7.620092 +og -7.632726 +ic -7.689602 +ve -7.695425 +_r -7.711422 +za -7.717185 +or -7.730648 +ji -7.762416 +il -7.779197 +it -7.785296 +bi -7.855574 +ju -7.872382 +ce -7.875427 +_g -7.882941 +_u -7.895151 +mo -7.912086 +go -7.918924 +el -7.926168 +aj -7.956478 +sa -7.956550 +lo -7.962014 +as -7.972119 +te -7.981013 +ar -7.988406 +et -8.041538 +on -8.042382 +ca -8.058489 +ac -8.075798 +tr -8.080001 +do -8.096389 +er -8.102350 +ik -8.143810 +me -8.145806 +am -8.146832 +es -8.152425 +az -8.159860 +em -8.168047 +n_ -8.178159 +ol -8.185391 +_j -8.219902 +ob -8.245718 +zi -8.264662 +sv -8.266558 +ec -8.271515 +io -8.331569 +iz -8.366467 +_c -8.368040 +dn -8.396756 +ek -8.408625 +ih -8.427478 +ao -8.433031 +g_ -8.482911 +ru -8.483917 +de -8.502687 +t_ -8.504095 +h_ -8.507538 +j_ -8.523244 +se -8.533685 +k_ -8.536471 +ku -8.575030 +sl -8.597999 +ki -8.614520 +iv -8.633196 +ze -8.653614 +_a -8.656382 +gr -8.675829 +si -8.683474 +ot -8.713606 +mi -8.728886 +dr -8.736418 +r_ -8.748557 +bo -8.773357 +tv -8.780536 +d_ -8.785028 +oz -8.805085 +ke -8.820794 +le -8.824033 +us -8.838549 +pa -8.849945 +vr -8.871421 +uc -8.881044 +du -8.883415 +_l -8.919904 +ga -8.923387 +tu -8.923455 +ok -8.923640 +rv -8.938245 +zn -8.978403 +_h -8.996073 +nu -8.999660 +sn -9.005563 +su -9.050709 +vn -9.076358 +ba -9.088362 +ut -9.091526 +oc -9.124923 +ez -9.138299 +kr -9.140629 +eg -9.140766 +s_ -9.146703 +up -9.153416 +op -9.164292 +sp -9.191610 +lu -9.223599 +ir -9.243657 +br -9.262546 +c_ -9.318419 +ud -9.349184 +pi -9.391045 +gl -9.426575 +un -9.447926 +ug -9.498205 +ts -9.524148 +ur -9.543709 +pu -9.552626 +vl -9.587191 +jo -9.673973 +vj -9.678716 +hr -9.702179 +_f -9.720721 +eb -9.722064 +tn -9.726417 +uz -9.732633 +ck -9.738249 +ev -9.743830 +ap -9.752938 +ig -9.755863 +nt -9.767494 +rs -9.773581 +dj -9.782641 +mj -9.809107 +gu -9.824413 +zb -9.844486 +cu -9.865867 +rn -9.873161 +be -9.885587 +ns -9.909567 +cn -9.911107 +tk -9.912227 +ag -9.920698 +uj -9.952593 +sm -9.970797 +uk -9.996401 +pl -10.006587 +_e -10.019335 +zv -10.029695 +rz -10.048530 +kl -10.095765 +ub -10.108415 +ul -10.121043 +v_ -10.129169 +ln -10.150129 +rt -10.209264 +so -10.223054 +um -10.243765 +pe -10.247063 +zu -10.261724 +bl -10.265865 +id -10.273768 +gi -10.276731 +bu -10.299850 +dv -10.318324 +sr -10.348286 +sj -10.352452 +nc -10.374936 +mu -10.391326 +ds -10.407360 +vu -10.411853 +jn -10.440918 +nd -10.463364 +fi -10.493174 +z_ -10.516659 +ip -10.534020 +js -10.574012 +ge -10.615114 +kt -10.628206 +ho -10.649988 +kv -10.662345 +l_ -10.672408 +zo -10.681318 +zl -10.847217 +ml -10.865068 +zd -10.922602 +ep -10.942826 +bj -10.983241 +nk -10.997856 +tj -11.024607 +uv -11.024704 +rk -11.053426 +eu -11.070513 +ab -11.092572 +bn -11.109248 +zr -11.140960 +rd -11.153517 +dl -11.187270 +eo -11.217637 +pj -11.226417 +mn -11.233232 +au -11.289042 +zg -11.320036 +rg -11.335822 +cr -11.478909 +p_ -11.499397 +pn -11.516005 +ps -11.525354 +rl -11.531126 +zm -11.627985 +fo -11.635721 +lt -11.655425 +ha -11.669480 +tp -11.688026 +sc -11.695938 +rc -11.702869 +lm -11.708370 +fe -11.762103 +ks -11.787829 +rb -11.797998 +co -11.808907 +cj -11.814325 +jv -11.843024 +gd -11.862571 +kn -11.876218 +cl -11.879369 +kc -11.916222 +ng -11.925820 +ia -11.927631 +fr -11.947338 +pc -11.965531 +hv -11.982375 +b_ -11.998312 +he -12.009855 +ib -12.015198 +rm -12.023309 +oi -12.041551 +ah -12.044299 +jk -12.065040 +uo -12.098884 +vs -12.107112 +db -12.118693 +mr -12.126493 +ea -12.226662 +ms -12.256399 +mp -12.276893 +jb -12.283233 +hi -12.296792 +eh -12.356631 +fa -12.389303 +dm -12.411486 +ai -12.468246 +dz -12.472787 +tl -12.473422 +af -12.473845 +rh -12.486444 +pt -12.551592 +zj -12.573317 +ef -12.576553 +rp -12.588423 +of -12.595949 +ua -12.606878 +uh -12.608504 +km -12.622635 +dg -12.625043 +lb -12.643678 +th -12.670056 +mc -12.674611 +y_ -12.679242 +ej -12.716707 +hn -12.735938 +ht -12.846241 +vc -12.900770 +ls -12.953303 +rj -12.960405 +mb -12.976438 +sh -13.121053 +ou -13.191463 +hu -13.211005 +ch -13.264205 +nz -13.270075 +ll -13.270443 +nf -13.279111 +gn -13.338398 +_w -13.341292 +hd -13.392601 +oh -13.454237 +bz -13.477184 +jc -13.542960 +jm -13.560726 +jd -13.585675 +vd -13.589570 +ae -13.594050 +if -13.664911 +vk -13.751060 +ss -13.836272 +sd -13.849671 +ii -13.860148 +fu -13.966946 +f_ -13.972021 +oa -14.012966 +dp -14.044535 +jp -14.212947 +w_ -14.260182 +cv -14.268605 +ie -14.334548 +hs -14.380643 +nv -14.389602 +oo -14.416004 +wa -14.429491 +lk -14.431751 +yo -14.443313 +ft -14.468404 +ew -14.497593 +_y -14.591406 +ry -14.600161 +ue -14.602936 +md -14.615260 +jt -14.650003 +np -14.671452 +oe -14.681441 +sf -14.728657 +nn -14.736763 +tz -14.774860 +ei -14.778516 +ay -14.787961 +ui -14.806238 +tm -14.875309 +wi -14.880632 +ow -14.889077 +nr -14.900130 +tt -14.922493 +bv -14.931769 +ld -14.957590 +hl -14.983881 +rr -15.050206 +x_ -15.077292 +nl -15.083094 +ee -15.093466 +we -15.166868 +ny -15.175106 +lc -15.365049 +ph -15.372924 +fl -15.380445 +fs -15.388405 +ty -15.419058 +lf -15.438347 +mk -15.458734 +_x -15.469670 +td -15.625102 +nb -15.649285 +pk -15.653596 +tc -15.665153 +aa -15.692485 +wo -15.754914 +ly -15.762135 +ex -15.786646 +jz -15.893601 +tf -15.976363 +mm -16.047053 +ey -16.047683 +xi -16.082125 +gh -16.129367 +uf -16.130034 +lp -16.169265 +yj -16.180970 +lg -16.238079 +zz -16.246735 +ix -16.354095 +sb -16.359562 +jj -16.383226 +bb -16.387208 +gm -16.402441 +jl -16.438347 +bc -16.498239 +jg -16.549081 +my -16.594395 +oy -16.595316 +mt -16.652637 +ct -16.696432 +ff -16.701381 +lv -16.715330 +ux -16.742618 +gt -16.769393 +dy -16.775643 +nh -16.794557 +pp -16.849533 +xe -16.881755 +dh -16.895302 +ax -16.910122 +mf -16.951976 +fg -17.025790 +xv -17.030765 +tw -17.071193 +ya -17.072475 +ye -17.087947 +bm -17.106209 +wh -17.130034 +wn -17.158336 +lz -17.189984 +gg -17.195560 +bd -17.195560 +iu -17.206776 +uu -17.252535 +ws -17.267137 +yp -17.334740 +ys -17.368981 +mv -17.381636 +yl -17.412145 +pb -17.415394 +qu -17.421915 +aw -17.423550 +gv -17.440000 +jr -17.448296 +hy -17.449961 +ox -17.451628 +gs -17.526946 +pd -17.549973 +mw -17.608272 +xa -17.640228 +hm -17.647852 +yw -17.700390 +fb -17.752857 +dw -17.765241 +cc -17.848435 +by -17.852833 +sy -17.855038 +bs -17.870563 +hb -17.893035 +cs -17.948441 +rf -17.960257 +fp -17.962632 +wt -17.996296 +_q -18.025790 +xm -18.089243 +sw -18.094442 +kh -18.134043 +hz -18.230905 +xt -18.256900 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hun b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hun new file mode 100644 index 00000000000..3d073a93732 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hun @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 357372068 +#UNIQUE_BIGRAMS 621 +#UNSEEN_LOG_PROB -17.611999 +#EXPECTED_SCORE -7.607032 +t_ -5.377090 +el -5.874897 +n_ -5.984119 +sz -6.020221 +en -6.050450 +_e -6.078838 +et -6.087175 +k_ -6.136374 +_m -6.210750 +te -6.236439 +eg -6.257070 +al -6.326058 +_k -6.370608 +an -6.372021 +gy -6.391915 +er -6.416835 +le -6.483417 +ta -6.500643 +i_ -6.560081 +es -6.629518 +_a -6.658420 +_s -6.741921 +a_ -6.745113 +at -6.760561 +ze -6.771600 +to -6.785580 +me -6.790213 +ar -6.850869 +tt -6.852707 +_t -6.874852 +_v -6.889822 +ol -6.950966 +ko -6.958111 +s_ -6.968064 +_h -6.981644 +ek -6.984513 +ke -6.988641 +la -6.996527 +y_ -7.018927 +or -7.071350 +e_ -7.073547 +ne -7.095243 +l_ -7.098258 +ak -7.122850 +ve -7.136740 +ma -7.215022 +_n -7.223787 +re -7.263968 +_f -7.321460 +as -7.367429 +se -7.375779 +on -7.386589 +ag -7.387334 +em -7.402056 +os -7.403090 +va -7.431626 +o_ -7.435698 +lt -7.454532 +ot -7.477318 +be -7.490486 +za -7.501997 +ul -7.540740 +ra -7.583572 +zo -7.590885 +sa -7.596668 +ez -7.610151 +ny -7.616871 +in -7.654697 +nt -7.655578 +ba -7.662289 +oz -7.673390 +g_ -7.700517 +ok -7.701943 +_b -7.703293 +lo -7.706803 +mi -7.725972 +ha -7.732119 +na -7.736095 +ro -7.769979 +r_ -7.818941 +_l -7.823216 +ye -7.903346 +_i -7.910484 +ly -7.920279 +ka -7.938485 +rt -7.944068 +it -7.955986 +so -7.977323 +ja -8.008402 +ik -8.012744 +m_ -8.015493 +ll -8.039719 +am -8.040963 +de -8.056524 +og -8.061239 +ho -8.081141 +zt -8.082962 +az -8.173966 +bb -8.234718 +ki -8.247807 +fe -8.284371 +ge -8.287813 +is -8.314063 +do -8.322806 +_o -8.346458 +_p -8.368041 +cs -8.379110 +ga -8.383972 +_r -8.401358 +ab -8.423913 +_j -8.436602 +nd -8.463214 +ti -8.471788 +ya -8.476120 +_c -8.514016 +he -8.531443 +ad -8.536237 +fo -8.561781 +je -8.594223 +ap -8.623590 +_u -8.624412 +li -8.645277 +ni -8.655333 +ep -8.663566 +ev -8.664921 +ri -8.684049 +ut -8.691307 +zi -8.693857 +vo -8.735584 +ed -8.746676 +ig -8.780076 +da -8.782579 +ob -8.795973 +pe -8.821066 +b_ -8.821233 +ia -8.880882 +il -8.910123 +st -8.923315 +di -8.925520 +om -8.933273 +aj -8.950513 +ai -9.020759 +si -9.063458 +mo -9.070651 +vi -9.088706 +pa -9.104093 +_g -9.110034 +tu -9.115134 +ru -9.144214 +z_ -9.156049 +ss -9.166062 +go -9.168509 +od -9.170647 +ov -9.182639 +no -9.189918 +yo -9.203115 +d_ -9.220894 +mu -9.225103 +eb -9.274765 +ir -9.290769 +_d -9.293161 +ku -9.337464 +us -9.353732 +yi -9.369669 +po -9.385102 +nn -9.396987 +lm -9.399815 +rs -9.444919 +u_ -9.446403 +bo -9.525572 +ug -9.554081 +gi -9.581749 +lk -9.612274 +un -9.648576 +bi -9.668576 +nk -9.677064 +eh -9.684283 +rm -9.712486 +zu -9.728171 +iz -9.734530 +ci -9.744171 +iv -9.761965 +id -9.785973 +jo -9.854338 +ls -9.870473 +uk -9.891623 +mb -10.037155 +kk -10.076363 +fi -10.077200 +ld -10.111713 +ln -10.118734 +rd -10.136464 +hi -10.194331 +ej -10.199054 +dt -10.230462 +ur -10.260369 +ud -10.260374 +ng -10.275800 +pi -10.300993 +yu -10.307035 +su -10.335076 +ts -10.353359 +nc -10.381640 +io -10.400353 +av -10.408362 +ei -10.417416 +zn -10.417719 +lu -10.474633 +lj -10.480470 +tr -10.485872 +rv -10.494716 +ce -10.528783 +im -10.556961 +rr -10.557746 +tk -10.561054 +zd -10.583792 +pu -10.593820 +lv -10.595131 +zs -10.620099 +tv -10.680446 +rn -10.682687 +tj -10.699049 +lg -10.705385 +th -10.730598 +tl -10.737535 +ju -10.745276 +p_ -10.765267 +gu -10.781220 +nb -10.794394 +nu -10.821490 +ac -10.825404 +zz -10.849901 +tb -10.865422 +pr -10.890239 +jd -10.933332 +_z -10.934256 +kb -10.958761 +op -10.961383 +du -10.990713 +gj -11.019067 +lh -11.022683 +rk -11.026737 +bu -11.033863 +ys -11.061429 +yt -11.078020 +ah -11.100571 +kt -11.115362 +rz -11.127323 +fa -11.132008 +uj -11.150023 +au -11.159253 +gr -11.161452 +kr -11.185612 +sm -11.207755 +oj -11.225732 +ks -11.276461 +ie -11.279940 +um -11.288670 +rc -11.295533 +tn -11.307414 +rg -11.308355 +dj -11.313915 +sk -11.314388 +iu -11.329767 +zl -11.332825 +ec -11.362801 +nl -11.375219 +ds -11.377934 +rb -11.396348 +gb -11.402936 +jt -11.409290 +oi -11.442016 +gs -11.444477 +mz -11.445761 +br -11.487344 +ns -11.518318 +v_ -11.538987 +gn -11.540153 +uz -11.548835 +gl -11.552698 +rj -11.553572 +pl -11.569053 +eu -11.588266 +pj -11.604101 +yb -11.612837 +fu -11.629419 +gg -11.674746 +mm -11.684134 +sb -11.698365 +ch -11.724492 +mp -11.731888 +vu -11.733839 +ic -11.735958 +rl -11.738549 +yz -11.747127 +oe -11.753232 +ml -11.759055 +gt -11.789485 +gh -11.797265 +ef -11.810617 +fr -11.811661 +uv -11.829504 +pt -11.842447 +zb -11.851937 +c_ -11.856870 +kl -11.897845 +ib -11.925106 +dn -11.942892 +dv -11.963977 +oc -12.041602 +lf -12.054475 +ea -12.067949 +jl -12.082005 +jn -12.086527 +rh -12.091505 +ca -12.091769 +yv -12.098800 +gf -12.106131 +lb -12.116760 +pc -12.124797 +ip -12.141680 +pp -12.153605 +hu -12.154690 +yn -12.173719 +nz -12.191321 +oa -12.194724 +zk -12.206390 +dd -12.240170 +sp -12.242888 +of -12.253059 +dr -12.256275 +kn -12.271982 +tm -12.346384 +co -12.350257 +gk -12.391460 +zv -12.404861 +yr -12.422903 +oh -12.435438 +bs -12.470979 +zg -12.473066 +gv -12.481814 +nf -12.485861 +h_ -12.489849 +ij -12.503310 +gz -12.522326 +if -12.524513 +dm -12.586725 +zm -12.607738 +ub -12.629751 +nh -12.636470 +kv -12.645236 +lc -12.654811 +bl -12.656999 +j_ -12.686980 +sn -12.689004 +sr -12.698848 +zp -12.702180 +ym -12.722553 +ps -12.730872 +ua -12.737363 +pv -12.747766 +_w -12.749016 +af -12.761407 +yh -12.795930 +f_ -12.889167 +db -12.901099 +sv -12.916497 +zr -13.048033 +rf -13.057397 +up -13.061084 +dk -13.074881 +eo -13.086282 +ue -13.121897 +yk -13.127811 +vb -13.128353 +vt -13.154470 +sl -13.183132 +ii -13.196674 +lr -13.203703 +mt -13.205380 +ms -13.251129 +ih -13.276341 +yj -13.282402 +ty -13.302368 +ck -13.344241 +sh -13.348193 +ex -13.374962 +zh -13.387582 +sg -13.401188 +hn -13.418233 +gm -13.435974 +yl -13.439917 +pz -13.468780 +bd -13.484981 +jr -13.486000 +nv -13.493662 +dh -13.513400 +mr -13.513967 +ou -13.520404 +x_ -13.536095 +ee -13.552347 +km -13.604541 +uc -13.611952 +tc -13.621383 +ae -13.658225 +sc -13.696675 +vv -13.749405 +hh -13.805521 +lz -13.816953 +uh -13.819694 +oo -13.823435 +js -13.850251 +nr -13.854012 +kc -13.874420 +xi -13.874905 +wa -13.942765 +vj -13.956112 +w_ -14.026248 +kh -14.037541 +cc -14.064815 +jz -14.068556 +rp -14.071055 +wi -14.074533 +ph -14.077531 +mn -14.197015 +ry -14.213716 +ay -14.258825 +tf -14.263898 +bk -14.297239 +ew -14.304981 +pk -14.326301 +nj -14.342479 +yf -14.395521 +dz -14.451312 +aa -14.452307 +ao -14.463846 +ji -14.466037 +mc -14.479345 +ft -14.491755 +_y -14.499682 +vr -14.535376 +lp -14.549246 +jj -14.556913 +kf -14.562762 +fj -14.564033 +bn -14.566578 +fl -14.567656 +pn -14.585509 +gp -14.593670 +gd -14.626276 +nm -14.632722 +sf -14.657547 +ow -14.669279 +we -14.702906 +mj -14.736344 +np -14.760119 +ey -14.764494 +ox -14.780424 +bj -14.802980 +md -14.888677 +ax -14.893830 +sd -14.899618 +kj -14.912508 +cl -15.061222 +cb -15.095722 +cz -15.098268 +_x -15.145626 +ui -15.146504 +jb -15.160925 +cr -15.163294 +vs -15.190660 +dl -15.218710 +dy -15.260567 +ff -15.282120 +tp -15.319764 +wo -15.358417 +mf -15.371877 +yd -15.390831 +cu -15.397611 +jc -15.412147 +pb -15.517655 +mh -15.525821 +ix -15.545187 +hr -15.550796 +ct -15.554094 +dp -15.665708 +oy -15.666547 +sy -15.690898 +zf -15.738439 +hl -15.804597 +cv -15.885130 +df -15.955727 +vn -15.992102 +tg -16.023758 +mk -16.026720 +xu -16.036726 +kz -16.044891 +yg -16.050086 +ht -16.056125 +uf -16.057501 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hye b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hye new file mode 100644 index 00000000000..5d65164a488 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/hye @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 132005718 +#UNIQUE_BIGRAMS 1664 +#UNSEEN_LOG_PROB -15.700500 +#EXPECTED_SCORE -7.906976 +ու -4.737288 +ան -4.973835 +ն_ -5.300669 +ար -5.667880 +եր -5.773436 +_ա -5.854550 +ի_ -5.899198 +կա -5.968654 +_հ -6.100873 +ակ -6.222251 +այ -6.247621 +մ_ -6.390390 +ւմ -6.392011 +րա -6.426201 +հա -6.507208 +նե -6.525823 +մա -6.532491 +ին -6.557481 +ը_ -6.558189 +նա -6.586050 +_մ -6.669006 +ել -6.747188 +որ -6.789658 +տա -6.871164 +ր_ -6.946459 +ամ -7.018759 +_կ -7.021083 +ւթ -7.028069 +լ_ -7.035054 +րի -7.043953 +րո -7.052724 +թյ -7.082296 +ւն -7.107692 +յա -7.113102 +վա -7.228103 +յո -7.264656 +նո -7.337747 +_ն -7.354369 +ատ -7.392441 +աս -7.402249 +պա -7.557199 +ավ -7.560019 +_ո -7.597692 +նի -7.624683 +ց_ -7.645885 +րե -7.650792 +են -7.702719 +_տ -7.754101 +_ե -7.799621 +_գ -7.809845 +_պ -7.822298 +_բ -7.850703 +ստ -7.887956 +տե -7.894199 +_ի -7.903510 +_դ -7.935024 +ետ -7.936322 +վե -7.955485 +_ս -7.982550 +աց -7.989436 +առ -7.992878 +մե -8.006178 +վո -8.027413 +ս_ -8.049634 +ող -8.062741 +նք -8.068492 +ով -8.107287 +ք_ -8.114902 +ած -8.134095 +բա -8.161749 +ւ_ -8.209454 +խա -8.231208 +սա -8.234749 +լո -8.239864 +կո -8.240459 +րը -8.245668 +մի -8.257646 +_վ -8.265474 +տո -8.267739 +վ_ -8.280525 +եւ -8.291824 +տ_ -8.308571 +պե -8.335523 +կ_ -8.339032 +իր -8.378253 +լի -8.378827 +քա -8.384605 +ես -8.397877 +ծ_ -8.399087 +նը -8.399404 +րդ -8.401206 +յն -8.457276 +ագ -8.484630 +նն -8.501274 +հե -8.505664 +ալ -8.515949 +դա -8.536884 +ահ -8.538306 +ա_ -8.551223 +յի -8.554322 +րկ -8.557893 +գո -8.589670 +գա -8.615114 +եղ -8.633343 +ռա -8.642023 +ապ -8.654612 +ղա -8.710287 +ազ -8.747554 +նդ -8.767457 +աղ -8.822542 +_ք -8.832308 +_թ -8.853046 +իս -8.883903 +կե -8.898260 +ից -8.903317 +աշ -8.928773 +ղ_ -8.931466 +_շ -8.938575 +_խ -8.956114 +_լ -8.958651 +սի -8.961215 +իա -8.961324 +շա -8.990750 +նց -8.992213 +դր -9.060864 +ախ -9.090466 +ցո -9.102004 +եկ -9.115726 +ոն -9.117091 +ըն -9.118925 +ւր -9.151868 +դե -9.153834 +տի -9.166037 +_ը -9.193142 +նչ -9.207182 +կի -9.210085 +եց -9.219254 +դի -9.231585 +դ_ -9.239174 +բե -9.240806 +ցա -9.246126 +մո -9.259681 +քի -9.262410 +ւյ -9.295144 +րծ -9.295844 +գր -9.296765 +մն -9.306157 +րտ -9.327899 +ադ -9.355505 +աք -9.369092 +տն -9.371211 +հո -9.394258 +ւս -9.395332 +մբ -9.404895 +ցի -9.406971 +սո -9.415095 +նկ -9.418539 +յս -9.423604 +սկ -9.446396 +սե -9.449052 +նր -9.452854 +րվ -9.460849 +_փ -9.464805 +_օ -9.471214 +ջա -9.482068 +րգ -9.490903 +_չ -9.504612 +հի -9.514234 +նվ -9.531876 +տր -9.544356 +խո -9.549474 +թա -9.570161 +ռո -9.608184 +րն -9.624094 +գի -9.624361 +փո -9.626295 +ղո -9.645922 +քո -9.648131 +ցն -9.669955 +քն -9.683560 +աբ -9.685943 +ժա -9.719411 +ճա -9.727245 +ծա -9.728107 +բո -9.748804 +ոս -9.764108 +իմ -9.764260 +իկ -9.769927 +աջ -9.780509 +շխ -9.788886 +նտ -9.792710 +րձ -9.794698 +բ_ -9.803764 +դո -9.808195 +թե -9.820822 +ցե -9.847678 +_զ -9.848474 +ջ_ -9.856780 +օր -9.863036 +իտ -9.879567 +ղե -9.880474 +վի -9.880875 +_ժ -9.881783 +լա -9.891518 +_ծ -9.905946 +ծո -9.906880 +նգ -9.913926 +շտ -9.915266 +զա -9.928603 +եմ -9.930180 +սն -9.935703 +րք -9.950215 +րջ -9.958227 +տվ -9.961460 +կր -9.967728 +յդ -10.014395 +րբ -10.015514 +կն -10.037054 +ոլ -10.049706 +գե -10.052315 +ձե -10.084894 +եռ -10.092523 +րմ -10.112988 +զմ -10.140592 +վր -10.143135 +_ձ -10.150967 +թո -10.160404 +իջ -10.194832 +չ_ -10.204007 +ոց -10.222404 +հր -10.243315 +չե -10.273582 +յլ -10.281599 +իշ -10.281817 +րհ -10.286000 +յտ -10.303310 +րց -10.346412 +ջո -10.352901 +տք -10.360166 +յց -10.365000 +քե -10.393824 +_ռ -10.405695 +ե_ -10.407698 +րպ -10.425399 +փա -10.444354 +քը -10.444918 +ջի -10.451438 +ռն -10.472277 +գն -10.475884 +ձա -10.489583 +_է -10.489992 +ւց -10.510299 +կց -10.517282 +մը -10.526780 +եջ -10.526796 +աժ -10.536746 +ւղ -10.541658 +ո_ -10.546030 +թվ -10.556937 +չո -10.571882 +նձ -10.573213 +ղի -10.576280 +ւա -10.585621 +լե -10.594533 +սպ -10.601189 +թի -10.603245 +աե -10.610181 +_ց -10.610608 +ոտ -10.641298 +շո -10.643097 +զո -10.659496 +եք -10.663585 +նշ -10.668291 +ցվ -10.670943 +իվ -10.696161 +գտ -10.703031 +նթ -10.751892 +եպ -10.769125 +ափ -10.781999 +բն -10.783655 +պի -10.784677 +_ֆ -10.793476 +եզ -10.812199 +կը -10.814854 +իք -10.816154 +չա -10.817849 +զգ -10.821247 +ոշ -10.821939 +պո -10.835235 +զի -10.857023 +նս -10.862100 +րս -10.875691 +ոխ -10.883041 +իչ -10.886389 +նյ -10.891030 +շե -10.918203 +_ճ -10.923669 +եթ -10.961133 +յր -10.979839 +ւո -10.990383 +րժ -10.995038 +պր -11.017880 +ւտ -11.023648 +չպ -11.031839 +կվ -11.037893 +կտ -11.045265 +եծ -11.064423 +ժե -11.072120 +խն -11.072732 +նպ -11.074333 +ծե -11.089090 +ղմ -11.099628 +ժո -11.127965 +օգ -11.132031 +րյ -11.140098 +շր -11.141702 +աթ -11.153903 +իո -11.216319 +հն -11.219886 +տմ -11.231533 +րթ -11.242533 +քր -11.250952 +զ_ -11.273907 +րզ -11.293771 +նմ -11.295116 +էի -11.337787 +աճ -11.356837 +ռե -11.363963 +սյ -11.367251 +ւկ -11.373442 +իլ -11.376577 +ծի -11.380389 +հ_ -11.382897 +թ_ -11.392208 +հմ -11.399452 +կյ -11.401964 +նհ -11.404717 +ցմ -11.415812 +մտ -11.460603 +յթ -11.464334 +ղջ -11.466838 +ջե -11.468045 +յք -11.489756 +իճ -11.529459 +յե -11.550219 +դյ -11.555177 +սխ -11.565541 +յ_ -11.573313 +ձն -11.579253 +լն -11.587974 +մր -11.612027 +ոգ -11.642066 +տը -11.656847 +իպ -11.662079 +ցր -11.665981 +շն -11.667437 +ղծ -11.673136 +մյ -11.684351 +ռ_ -11.705365 +գլ -11.709861 +ծն -11.723434 +շվ -11.729136 +ոք -11.740048 +պք -11.752551 +ղն -11.757689 +ւժ -11.757765 +սը -11.762579 +_ջ -11.770385 +ւշ -11.773136 +եխ -11.778386 +գյ -11.795417 +ւլ -11.796699 +գ_ -11.800281 +սվ -11.807392 +ֆի -11.807627 +_ղ -11.813634 +ցք -11.836554 +աչ -11.841358 +կս -11.841639 +ծք -11.843124 +աձ -11.843967 +զբ -11.869789 +ձր -11.898333 +ոմ -11.902762 +յմ -11.903306 +ոչ -11.925496 +ղթ -11.926942 +չն -11.930649 +տկ -11.948162 +_յ -11.951665 +ոյ -11.959739 +դն -11.961089 +ջն -11.962441 +ձ_ -11.983972 +խ_ -11.987562 +դպ -11.992497 +չի -12.009159 +խմ -12.009385 +e_ -12.014078 +լր -12.014123 +եշ -12.017337 +եա -12.022966 +տպ -12.026381 +բի -12.040906 +պ_ -12.041413 +սօ -12.047414 +ծը -12.054696 +շ_ -12.082250 +ծվ -12.111017 +զե -12.112807 +սք -12.116005 +ծր -12.125008 +իզ -12.163046 +եվ -12.174620 +տղ -12.191441 +շի -12.203556 +նջ -12.207893 +ոդ -12.208048 +t_ -12.218010 +_c -12.225946 +մշ -12.227046 +on -12.241898 +լը -12.249275 +ժի -12.249328 +րչ -12.258188 +ռք -12.284718 +գս -12.295061 +ֆո -12.300950 +բյ -12.308137 +լխ -12.325983 +վտ -12.328286 +տճ -12.364424 +դհ -12.371877 +զր -12.372805 +դը -12.379544 +յկ -12.383335 +ղղ -12.385204 +ոպ -12.400131 +ոկ -12.406170 +ւե -12.417066 +ղվ -12.422816 +ցը -12.428709 +քս -12.440385 +ռի -12.463902 +ne -12.466931 +դկ -12.478859 +ըս -12.486052 +խի -12.487620 +սր -12.496750 +ւի -12.510650 +ւծ -12.517042 +կչ -12.550219 +վյ -12.564978 +ւհ -12.566634 +իգ -12.571549 +ւզ -12.573013 +լս -12.573013 +ձո -12.576747 +ժ_ -12.590635 +եփ -12.591107 +ձի -12.600646 +դվ -12.601801 +եո -12.628611 +իե -12.639240 +կլ -12.643779 +բր -12.660168 +n_ -12.663000 +խվ -12.682769 +կմ -12.720333 +է_ -12.721661 +ոփ -12.741582 +պվ -12.759052 +ոհ -12.760796 +թն -12.769545 +ճի -12.770919 +լմ -12.780576 +վհ -12.808685 +խե -12.809784 +հպ -12.820258 +ղը -12.821682 +գծ -12.839515 +տյ -12.842562 +չէ -12.853763 +ղբ -12.866603 +co -12.880051 +ցկ -12.886244 +էր -12.889640 +մս -12.889640 +զվ -12.902553 +ւէ -12.913052 +մք -12.927709 +զդ -12.930521 +թը -12.931716 +վճ -12.935564 +խտ -12.941054 +ժմ -12.965322 +իթ -12.967947 +փե -12.979817 +te -12.981318 +եի -12.988495 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ibo b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ibo new file mode 100644 index 00000000000..03140a81285 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ibo @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 13855836 +#UNIQUE_BIGRAMS 594 +#UNSEEN_LOG_PROB -18.843430 +#EXPECTED_SCORE -7.247018 +e_ -4.333551 +a_ -4.802931 +_n -5.044170 +u_ -5.050433 +i_ -5.079680 +_a -5.332600 +_o -5.455992 +_i -5.639493 +o_ -5.792156 +_e -5.868599 +_m -6.010268 +ke -6.427710 +ny -6.501233 +ch -6.571022 +er -6.607281 +an -6.704541 +re -6.706693 +kw -6.728753 +di -6.737271 +ru -6.743996 +he -6.765415 +ma -6.773640 +we -6.791441 +_u -6.876040 +nd -6.969455 +nk -6.972303 +ak -7.040845 +ar -7.042299 +wu -7.043562 +gb -7.058280 +hi -7.067440 +ta -7.081459 +me -7.096799 +wa -7.101095 +nw -7.119741 +ri -7.131898 +bu -7.153794 +si -7.166886 +or -7.209800 +ka -7.220072 +ik -7.309883 +on -7.373966 +tu -7.380857 +ye -7.393214 +ra -7.461657 +ur -7.465903 +ba -7.521599 +ih -7.546551 +_k -7.595110 +ha -7.612182 +_b -7.654079 +gh -7.708728 +ah -7.736593 +in -7.739795 +be -7.764985 +gw -7.782943 +pu -7.814894 +_s -7.826239 +ot -7.831542 +hu -7.833346 +it -7.872900 +ut -7.895606 +kp -7.912616 +en -7.935298 +ag -7.965273 +ji -7.965871 +eg -7.978421 +mm -8.004709 +te -8.051455 +_g -8.064385 +yi -8.069438 +ro -8.072967 +ia -8.087961 +ok -8.088868 +ir -8.089860 +le -8.107872 +na -8.122220 +_c -8.140908 +em -8.216660 +uk -8.234926 +ko -8.252950 +at -8.267957 +al -8.284127 +ek -8.286921 +zi -8.300349 +la -8.300809 +eb -8.303570 +ug -8.305907 +ti -8.313440 +ic -8.314102 +ho -8.326917 +ig -8.397701 +ul -8.400725 +ad -8.418748 +og -8.430483 +ee -8.436747 +bi -8.441988 +du -8.447575 +bo -8.463365 +et -8.477547 +as -8.496262 +_d -8.557867 +_t -8.562425 +um -8.582362 +s_ -8.583680 +so -8.607037 +is -8.612610 +ng -8.644131 +am -8.691945 +es -8.698418 +oz -8.700799 +de -8.706834 +_p -8.708619 +zo -8.710930 +ni -8.716481 +ga -8.747963 +om -8.757350 +aa -8.763672 +mg -8.768295 +ob -8.772159 +go -8.780011 +nu -8.786019 +ac -8.808951 +li -8.829883 +el -8.850787 +pa -8.869853 +nt -8.888828 +il -8.904260 +_j -8.908757 +mu -8.910109 +ne -8.919658 +ab -8.922939 +od -8.923343 +lo -8.924051 +do -8.941914 +os -8.956323 +zu -8.959793 +oc -8.972708 +to -9.011840 +un -9.015230 +_w -9.040886 +wo -9.051455 +ya -9.080022 +ez -9.081093 +us -9.108648 +n_ -9.121234 +ku -9.126287 +ai -9.136623 +ij -9.155619 +t_ -9.158293 +sa -9.185923 +uo -9.228073 +po -9.235711 +l_ -9.281436 +no -9.323978 +im -9.325447 +ol -9.378932 +oo -9.392444 +da -9.408770 +id -9.448157 +ib -9.459255 +m_ -9.474210 +fo -9.475544 +pe -9.507017 +lu -9.518426 +mk -9.526236 +nn -9.554927 +mb -9.555788 +_f -9.574958 +gu -9.585079 +st -9.591491 +_r -9.592214 +ed -9.605130 +ge -9.626534 +ie -9.748949 +uz -9.765075 +mi -9.774893 +ii -9.775441 +ej -9.776901 +_h -9.788917 +ep -9.797231 +ec -9.801592 +se -9.817006 +r_ -9.827280 +oh -9.850066 +az -9.865329 +y_ -9.889914 +io -9.910810 +ns -9.916032 +je -9.945605 +nc -9.945605 +op -9.986849 +d_ -10.020087 +ub -10.024093 +iz -10.098053 +ay -10.127684 +ju -10.135276 +ap -10.141966 +_l -10.155440 +th -10.166407 +ki -10.171441 +nj -10.178061 +af -10.191026 +fe -10.200674 +mo -10.222900 +yo -10.311288 +ud -10.347322 +ew -10.379001 +mp -10.398123 +ow -10.427791 +gi -10.471029 +k_ -10.482603 +su -10.487229 +_z -10.505730 +jo -10.516976 +co -10.553752 +up -10.600677 +ea -10.625629 +va -10.660932 +iw -10.710842 +ca -10.728223 +pi -10.733000 +uw -10.735661 +h_ -10.786617 +ze -10.810914 +ce -10.817100 +c_ -10.825767 +ip -10.826712 +eh -10.834867 +uf -10.856905 +uc -10.896846 +nl -10.904410 +ef -10.918448 +tr -10.935068 +sh -10.936292 +nr -11.019006 +ov -11.025720 +g_ -11.038147 +vi -11.061990 +of -11.086460 +oj -11.106753 +_v -11.146798 +fr -11.171321 +pr -11.188472 +ua -11.193340 +fu -11.213721 +ll -11.307985 +aj -11.314865 +fa -11.356030 +ve -11.383306 +ou -11.398404 +tw -11.407143 +nz -11.407143 +oa -11.429370 +uu -11.435701 +aw -11.521867 +ja -11.534166 +ct -11.534784 +bl -11.572657 +pl -11.573291 +ss -11.597286 +eo -11.599869 +_y -11.608296 +wi -11.638518 +ck -11.662619 +nh -11.669047 +p_ -11.684043 +rs -11.697121 +ft -11.743137 +br -11.773799 +rt -11.787721 +ue -11.828794 +ci -11.874585 +ao -11.908607 +cr -11.916233 +mf -11.960778 +fi -11.989704 +ei -12.026154 +za -12.073387 +au -12.080134 +iy -12.082842 +sp -12.083745 +ts -12.090087 +ty -12.094178 +uj -12.128733 +ht -12.159841 +gl -12.173244 +uh -12.183378 +b_ -12.183378 +ry -12.238161 +py -12.255366 +bb -12.304557 +iv -12.323111 +bc -12.328992 +oi -12.332747 +rn -12.370844 +w_ -12.423066 +yr -12.473692 +tc -12.490970 +kr -12.515756 +dr -12.534784 +if -12.554065 +rc -12.557827 +pt -12.594063 +gr -12.613507 +tt -12.629253 +ex -12.631233 +ph -12.665999 +lt -12.671422 +oy -12.685757 +rd -12.719068 +oe -12.735306 +ev -12.773435 +sy -12.776353 +ly -12.816349 +ae -12.821615 +ey -12.857484 +ms -12.903811 +ui -12.940992 +rm -12.962439 +iu -13.037490 +kl -13.082842 +yl -13.126869 +x_ -13.151290 +bs -13.173244 +lv -13.187743 +sc -13.215205 +ld -13.293538 +cl -13.318849 +rk -13.352214 +pp -13.364241 +rl -13.393074 +hn -13.432820 +cu -13.454864 +rg -13.463071 +av -13.479627 +eu -13.506033 +sm -13.518197 +wn -13.525545 +ks -13.536638 +ys -13.599869 +ps -13.599869 +ws -13.622015 +f_ -13.641841 +yp -13.643173 +rr -13.661944 +mw -13.672781 +dg -13.680963 +qu -13.686443 +ls -13.691945 +sl -13.716963 +wh -13.796212 +ds -13.803638 +wd -13.832207 +zr -13.870681 +sk -13.931200 +vo -13.955806 +rp -13.962439 +cy -13.994370 +cs -14.014907 +gy -14.028762 +rv -14.035740 +kt -14.055105 +hr -14.076532 +nb -14.092813 +rb -14.098282 +fl -14.135276 +mn -14.137151 +nm -14.173244 +lm -14.222153 +_q -14.224145 +sw -14.254349 +sd -14.270720 +cc -14.314599 +dp -14.438588 +pc -14.487976 +pd -14.589564 +lb -14.615466 +ff -14.668708 +ax -14.729637 +md -14.743851 +tn -14.758206 +td -14.761094 +nf -14.778547 +yu -14.793253 +lc -14.817100 +gs -14.817100 +xa -14.826145 +dd -14.829173 +rf -14.850546 +ox -14.887940 +gt -14.897442 +xt -14.982523 +kn -15.013184 +pm -15.016631 +tl -15.040996 +hy -15.044510 +ix -15.062212 +yt -15.094634 +uy -15.112966 +dy -15.131533 +tp -15.165570 +yb -15.220165 +tm -15.248257 +nv -15.252315 +sq -15.256385 +xp -15.264559 +v_ -15.281047 +lg -15.310362 +yc -15.314599 +z_ -15.366438 +xi -15.406578 +gn -15.438588 +dh -15.438588 +my -15.438588 +fc -15.461896 +xe -15.466603 +xc -15.480816 +fs -15.485586 +kj -15.499989 +sb -15.514537 +xy -15.529234 +cn -15.529234 +dm -15.559083 +ky -15.559083 +_x -15.564119 +kc -15.574243 +dn -15.579332 +dl -15.625958 +yw -15.652528 +gm -15.663294 +dc -15.690567 +sn -15.718366 +lp -15.793253 +mc -15.829173 +lw -15.841347 +sx -15.884787 +lf -15.910209 +dv -15.923091 +zh -15.929575 +bm -15.942631 +tb -15.989281 +sr -15.989281 +gd -16.002891 +lk -16.087366 +pg -16.101939 +cp -16.101939 +js -16.124078 +hm -16.139028 +ml -16.146562 +hg -16.177096 +ux -16.192609 +df -16.281047 +gp -16.331673 +hb -16.340286 +tf -16.375262 +vu -16.384140 +dw -16.384140 +by -16.438588 +eq -16.504822 +ym -16.534166 +sv -16.554065 +cz -16.574243 +ql -16.574243 +gz -16.574243 +vd -16.584439 +lh -16.605049 +gg -16.605049 +wl -16.625958 +np -16.625958 +kv -16.657901 +wb -16.701623 +hs -16.758206 +yg -16.781476 +hd -16.781476 +yd -16.929575 +hc -16.969103 +tz -17.065779 +yn -17.080134 +xm -17.139028 +mt -17.139028 +zd -17.154135 +kk -17.169402 +zy -17.169402 +zz -17.216196 +qi -17.232137 +hp -17.248257 +yh -17.264559 +bh -17.281047 +bd -17.297726 +tv -17.402062 +sg -17.476063 +jb -17.554065 +rh -17.554065 +xu -17.574243 +db -17.594707 +fd -17.636528 +wt -17.636528 +gk -17.657901 +wk -17.657901 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ind b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ind new file mode 100644 index 00000000000..8beea5b6773 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ind @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 664368827 +#UNIQUE_BIGRAMS 656 +#UNSEEN_LOG_PROB -19.118892 +#EXPECTED_SCORE -7.194914 +an -4.464075 +n_ -5.045242 +a_ -5.185717 +ng -5.501717 +i_ -5.813033 +er -5.881513 +_d -5.913898 +da -5.921476 +_m -5.928946 +ka -5.939569 +en -5.954249 +_s -6.090311 +_p -6.148280 +la -6.284466 +ar -6.288323 +ta -6.294567 +me -6.309730 +ya -6.336431 +_k -6.440602 +at -6.501040 +_b -6.504150 +_t -6.549744 +ga -6.563556 +in -6.574288 +ak -6.578789 +ra -6.586173 +g_ -6.616295 +se -6.667676 +ah -6.748033 +al -6.768721 +pe -6.800490 +sa -6.856580 +ma -6.863631 +h_ -6.888134 +pa -6.889503 +_a -6.896756 +di -6.908709 +ba -6.951647 +as -6.952247 +un -7.013807 +k_ -7.017404 +si -7.031982 +t_ -7.055121 +te -7.063151 +em -7.097300 +am -7.109379 +tu -7.156631 +na -7.171734 +ri -7.181955 +be -7.197155 +u_ -7.307716 +ti -7.346687 +ha -7.379786 +ny -7.386375 +ke -7.400833 +ad -7.432863 +el -7.433210 +_i -7.491415 +r_ -7.494754 +nt -7.521310 +_y -7.626053 +s_ -7.632687 +it -7.640159 +ik -7.660510 +ia -7.672198 +uk -7.691772 +ja -7.729942 +ai -7.758799 +is -7.776917 +li -7.811998 +ni -7.824874 +ap -7.856227 +l_ -7.940453 +_l -7.994866 +ua -8.016227 +m_ -8.019476 +nd -8.033371 +bu -8.038408 +eb -8.054093 +de -8.075203 +ru -8.139319 +il -8.154737 +_j -8.176854 +es -8.210473 +_h -8.235232 +mb -8.263630 +us -8.281305 +_u -8.282659 +ut -8.355305 +wa -8.355727 +re -8.378197 +ek -8.384883 +su -8.386162 +ku -8.396454 +on -8.473153 +mu -8.483623 +ag -8.511803 +mi -8.517401 +ur -8.538981 +ki -8.541906 +or -8.542366 +et -8.556052 +mp -8.592297 +le -8.596741 +lu -8.618143 +bi -8.622810 +gi -8.680222 +rt -8.698538 +ju -8.711727 +um -8.724058 +id -8.742957 +_r -8.752244 +ir -8.795701 +pu -8.848839 +ko -8.849352 +gg -8.860567 +ul -8.870724 +du -8.935961 +ol -8.937500 +gu -8.956375 +ca -8.957205 +ne -8.959087 +_n -8.971515 +ih -8.991279 +im -9.012595 +ep -9.064480 +st -9.085685 +ay -9.117755 +aa -9.125986 +pi -9.143142 +_o -9.162905 +gk -9.170016 +rs -9.177798 +nj -9.186134 +au -9.246734 +ro -9.248083 +p_ -9.267988 +ge -9.331020 +hi -9.367460 +ud -9.372584 +_c -9.386136 +hu -9.394336 +_g -9.400966 +pr -9.402898 +up -9.438160 +ab -9.442046 +ed -9.491627 +eh -9.560462 +uh -9.619702 +e_ -9.629211 +to -9.644852 +rk -9.651255 +aw -9.668408 +_w -9.682296 +po -9.683067 +eg -9.687522 +nu -9.698068 +ib -9.700758 +tr -9.777952 +aj -9.818322 +om -9.823666 +lo -9.880299 +ks -9.910416 +rn -9.917284 +ot -9.923190 +nc -9.934967 +rb -9.940142 +ug -9.996903 +_e -10.020306 +ip -10.034604 +_f -10.035629 +o_ -10.081140 +ns -10.083066 +kt -10.098172 +ok -10.125587 +do -10.127512 +rg -10.180989 +rj -10.221493 +os -10.332283 +d_ -10.336396 +mo -10.365118 +rm -10.406407 +ec -10.489807 +nn -10.492996 +no -10.530979 +rl -10.535452 +ji -10.540052 +je -10.554672 +ub -10.565616 +rd -10.583948 +ce -10.608498 +io -10.655404 +ig -10.677050 +so -10.697639 +ye -10.726043 +ej -10.743044 +sk -10.768788 +od -10.810676 +fi -10.822099 +ci -10.828414 +rh -10.834258 +if -10.836810 +tk -10.843095 +ac -10.873640 +kn -10.902542 +og -10.911900 +b_ -10.936683 +uj -10.960104 +kh -11.000262 +hk -11.008492 +go -11.031385 +bo -11.054166 +fa -11.078962 +ob -11.083012 +cu -11.126014 +ea -11.126451 +f_ -11.132800 +wi -11.148178 +nk -11.150506 +sy -11.154698 +op -11.158257 +ve -11.161099 +gr -11.168377 +y_ -11.212022 +he -11.232134 +ui -11.274094 +vi -11.285654 +ew -11.331058 +gh -11.344502 +co -11.346463 +ho -11.396708 +rp -11.417745 +hw -11.428017 +gs -11.446559 +eo -11.454112 +fo -11.465145 +kr -11.497917 +sp -11.552319 +tn -11.554212 +lk -11.561124 +ic -11.577238 +pk -11.605886 +ll -11.617534 +nf -11.634556 +_v -11.656140 +yu -11.740285 +iv -11.845290 +fe -11.857680 +af -11.866281 +sn -11.878624 +sm -11.878795 +ij -11.907355 +ie -11.918786 +iu -11.964388 +pl -11.975408 +oh -11.985666 +ae -11.997045 +rc -12.030403 +kl -12.032622 +va -12.040888 +ch -12.048457 +sw -12.048659 +mn -12.084124 +hn -12.119052 +sl -12.133117 +pt -12.141737 +ml -12.164365 +ry -12.180685 +lm -12.204398 +bl -12.252381 +ov -12.328654 +yi -12.333084 +yo -12.340904 +br -12.348585 +ln -12.373926 +oa -12.400706 +kk -12.410651 +th -12.454331 +sh -12.473716 +ei -12.501539 +dr -12.524398 +ss -12.526382 +lt -12.552691 +ps -12.569885 +gn -12.592077 +jo -12.594546 +ev -12.606915 +ii -12.677739 +eu -12.680930 +iw -12.748316 +ou -12.748675 +oo -12.758918 +ef -12.802326 +np -12.828339 +we -12.842082 +za -12.945020 +ow -12.956125 +uc -12.989732 +of -12.996140 +mk -13.000351 +oy -13.097899 +dp -13.167778 +kp -13.186726 +ee -13.188022 +fr -13.211373 +wo -13.281729 +ms -13.319967 +fu -13.323480 +rw -13.348653 +bk -13.365674 +oi -13.371553 +_z -13.376510 +c_ -13.403198 +pn -13.404610 +ft -13.435167 +gl -13.473160 +zi -13.473259 +ky -13.479198 +nv -13.500457 +pp -13.519787 +hl -13.527305 +w_ -13.528254 +mm -13.534399 +km -13.564968 +ue -13.566232 +oc -13.645854 +az -13.677654 +ts -13.687161 +rv -13.692786 +iz -13.694080 +nl -13.697884 +fl -13.708688 +tt -13.722153 +ty -13.728447 +vo -13.737880 +ck -13.754830 +ls -13.761208 +sr -13.774110 +bs -13.803739 +ct -13.806225 +tl -13.811523 +ld -13.814742 +ey -13.894244 +dh -13.903033 +sc -13.936756 +hm -13.982385 +av -13.989820 +pd -14.014225 +hr -14.022007 +lb -14.064644 +dk -14.065761 +bb -14.074276 +ht -14.076188 +ph -14.083031 +ly -14.084539 +bd -14.107928 +rr -14.124278 +iy -14.138620 +wu -14.184864 +pb -14.218207 +oe -14.245658 +bn -14.262802 +ys -14.311907 +dm -14.313144 +tp -14.314427 +rf -14.325218 +tm -14.331695 +cl -14.352077 +bp -14.387708 +kb -14.408524 +uf -14.422332 +ex -14.442256 +dd -14.446371 +sd -14.465845 +gy -14.509089 +ws -14.535920 +v_ -14.540828 +dy -14.553566 +x_ -14.554872 +lr -14.605345 +dw -14.608542 +sb -14.635205 +ff -14.646800 +by -14.666712 +z_ -14.666769 +bm -14.679021 +qu -14.690688 +gm -14.740404 +bt -14.766615 +tb -14.794052 +cr -14.802466 +sj -14.872976 +tw -14.876238 +kw -14.962212 +ds -14.967768 +_q -15.002274 +hy -15.022224 +lp -15.070722 +ez -15.082428 +dj -15.097423 +uw -15.135372 +pm -15.138189 +dn -15.184905 +ao -15.217628 +oj -15.237540 +cc -15.245110 +lh -15.308642 +bj -15.382875 +sf -15.446128 +_x -15.449720 +md -15.486233 +zo -15.500557 +zu -15.552939 +mr -15.554088 +nm -15.555238 +dl -15.575772 +my -15.583428 +gt -15.606644 +ax -15.626281 +ze -15.654564 +wn -15.690171 +nh -15.692469 +dg -15.715653 +xi -15.728801 +gj -15.732816 +mt -15.738384 +uz -15.741593 +nb -15.744570 +fs -15.796521 +lf -15.916837 +bh -15.951645 +uy -15.991269 +nz -16.008344 +uu -16.063343 +hs -16.072442 +tc -16.098718 +lg -16.126102 +wt -16.184581 +kj -16.220775 +aq -16.279503 +xp -16.308642 +dz -16.312349 +mf -16.315533 +zh -16.329950 +dv -16.365811 +fk -16.367646 +kd -16.374272 +nr -16.389291 +uv -16.400894 +cy -16.405599 +gp -16.413918 +td -16.420760 +cp -16.425148 +iq -16.427443 +ox -16.454490 +py -16.484042 +nw -16.509748 +wh -16.536952 +bc -16.544819 +uo -16.576090 +hp -16.613704 +lv -16.616538 +tj -16.627050 +zz -16.633658 +ix -16.645854 +db -16.667616 +hd -16.692469 +cs -16.731634 +qa -16.756903 +xa -16.774810 +pg -16.816809 +q_ -16.821328 +tv -16.824097 +sg -16.856713 +lc -16.889820 +tf -16.944917 +j_ -16.949857 +gd -16.975093 +pj -16.997649 +yb -17.019697 +yl -17.023742 +yr -17.042087 +fg -17.088241 +tg -17.137484 +ym -17.159568 +xe -17.185876 +ww -17.186199 +qi -17.192365 +xy -17.246713 +vr -17.252466 +fp -17.310936 +ux -17.321567 +mh -17.336584 +rz -17.365261 +hg -17.440130 +sq -17.521548 +yw -17.527690 +dt -17.530154 +yp -17.533034 +jr -17.534682 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/isl b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/isl new file mode 100644 index 00000000000..948e51efcfe --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/isl @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 450191469 +#UNIQUE_BIGRAMS 581 +#UNSEEN_LOG_PROB -19.457510 +#EXPECTED_SCORE -7.548306 +r_ -5.419775 +a_ -5.464132 +_s -5.540983 +th -5.590383 +_t -5.772358 +i_ -5.936149 +n_ -5.963761 +_n -6.069543 +d_ -6.083322 +ar -6.092499 +in -6.197144 +_f -6.252673 +_h -6.304614 +he -6.321372 +_v -6.464514 +er -6.556603 +m_ -6.562172 +_a -6.592750 +ve -6.600121 +st -6.605681 +en -6.621801 +nn -6.656460 +an -6.683469 +id -6.690433 +ha -6.711352 +t_ -6.791510 +_m -6.840068 +_e -6.860426 +ra -6.888180 +ur -6.901787 +ei -6.908389 +ir -6.932468 +u_ -6.944972 +ti -6.998839 +_l -7.004388 +ri -7.031727 +na -7.041469 +et -7.085288 +di -7.100614 +ad -7.122802 +ta -7.139843 +da -7.169465 +ae -7.195915 +g_ -7.237452 +ng -7.259809 +um -7.295445 +ke -7.308932 +vi -7.317560 +nd -7.354942 +h_ -7.360656 +se -7.360982 +la -7.401175 +il -7.453894 +al -7.531022 +eg -7.550705 +ga -7.571247 +du -7.579667 +ni -7.611356 +nk -7.618290 +un -7.637456 +_b -7.648484 +or -7.666638 +ma -7.669132 +ki -7.683352 +s_ -7.685870 +le -7.710274 +em -7.711228 +at -7.738229 +_g -7.774242 +tt -7.792613 +va -7.808225 +ja -7.850527 +ed -7.872784 +ns -7.900785 +nu -7.902680 +l_ -7.909278 +o_ -7.911297 +li -7.945944 +nv -7.948547 +_k -7.968587 +sk -8.005976 +me -8.016744 +si -8.022413 +kk -8.062729 +fa -8.098723 +am -8.105037 +rd -8.107423 +ek -8.111183 +sa -8.129021 +nh -8.172464 +ka -8.178604 +ef -8.190843 +jo -8.208583 +re -8.212070 +is -8.218078 +yr -8.259218 +ru -8.274348 +mi -8.278724 +ss -8.294030 +es -8.294645 +tu -8.297505 +ll -8.302310 +af -8.312649 +eo -8.328328 +on -8.348278 +hv -8.364223 +gi -8.390556 +gu -8.395295 +fo -8.404310 +f_ -8.407559 +fn -8.413953 +fr -8.447331 +ag -8.496722 +ik -8.508077 +el -8.510052 +_r -8.575954 +_o -8.608571 +ol -8.613710 +e_ -8.615417 +fy -8.660813 +it -8.667831 +_u -8.705470 +ft -8.724330 +lu -8.729268 +ld -8.729760 +fe -8.735108 +fi -8.753291 +ko -8.778224 +ig -8.794690 +ok -8.806007 +vo -8.852743 +rs -8.858779 +ku -8.859363 +ge -8.865740 +as -8.949026 +fu -8.984041 +od -9.003042 +rn -9.044948 +og -9.052034 +au -9.062421 +ba -9.110139 +sf -9.140012 +hu -9.143343 +sn -9.156859 +to -9.158886 +_d -9.160185 +sl -9.178570 +ak -9.197661 +_i -9.233945 +ot -9.260563 +ho -9.270982 +im -9.294213 +te -9.298025 +kf -9.314505 +yn -9.328342 +ey -9.329483 +sv -9.339233 +lo -9.365118 +rf -9.392526 +rt -9.404002 +ee -9.406451 +os -9.419128 +ud -9.421062 +hf -9.424075 +so -9.468522 +om -9.469391 +k_ -9.471351 +vf -9.473142 +ds -9.490218 +fp -9.533966 +us -9.596385 +lk -9.605748 +gr -9.615099 +of -9.617707 +mu -9.634920 +ne -9.639038 +kr -9.676912 +rk -9.678624 +be -9.717461 +gn -9.781194 +ut -9.784692 +pp -9.785659 +ju -9.787238 +dr -9.791929 +rr -9.792810 +up -9.792915 +su -9.796901 +ug -9.797488 +ro -9.808949 +ul -9.859797 +lv -9.873866 +hg -9.883865 +fl -9.902812 +lh -9.907000 +_j -9.918987 +lt -9.921549 +hi -9.929589 +gs -9.955866 +nt -9.962767 +rg -9.977763 +bo -10.013955 +br -10.040669 +gt -10.047211 +no -10.069369 +tj -10.072430 +tr -10.094102 +kv -10.106697 +hr -10.130650 +hj -10.142435 +mt -10.159666 +gj -10.178056 +kj -10.184940 +lj -10.206812 +gg -10.228329 +sj -10.235346 +fj -10.239927 +mo -10.295336 +ph -10.314564 +_y -10.329042 +_p -10.347090 +if -10.362833 +rl -10.370886 +ls -10.403147 +pa -10.442031 +fs -10.468363 +hl -10.491138 +gd -10.497698 +ms -10.499228 +gl -10.522230 +ah -10.528058 +bu -10.550966 +p_ -10.593951 +lf -10.600529 +go -10.619577 +av -10.658765 +ly -10.686102 +ys -10.694213 +yt -10.697200 +kl -10.697924 +kn -10.763143 +yf -10.763722 +de -10.768902 +rj -10.773051 +ky -10.778000 +sm -10.799417 +my -10.836246 +kt -10.837377 +tl -10.844979 +sg -10.848602 +sp -10.910123 +ip -10.931325 +bi -10.934488 +lg -10.940194 +do -10.958112 +pu -10.973597 +ny -10.980088 +tv -10.985718 +pi -11.008848 +mm -11.026053 +ab -11.046866 +op -11.052402 +pk -11.058137 +fd -11.067473 +uk -11.086609 +yk -11.095080 +dl -11.113188 +by -11.116351 +oh -11.155698 +ov -11.157783 +nl -11.158267 +pt -11.192102 +yg -11.192387 +ks -11.229809 +mb -11.230572 +yl -11.230826 +rm -11.275273 +sy -11.294108 +_c -11.408559 +gh -11.442379 +bl -11.473397 +hy -11.479900 +tn -11.497656 +ua -11.562231 +ep -11.585511 +yj -11.609503 +hs -11.613715 +lm -11.630747 +bj -11.658946 +ry -11.690819 +aa -11.699106 +dv -11.721711 +ev -11.740229 +mk -11.742511 +pr -11.798030 +pl -11.817757 +gm -11.896754 +ap -11.916204 +ml -11.930605 +tf -11.948859 +dn -11.953771 +mj -11.968747 +dd -11.984867 +gv -12.047774 +ui -12.049662 +nf -12.103517 +dj -12.121696 +dh -12.126331 +md -12.150910 +po -12.164498 +pe -12.175663 +ai -12.192964 +yd -12.210400 +mf -12.216197 +ts -12.226742 +db -12.239485 +ty -12.239935 +rv -12.294479 +nm -12.324638 +pv -12.325214 +vm -12.365316 +ym -12.365604 +sb -12.366433 +rb -12.379146 +co -12.383283 +pn -12.433435 +sd -12.477659 +mn -12.481594 +sh -12.518893 +xe -12.533467 +mr -12.534760 +df -12.536834 +mh -12.541373 +y_ -12.545965 +rp -12.573849 +dg -12.609173 +iu -12.610194 +nx -12.626962 +km -12.732117 +ib -12.732117 +rh -12.753335 +gk -12.778017 +io -12.812849 +eu -12.815757 +lb -12.829454 +dm -12.898148 +nj -12.911492 +nb -12.935066 +ob -12.950380 +ex -12.951979 +uv -12.971794 +fb -12.992120 +ej -12.996592 +vr -13.015493 +nr -13.062311 +ps -13.105097 +ec -13.106340 +ic -13.117575 +oo -13.136871 +ct -13.138893 +ji -13.195036 +ax -13.207229 +ou -13.215862 +dt -13.218181 +sr -13.219373 +cl -13.220290 +ce -13.230479 +x_ -13.251014 +tk -13.263219 +fh -13.331410 +lr -13.343018 +eb -13.345317 +ia -13.363610 +eh -13.366585 +dy -13.388549 +ln -13.425304 +xt -13.427527 +fk -13.452815 +ch -13.521321 +dk -13.524943 +b_ -13.533695 +ub -13.567376 +ih -13.635929 +_w -13.639523 +iv -13.642348 +v_ -13.655934 +yp -13.666645 +mv -13.726329 +lp -13.729939 +je -13.749093 +tm -13.763995 +gf -13.810937 +bb -13.841423 +uf -13.894605 +ea -13.958418 +tg -13.964092 +ie -13.997561 +tb -14.064560 +vu -14.089203 +ca -14.090936 +mg -14.192934 +ac -14.210506 +fv -14.262652 +py -14.273019 +oa -14.455729 +fg -14.507036 +hn -14.511146 +ff -14.543916 +rc -14.578232 +bv -14.633117 +ay -14.743852 +fm -14.750020 +gb -14.758788 +c_ -14.786686 +w_ -14.848969 +ox -14.881681 +bh -14.943649 +yo -14.971794 +ew -14.983893 +xa -14.993165 +nc -15.007238 +wa -15.012842 +pd -15.013690 +ow -15.030967 +bk -15.033114 +we -15.043358 +ht -15.105832 +pf -15.108207 +wi -15.126202 +ue -15.133210 +kh -15.135285 +ci -15.171961 +gy -15.245371 +pj -15.261770 +tp -15.301596 +uu -15.302631 +ck -15.323636 +uh -15.340024 +pb -15.340556 +yi -15.419113 +sc -15.488134 +xi -15.540018 +cg -15.566987 +pg -15.586723 +oi -15.670986 +hm -15.730548 +z_ -15.753202 +oe -15.773343 +kd -15.828591 +wo -15.846418 +cr -15.873289 +ii -15.941026 +oc -15.948910 +bs -15.965629 +mp -16.059682 +pm -16.098280 +ez -16.145586 +ux -16.258375 +bm -16.400558 +oy -16.429682 +td -16.513842 +yv -16.710477 +ya -16.819297 +tw -16.928980 +wn -16.982751 +ws -16.982751 +xu -17.050300 +xp -17.135861 +az -17.162881 +uc -17.177057 +nz -17.200999 +iz -17.243629 +xl -17.246616 +bc -17.326530 +za -17.367127 +aj -17.425163 +cs -17.426291 +lc -17.472751 +cc -17.490345 +ye -17.520154 +uo -17.634176 +j_ -17.660491 +jp -17.690003 +yb -17.694075 +zt -17.706359 +fc -17.768684 +mc -17.813011 +wh -17.851904 +cu -17.863320 +tc -17.902828 +cd -17.918621 +bn -17.948302 +zz -17.980263 +_z -18.003654 +kg -18.006183 +uy -18.044657 +vd -18.073538 +wc -18.103912 +yw -18.106623 +kb -18.160062 +rw -18.160062 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ita b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ita new file mode 100644 index 00000000000..ac2cc4397be --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ita @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 474777043 +#UNIQUE_BIGRAMS 583 +#UNSEEN_LOG_PROB -20.048920 +#EXPECTED_SCORE -7.247035 +e_ -4.773442 +o_ -4.903442 +a_ -5.046351 +i_ -5.535118 +_c -5.780482 +on -5.937535 +_p -5.958144 +_s -5.962892 +er -6.072132 +_d -6.094868 +re -6.184733 +co -6.281615 +ta -6.285398 +to -6.326718 +en -6.449475 +ri -6.452346 +te -6.478116 +ra -6.523963 +de -6.545138 +an -6.553427 +_a -6.562827 +nt -6.567004 +el -6.597716 +at -6.607893 +al -6.661937 +ne -6.662425 +ti -6.670702 +io -6.728158 +no -6.746725 +st -6.776112 +or -6.843831 +ar -6.847869 +ll -6.879020 +li -6.911644 +in -6.927245 +es -6.971113 +la -7.048340 +le -7.100471 +ro -7.116023 +ia -7.155670 +tt -7.159104 +_n -7.179340 +_m -7.188373 +na -7.202943 +ch -7.254262 +me -7.264025 +tr -7.269382 +ca -7.302698 +pe -7.309029 +si -7.330671 +_r -7.362493 +it -7.362845 +se -7.368222 +ic -7.378324 +_t -7.399883 +pr -7.409201 +ni -7.418766 +ol -7.467592 +l_ -7.469982 +so -7.483379 +_f -7.545126 +po -7.548127 +di -7.550248 +ma -7.613408 +he -7.622231 +ci -7.655867 +ve -7.731243 +ss -7.744406 +is -7.774898 +_g -7.790740 +_v -7.790902 +zi -7.791874 +un -7.849942 +n_ -7.855879 +nd -7.869289 +_i -7.913204 +os -7.920113 +do -7.924155 +gi -7.946551 +_e -8.012591 +pa -8.017989 +ce -8.035707 +qu -8.042835 +et -8.045204 +lo -8.069235 +vi -8.085092 +r_ -8.086460 +om -8.087289 +ie -8.106816 +am -8.120220 +mo -8.159327 +sa -8.171457 +_l -8.173427 +da -8.186251 +_u -8.197201 +su -8.249194 +va -8.282283 +im -8.285745 +sc -8.295946 +as -8.302094 +rt -8.328203 +mi -8.333735 +pi -8.336570 +_q -8.374058 +fi -8.493241 +ue -8.494144 +iv -8.535477 +ua -8.538239 +ut -8.543671 +il -8.547128 +eg -8.548594 +nc -8.560538 +gl -8.577689 +tu -8.581236 +_b -8.630224 +ag -8.632005 +_o -8.644417 +lt -8.665106 +az -8.665809 +mp -8.710527 +ot -8.737053 +em -8.749517 +ur -8.753135 +uo -8.816039 +ec -8.821648 +vo -8.860749 +av -8.885055 +ov -8.891864 +op -8.892095 +cc -8.906008 +nn -8.921832 +sp -8.975088 +za -8.988831 +bi -9.026342 +rs -9.115007 +iu -9.119285 +oc -9.128596 +ui -9.149727 +ed -9.155756 +hi -9.156770 +ac -9.172933 +nz -9.184957 +ir -9.217655 +iz -9.264387 +ge -9.265610 +cu -9.301439 +og -9.314863 +gg -9.316549 +ei -9.341004 +ig -9.368747 +fo -9.369698 +fa -9.370671 +ap -9.409723 +us -9.437887 +gr -9.462735 +fe -9.469179 +pp -9.481445 +ul -9.488105 +ad -9.491329 +du -9.491883 +ev -9.493319 +lu -9.496022 +rr -9.505008 +ns -9.508791 +id -9.514678 +rm -9.516981 +zz -9.550913 +ba -9.562357 +ga -9.610195 +gn -9.624091 +nu -9.625223 +rn -9.630389 +rc -9.663651 +go -9.737114 +ng -9.767099 +be -9.779178 +od -9.825882 +pu -9.881146 +gu -9.920394 +if -9.949030 +bb -9.976113 +ea -9.996064 +br -9.997800 +cr -10.001256 +rd -10.042029 +ru -10.044418 +ip -10.092553 +ab -10.103420 +u_ -10.105480 +ff -10.108507 +fr -10.139814 +mb -10.140040 +ib -10.161241 +ai -10.203913 +um -10.418382 +mm -10.452528 +mu -10.472707 +up -10.500707 +ez -10.509699 +nf -10.510891 +oi -10.527776 +uc -10.564518 +s_ -10.568661 +au -10.657390 +cl -10.724365 +t_ -10.731028 +lm -10.733602 +eb -10.745358 +bo -10.780449 +zo -10.783585 +dr -10.795239 +fu -10.803682 +pl -10.846726 +ud -10.860541 +bl -10.874119 +d_ -10.882396 +af -10.893405 +_h -10.916460 +lc -10.922433 +ub -10.925781 +ha -10.960116 +rg -11.001866 +ob -11.109161 +nv -11.131265 +bu -11.134794 +rv -11.148193 +eo -11.167305 +of -11.219933 +y_ -11.227811 +rl -11.305859 +uz -11.347527 +ug -11.349929 +ze -11.354130 +ep -11.402995 +ef -11.417027 +m_ -11.551258 +vv -11.679700 +rz -11.681725 +_j -11.856271 +ae -11.858052 +gh -11.862186 +nq -11.903556 +sv -11.906376 +sf -12.013007 +_w -12.101068 +sq -12.127270 +eu -12.147021 +k_ -12.202412 +vu -12.222544 +vr -12.226937 +ho -12.290319 +rp -12.381336 +g_ -12.399395 +th -12.400397 +ld -12.432472 +sm -12.469856 +dd -12.471235 +ou -12.535359 +_z -12.537634 +lv -12.548188 +ee -12.558691 +h_ -12.566153 +oo -12.571562 +fl -12.694882 +uf -12.780716 +rb -12.793560 +ii -12.806476 +cn -12.874718 +cq -12.918605 +lp -12.969049 +ck -12.983594 +sh -13.004567 +ls -13.009370 +lg -13.023216 +oz -13.025621 +ke -13.075478 +wa -13.096803 +oe -13.102217 +_k -13.124106 +eq -13.125870 +b_ -13.201854 +p_ -13.205408 +_y -13.255967 +rf -13.264613 +lb -13.283819 +c_ -13.308238 +jo -13.346562 +ay -13.390622 +we -13.508198 +ja -13.514620 +w_ -13.515083 +ow -13.589804 +rk -13.598485 +x_ -13.602750 +ry -13.701789 +sl -13.739113 +wi -13.776617 +ya -13.791318 +ao -13.823468 +ki -13.839280 +_x -13.861406 +ew -13.882545 +ah -13.936550 +oa -13.991219 +yo -14.008794 +f_ -14.043879 +ey -14.144240 +lf -14.242299 +sb -14.252463 +nk -14.260671 +ty -14.263896 +ok -14.295320 +ju -14.302486 +ex -14.308639 +ph -14.331135 +z_ -14.331761 +tl -14.356344 +wo -14.360300 +ct -14.417799 +je -14.422062 +ly -14.437756 +oh -14.450218 +ka -14.472495 +ps -14.497440 +xi -14.508375 +hn -14.554572 +nl -14.629304 +ak -14.669281 +ft -14.681047 +ny -14.698311 +tw -14.754736 +lz -14.811186 +zu -14.818279 +uv -14.911470 +hu -14.919735 +tc -14.919923 +tn -14.978950 +tm -14.988895 +ts -15.028665 +dy -15.063931 +ix -15.075951 +nr -15.090084 +ik -15.138255 +xv -15.142645 +oy -15.155452 +ax -15.195711 +ht -15.247845 +sk -15.287157 +ds -15.292878 +aw -15.315252 +dl -15.352270 +ms -15.367733 +ws -15.405745 +pd -15.411694 +iq -15.414345 +by -15.431431 +wn -15.438161 +v_ -15.454441 +ox -15.515189 +dn -15.532512 +hr -15.557939 +my -15.582331 +ky -15.592354 +sr -15.599729 +cy -15.634704 +yl -15.693875 +ys -15.719551 +ks -15.722013 +ye -15.752051 +sy -15.763330 +ko -15.850772 +dq -15.875769 +sg -15.969365 +dv -16.055939 +gm -16.057182 +lk -16.110793 +ek -16.117475 +xe -16.120935 +sn -16.135737 +dw -16.136174 +dg -16.167368 +xt -16.245718 +ji -16.272890 +pt -16.335337 +sw -16.377919 +ln -16.407725 +bs -16.443567 +kr -16.455260 +yn -16.469804 +kl -16.485889 +yd -16.591454 +ux -16.600484 +vd -16.614136 +xa -16.635323 +aq -16.636870 +nh -16.664381 +pc -16.695003 +gt -16.722341 +wr -16.725630 +gy -16.742857 +cs -16.789940 +gs -16.875403 +ww -16.877231 +wh -16.891938 +ku -16.900091 +bn -16.976009 +sd -16.996126 +nb -17.071131 +hm -17.086696 +bc -17.113160 +uk -17.124404 +tb -17.182430 +pz -17.185144 +xp -17.186957 +tz -17.227417 +tp -17.234429 +fg -17.245246 +hy -17.269046 +aj -17.319843 +ml -17.321335 +kh -17.341885 +bm -17.375592 +dh -17.380768 +oq -17.422863 +dm -17.428212 +kk -17.509792 +cd -17.524040 +vy -17.539008 +cg -17.543064 +xx -17.544225 +py -17.561755 +hd -17.590254 +nj -17.596263 +pn -17.605323 +yw -17.657768 +mc -17.671658 +yu -17.706981 +kn -17.720043 +gb -17.720699 +yp -17.757932 +q_ -17.774869 +aa -17.812147 +mw -17.821267 +mf -17.824085 +kt -17.851131 +eh -17.851850 +hl -17.870662 +ym -17.888247 +ej -17.897862 +oj -17.946926 +yr -17.959263 +gp -17.967028 +lh -18.004093 +xy -18.022584 +ij -18.093904 +uy -18.116179 +nm -18.117907 +df -18.149366 +cf -18.184239 +wl -18.204289 +dc -18.216270 +yt -18.249975 +hz -18.256621 +np -18.310922 +hs -18.314880 +wt -18.366321 +rq -18.368376 +iw -18.379731 +wy -18.406933 +lq -18.430358 +wf -18.510927 +yb -18.523467 +zy -18.541904 +mt -18.552380 +qa -18.554718 +wk -18.575934 +fc -18.584270 +pm -18.645255 +fy -18.670390 +cp -18.710235 +wb -18.719387 +j_ -18.720699 +vl -18.725960 +rx -18.741857 +tv -18.786501 +kf -18.789252 +rw -18.790629 +yc -18.801695 +cb -18.815648 +hw -18.833990 +fs -18.843964 +dz -18.894897 +zh -18.932411 +td -18.972488 +xb -19.013711 +lw -19.026635 +gc -19.039677 +xf -19.086273 +bh -19.132677 +yi -19.144955 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/jav b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/jav new file mode 100644 index 00000000000..83cd3c8cfb7 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/jav @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 8459939 +#UNIQUE_BIGRAMS 543 +#UNSEEN_LOG_PROB -19.162356 +#EXPECTED_SCORE -7.155660 +an -4.511530 +ng -4.569365 +a_ -5.232227 +n_ -5.285911 +g_ -5.339190 +ka -5.470749 +in -5.516376 +_k -5.855817 +_s -5.907801 +i_ -6.034746 +_i -6.107769 +sa -6.149155 +en -6.194683 +_p -6.270275 +un -6.325524 +la -6.415720 +ra -6.461394 +_d -6.477807 +ar -6.501498 +e_ -6.583545 +ak -6.592387 +ta -6.616832 +ga -6.663419 +na -6.675117 +_m -6.754846 +_a -6.772599 +ik -6.776276 +si -6.818479 +_l -6.834407 +al -6.872145 +_n -6.907003 +ma -7.002671 +_b -7.068144 +ni -7.077949 +_t -7.132465 +ba -7.132872 +pa -7.138748 +h_ -7.196382 +as -7.207681 +pu -7.231958 +at -7.235680 +di -7.264652 +er -7.264652 +ke -7.321208 +te -7.339570 +on -7.360071 +is -7.370221 +am -7.385395 +u_ -7.390107 +li -7.421687 +ne -7.450331 +s_ -7.481413 +ah -7.485228 +nt -7.503678 +ya -7.538003 +wa -7.554995 +ku -7.571282 +ad -7.576448 +r_ -7.577360 +_w -7.578956 +da -7.585886 +t_ -7.617619 +aw -7.668787 +ha -7.691697 +gg -7.722269 +dh -7.810436 +se -7.850651 +ri -7.856054 +k_ -7.863263 +ge -7.872065 +ki -7.895466 +l_ -7.903161 +gk -7.943438 +_u -7.945664 +wi -7.950128 +ti -7.965859 +nd -7.989891 +ur -8.016051 +pe -8.043189 +ut -8.047649 +gi -8.086152 +ip -8.136275 +tu -8.163153 +we -8.173307 +it -8.177745 +_j -8.242533 +o_ -8.284774 +_g -8.310097 +ek -8.343552 +em -8.349659 +me -8.371519 +ja -8.372253 +ro -8.412303 +re -8.414280 +mi -8.423150 +bu -8.428663 +de -8.436145 +mb -8.462251 +es -8.485961 +ia -8.500154 +pi -8.543337 +et -8.546713 +du -8.563583 +_r -8.578500 +hi -8.620972 +ag -8.626556 +il -8.637176 +or -8.641801 +el -8.645689 +m_ -8.654870 +ul -8.663764 +be -8.690992 +ap -8.702526 +ab -8.730286 +us -8.737110 +mu -8.769860 +um -8.796000 +le -8.805507 +ih -8.812697 +su -8.838539 +th -8.844327 +ca -8.847387 +uk -8.860406 +he -8.864295 +go -8.865806 +gu -8.878752 +uw -8.881726 +au -8.905571 +ru -8.920780 +ko -8.952787 +_c -8.973983 +ir -9.025396 +ay -9.029490 +bi -9.037353 +do -9.045080 +iy -9.051399 +lu -9.070985 +st -9.112104 +eb -9.151711 +ny -9.160174 +ed -9.166334 +wo -9.193434 +os -9.199036 +je -9.200339 +mp -9.252536 +ug -9.281852 +_o -9.329550 +ew -9.348102 +ep -9.367008 +ji -9.375365 +ju -9.391309 +lo -9.391538 +aj -9.392225 +pr -9.412884 +to -9.421863 +tr -9.453078 +ol -9.497009 +nu -9.540033 +ai -9.613339 +p_ -9.650272 +nj -9.669724 +om -9.697633 +im -9.719894 +hu -9.749093 +d_ -9.750121 +_e -9.769488 +eh -9.772021 +ok -9.834641 +rt -9.841195 +_f -9.847780 +ot -9.850769 +iw -9.865488 +ij -9.870747 +up -9.883094 +ud -9.894410 +eg -9.904672 +rg -9.942430 +id -9.972097 +bo -9.981894 +_h -9.986211 +ks -10.102135 +ig -10.176363 +wu -10.176560 +ac -10.193035 +ye -10.217394 +_y -10.218816 +ns -10.261300 +po -10.294539 +kt -10.307232 +gr -10.316553 +mo -10.376724 +ua -10.383087 +no -10.402806 +io -10.408821 +yu -10.440700 +ci -10.442835 +uh -10.445686 +fi -10.495039 +rn -10.565391 +gs -10.573684 +nc -10.610803 +ib -10.626084 +od -10.738129 +ce -10.794561 +op -10.856765 +so -10.864375 +og -10.917149 +ho -10.956933 +rs -10.998544 +kr -11.034578 +lm -11.051490 +uj -11.061295 +ub -11.070435 +b_ -11.102698 +aa -11.107205 +kl -11.152681 +ob -11.213744 +ea -11.234139 +ae -11.291117 +ej -11.322655 +_v -11.336259 +rm -11.362062 +vi -11.414629 +jr -11.436204 +ie -11.439042 +ve -11.446638 +yo -11.489634 +if -11.509384 +gl -11.576025 +sm -11.590677 +y_ -11.618289 +br -11.671809 +ey -11.711863 +ry -11.712435 +sr -11.727970 +rb -11.737255 +fe -11.799112 +fa -11.805202 +sp -11.825483 +f_ -11.826102 +rd -11.855501 +ic -11.890035 +oh -11.893924 +rk -11.894573 +dr -11.972611 +cu -11.976729 +co -11.976729 +ml -12.009401 +yi -12.042829 +sl -12.048597 +jo -12.085179 +iv -12.103073 +af -12.106829 +fo -12.115883 +ch -12.118156 +sw -12.129573 +iu -12.141851 +kh -12.220239 +ll -12.221867 +eo -12.230037 +va -12.253160 +sk -12.257328 +np -12.297112 +pl -12.300549 +mr -12.350438 +ec -12.374685 +rl -12.407663 +ov -12.415094 +ow -12.471119 +pt -12.474997 +tl -12.547670 +rj -12.557917 +oc -12.591203 +oy -12.632837 +ei -12.640439 +bl -12.675709 +uc -12.708435 +kn -12.765475 +lt -12.773811 +sh -12.775006 +sy -12.776202 +ss -12.841039 +ui -12.851084 +eu -12.854869 +ao -12.906307 +za -12.930067 +ou -12.939413 +_z -12.969189 +lb -13.013625 +dy -13.042110 +fr -13.063849 +kd -13.069701 +ef -13.093353 +ii -13.135699 +ky -13.144937 +nf -13.160467 +oa -13.165158 +ev -13.195232 +rw -13.216176 +nk -13.255660 +vo -13.275814 +ue -13.320472 +sn -13.354004 +gh -13.403037 +hy -13.416026 +fu -13.467251 +rc -13.486695 +hl -13.571347 +nn -13.632837 +of -13.637176 +oo -13.752473 +ty -13.771424 +mt -13.800328 +oe -13.817459 +c_ -13.822391 +av -13.875225 +ls -13.890682 +nw -13.898474 +fl -13.919459 +w_ -13.924753 +ps -13.932731 +nb -13.940753 +ts -13.951520 +ck -13.959648 +uy -13.978793 +gy -14.012216 +tn -14.043549 +az -14.043549 +kw -14.052214 +sj -14.072637 +ld -14.072637 +mm -14.084438 +x_ -14.099326 +tt -14.108334 +sc -14.117398 +ly -14.132633 +oj -14.135699 +zi -14.154235 +rp -14.214554 +lk -14.264023 +wr -14.287702 +hm -14.287702 +uf -14.336259 +rv -14.339790 +nl -14.364757 +ee -14.368360 +oi -14.404886 +ct -14.427253 +nr -14.442360 +ms -14.442360 +nm -14.457627 +gn -14.473057 +rr -14.508390 +ph -14.520363 +dd -14.573424 +nz -14.577588 +bj -14.590151 +by -14.594363 +ys -14.607074 +ht -14.628512 +bd -14.672366 +ft -14.708435 +hn -14.769042 +z_ -14.769042 +ff -14.778596 +dm -14.793047 +km -14.793047 +cl -14.877790 +rh -14.914184 +gm -14.962367 +iz -15.000989 +bn -15.006591 +cr -15.006591 +zo -15.023531 +bb -15.040672 +qu -15.087403 +dw -15.093353 +sf -15.141851 +bs -15.141851 +dn -15.154235 +my -15.204861 +ds -15.291117 +_q -15.297970 +pp -15.311776 +nv -15.368360 +ex -15.404886 +ze -15.412303 +v_ -15.449973 +ox -15.465321 +tm -15.504421 +dp -15.528400 +rf -15.552784 +ez -15.611336 +tw -15.611336 +zh -15.619898 +xi -15.637176 +hr -15.637176 +sb -15.699333 +lp -15.736091 +hk -15.736091 +bh -15.773811 +ux -15.783397 +tb -15.783397 +lf -15.783397 +_x -15.802762 +fk -15.812543 +lg -15.812543 +tv -15.903691 +kk -15.903691 +wh -15.935400 +mn -15.967822 +yr -16.000989 +lj -16.012216 +pb -16.023531 +yl -16.023531 +gj -16.069701 +gd -16.117398 +tz -16.141851 +kj -16.179326 +jl -16.217800 +kb -16.217800 +nh -16.270749 +xa -16.311776 +dj -16.325715 +hw -16.412303 +ix -16.473057 +uo -16.536482 +ax -16.552784 +lh -16.637176 +ws -16.654664 +zu -16.654664 +wn -16.672366 +tk -16.690288 +bc -16.726814 +wt -16.764288 +gc -16.822391 +tp -16.822391 +qi -16.822391 +yd -16.842291 +vy -16.862469 +uv -16.924753 +lr -17.012216 +lc -17.058020 +py -17.129573 +fs -17.154235 +dl -17.179326 +sq -17.204861 +mf -17.204861 +zz -17.230856 +dg -17.311776 +vu -17.339790 +aq -17.339790 +sd -17.427253 +yn -17.427253 +wy -17.457627 +oz -17.457627 +bk -17.457627 +pd -17.520363 +yb -17.552784 +pk -17.585951 +cy -17.585951 +lv -17.585951 +gt -17.654664 +yw -17.654664 +mk -17.726814 +qa -17.764288 +dv -17.802762 +vd -17.802762 +xv -17.802762 +dt -17.802762 +wl -17.842291 +xf -17.882933 +md -17.882933 +uz -17.882933 +cn -17.924753 +vr -17.924753 +iq -17.924753 +hs -17.924753 +dk -18.012216 +yp -18.012216 +tf -18.058020 +yt -18.058020 +mc -18.105325 +ww -18.154235 +bt -18.154235 +pm -18.154235 +cq -18.204861 +sg -18.204861 +j_ -18.204861 +bp -18.257328 +fg -18.368360 +hd -18.368360 +mh -18.368360 +ym -18.368360 +mg -18.427253 +ln -18.427253 +hf -18.488654 +tg -18.488654 +pn -18.488654 +tj -18.488654 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/jpn b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/jpn new file mode 100644 index 00000000000..551a0a4f287 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/jpn @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 77832878 +#UNIQUE_BIGRAMS 29182 +#UNSEEN_LOG_PROB -16.452637 +#EXPECTED_SCORE -10.946627 +の_ -5.674113 +_の -5.771118 +て_ -5.855409 +_て -5.917677 +_か -6.225042 +_た -6.357486 +か_ -6.366998 +る_ -6.385893 +_に -6.402564 +は_ -6.499134 +_を -6.508482 +に_ -6.511634 +を_ -6.515686 +_は -6.516378 +た_ -6.540248 +と_ -6.558215 +し_ -6.586982 +_と -6.731057 +_し -6.815888 +_な -6.934824 +い_ -7.005773 +ン_ -7.083386 +_い -7.170451 +ト_ -7.376461 +_ン -7.720439 +れ_ -7.767229 +ス_ -7.784752 +っ_ -7.830826 +_ま -7.924727 +_こ -7.939334 +_フ -7.942368 +_す -7.950396 +_イ -7.964180 +_も -7.965166 +り_ -7.974283 +う_ -8.018595 +も_ -8.051790 +ル_ -8.088059 +ら_ -8.108924 +イ_ -8.148797 +す_ -8.165554 +ク_ -8.219821 +ッ_ -8.266160 +_シ -8.269239 +_ッ -8.274854 +_テ -8.296743 +_れ -8.298481 +_さ -8.301203 +フ_ -8.311431 +する -8.357864 +き_ -8.451157 +な_ -8.484236 +_ス -8.501363 +シ_ -8.508507 +さ_ -8.569774 +_ト -8.621127 +け_ -8.691663 +いる -8.712739 +く_ -8.729116 +_リ -8.774979 +_あ -8.787685 +_ク -8.812322 +リ_ -8.816962 +タ_ -8.820528 +_日 -8.833855 +_よ -8.845054 +_ラ -8.850323 +_ハ -8.874600 +ラ_ -8.903397 +テ_ -8.944722 +から -9.026505 +_や -9.048430 +こと -9.049735 +_ア -9.054754 +_サ -9.069238 +_タ -9.077676 +日_ -9.107262 +め_ -9.119283 +ん_ -9.133892 +え_ -9.221049 +_き -9.231507 +_そ -9.241547 +や_ -9.280508 +_コ -9.343199 +_ヒ -9.390858 +_レ -9.408888 +ない -9.447775 +年_ -9.451118 +なと -9.468806 +ます -9.487405 +_ら -9.488816 +ア_ -9.543690 +_年 -9.553449 +用_ -9.566671 +_ロ -9.569457 +_お -9.580498 +月_ -9.595706 +いう -9.613005 +ロ_ -9.655022 +つ_ -9.660307 +_カ -9.663956 +人_ -9.668610 +_つ -9.671600 +_ル -9.680030 +ィ_ -9.681277 +ハ_ -9.688478 +_メ -9.688983 +_ホ -9.692613 +レ_ -9.692797 +ム_ -9.695484 +ヒ_ -9.695638 +_月 -9.707445 +ま_ -9.762923 +ット -9.774500 +_人 -9.789612 +_大 -9.813164 +_同 -9.819280 +てき -9.834278 +ある -9.867276 +れる -9.896976 +イン -9.903778 +ント -9.908260 +e_ -9.921519 +して -9.924074 +_う -9.957797 +サ_ -9.973104 +_ヘ -9.977619 +能_ -9.996070 +ュ_ -10.008579 +_っ -10.017466 +_ウ -10.024727 +わ_ -10.027046 +_セ -10.029795 +_マ -10.034745 +ョ_ -10.059830 +み_ -10.060918 +者_ -10.062919 +_ィ -10.072029 +_中 -10.103903 +_ョ -10.105005 +てす -10.106659 +とい -10.113460 +_発 -10.120562 +_キ -10.126020 +ショ -10.126559 +ティ -10.126600 +_一 -10.137645 +せ_ -10.138585 +_時 -10.171597 +社_ -10.179034 +_ケ -10.185301 +_わ -10.194329 +_対 -10.205776 +よ_ -10.224482 +へ_ -10.233224 +ち_ -10.238096 +本_ -10.241839 +_会 -10.248836 +上_ -10.261021 +_機 -10.262614 +こ_ -10.267770 +間_ -10.268776 +定_ -10.276020 +業_ -10.296761 +_上 -10.313434 +きる -10.319272 +_オ -10.322900 +_出 -10.327370 +的_ -10.336544 +ャ_ -10.337192 +ック -10.350779 +_今 -10.370372 +_ュ -10.372607 +_開 -10.375879 +よう -10.378689 +_ネ -10.378739 +_ソ -10.383534 +_ニ -10.388669 +とし -10.391007 +_ん -10.397943 +_自 -10.406018 +ョン -10.414064 +ウ_ -10.417355 +_く -10.417456 +なっ -10.429088 +_モ -10.449316 +_円 -10.456278 +ンク -10.457841 +円_ -10.470725 +_者 -10.478585 +_け -10.482398 +合_ -10.488563 +サイ -10.490029 +なる -10.491629 +カ_ -10.492804 +_チ -10.493098 +スト -10.503985 +_エ -10.514656 +ェ_ -10.515009 +_最 -10.527540 +_へ -10.540291 +s_ -10.546183 +ろ_ -10.547430 +コ_ -10.557591 +_ャ -10.560723 +_り -10.569964 +_ェ -10.579547 +_高 -10.582387 +イト -10.589795 +ニ_ -10.590252 +_新 -10.594058 +_見 -10.610511 +この -10.612338 +性_ -10.615475 +分_ -10.630177 +国_ -10.632852 +_行 -10.633529 +品_ -10.652469 +_s -10.672326 +_的 -10.677356 +_実 -10.683348 +その -10.684049 +_ほ -10.689273 +_本 -10.689886 +_え -10.696822 +化_ -10.698639 +会_ -10.711200 +ライ -10.724812 +チ_ -10.736972 +場_ -10.741059 +_万 -10.747800 +ッフ -10.753420 +後_ -10.758517 +メ_ -10.762311 +ナ_ -10.766244 +約_ -10.773267 +表_ -10.782346 +キ_ -10.787939 +フラ -10.788201 +万_ -10.788857 +_使 -10.791844 +また -10.803558 +ネ_ -10.819514 +ため -10.833179 +ケ_ -10.835886 +フロ -10.836666 +まて -10.841657 +回_ -10.843564 +コン -10.847998 +マ_ -10.854573 +度_ -10.859110 +_取 -10.860350 +中_ -10.871489 +_ち -10.874096 +_分 -10.875488 +ホ_ -10.876498 +_利 -10.878067 +時_ -10.884781 +お_ -10.887096 +_み -10.889028 +_以 -10.893569 +セ_ -10.896534 +_通 -10.900285 +_市 -10.919004 +日本 -10.940664 +方_ -10.944384 +_入 -10.946431 +ヘ_ -10.947200 +_w -10.953214 +_ミ -10.959291 +ヒス -10.962394 +もの -10.965059 +_事 -10.967692 +数_ -10.969067 +_せ -10.988782 +ンタ -10.994859 +サヒ -10.997585 +_め -10.998078 +によ -11.001722 +力_ -11.003281 +_前 -11.010184 +かっ -11.015009 +er -11.015316 +より -11.015930 +_作 -11.017812 +_電 -11.023512 +出_ -11.025597 +動_ -11.030590 +機能 -11.034473 +いて -11.044969 +内_ -11.048341 +_m -11.051642 +長_ -11.053611 +られ -11.056214 +_提 -11.059533 +大_ -11.061433 +_ワ -11.063296 +_国 -11.068900 +入_ -11.070095 +これ -11.076765 +つい -11.082019 +_可 -11.084835 +なか -11.085922 +行_ -11.089795 +売_ -11.112762 +まし -11.124219 +in -11.125295 +あり -11.125544 +体_ -11.125999 +利用 -11.132559 +理_ -11.153138 +t_ -11.166923 +それ -11.179464 +部_ -11.180539 +_ナ -11.189256 +_方 -11.194546 +手_ -11.195241 +_ユ -11.200510 +前_ -11.209085 +さん -11.210402 +_向 -11.211237 +ンス -11.214273 +タイ -11.214933 +_動 -11.216785 +可能 -11.217756 +える -11.221469 +_c -11.224438 +員_ -11.230215 +_約 -11.230794 +しい -11.242602 +_関 -11.242827 +_多 -11.244850 +報_ -11.254101 +よる -11.255641 +なく -11.258363 +事_ -11.258408 +同_ -11.260089 +n_ -11.266376 +_製 -11.271362 +_全 -11.271408 +_現 -11.276457 +_化 -11.277146 +_性 -11.279218 +要_ -11.286469 +_a -11.287488 +信_ -11.290456 +金_ -11.290549 +_地 -11.290874 +ミ_ -11.308771 +ひ_ -11.310089 +ラン -11.312255 +d_ -11.312491 +_回 -11.317828 +氏_ -11.320101 +につ -11.322615 +新_ -11.333910 +_i -11.348433 +_場 -11.355652 +_社 -11.357499 +トル -11.360128 +_内 -11.365596 +a_ -11.366036 +イル -11.367945 +話_ -11.370839 +気_ -11.377382 +発表 -11.379257 +画_ -11.380986 +ステ -11.383015 +リス -11.385988 +発_ -11.386583 +_東 -11.393997 +フレ -11.403405 +たい -11.404209 +_合 -11.407780 +_目 -11.408485 +ァ_ -11.409342 +_氏 -11.409594 +_手 -11.411663 +_受 -11.412168 +子_ -11.413836 +_特 -11.417989 +フト -11.420476 +on -11.422510 +_予 -11.425158 +生_ -11.433183 +フリ -11.437752 +_生 -11.437803 +_公 -11.439501 +応_ -11.440892 +供_ -11.447658 +r_ -11.448798 +たっ -11.453520 +_情 -11.458311 +o_ -11.465788 +式_ -11.467782 +_用 -11.481338 +スタ -11.482027 +ファ -11.484521 +_p -11.487871 +_選 -11.497861 +ける -11.503447 +情報 -11.514141 +_思 -11.516692 +点_ -11.519573 +_ァ -11.520008 +なり -11.522733 +格_ -11.523824 +ツ_ -11.528361 +たけ -11.529073 +_不 -11.529785 +って -11.531540 +_部 -11.532967 +_検 -11.546266 +ハイ -11.551375 +対応 -11.557731 +ネッ -11.562713 +シャ -11.563161 +目_ -11.573292 +_g -11.573292 +提供 -11.573970 +期_ -11.581108 +_込 -11.582472 +ユサ -11.586513 +クラ -11.587255 +オ_ -11.592626 +おり -11.597787 +ォ_ -11.598189 +_米 -11.615882 +込_ -11.618561 +_表 -11.622529 +_ォ -11.622763 +re -11.624342 +下_ -11.626626 +たり -11.629207 +ne -11.640229 +ワ_ -11.649370 +作_ -11.652409 +_企 -11.653424 +フル -11.653663 +te -11.654499 +ンテ -11.655157 +ソ_ -11.657131 +_調 -11.657430 +トラ -11.658628 +_世 -11.661808 +_数 -11.661928 +_無 -11.663791 +_後 -11.667464 +モ_ -11.679209 +イス -11.685850 +_下 -11.687072 +見_ -11.689335 +le -11.692031 +市_ -11.696761 +_当 -11.700642 +oo -11.708685 +_記 -11.709306 +面_ -11.717147 +_指 -11.718146 +載_ -11.719583 +_経 -11.720771 +_保 -11.733212 +_設 -11.734791 +l_ -11.737763 +_f -11.738966 +向け -11.739410 +_d -11.744425 +会社 -11.749203 +京_ -11.750671 +or -11.755726 +_代 -11.757843 +時間 -11.761056 +_n -11.763181 +シス -11.765437 +b_ -11.772942 +_広 -11.775149 +_必 -11.776124 +テル -11.776319 +界_ -11.776970 +_感 -11.777816 +成_ -11.780421 +ょ_ -11.783489 +ソフ -11.790301 +_付 -11.792075 +題_ -11.794970 +_主 -11.800711 +務_ -11.801174 +_強 -11.802365 +_連 -11.805016 +_持 -11.805945 +an -11.806675 +今回 -11.809733 +_金 -11.810465 +リン -11.815801 +_言 -11.816737 +_変 -11.817472 +明_ -11.819883 +_ひ -11.822767 +フォ -11.824178 +のて -11.825523 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kan b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kan new file mode 100644 index 00000000000..ad243b3bc41 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kan @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 5216409 +#UNIQUE_BIGRAMS 1215 +#UNSEEN_LOG_PROB -15.393596 +#EXPECTED_SCORE -7.718819 +ದ_ -4.355257 +ರ_ -4.867123 +_ಅ -5.437365 +ಗ_ -5.534150 +_ಮ -5.543536 +_ಕ -5.667013 +ವ_ -5.713479 +_ಹ -5.725150 +_ಸ -5.975315 +_ನ -6.078069 +ನ_ -6.145346 +ಳ_ -6.174655 +_ಬ -6.196658 +ಗಳ -6.223066 +ದರ -6.237914 +ಯ_ -6.245428 +ತ_ -6.331075 +_ಇ -6.338039 +ಲ_ -6.381880 +ವರ -6.476677 +ಕ_ -6.601481 +_ಎ -6.712739 +_ಆ -6.887771 +_ವ -6.897663 +ಅವ -6.966658 +_ತ -7.022484 +_ಒ -7.085882 +ಡ_ -7.183010 +_ಪ -7.266564 +ಇದ -7.311459 +ಹಗ -7.315859 +ಒದ -7.318152 +ತರ -7.345779 +_ಜ -7.419618 +ಕಡ -7.464439 +ವಗ -7.473798 +ರದ -7.490164 +ಕರ -7.523005 +_ರ -7.529326 +ಮಲ -7.536600 +ರವ -7.554789 +_ಗ -7.577854 +ರಗ -7.591178 +ಸರ -7.598610 +ಹದ -7.614294 +ಕಲ -7.632454 +_ದ -7.698766 +ವದ -7.731425 +ರತ -7.743584 +ಲವ -7.818583 +ಸದ -7.828859 +ಮರ -7.837247 +ಡದ -7.838449 +ಮದ -7.843903 +ನಡ -7.866316 +ಮಡ -7.877134 +_ಭ -7.904240 +ನಗ -7.929977 +ಎದ -7.931799 +ರಣ -7.972204 +ಸ_ -7.997566 +ಗದ -8.117025 +ಅದ -8.158542 +ಗರ -8.201616 +ಬಳ -8.220383 +ನಯ -8.237309 +ಸವ -8.237309 +ಬದ -8.264182 +ಪರ -8.269463 +ದಲ -8.281375 +ವನ -8.301565 +ನತ -8.306109 +ದದ -8.323724 +ಡವ -8.331632 +ಸಲ -8.332434 +ಮನ -8.338957 +ಹಸ -8.356073 +ಅಥ -8.360884 +ಥವ -8.375965 +ಹರ -8.387588 +ಸಮ -8.389999 +ಬರ -8.405858 +ಯವ -8.427119 +ಎಬ -8.434085 +ಬ_ -8.474438 +ಳಗ -8.480451 +ಜನ -8.493350 +ಲಗ -8.511302 +ಬಹ -8.515344 +ನದ -8.518485 +ನರ -8.519702 +ರಸ -8.524277 +ದವ -8.549443 +ದಗ -8.557131 +ತಯ -8.562454 +_ಯ -8.595130 +ಣ_ -8.597806 +ತಗ -8.628563 +ರಕ -8.630096 +ನಕ -8.645076 +ರಡ -8.658201 +ಆದ -8.675511 +ಅನ -8.679588 +ದನ -8.681517 +ಲಯ -8.684927 +ಲಕ -8.702794 +ಇವ -8.728255 +ತದ -8.728372 +_ಶ -8.736135 +_ಉ -8.773650 +ಳದ -8.773650 +ಗಡ -8.800035 +ಯತ -8.819645 +ಯದ -8.880779 +ರಯ -8.894665 +ಹಳ -8.903379 +ಮ_ -8.915081 +ನವ -8.943938 +ಡಯ -8.950354 +ಕದ -8.965483 +ಎರ -8.999051 +ಟ_ -8.999334 +ಧನ -9.013987 +ಹಲ -9.016277 +ಡಗ -9.065660 +ಭರ -9.076669 +ಯಗ -9.088063 +ಳಸ -9.099396 +ಪಡ -9.129750 +ಣಗ -9.160599 +ಇತ -9.164561 +ಗವ -9.191798 +ತನ -9.224182 +ಡಲ -9.228987 +ಸಗ -9.233142 +ಕಯ -9.290698 +ಜತ -9.291564 +_ಚ -9.292431 +ಕವ -9.298688 +ಗಗ -9.324522 +ಹ_ -9.375781 +ವಧ -9.378908 +ಕಗ -9.423222 +ಳಯ -9.430646 +ಭಗ -9.450826 +ಲಸ -9.456839 +ವಲ -9.478181 +ಳಕ -9.483516 +ಅತ -9.487878 +ಜವ -9.490065 +ಸಬ -9.504054 +ಸಕ -9.506466 +ಲದ -9.514737 +ರನ -9.567272 +ಧ_ -9.598449 +ಳವ -9.600595 +ಮತ -9.602529 +ಗತ -9.612669 +ಪದ -9.612669 +ಕಳ -9.615487 +ಕನ -9.623536 +ವವ -9.633827 +ವಳ -9.653516 +ತವ -9.663125 +ಲನ -9.670769 +ಹಡ -9.675511 +ಸಹ -9.702909 +ಯರ -9.710304 +ಡಸ -9.718902 +ಬಡ -9.743110 +ಜ_ -9.747620 +ರಜ -9.755009 +ಗಯ -9.757641 +ದಶ -9.762437 +ಸತ -9.782757 +ಪನ -9.790084 +ದಯ -9.808318 +ಕತ -9.812042 +ಡತ -9.843713 +ತಹ -9.865477 +ಶ_ -9.872978 +ಗಲ -9.880258 +ಬಗ -9.883128 +_ಏ -9.885219 +ಮಹ -9.891772 +ರರ -9.892035 +ನನ -9.895455 +ಚನ -9.897300 +ಆರ -9.905235 +ವಶ -9.909218 +ಬಧ -9.913213 +ತಮ -9.917218 +ಡಮ -9.920163 +_ಲ -9.924188 +ವತ -9.925801 +ಧರ -9.926608 +ಹತ -9.940945 +ಮಗ -9.970052 +ವಕ -9.988477 +ರಚ -10.001743 +ದಹ -10.009704 +ವಯ -10.030380 +ರಪ -10.040539 +ವಣ -10.049597 +ಭವ -10.057532 +ಬಕ -10.078013 +ಕಪ -10.078910 +ಧಕ -10.086709 +ತಕ -10.095457 +ದತ -10.130680 +ಆಗ -10.136583 +ಗಣ -10.151605 +ವಸ -10.154124 +ದಕ -10.157594 +ಶಲ -10.161074 +ತಡ -10.169330 +ಗನ -10.182769 +ಯಲ -10.213635 +ಸಧ -10.227494 +ಮಯ -10.232809 +ಷ_ -10.240484 +ಸನ -10.249546 +ಡನ -10.250556 +ಧಸ -10.251905 +ನಸ -10.272966 +ಪ_ -10.278795 +ಒಳ -10.283958 +ಬಲ -10.297817 +ವಹ -10.313217 +ಕಟ -10.315330 +ರಭ -10.316035 +ಆಧ -10.318152 +ಇರ -10.323458 +ಟಗ -10.337346 +ಅಧ -10.344879 +ಕಣ -10.347399 +ಹಕ -10.352453 +ಡಬ -10.357161 +ನಮ -10.386848 +ರಳ -10.401362 +ತಲ -10.439260 +ತಳ -10.448119 +ಜಕ -10.463267 +ಡರ -10.479365 +ಟಕ -10.497642 +ಣವ -10.510495 +ಅಮ -10.521429 +ಕಶ -10.527948 +ಮಸ -10.528764 +ಚರ -10.530809 +ಡಕ -10.563919 +ಶಷ -10.571895 +ಮಜ -10.574423 +ಉದ -10.593099 +ರಬ -10.598235 +ಶಗ -10.634706 +_ಖ -10.640875 +ಶತ -10.642642 +ನಟ -10.692574 +ಣದ -10.730603 +ಗಸ -10.740032 +ಭಷ -10.747620 +ಏಕ -10.748572 +ರಟ -10.767731 +ರಮ -10.818771 +ಪಯ -10.821771 +ಷಯ -10.825279 +ಯಜ -10.830810 +ಣಮ -10.840413 +ನಲ -10.840920 +ಆತ -10.843458 +ಸಯ -10.844475 +ಯಕ -10.853658 +_ಈ -10.874795 +_ಘ -10.876873 +_ಐ -10.891510 +ಘಟ -10.900469 +ಶದ -10.900998 +ಯನ -10.908952 +ಳತ -10.974776 +ಹಯ -10.976446 +ಶವ -10.993261 +ಣಯ -11.009134 +ರಹ -11.010274 +ವಮ -11.019431 +ಭಮ -11.035595 +ಆಡ -11.041413 +ಅಪ -11.044330 +ಹನ -11.045499 +ಶಧ -11.048425 +ಸಚ -11.067885 +ಚಸ -11.085807 +ಉಳ -11.098486 +ಗಮ -11.099093 +ಟನ -11.107001 +ಮಳ -11.108222 +ಮವ -11.110054 +ಬತ -11.126655 +ಆಟ -11.140324 +ಯಸ -11.144075 +ಶಕ -11.150976 +ಜಲ -11.155385 +ವಡ -11.156016 +ಇಡ -11.178275 +ಉಪ -11.178275 +ಹವ -11.195684 +ಆಕ -11.204795 +ಲಭ -11.210682 +ಮಕ -11.221868 +ಕಸ -11.225837 +ಬಯ -11.237810 +ತಬ -11.239146 +ಗಪ -11.247191 +ಸಶ -11.249209 +ಅರ -11.263417 +ಪಟ -11.275707 +ಥ_ -11.282580 +ಪಲ -11.304097 +ಚ_ -11.311811 +ಚಲ -11.313921 +ಉಟ -11.318152 +ಬಸ -11.323104 +ಸಭ -11.329495 +ಲತ -11.336630 +ವಭ -11.342364 +ಚತ -11.342364 +ಅಶ -11.347399 +ವಷ -11.351730 +ಡಳ -11.369913 +ಟದ -11.380198 +ರಲ -11.392785 +ದಳ -11.400988 +ಟವ -11.408487 +ಹಣ -11.420566 +ಮಚ -11.422842 +ಲಖ -11.435808 +ಅಗ -11.450439 +ಣಕ -11.469136 +ಪತ -11.478575 +ಜಗ -11.483318 +ಪಗ -11.492852 +ಣಸ -11.517775 +ಜರ -11.524277 +ನಜ -11.539015 +ಳರ -11.544788 +ಪಕ -11.552244 +ಖಲ -11.568111 +ಯಮ -11.575689 +ಆಹ -11.596949 +ತಸ -11.596949 +ಜಯ -11.602099 +ಕಥ -11.606404 +ರಧ -11.606404 +ಯಯ -11.619397 +ಐದ -11.643084 +ಸಘ -11.647514 +ಬಬ -11.655522 +ಫಲ -11.662677 +ನಪ -11.665369 +ದಬ -11.678907 +ವಜ -11.707295 +ದಸ -11.710999 +ಅಕ -11.736253 +ಲಪ -11.776437 +ಭಸ -11.793025 +ಕಕ -11.797941 +ಧಯ -11.800898 +ಚಯ -11.807822 +_ಧ -11.818771 +ಟಬ -11.821771 +ಈಗ -11.824778 +ಳನ -11.835856 +ದಖ -11.842950 +ಖರ -11.842950 +ಷವ -11.848039 +ಅಳ -11.858271 +ಟಯ -11.861355 +_ಫ -11.863414 +ಲಬ -11.865477 +ಪಸ -11.865477 +ಸಪ -11.874795 +ಧಗ -11.879997 +ಅಭ -11.924457 +_ಟ -11.928763 +ಅಡ -11.936331 +ನಧ -11.952682 +ಭ_ -11.977004 +ತತ -11.988196 +ಮಖ -11.992697 +ಸಸ -11.999476 +ಗಧ -12.010845 +ನಬ -12.013129 +ವಚ -12.024607 +ಖ_ -12.028068 +ಬಜ -12.041996 +ಷಧ -12.041996 +ನಭ -12.057238 +_ಓ -12.067885 +ಆಯ -12.076221 +ವಟ -12.078611 +ಷಗ -12.103954 +ಜಪ -12.108832 +ಣತ -12.110054 +ಭತ -12.112502 +_ಔ -12.138452 +ಹಮ -12.149719 +ದಡ -12.154754 +ತಪ -12.157279 +ಶರ -12.166149 +ಲಹ -12.168693 +ಕಷ -12.185343 +ಮಣ -12.217910 +ಕಮ -12.229817 +ಭಕ -12.236475 +ಜಡ -12.240484 +ಣಬ -12.257988 +ಪವ -12.260700 +ಶಸ -12.274336 +ಟರ -12.297817 +_ಊ -12.304797 +ಲಜ -12.306197 +_i -12.324522 +ಲರ -12.334486 +ಖಸ -12.338778 +ಸಟ -12.386848 +ಚಳ -12.392785 +ಮಪ -12.401736 +ಔಷ -12.406233 +ಮಬ -12.407735 +ಕಚ -12.425882 +ಭಟ -12.458200 +ಷರ -12.461316 +ಳಲ -12.464439 +ಗಟ -12.466003 +ಆನ -12.469136 +ಶಖ -12.497642 +ರಷ -12.525092 +ದಜ -12.534906 +ಎಲ -12.544788 +ಚಟ -12.548097 +ಏಳ -12.553074 +a_ -12.554737 +ಅಲ -12.556402 +ಚಕ -12.558069 +ಆಚ -12.559738 +ಘಷ -12.590112 +ಧವ -12.591818 +ಪಜ -12.605542 +ಐತ -12.621139 +ಖಕ -12.631631 +ಭನ -12.656414 +ಬವ -12.679815 +ಗಬ -12.687092 +ಜಸ -12.688917 +ಓದ -12.716573 +ಣನ -12.718436 +ಧದ -12.720301 +ಖನ -12.720301 +ರಶ -12.731543 +ಇಥ -12.735310 +ಪಚ -12.737197 +ಯಬ -12.740978 +ಶಯ -12.773529 +ಏರ -12.775467 +ಖಡ -12.800898 +ಟಲ -12.802873 +ನಖ -12.808814 +ಮಧ -12.812788 +ಇಳ -12.816774 +ಈತ -12.832826 +ಷಸ -12.838892 +ಗಕ -12.840920 +ಏನ -12.849059 +ಭಯ -12.859298 +ಇಲ -12.865477 +ಷಕ -12.869611 +ಎಟ -12.871682 +ಭದ -12.882084 +ನಶ -12.882084 +ಉಡ -12.894665 +ಖತ -12.898884 +ಭಜ -12.903115 +ಲಮ -12.918021 +ತಣ -12.948303 +ಕಬ -12.950491 +_ಡ -12.959274 +ದಪ -12.961479 +ಒಡ -12.972551 +ಷಣ -12.972551 +ಥಗ -13.004013 +ಡಹ -13.004013 +ಖಯ -13.010845 +ಹಜ -13.026913 +ಧತ -13.064327 +ಆವ -13.083404 +ಬನ -13.110054 +ಗಭ -13.112502 +ಜದ -13.117409 +ಪಳ -13.119869 +ಟಸ -13.154754 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kat b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kat new file mode 100644 index 00000000000..44a15f1a773 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kat @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 46202194 +#UNIQUE_BIGRAMS 1211 +#UNSEEN_LOG_PROB -15.693188 +#EXPECTED_SCORE -7.921733 +ს_ -5.031562 +ი_ -5.126163 +ის -5.391447 +ა_ -5.519935 +ებ -5.780868 +_მ -5.865696 +ლი -6.013471 +არ -6.174715 +სა -6.292323 +_ს -6.308186 +და -6.394339 +რი -6.546533 +ან -6.551703 +მა -6.581166 +_ა -6.607372 +ელ -6.609450 +ბი -6.700965 +ამ -6.773710 +ერ -6.839925 +ვე -6.884214 +_გ -6.913381 +რო -6.932371 +გა -6.948841 +ად -6.958335 +მი -7.062877 +ბა -7.066832 +_რ -7.110389 +რა -7.119844 +რე -7.128342 +ია -7.134647 +მო -7.179863 +ნა -7.188072 +_დ -7.202207 +ლე -7.216987 +ნი -7.224080 +ას -7.261795 +ულ -7.292903 +ავ -7.300853 +ე_ -7.308396 +თა -7.331908 +ში -7.333801 +დე -7.338468 +ვი -7.348021 +ო_ -7.356097 +ენ -7.390561 +ომ -7.394775 +ალ -7.396200 +_ი -7.405350 +ვა -7.414798 +რთ -7.445324 +მე -7.448937 +ობ -7.453681 +_შ -7.513359 +ლო -7.570950 +თ_ -7.598137 +შე -7.670670 +ინ -7.707791 +სი -7.729509 +ილ -7.733697 +ურ -7.744418 +ნე -7.744813 +ით -7.772914 +ლა -7.778923 +ნ_ -7.782060 +დ_ -7.801232 +ეს -7.812153 +ორ -7.814695 +_წ -7.849230 +ახ -7.940924 +_კ -8.024405 +_ე -8.050419 +დი -8.050593 +ტი -8.054481 +_თ -8.059912 +_პ -8.092935 +აც -8.135432 +ოს -8.143189 +ც_ -8.144867 +ონ -8.145698 +ემ -8.184162 +იდ -8.242138 +ხე -8.245716 +აქ -8.290897 +ირ -8.303184 +თი -8.307016 +_უ -8.327182 +ეთ -8.377428 +მ_ -8.390100 +აშ -8.397568 +ცი -8.439632 +კა -8.441585 +ევ -8.484077 +სე -8.496722 +ზე -8.527571 +ოლ -8.532096 +ოდ -8.533865 +თვ -8.550441 +ედ -8.588050 +ატ -8.624606 +_ხ -8.658891 +ქა -8.672319 +ტე -8.676925 +ხა -8.677271 +მდ -8.739011 +ნდ -8.752065 +იმ -8.766719 +იც -8.782968 +აზ -8.812806 +ათ -8.813157 +აი -8.827952 +სტ -8.830139 +ბე -8.830949 +ტა -8.833639 +გი -8.874105 +_ბ -8.874179 +_ნ -8.888921 +ყო -8.889809 +ოვ -8.896966 +აკ -8.900662 +ბუ -8.910576 +ბს -8.919575 +ნტ -8.923982 +რ_ -8.935035 +ტო -8.950138 +_ქ -8.951127 +ლ_ -8.984538 +გე -9.000506 +იტ -9.007416 +_ვ -9.018822 +წე -9.026634 +კო -9.043820 +პი -9.053643 +აგ -9.085163 +იო -9.088848 +_ფ -9.093241 +_ო -9.107019 +რუ -9.118567 +უნ -9.118706 +იე -9.122252 +ვლ -9.138878 +_ჩ -9.148646 +აა -9.159551 +აღ -9.180669 +დგ -9.188282 +იკ -9.195971 +წი -9.209125 +ეო -9.212567 +ვრ -9.233917 +ეგ -9.251262 +კუ -9.283961 +ნო -9.298890 +იუ -9.303065 +რმ -9.312564 +ხო -9.355754 +იი -9.364949 +ხვ -9.377397 +კი -9.404418 +ივ -9.405815 +ლს -9.434286 +ოგ -9.437747 +პრ -9.450384 +იყ -9.459258 +ეტ -9.461634 +პა -9.477730 +ყვ -9.478442 +რდ -9.488389 +დრ -9.488792 +რს -9.491330 +ქვ -9.505059 +ბო -9.506149 +თე -9.518463 +გო -9.518921 +აწ -9.520663 +_ტ -9.533564 +წლ -9.535139 +_ც -9.538665 +ტრ -9.541082 +სო -9.542851 +ცე -9.557459 +ტუ -9.588927 +ფი -9.589890 +სუ -9.607612 +ნს -9.611833 +ბრ -9.617562 +ლმ -9.621762 +ფე -9.622796 +მხ -9.627357 +გრ -9.632652 +ცხ -9.642102 +წა -9.677846 +ვს -9.686311 +ვნ -9.687469 +დო -9.693067 +ოფ -9.704615 +მთ -9.712296 +აუ -9.718386 +_ყ -9.722917 +ზი -9.747293 +უს -9.748340 +პო -9.755529 +აფ -9.769360 +ქტ -9.771405 +ხლ -9.777038 +ოე -9.782885 +ჩე -9.806458 +ღე -9.808501 +ლუ -9.843876 +უფ -9.853001 +შვ -9.856413 +რც -9.860935 +იგ -9.900379 +აბ -9.900498 +თხ -9.902499 +ექ -9.910952 +კვ -9.922633 +უმ -9.935784 +წყ -9.936978 +სხ -9.938233 +ქმ -9.963575 +ბლ -9.981488 +ფო -9.982057 +თუ -10.006580 +იშ -10.009410 +უკ -10.010344 +_ლ -10.033425 +მუ -10.040610 +გუ -10.048326 +_ძ -10.071390 +ცა -10.084960 +ფლ -10.088287 +ხი -10.099925 +რვ -10.106038 +რგ -10.159390 +კე -10.181993 +ზა -10.194159 +_ზ -10.213679 +რტ -10.229449 +ეუ -10.281199 +ფა -10.290868 +იზ -10.293099 +სთ -10.303006 +მს -10.305612 +აპ -10.313617 +_ჯ -10.326270 +აე -10.352566 +ვდ -10.354445 +ძლ -10.360426 +გ_ -10.368742 +სპ -10.374949 +ეხ -10.386480 +ეი -10.387777 +იქ -10.394570 +კლ -10.410887 +ოც -10.418003 +პე -10.418132 +გვ -10.429671 +შო -10.437660 +ჩა -10.443823 +ფრ -10.455702 +თო -10.469980 +შა -10.480649 +რჩ -10.481676 +მწ -10.484312 +აო -10.493691 +რძ -10.502452 +ეკ -10.503358 +მც -10.507262 +ეზ -10.521099 +უთ -10.545999 +უბ -10.552972 +ბშ -10.563046 +ზო -10.598336 +ეა -10.601681 +კრ -10.613911 +ყა -10.614107 +ნუ -10.631933 +დღ -10.635158 +ბდ -10.659497 +ხდ -10.670142 +ოი -10.672332 +სწ -10.672638 +ღვ -10.682406 +ვ_ -10.701310 +ჩვ -10.731678 +იწ -10.733857 +ცვ -10.747105 +სრ -10.752590 +უა -10.770205 +თქ -10.771351 +სკ -10.774903 +ოთ -10.784564 +ძა -10.797400 +ოპ -10.801072 +მპ -10.802577 +ეც -10.810687 +ოკ -10.827895 +ნც -10.829485 +ნგ -10.835179 +ქი -10.835236 +იღ -10.835521 +ლშ -10.837634 +მნ -10.838034 +ძე -10.858006 +ყე -10.861139 +ოა -10.872978 +ჯა -10.898500 +ღმ -10.905611 +უშ -10.915709 +აძ -10.918124 +ძი -10.919273 +დნ -10.926790 +ოქ -10.927398 +მლ -10.930625 +ხუ -10.930747 +მბ -10.936794 +წვ -10.949335 +ღა -10.970858 +ჯე -10.975817 +ეე -10.982562 +ოშ -10.993025 +ცნ -11.029112 +თლ -11.036373 +მშ -11.044529 +წო -11.052664 +ვშ -11.059645 +იხ -11.080524 +ხმ -11.086079 +პუ -11.091656 +_ჰ -11.094863 +ხს -11.107967 +ლდ -11.134256 +ქო -11.138685 +ეშ -11.139530 +ეყ -11.150845 +იძ -11.181993 +ხრ -11.183953 +უდ -11.217763 +ეფ -11.230762 +ოზ -11.237155 +ზრ -11.269011 +მრ -11.272637 +ლთ -11.273410 +ბ_ -11.291142 +ქს -11.316003 +იპ -11.331209 +ჭი -11.343166 +ქნ -11.345927 +ღი -11.351791 +შნ -11.412971 +უც -11.441520 +ნვ -11.467812 +ტყ -11.470468 +რშ -11.472596 +ნფ -11.474549 +რჯ -11.499195 +იფ -11.517570 +უხ -11.525464 +ნმ -11.531737 +_ღ -11.579388 +ქც -11.587822 +სმ -11.589745 +რნ -11.589938 +რკ -11.604352 +ქე -11.606006 +დვ -11.624520 +ჩნ -11.626888 +ვთ -11.646075 +ჩი -11.646375 +იბ -11.651790 +გზ -11.672434 +აყ -11.691104 +ოხ -11.698453 +სნ -11.717360 +თბ -11.738117 +ზღ -11.745496 +სშ -11.753021 +ოუ -11.753883 +ქრ -11.761994 +ნთ -11.799680 +ძო -11.799680 +აჩ -11.822796 +ლზ -11.824833 +ღო -11.834380 +სვ -11.839521 +დუ -11.854243 +რლ -11.858411 +ყნ -11.860152 +ოტ -11.869235 +შუ -11.874735 +ძვ -11.942682 +თშ -11.949952 +თმ -11.956887 +გლ -11.957136 +რქ -11.969229 +უძ -11.983068 +ბმ -11.987625 +უტ -11.997934 +აჭ -12.001642 +მყ -12.010891 +ცო -12.015796 +ლხ -12.019940 +ლბ -12.057913 +მკ -12.079239 +ჯგ -12.089817 +ფუ -12.095409 +ეპ -12.101434 +დმ -12.113837 +უწ -12.137559 +ზუ -12.148008 +ყი -12.165402 +ტს -12.176489 +ნშ -12.190725 +ცდ -12.193062 +ნძ -12.208201 +ფხ -12.209532 +სს -12.217838 +ნხ -12.251396 +მზ -12.252766 +თრ -12.253986 +მძ -12.264856 +ეწ -12.266393 +ჯი -12.279529 +ლტ -12.303322 +რპ -12.304743 +უქ -12.305375 +ბთ -12.306165 +ქუ -12.336529 +რბ -12.343490 +ნლ -12.367710 +ოყ -12.371677 +გმ -12.375986 +ღწ -12.389493 +ნკ -12.390834 +ჯვ -12.404651 +ვო -12.406006 +თს -12.412290 +ღლ -12.416722 +წმ -12.434070 +მღ -12.454957 +ჰქ -12.457764 +ტშ -12.461634 +ii -12.464455 +ბზ -12.476506 +ხბ -12.488299 +დშ -12.529612 +სც -12.532015 +ძღ -12.534421 +კმ -12.545579 +ხშ -12.547447 +ოჩ -12.549879 +_ჟ -12.550253 +უპ -12.572525 +მგ -12.579770 +ლგ -12.579962 +შრ -12.582258 +უ_ -12.617733 +აჯ -12.623436 +_ჭ -12.635108 +ჟი -12.635704 +ძუ -12.653902 +ძნ -12.663189 +რწ -12.669685 +შ_ -12.672536 +ტჩ -12.689143 +წრ -12.704485 +ჭო -12.708032 +გნ -12.721888 +ტ_ -12.725056 +ბჭ -12.737797 +ეძ -12.737797 +ოჯ -12.738437 +ჭე -12.791023 +წუ -12.823248 +მტ -12.832329 +ყუ -12.839635 +ოძ -12.849972 +იჭ -12.850203 +ოწ -12.853665 +ჯო -12.857368 +ჩრ -12.865268 +s_ -12.867367 +რზ -12.873681 +ვმ -12.899216 +თნ -12.902320 +პლ -12.913840 +ძრ -12.914081 +ჰყ -12.924482 +ნზ -12.924725 +ღნ -12.931052 +რხ -12.947484 +ვკ -12.950200 +უზ -12.950941 +მმ -12.952425 +მჯ -12.952920 +i_ -12.975377 +აჟ -12.986231 +ზნ -12.989275 +ვტ -13.001770 +ტკ -13.020329 +შშ -13.021368 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kaz b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kaz new file mode 100644 index 00000000000..8fe349a33e5 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kaz @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 31908620 +#UNIQUE_BIGRAMS 1216 +#UNSEEN_LOG_PROB -16.985780 +#EXPECTED_SCORE -7.715581 +н_ -4.996791 +_к -5.508763 +ы_ -5.555845 +ын -5.714334 +_б -5.912831 +ан -6.008390 +ар -6.044827 +_ж -6.111359 +а_ -6.255056 +ін -6.283270 +ал -6.292777 +і_ -6.300201 +да -6.303972 +_а -6.322173 +ка -6.378038 +та -6.394817 +ен -6.451567 +_т -6.615655 +нд -6.637600 +е_ -6.643857 +ды -6.709887 +ер -6.731975 +га -6.760986 +ла -6.768631 +де -6.795666 +к_ -6.798608 +лы -6.824247 +ты -6.865223 +ас -6.959654 +_с -6.999572 +р_ -7.038432 +ны -7.044269 +ке -7.073342 +_м -7.074200 +_о -7.135606 +ат -7.144457 +сы -7.151884 +аи -7.161005 +ыл -7.259257 +ті -7.278067 +ме -7.287742 +ра -7.325614 +ык -7.327850 +ол -7.338143 +ст -7.350560 +на -7.383067 +ба -7.387739 +ді -7.429646 +те -7.430636 +ет -7.434150 +не -7.435770 +п_ -7.452897 +ры -7.458445 +гы -7.469253 +ак -7.567204 +жа -7.588846 +кт -7.599566 +са -7.643920 +ір -7.704771 +аг -7.705761 +үр -7.722910 +ма -7.738526 +ге -7.741029 +рі -7.766419 +ле -7.814382 +лд -7.825888 +ек -7.830357 +ау -7.835057 +_д -7.857920 +ыс -7.866494 +_е -7.894418 +ад -7.911200 +ел -7.921774 +гі -7.967611 +бо -7.973326 +ес -7.979897 +ам -7.987692 +лі -8.016004 +_ө -8.025645 +тт -8.058078 +ні -8.060623 +рд -8.075865 +лг -8.092244 +и_ -8.102136 +ор -8.145059 +ре -8.167568 +тү -8.192257 +бі -8.192958 +л_ -8.204089 +у_ -8.234487 +кы -8.249599 +ік -8.261580 +он -8.266806 +іл -8.268590 +ыр -8.283671 +ко -8.292972 +рг -8.300778 +сі -8.324295 +_ү -8.341673 +кү -8.342040 +аз -8.342906 +_ш -8.416988 +с_ -8.439148 +кі -8.478875 +шы -8.491968 +ып -8.500883 +бе -8.525347 +за -8.551351 +ег -8.552115 +се -8.602014 +ша -8.627591 +ем -8.635391 +рт -8.636974 +ид -8.651209 +же -8.669319 +т_ -8.693112 +еи -8.694029 +үл -8.704950 +ия -8.721421 +иы -8.726563 +ші -8.743536 +уы -8.744156 +жы -8.755641 +ед -8.765543 +кө -8.793578 +ім -8.807043 +ым -8.818163 +_ә -8.822701 +со -8.826575 +ән -8.843382 +па -8.855876 +мы -8.861879 +өз -8.863290 +_н -8.896022 +ск -8.900552 +рл -8.901892 +_п -8.952177 +мд -8.963599 +бү -8.978962 +із -9.013034 +ыг -9.025892 +жә -9.032246 +ит -9.041103 +м_ -9.053518 +іс -9.067423 +зд -9.070896 +зі -9.084675 +тк -9.090209 +ру -9.155721 +нг -9.177600 +жү -9.188111 +_г -9.196389 +ап -9.230149 +ту -9.243873 +ыз -9.276616 +ил -9.301078 +з_ -9.340574 +аш -9.352199 +ше -9.361686 +ин -9.381966 +_р -9.395574 +ог -9.400730 +кс -9.411035 +ос -9.436686 +аб -9.456879 +ни -9.488521 +мі -9.507779 +ик -9.510480 +іп -9.513650 +зе -9.514146 +ез -9.524031 +иі -9.525097 +ои -9.537543 +то -9.552098 +үс -9.555326 +уг -9.589995 +еп -9.592471 +уд -9.600276 +нш -9.606431 +рн -9.644463 +өл -9.648340 +жо -9.675517 +ри -9.705554 +ми -9.709069 +ок -9.729919 +ие -9.742451 +үн -9.751348 +ха -9.764658 +үш -9.778688 +еу -9.782546 +ти -9.819041 +мү -9.835346 +_і -9.840270 +үи -9.896603 +нт -9.912420 +пт -9.922257 +_х -9.922652 +үм -9.935257 +лу -9.947125 +іг -9.963779 +_з -9.964411 +от -9.968347 +әс -9.999619 +ур -10.000730 +лм -10.025963 +рк -10.028180 +ом -10.029030 +рм -10.051358 +лт -10.087599 +әр -10.092676 +бы -10.095839 +өр -10.101192 +мә -10.119384 +ли -10.122404 +өн -10.152759 +зг -10.162053 +ул -10.177207 +уа -10.202077 +әл -10.224783 +си -10.232432 +пе -10.234664 +зы -10.240669 +рс -10.272807 +ци -10.287424 +_и -10.295186 +би -10.305334 +ро -10.320459 +өт -10.338084 +я_ -10.348716 +іш -10.388526 +об -10.403330 +яс -10.416926 +ял -10.418720 +су -10.429279 +ыт -10.435277 +ш_ -10.490666 +ло -10.495423 +үк -10.512229 +үз -10.538551 +нн -10.567076 +тр -10.589542 +сп -10.596736 +им -10.601013 +ис -10.608124 +өм -10.690009 +шк -10.690681 +_э -10.704346 +кш -10.716924 +мл -10.729075 +бө -10.740478 +ая -10.758379 +_ф -10.782944 +иг -10.785894 +еш -10.787332 +лк -10.835016 +уі -10.847292 +уш -10.883817 +бл -10.889124 +но -10.921818 +іт -10.923749 +зи -10.951326 +оп -10.978072 +ги -10.979715 +кк -10.986670 +гд -11.014741 +әд -11.016519 +еб -11.054191 +зм -11.062870 +тө -11.071406 +од -11.075304 +сө -11.083228 +_у -11.101093 +өп -11.109460 +ки -11.113461 +ум -11.138317 +пр -11.142706 +ль -11.146698 +по -11.154922 +тә -11.167035 +ир -11.174331 +лс -11.189245 +шт -11.193580 +ку -11.202609 +нб -11.210409 +сү -11.251155 +ди -11.251927 +му -11.266556 +дә -11.294107 +мо -11.311928 +ыш -11.327995 +пы -11.356401 +әу -11.361270 +фи -11.402289 +ну -11.406701 +ип -11.449052 +нс -11.518848 +мн -11.538887 +өс -11.581345 +кр -11.584674 +ун -11.590100 +жи -11.600311 +кп -11.622949 +үг -11.627376 +иц -11.643920 +мк -11.654084 +ян -11.657002 +их -11.661096 +рш -11.678478 +сш -11.682930 +оз -11.716163 +аж -11.730687 +зу -11.741948 +лш -11.760338 +мп -11.776902 +рб -11.818102 +_л -11.821534 +сс -11.844462 +гр -11.849459 +ио -11.858329 +әк -11.890926 +эк -11.904555 +вт -11.905595 +жі -11.913248 +сә -11.918839 +өк -11.923574 +уб -11.955540 +дү -11.961118 +ав -11.984012 +иб -11.989517 +үт -11.998739 +пі -12.011377 +лп -12.015489 +лл -12.017362 +ов -12.033952 +як -12.052269 +пу -12.054576 +нк -12.057464 +ев -12.059585 +яд -12.069073 +ве -12.083718 +нә -12.091590 +әи -12.103280 +в_ -12.107463 +c_ -12.114664 +_ы -12.114864 +пк -12.118278 +иа -12.129376 +еж -12.139948 +из -12.140561 +д_ -12.156779 +gc -12.165061 +ео -12.177783 +кә -12.193368 +әт -12.196973 +ь_ -12.210409 +ыд -12.257893 +бу -12.284263 +хи -12.287650 +яг -12.291271 +_в -12.300137 +өб -12.311813 +ус -12.344360 +см -12.346949 +ут -12.356164 +кл -12.387073 +уе -12.392168 +нц -12.401188 +ех -12.405597 +жө -12.413962 +до -12.424860 +іб -12.430090 +го -12.442620 +мш -12.447158 +иж -12.468780 +ив -12.477778 +мг -12.478036 +үд -12.486833 +_я -12.562488 +жу -12.563308 +ва -12.569341 +тш -12.576504 +мб -12.585924 +пи -12.585924 +ід -12.605797 +фр -12.619104 +мт -12.620812 +о_ -12.625375 +аф -12.646382 +др -12.654813 +ви -12.660071 +мм -12.662707 +зб -12.664761 +рж -12.699225 +өи -12.700128 +өш -12.700429 +ъе -12.701934 +ац -12.702838 +фе -12.723789 +це -12.727771 +хн -12.741948 +пс -12.754703 +фо -12.778331 +эл -12.793017 +g_ -12.825467 +үб -12.826781 +бъ -12.837661 +нү -12.842634 +тп -12.845294 +ах -12.859336 +гн -12.863711 +фа -12.866409 +гу -12.915517 +зо -12.975066 +бә -12.976158 +еа -12.984195 +гт -12.988966 +уз -12.997075 +оц -13.030354 +хы -13.049392 +гү -13.050542 +ук -13.098117 +пш -13.112060 +әж -13.114864 +эн -13.128161 +дж -13.159259 +тс -13.164231 +пп -13.271018 +кц -13.300365 +_p -13.305849 +pg -13.312733 +шу -13.315035 +рп -13.336856 +ши -13.340134 +тб -13.342950 +үж -13.352849 +ыб -13.366632 +ае -13.369979 +нө -13.378140 +мө -13.404861 +_n -13.428095 +мс -13.452216 +бр -13.459328 +ng -13.460857 +ии -13.463919 +ыи -13.475717 +үх -13.537274 +во -13.570991 +оф -13.575399 +ср -13.602700 +иш -13.633397 +сл -13.650155 +рз -13.671234 +лж -13.677737 +шо -13.704044 +_m -13.729612 +лю -13.737001 +шө -13.753766 +оя -13.773891 +cg -13.775792 +mc -13.787252 +i_ -13.826124 +г_ -13.836669 +хо -13.839317 +нм -13.853971 +әз -13.863374 +иф -13.868775 +ду -13.878275 +б_ -13.884416 +лб -13.902996 +үп -13.902996 +ю_ -13.911331 +ьм -13.933797 +әб -13.953746 +ух -13.958776 +ея -13.984195 +ют -13.998925 +ү_ -14.002630 +бд -14.013805 +че -14.020552 +_ч -14.025068 +еф -14.078820 +х_ -14.112060 +әм -14.141173 +_z -14.170887 +zw -14.170887 +wg -14.170887 +лн -14.177573 +уи -14.189351 +рө -14.201225 +ью -14.218359 +нж -14.218359 +ья -14.250604 +яу -14.264775 +ьд -14.281784 +ою -14.289007 +ье -14.314574 +зк -14.323817 +ii -14.323817 +пә -14.347184 +фу -14.349070 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/khm b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/khm new file mode 100644 index 00000000000..c2051abe376 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/khm @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 132788825 +#UNIQUE_BIGRAMS 1966 +#UNSEEN_LOG_PROB -14.385625 +#EXPECTED_SCORE -8.236691 +ន_ -4.452101 +ង_ -4.938953 +_ន -5.069852 +រ_ -5.659130 +_ម -5.668190 +ល_ -5.731050 +យ_ -5.767833 +_ដ -5.786212 +_ក -5.825118 +ក_ -5.825543 +_ប -5.905201 +នង -6.063151 +មន -6.244960 +ករ -6.315086 +បន -6.398084 +_ស -6.431403 +_ល -6.451532 +ត_ -6.502777 +_ទ -6.528642 +_ច -6.548110 +_រ -6.559347 +_ជ -6.562405 +លក -6.710895 +_អ -6.787958 +ម_ -6.922806 +ដល -7.024368 +_ព -7.087927 +ច_ -7.280285 +ព_ -7.375995 +_ត -7.497162 +e_ -7.563244 +មយ -7.565423 +ស_ -7.667137 +_ខ -7.710042 +ញ_ -7.738189 +_ហ -7.873935 +ប_ -7.912622 +រប -7.945712 +ដយ -7.993568 +ពល -7.995674 +_វ -7.995988 +_គ -8.141613 +s_ -8.183542 +ទ_ -8.191743 +n_ -8.225294 +_c -8.245942 +ចន -8.304343 +_ភ -8.329507 +er -8.349092 +ហយ -8.366991 +an -8.373526 +_s -8.400330 +on -8.404602 +សរ -8.433620 +នយ -8.443220 +ខ_ -8.544586 +_a -8.547888 +តម -8.556767 +ភព -8.558511 +រង -8.565479 +វ_ -8.598018 +d_ -8.605514 +a_ -8.622615 +in -8.633300 +នន -8.649260 +ជន -8.675793 +_p -8.679650 +ណ_ -8.711182 +ជត -8.713524 +r_ -8.715414 +_ឡ -8.716026 +កល -8.724181 +_m -8.728682 +ឡង -8.739363 +re -8.740055 +t_ -8.755411 +_យ -8.757855 +_t -8.769315 +ទង -8.778489 +បរ -8.788677 +ជ_ -8.817562 +សន -8.894761 +ចល -8.905011 +ne -8.906694 +គរ -8.940058 +ជង -8.967979 +or -8.978802 +ar -9.013430 +ng -9.016229 +រក -9.016859 +គ_ -9.018577 +l_ -9.021459 +en -9.025219 +te -9.036573 +ទត -9.038246 +មរ -9.064380 +y_ -9.080618 +រម -9.085609 +al -9.107616 +ទល -9.122048 +ដរ -9.122642 +ទក -9.126286 +ទទ -9.141196 +អច -9.148212 +_n -9.152911 +ពក -9.169907 +កព -9.221022 +me -9.235291 +ខង -9.242572 +le -9.257629 +_r -9.266742 +ma -9.271253 +_ឃ -9.278245 +សម -9.288488 +ជម -9.290901 +រយ -9.311898 +st -9.332869 +_l -9.347269 +_b -9.365578 +am -9.394410 +g_ -9.396173 +បក -9.411163 +at -9.418861 +ចណ -9.421005 +កន -9.459994 +o_ -9.464546 +_f -9.467835 +មខ -9.475673 +ea -9.487416 +ti -9.492181 +ធ_ -9.497543 +កស -9.498368 +អយ -9.500208 +លន -9.503965 +ពង -9.505568 +រស -9.513272 +ថ_ -9.528284 +_h -9.540846 +ra -9.547717 +ch -9.567712 +th -9.569502 +សខ -9.581763 +នទ -9.585105 +ពរ -9.593038 +es -9.598427 +he -9.602534 +_d -9.606281 +ri -9.606399 +ធន -9.620082 +ha -9.620125 +នវ -9.620236 +យង -9.622521 +io -9.624655 +ដង -9.628296 +តង -9.662023 +ro -9.662692 +li -9.665283 +_ផ -9.676228 +កត -9.687786 +ចញ -9.692875 +_k -9.726084 +na -9.734390 +_v -9.738115 +វញ -9.750714 +មក -9.752297 +អព -9.755759 +nt -9.758909 +រល -9.761952 +ដក -9.768959 +រណ -9.777945 +ដច -9.794020 +សហ -9.795815 +ce -9.804339 +ដ_ -9.808473 +ho -9.812141 +ic -9.821440 +ភ_ -9.827379 +ia -9.829098 +_i -9.842850 +co -9.845417 +ca -9.852471 +យយ -9.859549 +ហ_ -9.871997 +_w -9.872394 +អម -9.904730 +ខម -9.904907 +se -9.905678 +el -9.910689 +di -9.920521 +រជ -9.923567 +វត -9.930558 +k_ -9.935274 +មល -9.944130 +ad -9.951060 +លព -9.952394 +p_ -9.967837 +ou -9.973342 +ហត -9.985054 +_g -9.985351 +_ធ -9.988195 +ណង -9.995719 +ទន -10.000975 +nd -10.003616 +bo -10.006161 +រច -10.008990 +it -10.016083 +តល -10.049313 +ទស -10.051721 +ងរ -10.070150 +លខ -10.072874 +la -10.074232 +om -10.089421 +po -10.095815 +h_ -10.097790 +ភគ -10.097885 +ta -10.105669 +តណ -10.107322 +m_ -10.114772 +សភ -10.116411 +i_ -10.124053 +ឃញ -10.141668 +ភម -10.153944 +un -10.159128 +ew -10.169626 +vi -10.170815 +នគ -10.183014 +de -10.184430 +បល -10.187961 +សល -10.220879 +il -10.228732 +បង -10.230493 +_e -10.238385 +បយ -10.241999 +ដណ -10.246413 +បប -10.247192 +ណម -10.252166 +រព -10.263980 +នក -10.267873 +ed -10.270810 +lo -10.286776 +ac -10.287781 +យប -10.289099 +កឡ -10.297743 +as -10.303114 +ba -10.310822 +is -10.317946 +ve -10.325050 +mo -10.332092 +ឡយ -10.332778 +to -10.337915 +ចព -10.347155 +ណរ -10.349294 +os -10.362278 +oo -10.382628 +លង -10.403402 +id -10.405272 +_j -10.405346 +si -10.411829 +យក -10.416484 +ws -10.420558 +ជយ -10.422033 +ll -10.434864 +ផល -10.440651 +_o -10.454777 +_ថ -10.455616 +tr -10.457219 +ជវ -10.460185 +នត -10.466274 +ឃ_ -10.473640 +sa -10.481680 +us -10.492439 +mi -10.495777 +ai -10.499972 +អន -10.500003 +ទប -10.506263 +ដម -10.513312 +ge -10.552473 +កម -10.554727 +អ_ -10.571311 +_ឈ -10.574438 +pr -10.586784 +មស -10.587084 +da -10.587720 +វន -10.588355 +od -10.588857 +ol -10.591755 +ចរ -10.612646 +ណច -10.615217 +so -10.617314 +លយ -10.619125 +សណ -10.621211 +no -10.629466 +ur -10.629638 +ie -10.632722 +ខត -10.635260 +be -10.645248 +សស -10.648366 +c_ -10.653484 +ni -10.654552 +rt -10.656725 +ay -10.668277 +ជក -10.670436 +ph -10.670931 +pa -10.680457 +hi -10.681473 +ហរ -10.684242 +ពត -10.686908 +ហង -10.706563 +kh -10.707053 +លរ -10.713150 +កគ -10.720061 +ag -10.737836 +តច -10.745501 +អត -10.748675 +សក -10.756454 +mb -10.777296 +ផត -10.784292 +pe -10.784330 +វល -10.786651 +ut -10.787976 +វធ -10.801233 +ពស -10.809087 +ap -10.809711 +វង -10.814594 +em -10.820241 +គត -10.822443 +នម -10.826659 +do -10.842252 +et -10.843789 +ld -10.863794 +រន -10.863956 +ខស -10.866692 +លប -10.866834 +sh -10.877117 +_u -10.879960 +vo -10.881476 +សធ -10.882480 +បត -10.902805 +តន -10.909873 +fa -10.921817 +ជប -10.922702 +ចក -10.924158 +បស -10.927519 +នប -10.929869 +ss -10.929933 +វដ -10.937648 +គម -10.938991 +ជធ -10.940762 +នរ -10.947720 +ឡក -10.949459 +ns -10.951932 +ខណ -10.952039 +ot -10.952663 +លត -10.956458 +រវ -10.959311 +សវ -10.959419 +ពធ -10.963101 +មជ -10.964316 +ដប -10.964794 +នជ -10.965359 +hm -10.966967 +ពញ -10.972218 +បទ -10.983414 +អង -10.991067 +ec -10.993147 +_ណ -10.997494 +ឯក -11.000518 +ដអ -11.004687 +ke -11.015464 +យស -11.015892 +ចង -11.022589 +ថន -11.029385 +យព -11.031158 +បផ -11.036807 +គណ -11.039572 +រទ -11.040945 +ok -11.044061 +ទរ -11.052113 +មថ -11.056503 +នញ -11.059700 +សង -11.075955 +ពភ -11.080183 +បច -11.091802 +រត -11.092988 +ងម -11.103947 +pl -11.112196 +nc -11.120542 +fo -11.132907 +wi -11.143142 +oa -11.145995 +ជរ -11.153424 +នព -11.155455 +fi -11.158457 +mp -11.166226 +ឧស -11.172105 +ip -11.174439 +hu -11.189863 +rs -11.190904 +go -11.206481 +_ឯ -11.207920 +ៗ_ -11.217564 +su -11.229775 +ga -11.230584 +តត -11.232099 +គជ -11.233982 +តរ -11.239147 +gu -11.254380 +ផង -11.257702 +wo -11.263726 +ig -11.263780 +wa -11.266240 +ow -11.275286 +លម -11.292924 +លច -11.300330 +iv -11.300851 +ab -11.302167 +ry -11.316006 +គល -11.326319 +yo -11.327408 +ណក -11.328497 +dr -11.331742 +ចជ -11.331966 +ua -11.349606 +ងក -11.352103 +ធរ -11.353296 +op -11.358479 +នដ -11.360648 +up -11.368125 +u_ -11.369964 +pp -11.373707 +កង -11.374168 +ev -11.378963 +ct -11.379397 +ងថ -11.382933 +ឡ_ -11.387321 +ov -11.395171 +_y -11.397396 +ex -11.399097 +_ឆ -11.399156 +ទញ -11.400888 +ងត -11.401094 +ងដ -11.401799 +ki -11.406953 +rd -11.413161 +we -11.418861 +ue -11.420647 +មហ -11.422942 +ងល -11.423300 +រហ -11.424853 +rl -11.431979 +អណ -11.437152 +មត -11.438327 +ck -11.439292 +ak -11.450134 +វក -11.454152 +eb -11.461732 +w_ -11.474165 +cr -11.485709 +_ង -11.487299 +តក -11.488423 +va -11.491203 +ខភ -11.498541 +ir -11.500050 +យម -11.505631 +im -11.509870 +អអ -11.513074 +ee -11.516921 +ចទ -11.519311 +មទ -11.532510 +នស -11.535120 +ហក -11.536959 +យន -11.541939 +វយ -11.546124 +ផន -11.550843 +កដ -11.558294 +ខន -11.558851 +ងន -11.562231 +មដ -11.564236 +ទហ -11.565718 +យវ -11.568751 +tu -11.570005 +um -11.577457 +នល -11.577955 +ci -11.587452 +រដ -11.588355 +ណន -11.588455 +កក -11.597854 +cu -11.598561 +tt -11.603996 +កហ -11.607823 +ជព -11.620321 +bi -11.628038 +ru -11.636868 +oc -11.641234 +ទវ -11.643144 +ts -11.645196 +ដន -11.648191 +ty -11.657778 +កប -11.661328 +gi -11.662595 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kin b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kin new file mode 100644 index 00000000000..fe98a07ea96 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kin @@ -0,0 +1,406 @@ +#TOTAL_BIGRAMS 2030940 +#UNIQUE_BIGRAMS 402 +#UNSEEN_LOG_PROB -17.631788 +#EXPECTED_SCORE -7.042766 +a_ -4.297864 +e_ -5.002796 +o_ -5.079335 +i_ -5.283779 +an -5.588761 +_a -5.673634 +_i -5.700386 +_b -5.714565 +ba -5.774468 +ri -5.972461 +_u -5.983655 +_k -6.022378 +_n -6.054170 +ya -6.154030 +ra -6.319643 +ur -6.335216 +ar -6.371163 +ab -6.488405 +ku -6.515444 +_m -6.521501 +ga -6.544922 +ng -6.622589 +wa -6.645804 +nd -6.658306 +mu -6.672076 +ko -6.687662 +bi -6.759806 +um -6.859474 +ka -6.866005 +ma -6.875649 +ub -6.897841 +ir -6.964854 +yo -6.979033 +in -6.991905 +ha -7.024365 +u_ -7.035692 +sh -7.075282 +ye -7.079984 +re -7.099335 +am -7.105387 +uk -7.131046 +ib -7.153726 +_g -7.190608 +er -7.206467 +gi -7.229735 +ta -7.239578 +ug -7.243986 +im -7.261973 +we -7.327893 +bu -7.328578 +ik -7.334184 +na -7.335790 +it -7.366642 +za -7.391832 +ak -7.402729 +by -7.430889 +gu -7.436170 +ag -7.440729 +go -7.469145 +ny -7.538106 +cy -7.577726 +at -7.614005 +bo -7.624200 +or -7.624340 +bw -7.625322 +ki -7.627989 +_y -7.640550 +zi -7.669470 +_r -7.672945 +nt -7.694267 +da -7.715013 +tu -7.744263 +ru -7.793372 +ah -7.799214 +te -7.828626 +yi -7.849773 +rw -7.913255 +iy -7.916512 +_c -7.921843 +mb -7.927539 +un -7.929789 +se -7.942838 +ze -7.972863 +di -8.023903 +us -8.032061 +ne -8.032991 +ge -8.069927 +ho -8.083930 +az -8.087017 +as -8.142141 +wo -8.170923 +en -8.175023 +hi -8.185532 +is -8.217526 +ez -8.243340 +mw -8.262845 +vu -8.283724 +iz -8.315959 +on -8.339466 +wi -8.344076 +ig -8.362895 +_h -8.370163 +sa -8.405617 +ro -8.445674 +nz -8.459111 +me -8.461863 +ut -8.471160 +ih -8.552037 +ni -8.607202 +uz -8.638567 +si -8.640550 +mi -8.643672 +ns -8.652506 +yu -8.665427 +jy -8.688101 +be -8.709650 +hu -8.747007 +ti -8.777231 +om -8.816405 +ob -8.848135 +kw -8.906592 +ry -8.964677 +he -8.966097 +_w -8.987932 +af -8.998430 +fi -9.008638 +mo -9.020763 +os -9.053982 +ts -9.064212 +av -9.071455 +ic -9.095735 +uy -9.208462 +no -9.215202 +zo -9.272917 +_t -9.275116 +de -9.278641 +uh -9.299080 +ay -9.299975 +et -9.377704 +je -9.392906 +nj -9.458861 +ke -9.505084 +es -9.524309 +nk -9.629536 +fa -9.636304 +so -9.640266 +oz -9.645946 +ek -9.726100 +aw -9.770701 +du -9.771322 +tw -9.797633 +_p -9.798898 +mv -9.809695 +to -9.851741 +ac -9.855684 +pa -9.857660 +mp -9.871567 +eg -9.901148 +_s -9.901828 +nu -9.944587 +ey -9.954421 +uf -9.965742 +gw -9.977152 +va -9.979302 +aj -9.985769 +my -9.987932 +ij -10.000247 +_d -10.102748 +_z -10.105093 +li -10.131145 +uv -10.166630 +em -10.252410 +ok -10.274236 +id -10.391474 +if -10.422335 +hy -10.433098 +fu -10.464872 +zw -10.535864 +pe -10.583029 +n_ -10.643103 +ew -10.654508 +ad -10.674106 +cu -10.721295 +uj -10.730921 +_e -10.751592 +do -10.780039 +le -10.833478 +ej -10.839974 +el -10.859639 +uw -10.913426 +_j -10.917543 +po -10.990820 +zu -11.024458 +iv -11.031875 +y_ -11.081811 +eb -11.097291 +r_ -11.117666 +l_ -11.146361 +su -11.152816 +il -11.261973 +vi -11.274236 +jo -11.326182 +al -11.329835 +ot -11.346386 +eh -11.361259 +ci -11.374400 +s_ -11.434080 +_f -11.436047 +ol -11.439989 +ap -11.455864 +ip -11.512847 +ce -11.559253 +ca -11.589582 +uc -11.620561 +dw -11.682253 +pu -11.686930 +oy -11.715311 +pi -11.781289 +d_ -11.853054 +_l -11.860959 +pr -11.866253 +ea -11.879575 +_v -11.898434 +ty -11.928577 +ch -11.967874 +wu -12.008272 +ow -12.011202 +st -12.037837 +nn -12.043823 +ud -12.049834 +ja -12.071073 +nw -12.071073 +iw -12.136733 +co -12.284831 +ji -12.339006 +ia -12.346386 +ju -12.461863 +dy -12.473936 +br -12.502505 +dd -12.523264 +t_ -12.527451 +jw -12.557111 +ep -12.591772 +fo -12.609420 +oh -12.618326 +ls -12.654508 +au -12.654508 +ed -12.682253 +c_ -12.682253 +_o -12.686930 +og -12.686930 +h_ -12.724897 +ul -12.734548 +fr -12.763892 +rt -12.829595 +ss -12.855684 +b_ -12.860959 +p_ -12.925810 +th -12.953716 +pf -12.965032 +m_ -12.970723 +mf -12.987932 +io -13.017078 +hw -13.022979 +yw -13.064973 +up -13.071073 +ve -13.083351 +ll -13.146361 +la -13.152816 +ou -13.192165 +od -13.295505 +sp -13.339006 +tr -13.353803 +ai -13.368754 +fp -13.376287 +cl -13.469900 +ph -13.510773 +ie -13.570012 +aa -13.604988 +k_ -13.613866 +sw -13.677592 +lo -13.686930 +ef -13.754044 +op -13.773807 +ue -13.845192 +ev -13.855684 +kr -13.909322 +f_ -13.942489 +ov -13.965032 +ec -14.058898 +sy -14.198829 +rn -14.212249 +bc -14.267216 +hh -14.324360 +ee -14.339006 +wn -14.399127 +nc -14.477983 +oi -14.494285 +bb -14.510773 +kd -14.578677 +dr -14.578677 +rd -14.578677 +gr -14.578677 +rv -14.613866 +oc -14.649935 +g_ -14.668314 +mm -14.763892 +oj -14.783791 +uu -14.783791 +hn -14.803969 +oo -14.866253 +tt -14.909322 +sc -14.909322 +rl -14.976436 +bl -15.022979 +lu -15.071073 +fe -15.095735 +rc -15.095735 +eo -15.172356 +vo -15.253276 +lb -15.339006 +df -15.339006 +nl -15.368754 +of -15.399127 +ct -15.399127 +hr -15.399127 +w_ -15.430154 +pl -15.461863 +rr -15.527451 +rp -15.561399 +sr -15.561399 +ae -15.561399 +yb -15.596164 +ck -15.631788 +ex -15.631788 +ld -15.668314 +sk -15.705789 +tb -15.705789 +db -15.744263 +qu -15.824433 +lm -15.866253 +x_ -15.866253 +rs -15.953716 +dm -16.046826 +cw -16.095735 +v_ -16.146361 +cr -16.146361 +lt -16.146361 +py -16.198829 +tc -16.309860 +iq -16.309860 +sm -16.309860 +lc -16.368754 +ps -16.430154 +sd -16.430154 +fl -16.494285 +nx -16.494285 +hp -16.494285 +tx -16.494285 +xl -16.494285 +sb -16.494285 +wh -16.494285 +pp -16.494285 +yv -16.494285 +gg -16.561399 +cp -16.783791 +kn -16.783791 +wl -16.783791 +np -16.783791 +pc -16.783791 +xi -16.866253 +sl -16.866253 +lg -16.953716 +eu -16.953716 +rb -16.953716 +td -17.046826 +gn -17.146361 +ms -17.146361 +ds -17.146361 +ym -17.146361 +pt -17.253276 +bd -17.253276 +ys -17.253276 +hl -17.253276 +ff -17.253276 +fs -17.253276 +ei -17.368754 +_x -17.368754 +xa -17.368754 +ui -17.368754 +pm -17.494285 +xp -17.494285 +vb -17.494285 +ly -17.494285 +z_ -17.494285 +gh -17.494285 +mg -17.494285 +iu -17.631788 +bm -17.631788 +cq -17.631788 +rg -17.631788 +gb -17.631788 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kir b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kir new file mode 100644 index 00000000000..b34593662c2 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kir @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 47074473 +#UNIQUE_BIGRAMS 8642 +#UNSEEN_LOG_PROB -17.378923 +#EXPECTED_SCORE -8.488096 +н_ -5.167051 +_к -5.611328 +_б -5.846385 +ан -5.920432 +ар -6.101796 +а_ -6.129233 +_а -6.173562 +_ж -6.320869 +ал -6.484166 +ын -6.508347 +да -6.581275 +_т -6.663349 +ка -6.673712 +ы_ -6.680618 +ен -6.684046 +та -6.721114 +_м -6.782057 +нд -6.821715 +и_ -6.823824 +к_ -6.832265 +ат -6.926236 +ин -6.962336 +п_ -7.045745 +жа -7.146163 +р_ -7.179192 +т_ -7.182487 +аи -7.199787 +ер -7.232924 +га -7.253418 +ба -7.262552 +у_ -7.326552 +ма -7.351335 +ра -7.368290 +ла -7.374465 +ке -7.389346 +на -7.426928 +де -7.491003 +ол -7.503506 +_с -7.506228 +уу -7.541387 +е_ -7.566374 +кы -7.582449 +_э -7.606885 +рд -7.683409 +ил -7.688104 +лы -7.688641 +би -7.727819 +ун -7.741704 +ыл -7.750877 +үн -7.773150 +ир -7.801873 +те -7.812788 +ды -7.819612 +бо -7.830391 +гы -7.844515 +ты -7.851817 +ту -7.859156 +кт -7.866719 +ти -7.877381 +_о -7.884533 +не -7.905256 +ны -7.922908 +_д -7.929871 +ак -7.935773 +ме -7.938725 +ык -7.943862 +ры -7.947625 +ү_ -7.976033 +лд -7.976187 +рг -7.986900 +ле -7.990769 +лг -7.997403 +ет -8.002259 +ор -8.030516 +ек -8.049625 +ур -8.058079 +лу -8.079374 +л_ -8.124700 +он -8.131139 +ул -8.131715 +ко -8.179401 +ыр -8.194252 +ри -8.194341 +тү -8.231321 +_ч -8.249654 +ас -8.249869 +тт -8.255309 +ге -8.259538 +аг -8.273866 +аш -8.286729 +_ө -8.290188 +сы -8.303518 +ли -8.305582 +ик -8.320367 +ам -8.325362 +кө -8.342828 +ки -8.393838 +бе -8.404609 +_и -8.411584 +ни -8.425784 +ди -8.439657 +ии -8.443962 +_у -8.490050 +ги -8.500733 +ит -8.501666 +са -8.506830 +ру -8.548449 +шт -8.572084 +ел -8.576231 +ст -8.588023 +ия -8.600638 +ө_ -8.604534 +ыз -8.611038 +з_ -8.633561 +ку -8.634085 +оо -8.640320 +өн -8.658248 +ми -8.659809 +то -8.688730 +үү -8.729620 +аа -8.742728 +өр -8.776237 +о_ -8.801271 +чы -8.826705 +из -8.830635 +ып -8.850713 +ок -8.858559 +ре -8.879479 +м_ -8.881097 +иш -8.939078 +_п -8.944957 +ду -8.956984 +тк -8.989219 +ну -9.008914 +ук -9.015735 +ег -9.022700 +шк -9.050852 +бу -9.056846 +си -9.061211 +өт -9.065244 +го -9.070309 +им -9.075921 +эл -9.079349 +ча -9.092155 +ыш -9.102175 +рү -9.105801 +рт -9.109199 +аз -9.114472 +ад -9.120515 +өл -9.171064 +су -9.181935 +дө -9.206874 +аб -9.212444 +до -9.218256 +ло -9.223339 +үр -9.225374 +жы -9.244208 +мд -9.249123 +ш_ -9.281218 +еп -9.287524 +кү -9.330542 +па -9.347153 +ис -9.355741 +нч -9.358535 +ап -9.361273 +зд -9.365998 +ша -9.370717 +же -9.372524 +жо -9.384150 +лү -9.391768 +_日 -9.394942 +өз -9.400440 +ыг -9.413924 +дү -9.431338 +нү -9.440636 +со -9.451324 +иы -9.453277 +ид -9.466616 +уш -9.478591 +ро -9.488001 +ым -9.496322 +гө -9.505671 +за -9.506964 +_н -9.530795 +мы -9.541261 +ог -9.547623 +ес -9.548839 +нт -9.560432 +_浙 -9.560803 +се -9.565393 +рк -9.565718 +шы -9.584748 +чу -9.597322 +_工 -9.609505 +уп -9.610079 +тө -9.615695 +му -9.639843 +чи -9.649533 +日_ -9.649779 +_在 -9.650050 +_р -9.653008 +ош -9.657060 +江_ -9.657456 +_ш -9.657654 +在_ -9.662117 +浙江 -9.663856 +工_ -9.682849 +_江 -9.689894 +еи -9.692884 +үл -9.703066 +че -9.712703 +иг -9.719947 +чү -9.726215 +ип -9.749531 +үч -9.785836 +от -9.790605 +гү -9.793893 +ыи -9.801668 +зи -9.815077 +нг -9.827165 +эм -9.827666 +шу -9.853716 +уг -9.862219 +ем -9.862933 +ци -9.877647 +ою -9.911190 +гу -9.938296 +нө -9.941246 +с_ -9.948041 +шо -9.960720 +ус -9.987444 +эк -9.988938 +зы -9.990652 +_ү -9.992399 +рл -10.006390 +рм -10.010494 +зү -10.015941 +бы -10.044268 +үк -10.045725 +ом -10.056715 +үш -10.071084 +пт -10.089999 +ои -10.097668 +的_ -10.101299 +зг -10.139955 +үз -10.141997 +я_ -10.151726 +ез -10.152947 +ая -10.174425 +_的 -10.175984 +оп -10.191561 +уч -10.195473 +лө -10.213117 +ял -10.221582 +_月 -10.244003 +月_ -10.257708 +ос -10.283641 +_г -10.297151 +лк -10.310788 +юн -10.314686 +мү -10.322671 +яс -10.335642 +сү -10.335998 +үг -10.348011 +网_ -10.399529 +пр -10.425299 +ши -10.456051 +ич -10.464990 +ыс -10.466941 +ач -10.468981 +үп -10.484353 +өө -10.489675 +бө -10.493735 +ск -10.496034 +кк -10.497407 +мл -10.519010 +_人 -10.520180 +ут -10.535473 +өк -10.535746 +лб -10.547119 +ян -10.549689 +报_ -10.550700 +сс -10.555996 +年_ -10.558767 +人_ -10.565532 +_报 -10.580002 +_年 -10.586868 +рс -10.589085 +үи -10.589368 +ев -10.598843 +_ф -10.601126 +рө -10.615719 +мө -10.632600 +уб -10.648795 +уз -10.654563 +_ы -10.656837 +рб -10.682597 +чө -10.695194 +нк -10.704471 +тр -10.707492 +бү -10.716952 +шү -10.718294 +об -10.720671 +но -10.721861 +线_ -10.732407 +现_ -10.735277 +日报 -10.743450 +_现 -10.749347 +在线 -10.751141 +ше -10.765901 +位_ -10.768839 +зс -10.778012 +мо -10.782431 +сө -10.788164 +ум -10.795064 +工人 -10.796099 +жү -10.807093 +_位 -10.813642 +现在 -10.814360 +人日 -10.815851 +置_ -10.816127 +江工 -10.816127 +_职 -10.820055 +在的 -10.829003 +位置 -10.829616 +的位 -10.831570 +报网 -10.831682 +_您 -10.832296 +您现 -10.832296 +职工 -10.838680 +кс -10.845939 +в_ -10.851874 +үм -10.854538 +өм -10.864674 +өп -10.865417 +职_ -10.870802 +工在 -10.893825 +江职 -10.895809 +ов -10.914025 +кш -10.920486 +еш -10.929005 +зм -10.964941 +ээ -10.968990 +ва -10.974221 +ио -11.001858 +өг -11.009672 +лт -11.028690 +чо -11.038519 +од -11.068942 +по -11.069535 +еб -11.081971 +оз -11.093175 +км -11.099550 +бл -11.150193 +лс -11.156195 +кч -11.176835 +уд -11.179747 +_з -11.189591 +эс -11.217635 +ед -11.233708 +өс -11.241330 +еч -11.243112 +мг -11.274958 +см -11.280360 +зу -11.297691 +иц -11.312814 +国_ -11.317030 +сп -11.319926 +эр -11.328886 +_中 -11.341078 +лм -11.343863 +ь_ -11.377387 +иб -11.394694 +жу -11.403384 +пу -11.453213 +пе -11.476865 +ль -11.489763 +ую -11.494088 +ыт -11.499047 +пи -11.499491 +нс -11.525546 +өи -11.545286 +кр -11.555673 +ие -11.556781 +аж -11.569020 +чк -11.579580 +иа -11.612692 +ч_ -11.628713 +шч -11.687946 +жө -11.690780 +ыя -11.697177 +фи -11.734807 +гр -11.750138 +үт -11.753309 +_л -11.785728 +лл -11.822773 +үс -11.831235 +мк -11.849214 +ве -11.886555 +ав -11.889342 +оч -11.895868 +бр -11.915978 +өч -11.928467 +中国 -11.932295 +_в -11.934093 +中_ -11.970895 +вд -11.973481 +ят -11.987724 +чт -12.003241 +лч -12.034785 +фе -12.036587 +ив -12.042521 +пк -12.058250 +рн -12.072040 +мб -12.089030 +рч -12.108251 +мп -12.139713 +лп -12.144007 +д_ -12.163980 +як -12.184518 +өб -12.185517 +_国 -12.218877 +пы -12.227522 +жи -12.230612 +юм -12.253924 +_х -12.257821 +эн -12.293993 +фо -12.301554 +уя -12.310555 +эг -12.329201 +会_ -12.333307 +ае -12.344102 +рш -12.345377 +ыб -12.347772 +оц -12.361253 +нб -12.361253 +_新 -12.384662 +вр -12.395684 +ха -12.446269 +яд -12.446954 +өд -12.465207 +_纵 -12.465727 +纵横 -12.465727 +横_ -12.465727 +_横 -12.466768 +横中 -12.466768 +ых -12.488970 +кл -12.490027 +_一 -12.493381 +мч -12.522657 +өш -12.525726 +яг -12.529345 +нц -12.535700 +зб -12.539344 +зе -12.544645 +мн -12.554751 +во -12.566973 +市_ -12.576675 +ео -12.606372 +үд -12.607519 +зө -12.616729 +яб -12.637473 +_网 -12.649829 +业_ -12.664875 +_b -12.666269 +еа -12.728761 +_е -12.751617 +кц -12.766274 +州_ -12.770551 +мм -12.779573 +тч -12.782377 +яр -12.790605 +бд -12.796916 +це -12.802160 +уи -12.809841 +_会 -12.822440 +ви -12.824439 +юш -12.829115 +фр -12.836717 +_市 -12.837166 +зо -12.863418 +新_ -12.876956 +ац -12.902071 +рь -12.915741 +иү -12.919061 +动_ -12.927154 +тм -12.932415 +эт -12.934332 +n_ -12.941065 +一_ -12.957304 +_发 -12.964390 +юл -12.972249 +эп -12.981638 +_я -12.995337 +пс -12.999095 +фа -13.023896 +аф -13.038519 +кэ -13.040325 +ех -13.046016 +тп -13.051990 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/knn b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/knn new file mode 100644 index 00000000000..a14896d1d46 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/knn @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 1418659 +#UNIQUE_BIGRAMS 1128 +#UNSEEN_LOG_PROB -13.454647 +#EXPECTED_SCORE -8.494818 +_ا -5.302152 +ي_ -5.682618 +ه_ -5.893548 +د_ -5.947691 +ن_ -6.034950 +اي -6.085572 +ت_ -6.157357 +ان -6.299666 +_ب -6.329043 +i_ -6.334613 +_م -6.481537 +ار -6.552785 +e_ -6.611436 +ر_ -6.643509 +را -6.823804 +ha -6.837462 +ست -6.853484 +a_ -6.884028 +_د -6.989788 +ين -7.094578 +اس -7.096107 +r_ -7.137461 +_ك -7.164050 +ar -7.173414 +_h -7.187428 +ري -7.248744 +_k -7.286032 +_ت -7.293511 +دا -7.301189 +يد -7.408882 +_a -7.431349 +_ن -7.433106 +ام -7.476638 +م_ -7.493032 +ai -7.502959 +ند -7.548495 +_ش -7.561691 +ده -7.624922 +ل_ -7.641681 +_س -7.655557 +_t -7.659663 +ما -7.709453 +_m -7.729816 +رد -7.777662 +hi -7.794045 +_ه -7.803556 +_b -7.819088 +n_ -7.820237 +ني -7.829460 +aa -7.870756 +ال -7.891856 +ود -7.894758 +_خ -7.909597 +با -8.017453 +ka -8.019035 +ا_ -8.019035 +تر -8.060785 +بر -8.062687 +اش -8.089306 +ah -8.090691 +اد -8.099032 +_ر -8.100985 +سا -8.115296 +ها -8.130320 +ور -8.174589 +شت -8.178414 +in -8.197393 +مي -8.227252 +يا -8.236424 +دي -8.247508 +وا -8.249054 +_p -8.256187 +_s -8.268992 +ير -8.300387 +_d -8.304239 +رو -8.315535 +an -8.326919 +لا -8.368998 +h_ -8.374388 +te -8.408882 +اه -8.415116 +_ج -8.451322 +سي -8.479720 +er -8.486999 +ya -8.499090 +ش_ -8.507208 +ك_ -8.519471 +_پ -8.521711 +لي -8.536739 +na -8.565732 +نا -8.587082 +ra -8.596105 +تا -8.606770 +در -8.629547 +_گ -8.634388 +ز_ -8.642900 +_ع -8.666672 +اك -8.669153 +at -8.673299 +از -8.694629 +خو -8.703505 +وي -8.718848 +كن -8.737826 +t_ -8.747409 +ba -8.756616 +شد -8.760139 +ne -8.768985 +ma -8.780566 +_j -8.783700 +ta -8.785045 +_ح -8.789988 +_l -8.794496 +_ف -8.814960 +كا -8.823689 +ات -8.826456 +ch -8.826918 +th -8.847850 +me -8.852073 +sa -8.876719 +_و -8.887756 +اب -8.906666 +مر -8.927311 +گذ -8.928302 +hu -8.951777 +مو -8.953288 +ذا -8.959350 +bh -8.969000 +pa -8.989013 +re -8.991599 +_n -8.993153 +كر -8.995227 +ur -8.999905 +ته -8.999905 +گر -9.006167 +_ز -9.018244 +يم -9.022468 +بگ -9.044853 +بي -9.057260 +يت -9.075249 +نه -9.076347 +_r -9.094022 +ب_ -9.105740 +زي -9.113605 +_ق -9.116424 +ap -9.142625 +هم -9.142625 +ay -9.170481 +m_ -9.185206 +تو -9.188762 +_c -9.199484 +he -9.215718 +is -9.218139 +ho -9.219957 +ga -9.226643 +يل -9.227862 +دن -9.233360 +ون -9.241955 +ja -9.244420 +رس -9.256809 +نو -9.262419 +ره -9.276225 +it -9.286349 +au -9.288256 +ye -9.292713 +l_ -9.316507 +مه -9.318453 +d_ -9.326919 +k_ -9.330188 +oo -9.332809 +ki -9.334777 +يش -9.338723 +al -9.340040 +تي -9.348634 +شو -9.355945 +_f -9.374725 +di -9.375400 +جا -9.380814 +يس -9.391019 +بو -9.392386 +فت -9.405429 +ول -9.411649 +ad -9.413035 +am -9.467430 +la -9.467430 +وز -9.468870 +iy -9.478269 +_u -9.478269 +kh -9.490653 +يي -9.501668 +ac -9.528455 +ab -9.529206 +يك -9.551926 +da -9.554218 +قا -9.557279 +_g -9.566503 +زا -9.582007 +bo -9.589039 +من -9.594532 +um -9.604789 +اف -9.632773 +او -9.636005 +س_ -9.640057 +فر -9.640868 +sh -9.649827 +us -9.658019 +يز -9.676208 +گي -9.677040 +دو -9.706476 +نم -9.712435 +شا -9.737392 +گا -9.750472 +fa -9.752225 +مت -9.753102 +رت -9.755737 +s_ -9.776101 +ri -9.776992 +وس -9.792240 +_چ -9.792240 +اخ -9.798566 +_ص -9.801285 +le -9.801285 +ع_ -9.810388 +ce -9.812215 +ok -9.814960 +se -9.817711 +wi -9.825994 +خا -9.828766 +un -9.831543 +اع -9.832470 +بل -9.840839 +هد -9.844574 +eb -9.874808 +يه -9.883427 +دد -9.902767 +ag -9.904715 +ti -9.904715 +ق_ -9.911555 +نگ -9.916460 +مد -9.917443 +هر -9.923356 +گ_ -9.927311 +عا -9.943242 +ak -9.947252 +o_ -9.949261 +li -9.982826 +tt -9.986948 +tw -9.989013 +on -9.993153 +زم -10.001468 +رف -10.002511 +هي -10.041634 +خت -10.056718 +مع -10.063231 +en -10.074153 +مل -10.082950 +nd -10.084053 +نت -10.094022 +نش -10.107421 +ف_ -10.109667 +جه -10.119815 +هن -10.119815 +_ي -10.124348 +il -10.126620 +ei -10.134600 +كي -10.134600 +شي -10.139180 +تن -10.143775 +ke -10.153008 +اق -10.161136 +پي -10.165801 +_ط -10.176353 +tu -10.195305 +تم -10.200082 +de -10.212095 +_y -10.224208 +فا -10.226643 +wa -10.227862 +وب -10.232748 +يگ -10.240109 +كش -10.242571 +ni -10.245037 +b_ -10.253702 +رن -10.257432 +حا -10.258677 +رش -10.267424 +رب -10.269933 +رك -10.282544 +ط_ -10.285080 +له -10.288891 +كت -10.290164 +رگ -10.305526 +عل -10.306813 +eh -10.317155 +شر -10.353947 +دس -10.370007 +pn -10.371354 +_w -10.375400 +اگ -10.376752 +ko -10.376752 +شه -10.380814 +مح -10.380814 +st -10.395807 +وم -10.395807 +نس -10.397177 +زن -10.402673 +رم -10.406809 +as -10.410957 +جو -10.412342 +_i -10.419288 +شم -10.447412 +صو -10.447412 +بن -10.460248 +مس -10.463117 +دم -10.474647 +سر -10.480447 +لم -10.499458 +پا -10.506838 +قي -10.533721 +نج -10.542795 +ky -10.548876 +جم -10.550400 +يو -10.551926 +عه -10.558046 +اح -10.558046 +وش -10.559579 +اط -10.564191 +طر -10.565732 +ir -10.573459 +سل -10.581228 +شك -10.582787 +به -10.585910 +mu -10.587473 +sk -10.590606 +وه -10.600046 +بت -10.609548 +si -10.627132 +وج -10.633580 +في -10.641681 +عي -10.646563 +شن -10.649827 +اج -10.672884 +نن -10.679540 +قر -10.694629 +p_ -10.696316 +mi -10.716708 +ot -10.720134 +زه -10.739129 +غا -10.756616 +rt -10.772538 +لو -10.830617 +ح_ -10.832470 +ek -10.841772 +مش -10.841772 +nk -10.847382 +عت -10.849257 +oh -10.851134 +اص -10.864344 +nt -10.864344 +تب -10.870042 +لب -10.871947 +سو -10.879590 +گو -10.881508 +ng -10.887275 +og -10.896938 +خي -10.900821 +عر -10.908619 +تح -10.914496 +ku -10.920397 +كس -10.922369 +قد -10.926321 +خش -10.938245 +نظ -10.948256 +uj -10.958338 +يق -10.960363 +تل -10.962391 +مك -10.964421 +كل -10.976665 +g_ -10.980769 +مق -10.982826 +jh -10.984885 +عد -10.984885 +_ل -11.005644 +عم -11.007736 +تص -11.011930 +بخ -11.030955 +كو -11.035217 +بس -11.039492 +وع -11.041634 +جر -11.041634 +os -11.043779 +رز -11.056718 +خد -11.058886 +uc -11.063231 +گز -11.065409 +قه -11.071962 +نق -11.074153 +do -11.076347 +es -11.098475 +دگ -11.100706 +چن -11.105180 +قل -11.114168 +u_ -11.118684 +سب -11.127757 +شگ -11.132316 +پر -11.148384 +تف -11.150694 +et -11.159972 +aj -11.171654 +كم -11.174002 +بع -11.178709 +ee -11.190544 +جد -11.192922 +رخ -11.192922 +بد -11.195305 +y_ -11.204875 +مج -11.221777 +تق -11.221777 +عن -11.229082 +ج_ -11.229082 +تش -11.231525 +or -11.233973 +lo -11.236424 +gi -11.238880 +ro -11.238880 +سط -11.261171 +سم -11.261171 +صا -11.268678 +ut -11.273705 +يب -11.278749 +rn -11.291438 +نر -11.301670 +aw -11.309392 +زش -11.311975 +نب -11.314563 +oi -11.317155 +ge -11.324961 +كه -11.332809 +خص -11.345984 +نك -11.359281 +صل -11.378105 +pk -11.378105 +pe -11.386248 +تخ -11.391702 +هس -11.405429 +سن -11.408190 +هو -11.416506 +to -11.419288 +بز -11.419288 +طل -11.424869 +طا -11.438917 +يع -11.447412 +be -11.450254 +لت -11.453103 +du -11.453103 +حم -11.458816 +ol -11.464553 +يج -11.470312 +جن -11.473200 +اغ -11.484812 +زد -11.484812 +قت -11.496517 +ht -11.499458 +j_ -11.502406 +هش -11.502406 +ضو -11.517233 +چه -11.529206 +زر -11.532215 +مط -11.544313 +رج -11.547353 +گف -11.556513 +حي -11.559579 +حق -11.562652 +ji -11.568818 +تع -11.571910 +حو -11.575010 +حت -11.581228 +و_ -11.581228 +مخ -11.593746 +خب -11.600046 +صي -11.600046 +حد -11.609548 +ص_ -11.615917 +ss -11.619113 +tn -11.625525 +ik -11.635197 +رح -11.658019 +_غ -11.661309 +ضا -11.667912 +fi -11.674545 +su -11.684552 +خر -11.687904 +وق -11.698004 +يف -11.704777 +_z -11.732193 +فز -11.735657 +ظا -11.735657 +ظر -11.742609 +ضر -11.760139 +ul -11.785045 +فق -11.792240 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kor b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kor new file mode 100644 index 00000000000..2ddbba9366e --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kor @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 167719513 +#UNIQUE_BIGRAMS 32020 +#UNSEEN_LOG_PROB -16.358472 +#EXPECTED_SCORE -11.725318 +다_ -6.138755 +이_ -6.380847 +는_ -6.465397 +_이 -6.485571 +에_ -6.555554 +의_ -6.714487 +을_ -6.833818 +_하 -6.926824 +로_ -7.011976 +고_ -7.190097 +하_ -7.244171 +은_ -7.451257 +한_ -7.548890 +가_ -7.549818 +서_ -7.576543 +_지 -7.594258 +_에 -7.602764 +를_ -7.613649 +지_ -7.627927 +_대 -7.743567 +_사 -7.782962 +_으 -7.809528 +_기 -7.860112 +으_ -7.867622 +도_ -7.998425 +기_ -8.007952 +_있 -8.040209 +리_ -8.051502 +_시 -8.137649 +_가 -8.206673 +_정 -8.226751 +으로 -8.238536 +_리 -8.245766 +_자 -8.292479 +_수 -8.299414 +인_ -8.307640 +에서 -8.321024 +해_ -8.344861 +_한 -8.356675 +_다 -8.363991 +_전 -8.399379 +_그 -8.412192 +어_ -8.428683 +_들 -8.461449 +라_ -8.469937 +_아 -8.488681 +_했 -8.501611 +_부 -8.556892 +들_ -8.562943 +자_ -8.579311 +시_ -8.580249 +_인 -8.582816 +사_ -8.619025 +스_ -8.621693 +_라 -8.638374 +과_ -8.655848 +_주 -8.657955 +_스 -8.666916 +_어 -8.713549 +_보 -8.716850 +나_ -8.750767 +했_ -8.788127 +_국 -8.789359 +수_ -8.809151 +장_ -8.814807 +_되 -8.838155 +_상 -8.869282 +_것 -8.880935 +_제 -8.882091 +국_ -8.894162 +_장 -8.898240 +대_ -8.902285 +_로 -8.930429 +_도 -8.934998 +했다 -8.937552 +적_ -8.937653 +게_ -8.948300 +_서 -8.956477 +_나 -8.958538 +정_ -8.966671 +_고 -8.968277 +부_ -8.972730 +_었 -8.987014 +_의 -8.997818 +었_ -9.006804 +_성 -9.045287 +_일 -9.069341 +_적 -9.112412 +아_ -9.115028 +_해 -9.120356 +_동 -9.130305 +_경 -9.137603 +_위 -9.142635 +_공 -9.143345 +_원 -9.160560 +있다 -9.161427 +와_ -9.174444 +원_ -9.233769 +_구 -9.235168 +상_ -9.256043 +며_ -9.265123 +면_ -9.266008 +_중 -9.270548 +_선 -9.281769 +전_ -9.283317 +성_ -9.299915 +하는 -9.305489 +_문 -9.306185 +_조 -9.379499 +제_ -9.382665 +_유 -9.397475 +_비 -9.398032 +_신 -9.408072 +되_ -9.425338 +_등 -9.447220 +_관 -9.471056 +_소 -9.475085 +화_ -9.485702 +주_ -9.516670 +_연 -9.524859 +_미 -9.528723 +이다 -9.536425 +구_ -9.546534 +하고 -9.553045 +만_ -9.556798 +_마 -9.568069 +_화 -9.580140 +_회 -9.602722 +었다 -9.603070 +_모 -9.638159 +동_ -9.649623 +_오 -9.652140 +_계 -9.653234 +_우 -9.655890 +회_ -9.675282 +_여 -9.700640 +보_ -9.717639 +_무 -9.719045 +_세 -9.720118 +치_ -9.732607 +_였 -9.733932 +_통 -9.747163 +용_ -9.748013 +한다 -9.757691 +_영 -9.771164 +문_ -9.774799 +_교 -9.775448 +_치 -9.777598 +트_ -9.778202 +_학 -9.781552 +_내 -9.784553 +니_ -9.791229 +였_ -9.795840 +_당 -9.807254 +여_ -9.815625 +러_ -9.817929 +_용 -9.820680 +_거 -9.837447 +_발 -9.840322 +드_ -9.846304 +계_ -9.882416 +선_ -9.883813 +_방 -9.884536 +르_ -9.909964 +_개 -9.918430 +_생 -9.944222 +_명 -9.945358 +할_ -9.960680 +된_ -9.961571 +_후 -9.972566 +_는 -9.996899 +_과 -10.002977 +_니 -10.003824 +_진 -10.015743 +학_ -10.028875 +_만 -10.029450 +우_ -10.030242 +_실 -10.031339 +려_ -10.040433 +_러 -10.048891 +신_ -10.055906 +진_ -10.074206 +_르 -10.076461 +_민 -10.081373 +_행 -10.105592 +_역 -10.111232 +_작 -10.123318 +_재 -10.126007 +그_ -10.135226 +_단 -10.144085 +_드 -10.151570 +터_ -10.169498 +력_ -10.169884 +_프 -10.175543 +_분 -10.184975 +_반 -10.186297 +당_ -10.187811 +_현 -10.190211 +_때 -10.191156 +_면 -10.194277 +_결 -10.200873 +_않 -10.213227 +_트 -10.222323 +일_ -10.225780 +후_ -10.226717 +간_ -10.230309 +_안 -10.241303 +데_ -10.242418 +위_ -10.256806 +하였 -10.277113 +_산 -10.290862 +_입 -10.294498 +_말 -10.294898 +_체 -10.303395 +였다 -10.306147 +_타 -10.307924 +산_ -10.317815 +교_ -10.319880 +업_ -10.323359 +지만 -10.323767 +_출 -10.333101 +있는 -10.340722 +체_ -10.341135 +되었 -10.341626 +명_ -10.346781 +_바 -10.354823 +소_ -10.366552 +비_ -10.386471 +_요 -10.401902 +민_ -10.410915 +_업 -10.414890 +던_ -10.416676 +_차 -10.422449 +_없 -10.424008 +야_ -10.438784 +행_ -10.441581 +_최 -10.441617 +_표 -10.458426 +재_ -10.460716 +_려 -10.466084 +관_ -10.467849 +이라 -10.491505 +식_ -10.492285 +_포 -10.492422 +_물 -10.497237 +_노 -10.497760 +조_ -10.500075 +마_ -10.503467 +_남 -10.504467 +_간 -10.519540 +무_ -10.521435 +공_ -10.525233 +단_ -10.527758 +안_ -10.537595 +_받 -10.542885 +운_ -10.549045 +등_ -10.561158 +것이 -10.564658 +_강 -10.565714 +히_ -10.568089 +_레 -10.578784 +중_ -10.594632 +미_ -10.600470 +_식 -10.601485 +_음 -10.607324 +다는 -10.620494 +_설 -10.628288 +_파 -10.640182 +역_ -10.642820 +_각 -10.654336 +타_ -10.655376 +대한 -10.659976 +_았 -10.675873 +_운 -10.676169 +통_ -10.676281 +_불 -10.677830 +_양 -10.681132 +거_ -10.689900 +하여 -10.690156 +_력 -10.693386 +았_ -10.697280 +내_ -10.709586 +으며 -10.711777 +요_ -10.719908 +_예 -10.720778 +_된 -10.723176 +_감 -10.725169 +에게 -10.728960 +_함 -10.731780 +음_ -10.735809 +하지 -10.736043 +들이 -10.737085 +_직 -10.746483 +_임 -10.751160 +난_ -10.756448 +_본 -10.760650 +_건 -10.764460 +_금 -10.768056 +_두 -10.781832 +표_ -10.784530 +_또 -10.786534 +_군 -10.808642 +다고 -10.809645 +세_ -10.810819 +면서 -10.811468 +작_ -10.817293 +라고 -10.821458 +른_ -10.829685 +유_ -10.833086 +에는 -10.833384 +영_ -10.840338 +연_ -10.840354 +_호 -10.841143 +두_ -10.841158 +_심 -10.841742 +_승 -10.844966 +본_ -10.849118 +각_ -10.852438 +_법 -10.858430 +적인 -10.862140 +_속 -10.864415 +속_ -10.874053 +경_ -10.876331 +반_ -10.876557 +레_ -10.877382 +생_ -10.880022 +_점 -10.884617 +_따 -10.887173 +래_ -10.892755 +물_ -10.897604 +록_ -10.903969 +서는 -10.920996 +_처 -10.923517 +_합 -10.924837 +오_ -10.929342 +_태 -10.941894 +_매 -10.950073 +실_ -10.952918 +_같 -10.955546 +금_ -10.958692 +방_ -10.961314 +된다 -10.962961 +_김 -10.962979 +적으 -10.963288 +_독 -10.968260 +_종 -10.978203 +_열 -10.991924 +_카 -10.996943 +_크 -10.999864 +_특 -11.001345 +격_ -11.013296 +_외 -11.015984 +프_ -11.016839 +권_ -11.034557 +하기 -11.041103 +_배 -11.041629 +_달 -11.042246 +_권 -11.045878 +_까 -11.046060 +분_ -11.056355 +_추 -11.063627 +_올 -11.072529 +임_ -11.073270 +_평 -11.079399 +_청 -11.081951 +린_ -11.091887 +들은 -11.093728 +하게 -11.096550 +_터 -11.097361 +_격 -11.107479 +군_ -11.111127 +_투 -11.111451 +_많 -11.118204 +_피 -11.118873 +_더 -11.124163 +람_ -11.129491 +한국 -11.130165 +_약 -11.133022 +_판 -11.134007 +_데 -11.143997 +발_ -11.146940 +_형 -11.159556 +양_ -11.161446 +법_ -11.162826 +_초 -11.166282 +_은 -11.176937 +까_ -11.199901 +크_ -11.210645 +직_ -11.213788 +호_ -11.229627 +_버 -11.235567 +_야 -11.235795 +_활 -11.237207 +_토 -11.239888 +차_ -11.244764 +_변 -11.253579 +_키 -11.256522 +령_ -11.265748 +것_ -11.268462 +미국 -11.283049 +_게 -11.285731 +키_ -11.287472 +심_ -11.287515 +것으 -11.290981 +약_ -11.296055 +까지 -11.299584 +때문 -11.306714 +모_ -11.307869 +_디 -11.308174 +았다 -11.309855 +라는 -11.314295 +_목 -11.315938 +_람 -11.322136 +_능 -11.330175 +출_ -11.330972 +합_ -11.331394 +_와 -11.340064 +_북 -11.341269 +_집 -11.343771 +_언 -11.350471 +_박 -11.357180 +위해 -11.362334 +입_ -11.362447 +들의 -11.365145 +_알 -11.369462 +울_ -11.373245 +는데 -11.379809 +하면 -11.395689 +_코 -11.398009 +되어 -11.400797 +_증 -11.404896 +_루 -11.409802 +_준 -11.410973 +_확 -11.416747 +부터 -11.417617 +승_ -11.427084 +루_ -11.428529 +설_ -11.431853 +_래 -11.434778 +론_ -11.435755 +점_ -11.436399 +능_ -11.442921 +련_ -11.445342 +카_ -11.448752 +이후 -11.453786 +건_ -11.463009 +후보 -11.477481 +_저 -11.485154 +하다 -11.485721 +_복 -11.490094 +종_ -11.492620 +_석 -11.493066 +그는 -11.499676 +들어 -11.503692 +사람 -11.506618 +란_ -11.509047 +태_ -11.512360 +름_ -11.512486 +그러 -11.522396 +_참 -11.533471 +립_ -11.534746 +_총 -11.536610 +토_ -11.541346 +기도 -11.547898 +형_ -11.549573 +석_ -11.550166 +일본 -11.552671 +_향 -11.554506 +있었 -11.556681 +준_ -11.557355 +_항 -11.566379 +_메 -11.570402 +료_ -11.579587 +술_ -11.580509 +더_ -11.585100 +문에 -11.592173 +_졌 -11.595257 +질_ -11.600323 +_못 -11.605085 +_근 -11.620575 +날_ -11.621632 +_령 -11.633143 +디_ -11.639249 +품_ -11.642353 +개_ -11.643480 +졌_ -11.643976 +포_ -11.646318 +같은 -11.654114 +사용 -11.658807 +_난 -11.662985 +_환 -11.663264 +정부 -11.663599 +되는 -11.680496 +_됐 -11.687033 +_협 -11.688083 +_질 -11.691493 +대해 -11.695795 +_천 -11.703577 +러나 -11.707484 +청_ -11.716864 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kur b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kur new file mode 100644 index 00000000000..8d9bbb12f48 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/kur @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 455132956 +#UNIQUE_BIGRAMS 1055 +#UNSEEN_LOG_PROB -17.235791 +#EXPECTED_SCORE -7.328046 +e_ -4.658197 +n_ -4.883301 +a_ -5.377412 +in -5.453219 +er -5.563133 +an -5.625318 +_d -5.832403 +ir -6.019549 +en -6.028080 +_b -6.079371 +di -6.123109 +i_ -6.183711 +_h -6.233584 +ye -6.286903 +re -6.384759 +ne -6.388397 +ri -6.414471 +ar -6.474504 +ti -6.474900 +r_ -6.486782 +_k -6.493524 +iy -6.511150 +_s -6.538253 +he -6.610831 +ki -6.658560 +de -6.674131 +ya -6.686738 +na -6.729046 +ek -6.745055 +we -6.770552 +ke -6.802247 +se -6.838057 +st -6.859026 +be -6.878186 +bi -6.937885 +is -6.955068 +te -7.010586 +ra -7.032562 +es -7.067991 +_n -7.085610 +_p -7.108120 +me -7.196543 +da -7.253846 +_m -7.260869 +ni -7.291473 +_x -7.308554 +ma -7.329511 +ik -7.354148 +_t -7.367188 +_g -7.389017 +_r -7.402297 +_c -7.419278 +xw -7.445657 +_a -7.490379 +ur -7.491573 +em -7.493757 +ey -7.527391 +le -7.539939 +sa -7.595750 +et -7.614761 +_w -7.619148 +ta -7.649394 +ha -7.678943 +al -7.692304 +at -7.709175 +ro -7.738120 +rd -7.746928 +nd -7.752869 +el -7.756355 +ku -7.771225 +k_ -7.794043 +t_ -7.821000 +ka -7.828785 +si -7.849948 +_e -7.853253 +la -7.853664 +ge -7.898025 +mi -7.898107 +ba -7.904090 +pe -7.935825 +av -7.965191 +as -8.006404 +wa -8.038891 +ev -8.108447 +ve -8.113182 +ew -8.138593 +ay -8.140802 +li -8.175331 +ed -8.191662 +il -8.219162 +va -8.286125 +am -8.308316 +gi -8.309074 +un -8.314039 +ib -8.350300 +hi -8.374253 +ez -8.382641 +zi -8.404742 +bu -8.415865 +d_ -8.450602 +_y -8.453656 +ca -8.558974 +za -8.566287 +ih -8.567130 +im -8.571179 +ce -8.587916 +ci -8.653020 +eb -8.657725 +_z -8.659856 +pi -8.710654 +m_ -8.712993 +_i -8.768273 +rk -8.785676 +rt -8.806096 +eh -8.810650 +s_ -8.820843 +or -8.822069 +ji -8.823351 +iv -8.823525 +vi -8.907866 +pa -8.983677 +wi -8.996171 +l_ -9.023855 +ko -9.059762 +go -9.076129 +ok -9.089122 +ze -9.125821 +id -9.153696 +az -9.161127 +ng -9.173959 +ja -9.215438 +ak -9.215615 +v_ -9.315431 +_l -9.340942 +ej -9.383522 +ad -9.387548 +yi -9.387696 +oj -9.390642 +u_ -9.433654 +on -9.437196 +_f -9.444570 +ot -9.453603 +it -9.464882 +_q -9.476214 +xe -9.531233 +ij -9.548986 +fe -9.566821 +aw -9.600820 +_v -9.635273 +qe -9.647610 +ol -9.708948 +om -9.734973 +ax -9.743820 +ga -9.780979 +ex -9.785719 +xi -9.789572 +_j -9.794619 +je -9.813965 +su -9.828011 +rs -9.845638 +sk -9.846767 +uy -9.865987 +mu -9.867428 +iz -9.873300 +ef -9.918985 +us -9.926051 +xa -9.973119 +o_ -9.991326 +ab -10.002826 +wl -10.016794 +ru -10.028171 +rm -10.039546 +gu -10.075907 +do -10.088228 +ix -10.095246 +ah -10.124203 +ag -10.137806 +rb -10.139307 +ig -10.148456 +du -10.195116 +ov -10.205121 +nc -10.232183 +fi -10.267537 +um -10.274081 +tu -10.300501 +eg -10.311315 +qa -10.340515 +z_ -10.345847 +aj -10.443885 +lo -10.450333 +hu -10.456617 +_o -10.483486 +h_ -10.529765 +cu -10.538970 +sy -10.605096 +os -10.610052 +lb -10.643563 +nu -10.653873 +af -10.660727 +bo -10.676967 +to -10.695313 +ic -10.696423 +ec -10.712311 +ul -10.746859 +rx -10.749530 +yo -10.762528 +mo -10.795675 +g_ -10.838037 +ns -10.841377 +so -10.846277 +rg -10.870831 +uh -10.912357 +ud -10.925706 +aq -10.953572 +oz -10.953628 +po -10.961679 +xu -10.999305 +uk -11.007171 +nt -11.010156 +fa -11.041054 +op -11.056553 +oy -11.064569 +eq -11.071797 +mb -11.084723 +tr -11.100638 +kr -11.140341 +ry -11.250346 +ft -11.306530 +rh -11.337940 +qi -11.344536 +rw -11.348796 +rv -11.355151 +ws -11.357636 +j_ -11.360367 +lk -11.360883 +if -11.399589 +ap -11.456738 +ai -11.467433 +rc -11.469571 +lm -11.479828 +ny -11.502908 +fr -11.544115 +p_ -11.548780 +vb -11.548950 +vk -11.551327 +og -11.552250 +vd -11.568197 +nk -11.596894 +wr -11.622681 +no -11.636733 +ld -11.721605 +zg -11.727924 +uc -11.734703 +sm -11.751250 +rn -11.755244 +ip -11.768454 +gr -11.773550 +sl -11.778497 +ub -11.779388 +jn -11.792619 +co -11.817470 +iq -11.831022 +hm -11.841790 +sd -11.868032 +f_ -11.869917 +ho -11.876267 +iw -11.884716 +sp -11.891071 +yb -11.900480 +oc -11.905895 +pr -11.908001 +tp -11.954370 +rz -11.958603 +b_ -11.971441 +y_ -11.973237 +c_ -12.005417 +ut -12.013730 +ep -12.014989 +jm -12.022934 +yd -12.027228 +rl -12.098586 +vc -12.100519 +nb -12.136660 +mp -12.149463 +sn -12.200813 +ia -12.243751 +zm -12.262973 +hd -12.269296 +sg -12.303370 +ll -12.304749 +jo -12.342246 +vn -12.344487 +kk -12.359334 +xt -12.363504 +rp -12.378633 +ac -12.388389 +dy -12.421150 +lp -12.427998 +qu -12.451295 +mk -12.463435 +yn -12.464223 +kt -12.466087 +jh -12.483517 +sw -12.498480 +sr -12.498884 +tg -12.503184 +vg -12.514787 +dk -12.545895 +x_ -12.553193 +pk -12.555614 +ob -12.559819 +rf -12.561810 +th -12.574109 +ي_ -12.594059 +yv -12.601881 +tn -12.604366 +pl -12.623441 +_u -12.642022 +lu -12.647788 +ly -12.654557 +q_ -12.696444 +ks -12.728462 +br -12.745492 +up -12.748586 +zo -12.772673 +fo -12.811727 +ym -12.823236 +lt -12.847071 +vl -12.855574 +dp -12.868293 +hs -12.871876 +uj -12.873802 +w_ -12.912674 +tm -12.951995 +bd -12.957657 +lw -13.014674 +kd -13.020536 +cl -13.059053 +vp -13.080749 +ea -13.098600 +tk -13.104339 +kh -13.120931 +uz -13.172881 +rr -13.193816 +kb -13.195242 +hr -13.212590 +pd -13.229874 +pu -13.234480 +sb -13.242107 +rj -13.256273 +ck -13.260560 +zd -13.283386 +ae -13.293821 +tt -13.328845 +kw -13.374032 +dg -13.391979 +yp -13.397065 +od -13.403165 +ch -13.404126 +nf -13.425485 +xd -13.442111 +vs -13.456506 +pg -13.504767 +zk -13.528767 +dr -13.535414 +mz -13.538465 +cr -13.547811 +ht -13.557486 +py -13.579939 +kl -13.585462 +xo -13.611093 +vr -13.611172 +gl -13.622801 +ls -13.643542 +cb -13.655068 +td -13.658917 +sx -13.670030 +of -13.677320 +vo -13.695203 +_ي -13.716465 +qo -13.722794 +sh -13.723951 +lg -13.727946 +kn -13.764489 +yu -13.780190 +yt -13.791427 +ds -13.792236 +zu -13.798004 +ان -13.810337 +yr -13.814944 +kp -13.830421 +yw -13.835046 +eo -13.836715 +bl -13.850930 +oo -13.851632 +xs -13.884333 +xn -13.953151 +kc -13.984149 +ei -13.987646 +_ك -13.996064 +oh -14.023990 +sv -14.027162 +ee -14.032304 +fu -14.039012 +ن_ -14.041415 +uq -14.054677 +_ب -14.063551 +wc -14.067573 +mr -14.083333 +mm -14.105679 +nq -14.125201 +tb -14.140176 +zy -14.157391 +zr -14.175929 +tv -14.192510 +ms -14.232160 +hn -14.251505 +ss -14.253361 +jw -14.254290 +wq -14.264984 +ie -14.268170 +km -14.272492 +ux -14.277771 +و_ -14.283891 +_د -14.303050 +zb -14.327085 +ua -14.340240 +يا -14.386130 +kv -14.389936 +يە -14.397783 +hk -14.409669 +dl -14.430236 +ps -14.431426 +io -14.435494 +fs -14.440630 +dd -14.456079 +yl -14.457433 +xm -14.463865 +hp -14.470758 +ر_ -14.494780 +my -14.495512 +np -14.501675 +وو -14.509121 +uv -14.514824 +vt -14.521816 +fx -14.523905 +را -14.533120 +رد -14.553250 +jd -14.562961 +md -14.564650 +wn -14.573820 +nn -14.594765 +hw -14.601841 +ار -14.620085 +ug -14.621602 +vx -14.662036 +tc -14.667058 +vu -14.681728 +ا_ -14.686317 +ow -14.724938 +ys -14.752234 +gd -14.752934 +mt -14.774626 +ست -14.810610 +wd -14.814989 +ju -14.818557 +wm -14.822317 +gh -14.823052 +ور -14.851820 +yk -14.856702 +yy -14.884141 +bn -14.892215 +tl -14.906942 +uf -14.907039 +db -14.909573 +vy -14.909964 +cd -14.921919 +_س -14.924578 +كو -14.930208 +rq -14.942931 +aa -14.944130 +ە_ -14.949034 +تا -14.951945 +او -14.958995 +tw -14.971772 +jb -14.983020 +ue -14.985383 +dw -14.986000 +tx -14.993942 +ني -14.994459 +nx -15.002449 +gt -15.037625 +mh -15.044251 +lh -15.044893 +ري -15.056296 +ت_ -15.059107 +kg -15.066702 +ات -15.100603 +oa -15.101605 +ەو -15.116843 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/languages.lst b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/languages.lst new file mode 100644 index 00000000000..036bae92509 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/languages.lst @@ -0,0 +1,148 @@ +afr +amh +ara +asm +ast +aze +bak +ban +bel +ben +ben-rom +bih +bos +bre +bul +cat +ceb +ces +che +ckb +cmn +cym +dan +deu +div +ekk +ell +eng +epo +est +eus +fao +fas +fin +fra +fry +ful +gla +gle +glg +gom +gsw +gug +guj +hat +hau +heb +hin +hin-rom +hrv +hun +hye +ibo +ind +isl +ita +jav +jpn +kan +kat +kaz +khm +kin +kir +knn +kor +kur +lao +lat +lav +lim +lin +lit +ltz +lug +lvs +mal +mar +mhr +min +mkd +mlg +mlt +mon +mri +msa +mya +mya-zaw +nan +nds +nep +new +nld +nno +nob +nso +oci +ori +orm +pan +pes +pnb +pol +por +pus +quz +roh +ron +rus +san +sin +slk +slv +snd +som +spa +sqi +srd +srp +ssw +swa +swe +tam +tam-rom +tat +tel +tel-rom +tgk +tgl +tha +tsn +tuk +tur +uig +ukr +urd +urd-rom +uzb +vie +vol +war +wol +xho +yid +yor +zho-simp +zho-trad +zul diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lao b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lao new file mode 100644 index 00000000000..637cbed6f30 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lao @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 68902407 +#UNIQUE_BIGRAMS 1791 +#UNSEEN_LOG_PROB -14.059273 +#EXPECTED_SCORE -8.423977 +ນ_ -4.568451 +ດ_ -5.428590 +ງ_ -5.466559 +_ເ -5.469209 +_ແ -5.691915 +ານ -5.754901 +_ກ -5.761630 +ະ_ -5.845669 +ລະ -5.914105 +ກາ -6.049544 +ແລ -6.104340 +າ_ -6.121903 +ກ_ -6.205952 +_ສ -6.231922 +ອງ -6.476245 +ມ_ -6.529606 +ບ_ -6.572519 +_ຫ -6.931539 +_ຄ -6.997791 +_ລ -7.074884 +_ພ -7.111344 +ຍ_ -7.133055 +າມ -7.179460 +_ຂ -7.221105 +_ປ -7.309935 +ນາ -7.330854 +_ອ -7.365011 +ຂອ -7.423636 +ປະ -7.535856 +ສາ -7.566281 +ລາ -7.590968 +_ທ -7.594521 +າຍ -7.600770 +_ຈ -7.686495 +_ນ -7.687518 +ວ_ -7.696207 +_ບ -7.700172 +າດ -7.803664 +ສະ -7.818745 +ວນ -7.819190 +_ວ -7.844940 +ອນ -7.850530 +_ໂ -7.986449 +_ຊ -8.018135 +_ມ -8.027250 +_ຕ -8.056088 +ເປ -8.068749 +ລດ -8.113482 +ປນ -8.146611 +ພາ -8.160102 +ວາ -8.160246 +ທາ -8.182453 +ວງ -8.186558 +າກ -8.190003 +ຫວ -8.241029 +າວ -8.279093 +e_ -8.306135 +າງ -8.312392 +_ຮ -8.320348 +ະນ -8.354727 +ວຽ -8.365743 +ຈາ -8.366220 +ະຫ -8.394420 +ຊາ -8.396444 +ຄວ -8.401221 +ບນ -8.428209 +s_ -8.432022 +ມາ -8.475376 +ຫາ -8.508437 +ອກ -8.518829 +ທດ -8.546574 +ສດ -8.619260 +ອາ -8.623655 +ຄນ -8.630559 +ນະ -8.635630 +_s -8.637672 +ຫນ -8.655841 +ເກ -8.709753 +_ຍ -8.723105 +ກດ -8.732265 +in -8.741816 +ກບ -8.747141 +ກນ -8.766469 +ເມ -8.836425 +ນກ -8.864579 +er -8.868684 +ເທ -8.879836 +ທ_ -8.925235 +n_ -8.948497 +ະກ -8.954219 +ມນ -8.969493 +ດນ -8.971510 +ະຊ -8.978622 +ຍາ -8.990576 +ອບ -9.005564 +an -9.006791 +ຈນ -9.011960 +ກອ -9.013918 +on -9.043908 +າລ -9.074794 +_c -9.077506 +ສງ -9.080994 +ງາ -9.083548 +ຫລ -9.102333 +ເຮ -9.107129 +ວດ -9.107140 +ຫມ -9.108735 +ກມ -9.121040 +ເຫ -9.149427 +ບາ -9.157164 +ນທ -9.165893 +ຽດ -9.198503 +ທະ -9.201458 +ລກ -9.202124 +ມອ -9.215493 +ໃນ -9.218333 +t_ -9.218658 +_ດ -9.219631 +ລງ -9.228484 +_a -9.245693 +ຖະ -9.253531 +d_ -9.254580 +ຮບ -9.261194 +ຄາ -9.267052 +_m -9.281182 +ບດ -9.297150 +ລ_ -9.297994 +ນວ -9.300764 +ອດ -9.301213 +_ໃ -9.308098 +ດຖ -9.317553 +ພວ -9.328308 +ເລ -9.348149 +ຄອ -9.349705 +ເສ -9.351879 +r_ -9.357019 +ຕາ -9.383457 +_p -9.388921 +ຽນ -9.395365 +ຂວ -9.400096 +ນດ -9.400379 +ດາ -9.400577 +ຊນ -9.410560 +ຍງ -9.436933 +ສນ -9.447274 +re -9.456291 +_t -9.459030 +ແຂ -9.462822 +ໂດ -9.476689 +ດຍ -9.493132 +ວກ -9.498408 +es -9.502153 +ຽງ -9.502229 +ກະ -9.506243 +ງກ -9.530458 +ar -9.534629 +າຊ -9.534909 +al -9.540667 +ຫ_ -9.544258 +ເອ -9.545494 +_b -9.545744 +ະບ -9.551656 +ພດ -9.555779 +ປ_ -9.556015 +y_ -9.562555 +a_ -9.570111 +ma -9.586598 +_ງ -9.597344 +ະທ -9.624969 +te -9.624969 +or -9.630617 +ະມ -9.632045 +ະເ -9.634756 +ພນ -9.642082 +la -9.647647 +າບ -9.653571 +ຕ_ -9.661841 +ລບ -9.668948 +ຮອ -9.676467 +ຜນ -9.684611 +ສກ -9.684887 +ະພ -9.691035 +ກສ -9.702451 +l_ -9.704721 +o_ -9.709692 +_f -9.711095 +_ຜ -9.735466 +ຊວ -9.746359 +ເຂ -9.754963 +ພະ -9.772912 +ng -9.777885 +ນນ -9.787919 +ະລ -9.798247 +st -9.798546 +_ຖ -9.799254 +ເດ -9.801196 +en -9.809326 +ສອ -9.826923 +at -9.830446 +ເຊ -9.832201 +ດທ -9.863964 +ງວ -9.864960 +ມດ -9.874519 +_l -9.875050 +ເວ -9.879579 +le -9.893393 +as -9.893871 +ra -9.911084 +ກລ -9.912153 +ທງ -9.914900 +ne -9.921139 +ຫະ -9.932572 +ti -9.936691 +ມະ -9.945893 +_h -9.950195 +se -9.962635 +ພກ -9.971794 +ທນ -9.976131 +_d -9.978812 +ະຈ -9.984698 +ຄມ -9.985525 +ະສ -9.987182 +ນເ -10.001641 +ri -10.002092 +g_ -10.010447 +ຈດ -10.015185 +ອອ -10.029929 +ຂາ -10.047637 +ປກ -10.049921 +_ໄ -10.070149 +າທ -10.076556 +ha -10.100195 +ະຄ -10.105283 +ນອ -10.109857 +ບລ -10.110620 +_r -10.116814 +ນຕ -10.121868 +_g -10.134593 +ro -10.137750 +ic -10.149665 +_i -10.190455 +nt -10.215903 +ຮດ -10.218246 +ນຫ -10.220568 +na -10.231527 +ຕະ -10.241200 +ດສ -10.242874 +ເພ -10.243787 +ນຍ -10.253518 +ບບ -10.256435 +ເຕ -10.260924 +ທກ -10.262004 +_w -10.266613 +ຖາ -10.270074 +ss -10.271160 +ຊມ -10.280869 +ຄະ -10.295400 +າເ -10.300962 +is -10.308006 +ຂດ -10.322062 +າຫ -10.324450 +co -10.334823 +ສມ -10.338533 +ຍກ -10.338804 +h_ -10.342415 +li -10.353001 +nd -10.356868 +ນສ -10.357994 +ຫງ -10.358956 +ຖງ -10.366206 +ຊ_ -10.368225 +ດອ -10.375299 +sh -10.380844 +ເຖ -10.389722 +ສປ -10.393772 +_v -10.398286 +me -10.413084 +to -10.413284 +io -10.422450 +າຄ -10.425039 +ch -10.428180 +ະຕ -10.444485 +th -10.447259 +ຕກ -10.447435 +ta -10.448605 +ຮງ -10.452443 +ມສ -10.453500 +ທບ -10.455791 +ປປ -10.472413 +ຕນ -10.473902 +ຮາ -10.481964 +it -10.481964 +ງຄ -10.484362 +_n -10.485652 +ປດ -10.488748 +ນລ -10.488868 +he -10.500802 +າສ -10.503627 +i_ -10.522752 +p_ -10.526638 +ດກ -10.528554 +ຊຍ -10.543289 +ດຕ -10.545447 +ນປ -10.548015 +ou -10.557860 +el -10.560482 +ວລ -10.566598 +ໂລ -10.575868 +ອ_ -10.576763 +ງສ -10.585520 +ລອ -10.587130 +ho -10.587581 +de -10.591614 +ລວ -10.593166 +am -10.603749 +ສບ -10.610182 +ສ_ -10.614574 +ະຍ -10.615198 +າະ -10.616841 +ງຫ -10.621847 +ca -10.630020 +ງນ -10.633209 +ງຈ -10.663351 +om -10.664812 +_ຢ -10.672036 +lo -10.680843 +ni -10.693166 +ປາ -10.745478 +ແນ -10.750267 +ຫດ -10.751674 +si -10.767537 +_k -10.774305 +ce -10.779964 +ac -10.781547 +ຕວ -10.782986 +ຈບ -10.786495 +_e -10.795175 +mi -10.796701 +ea -10.797931 +ba -10.799758 +ນບ -10.809120 +ຂະ -10.822821 +k_ -10.823201 +ກງ -10.826087 +em -10.827113 +c_ -10.829169 +ad -10.830350 +ບກ -10.831876 +ເງ -10.834321 +ຽກ -10.843179 +ໂຄ -10.850235 +ap -10.866992 +m_ -10.869183 +us -10.876919 +ie -10.879362 +ຕດ -10.879915 +ເຈ -10.888264 +pe -10.891085 +oo -10.893114 +ແດ -10.897541 +ຮກ -10.897740 +ge -10.898020 +ໄຊ -10.902302 +ະດ -10.912355 +pa -10.916072 +ດງ -10.916234 +et -10.921829 +tr -10.930875 +ve -10.938128 +hi -10.938868 +un -10.944716 +ed -10.950091 +ຊກ -10.977228 +ເນ -10.995964 +_j -11.002651 +sa -11.003897 +po -11.008246 +ຕອ -11.008548 +ມກ -11.013301 +di -11.014427 +vi -11.026867 +ຖກ -11.030717 +ວທ -11.034138 +າປ -11.034314 +ລຍ -11.042107 +ຄງ -11.045068 +os -11.048435 +ai -11.058001 +bo -11.061934 +ກວ -11.063681 +ia -11.065385 +_o -11.068934 +mo -11.070104 +id -11.083628 +ur -11.086812 +ແບ -11.096316 +so -11.096453 +ll -11.101689 +ຈກ -11.112450 +il -11.113563 +ແຜ -11.114677 +ol -11.118722 +_ຟ -11.124741 +ki -11.130081 +ແຮ -11.134357 +ຍະ -11.137562 +ຮຽ -11.139072 +ec -11.141529 +ວມ -11.147074 +ທລ -11.179050 +ມງ -11.181382 +ງພ -11.201803 +ld -11.211552 +ງທ -11.217822 +ຊຽ -11.226877 +ໄລ -11.228434 +ov -11.236343 +ນງ -11.238466 +ດບ -11.238769 +wo -11.249792 +ະວ -11.249894 +ຕບ -11.262544 +ut -11.267800 +be -11.268936 +ns -11.269815 +ir -11.278839 +dr -11.298218 +ໂທ -11.298692 +rt -11.303712 +pr -11.312578 +ຊອ -11.319411 +cr -11.325470 +ຣ_ -11.325846 +ດເ -11.327783 +ຄດ -11.341899 +rl -11.342714 +_u -11.346853 +ຫຍ -11.346907 +ບປ -11.359286 +wi -11.363859 +ck -11.368446 +lt -11.372771 +ke -11.373826 +ເຄ -11.382464 +do -11.383807 +ນຊ -11.385542 +ໄຫ -11.389188 +ນໂ -11.396338 +ອະ -11.409207 +ປງ -11.412000 +ຕລ -11.417774 +go -11.422766 +ກຄ -11.433729 +ໂຮ -11.434888 +ດຫ -11.439999 +bi -11.447113 +tt -11.454263 +og -11.455086 +ະຖ -11.456144 +ດດ -11.465232 +ts -11.472949 +ru -11.475928 +ງປ -11.482443 +ຣາ -11.486643 +າພ -11.489590 +vo -11.494596 +ot -11.497197 +ຽວ -11.498529 +nc -11.516880 +ຟກ -11.523892 +ລສ -11.528833 +ດຈ -11.539700 +ນພ -11.539825 +ຣະ -11.547200 +no -11.553354 +ຍດ -11.566185 +ow -11.581825 +ag -11.589031 +ຫອ -11.594460 +_ຣ -11.602511 +ip -11.604400 +ະແ -11.607402 +da -11.608906 +ພບ -11.614935 +ທອ -11.634972 +ແຫ -11.636638 +ry -11.637438 +ມພ -11.644124 +ນມ -11.644325 +fl -11.654347 +rs -11.656441 +ຈງ -11.660027 +ກກ -11.660841 +ຣກ -11.661994 +sp -11.665798 +ສຍ -11.672139 +ຢາ -11.717885 +ງເ -11.732916 +b_ -11.734199 +mp -11.734627 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lat b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lat new file mode 100644 index 00000000000..1c4e0dd1561 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lat @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 6624265 +#UNIQUE_BIGRAMS 569 +#UNSEEN_LOG_PROB -19.019661 +#EXPECTED_SCORE -7.388014 +s_ -5.156552 +m_ -5.512820 +e_ -5.740047 +t_ -5.885237 +a_ -5.913720 +er -6.098638 +um -6.121413 +us -6.137468 +_a -6.168071 +es -6.260452 +_c -6.266911 +an -6.294493 +_s -6.314045 +_e -6.351257 +_p -6.363183 +it -6.364942 +at -6.374993 +is -6.393769 +ri -6.444611 +tu -6.467884 +ae -6.479847 +st -6.500404 +ti -6.514890 +in -6.541645 +o_ -6.559550 +ta -6.659065 +li -6.680775 +ia -6.691832 +nt -6.707839 +ra -6.730602 +i_ -6.743543 +on -6.871579 +_i -6.897103 +te -6.903372 +ni -6.912106 +_m -6.919838 +or -6.972692 +no -6.983923 +ic -6.988783 +qu -7.024008 +co -7.028948 +re -7.074132 +di -7.080455 +ci -7.090571 +ar -7.115149 +en -7.130966 +_d -7.237166 +r_ -7.250469 +un -7.261855 +_f -7.264531 +io -7.312088 +em -7.340751 +ca -7.360300 +am -7.412440 +cu -7.436911 +si -7.465804 +ur -7.472286 +ne -7.476236 +la -7.489913 +_n -7.494029 +ma -7.495561 +ru -7.537027 +na -7.560639 +om -7.578803 +iu -7.603326 +ec -7.630732 +_q -7.635488 +nn -7.648626 +pe -7.650069 +vi -7.689987 +se -7.704678 +al -7.736280 +_v -7.739255 +su -7.752110 +_r -7.757755 +ul -7.760114 +ua -7.766170 +de -7.779076 +mi -7.794369 +pr -7.806019 +ui -7.857015 +_h -7.888665 +_l -7.901262 +_t -7.922134 +ct -7.930824 +ol -7.943153 +ve -7.996049 +ro -7.998442 +im -8.051652 +po -8.057210 +ie -8.068274 +le -8.069385 +et -8.118959 +nc -8.126060 +nu -8.138158 +me -8.153331 +el -8.193571 +_o -8.193699 +mu -8.214767 +ce -8.255252 +tr -8.278056 +to -8.312400 +pa -8.320384 +mo -8.332759 +ns -8.365929 +ib -8.397601 +_u -8.408734 +bu -8.411550 +oc -8.432616 +os -8.465881 +ll -8.471822 +il -8.496072 +ed -8.505698 +rt -8.521617 +gi -8.540145 +lo -8.591979 +nd -8.606336 +lu -8.613569 +ac -8.629265 +_g -8.685452 +sa -8.722139 +ii -8.724348 +as -8.758651 +ha -8.760350 +be -8.775827 +ep -8.815604 +eg -8.824068 +pu -8.833774 +op -8.873264 +ap -8.885160 +ss -8.886087 +so -8.897674 +ut -8.924408 +sc -8.928010 +ue -8.940047 +ho -8.942724 +hi -8.960407 +du -8.961384 +au -8.968131 +pi -8.990776 +d_ -9.001676 +ab -9.004246 +n_ -9.026674 +ge -9.033506 +od -9.042665 +id -9.067573 +mp -9.141783 +bi -9.172117 +_b -9.177656 +iv -9.188035 +uo -9.191469 +ip -9.218590 +rc -9.223138 +ov -9.224831 +ba -9.269967 +ir -9.296700 +ng -9.322543 +fi -9.323520 +fe -9.347864 +ch -9.387866 +c_ -9.390495 +cr -9.397528 +fa -9.410216 +eb -9.419730 +fu -9.441674 +ex -9.457818 +ot -9.459043 +ud -9.479886 +do -9.509264 +pt -9.537957 +rs -9.550805 +gr -9.556697 +mm -9.560968 +da -9.571866 +ea -9.574521 +ad -9.579344 +br -9.624358 +rb -9.626594 +ei -9.632114 +gn -9.632806 +th -9.647403 +va -9.652302 +ag -9.664268 +gu -9.707498 +fr -9.764511 +he -9.775158 +ig -9.778789 +cc -9.782620 +rm -9.832582 +ub -9.843545 +oe -9.877559 +lt -9.880020 +av -9.886396 +fo -9.988894 +uc -10.006708 +l_ -10.012095 +pp -10.013671 +eo -10.045309 +pl -10.049919 +sp -10.058951 +ga -10.084499 +cl -10.116297 +xi -10.201179 +u_ -10.314201 +ph -10.331776 +eu -10.396940 +rr -10.448658 +og -10.532947 +rn -10.565911 +ef -10.614593 +ob -10.624874 +bo -10.639391 +bl -10.641825 +up -10.691022 +rd -10.714983 +x_ -10.726007 +vo -10.742330 +hu -10.752814 +rg -10.766786 +iq -10.769825 +ev -10.804850 +mn -10.807970 +mb -10.826044 +gl -10.896117 +uu -10.948092 +ai -10.972828 +ps -10.990001 +if -11.007829 +tt -11.075776 +rv -11.190705 +ux -11.200410 +bs -11.218460 +go -11.230969 +ax -11.237264 +ug -11.244115 +tq -11.293007 +sq -11.297933 +ff -11.406072 +nv -11.406664 +vu -11.454147 +eq -11.464572 +xe -11.541686 +fl -11.565911 +xt -11.632806 +mq -11.781278 +nf -11.847152 +oa -11.856813 +aq -11.901106 +hr -11.962361 +lp -12.038193 +oq -12.045539 +y_ -12.144615 +dr -12.160480 +of -12.170485 +rh -12.172494 +dm -12.305080 +ty -12.377399 +lg -12.387866 +xo -12.390202 +rp -12.426908 +dh -12.443796 +ix -12.447441 +uv -12.478177 +yr -12.508312 +ox -12.591895 +ys -12.656514 +sy -12.669225 +b_ -12.700776 +ou -12.730071 +xp -12.741957 +lv -12.758462 +sd -12.881252 +_x -12.919548 +py -12.958889 +rf -12.960624 +nq -12.965842 +_k -13.020893 +lm -13.037277 +xa -13.057558 +oh -13.098996 +ls -13.108582 +_w -13.110507 +yt -13.129898 +lb -13.155503 +rl -13.183596 +ym -13.230969 +xs -13.275625 +af -13.337401 +_j -13.387866 +ld -13.423315 +ih -13.476935 +my -13.509582 +g_ -13.530046 +nr -13.569217 +dd -13.656514 +k_ -13.679189 +sl -13.731551 +eh -13.734516 +uf -13.743450 +h_ -13.798242 +oi -13.829606 +yp -13.864913 +ka -13.887839 +hy -13.958889 +ly -13.976334 +dv -14.037277 +gy -14.100908 +ry -14.120170 +sm -14.155503 +lc -14.159483 +pd -14.171489 +sv -14.183596 +oo -14.220537 +_z -14.237264 +jo -14.267012 +tl -14.271312 +cy -14.332899 +wa -14.337401 +p_ -14.364708 +bb -14.401941 +by -14.401941 +hn -14.411401 +za -14.411401 +sh -14.416155 +xu -14.425709 +zi -14.474454 +yc -14.530046 +xv -14.577180 +cd -14.642521 +nx -14.659329 +ny -14.670644 +sb -14.676335 +xc -14.676335 +ln -14.699327 +az -14.716814 +ko -14.716814 +ms -14.746440 +wi -14.764511 +ts -14.884542 +ee -14.951970 +mt -14.951970 +ok -14.958889 +yl -14.965842 +rk -14.972828 +bt -14.986904 +ck -15.008277 +zo -15.015473 +ew -15.081900 +ow -15.089473 +gm -15.104740 +ah -15.120170 +cf -15.143629 +yz -15.143629 +wo -15.167476 +nl -15.199897 +ja -15.199897 +ki -15.199897 +cn -15.216385 +ke -15.233064 +ze -15.275625 +we -15.293007 +ds -15.319479 +df -15.392542 +w_ -15.411401 +bn -15.509582 +iz -15.550805 +hl -15.561297 +lf -15.593240 +tc -15.614935 +dp -15.614935 +gd -15.636961 +yo -15.636961 +ak -15.670644 +yn -15.682049 +ay -15.693545 +v_ -15.764511 +z_ -15.764511 +fd -15.776686 +ya -15.776686 +_y -15.801348 +mv -15.813839 +dt -15.972828 +bh -15.986904 +ao -15.986904 +sk -16.001117 +je -16.001117 +ви -16.044619 +ич -16.044619 +ч_ -16.044619 +ks -16.074366 +ez -16.074366 +dg -16.089473 +ey -16.089473 +oy -16.135767 +sr -16.167476 +tg -16.167476 +bd -16.183596 +gt -16.199897 +dy -16.267012 +f_ -16.337401 +rx -16.373927 +xf -16.430510 +ht -16.469504 +rq -16.489404 +ky -16.530046 +wh -16.550805 +gh -16.593240 +ан -16.636961 +aw -16.682049 +tm -16.705133 +dn -16.705133 +md -16.728592 +tz -16.752438 +nb -16.752438 +nk -16.776686 +wr -16.826439 +bf -16.851974 +ов -16.877969 +xx -16.904441 +ju -16.904441 +dc -16.958889 +wn -16.986904 +cs -17.015473 +ва -17.015473 +ил -17.015473 +vl -17.044619 +cq -17.074366 +ол -17.104740 +yb -17.104740 +dq -17.135767 +tf -17.135767 +aa -17.167476 +vp -17.199897 +ев -17.233064 +аи -17.267012 +yd -17.301777 +sn -17.337401 +dl -17.337401 +_а -17.373927 +ал -17.373927 +ле -17.373927 +ек -17.373927 +кс -17.373927 +са -17.373927 +нд -17.373927 +др -17.373927 +и_ -17.373927 +sg -17.411401 +mc -17.411401 +yg -17.411401 +dw -17.411401 +_н -17.449876 +ни -17.449876 +ик -17.449876 +ко -17.449876 +ла -17.449876 +rw -17.449876 +db -17.530046 +gs -17.530046 +kl -17.571866 +bv -17.614935 +ye -17.614935 +lh -17.659329 +ws -17.659329 +hm -17.705133 +zu -17.705133 +gg -17.752438 +fg -17.752438 +οτ -17.752438 +ро -17.851974 +fh -17.904441 +ft -17.904441 +zz -17.958889 +_и -17.958889 +ив -17.958889 +wl -17.958889 +_м -17.958889 +ми -17.958889 +их -17.958889 +ха -17.958889 +xh -18.015473 +lz -18.015473 +aj -18.015473 +nh -18.015473 +ek -18.015473 +_в -18.074366 +ас -18.074366 +си -18.074366 +sf -18.074366 +р_ -18.135767 +tp -18.199897 +ml -18.199897 +hs -18.199897 +ае -18.267012 +ль -18.267012 +lk -18.411401 +bc -18.489404 +kp -18.571866 +oz -18.659329 +tv -18.659329 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lav b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lav new file mode 100644 index 00000000000..b6b7ac6c3b3 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lav @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 84148414 +#UNIQUE_BIGRAMS 549 +#UNSEEN_LOG_PROB -20.310420 +#EXPECTED_SCORE -7.393758 +s_ -4.663679 +a_ -5.317800 +ie -5.385640 +as -5.565581 +u_ -5.802207 +_p -5.831285 +i_ -5.958040 +ta -6.162649 +ar -6.213913 +_v -6.245735 +_s -6.278124 +ja -6.305182 +ti -6.416449 +ka -6.416853 +pa -6.552049 +es -6.553970 +na -6.566520 +sa -6.569333 +at -6.609212 +_t -6.649009 +la -6.752492 +_a -6.759084 +_k -6.778032 +ai -6.810737 +vi -6.816874 +ri -6.826274 +va -6.852868 +_l -6.871519 +st -6.907110 +ma -6.920170 +_n -6.921360 +ra -6.926617 +t_ -6.951913 +is -6.956209 +li -6.970131 +m_ -7.019221 +in -7.024716 +ad -7.051054 +al -7.084692 +_i -7.120904 +da -7.124397 +ij -7.136215 +et -7.160353 +en -7.167020 +em -7.192954 +_d -7.240250 +_b -7.291379 +au -7.300163 +ga -7.304293 +am -7.355307 +an -7.365300 +ne -7.407939 +tu -7.410795 +ek -7.414184 +ci -7.416853 +ni -7.439703 +ik -7.464474 +ak -7.477944 +us -7.486223 +si -7.489735 +_m -7.500666 +ve -7.502677 +er -7.535496 +ba -7.584597 +aj -7.588835 +di -7.596019 +um -7.604156 +ju -7.637493 +pi -7.645009 +ku -7.651930 +it -7.665065 +ot -7.691246 +re -7.700938 +zi -7.755413 +el -7.777908 +dz -7.809029 +te -7.839936 +_g -7.860735 +no -7.864733 +me -7.889640 +iz -7.902107 +r_ -7.905099 +ib -7.924403 +ur -8.057133 +ko -8.069989 +sk -8.071524 +av -8.101927 +e_ -8.105403 +ej -8.127216 +_r -8.127906 +_j -8.145722 +ks -8.155921 +de -8.165719 +pr -8.206023 +pe -8.209226 +mu -8.232530 +ro -8.251053 +ed -8.259934 +os -8.266053 +ei -8.271807 +id -8.282711 +mi -8.282887 +tr -8.305643 +bi -8.329220 +ir -8.335498 +ts -8.382573 +_c -8.400739 +il -8.415199 +bu -8.433042 +le -8.459945 +ut -8.465388 +ec -8.532157 +ap -8.548747 +to -8.551768 +ru -8.623881 +se -8.631748 +ls -8.679994 +ce -8.681415 +be -8.685461 +ms -8.691437 +on -8.696784 +rt -8.698628 +kt -8.699148 +ze -8.745232 +su -8.753192 +ig -8.755562 +nu -8.765548 +rs -8.784905 +o_ -8.797849 +om -8.821335 +k_ -8.846566 +or -8.856982 +nt -8.877349 +ac -8.880634 +ev -8.924137 +do -8.952106 +sp -8.953704 +im -8.955405 +iv -8.985540 +ki -9.005994 +kl -9.009515 +ol -9.017445 +ud -9.023223 +so -9.042775 +lu -9.045109 +ns -9.071810 +du -9.092373 +z_ -9.120115 +ab -9.163678 +un -9.201918 +n_ -9.204899 +_z -9.219614 +d_ -9.236537 +_u -9.242330 +az -9.244564 +_e -9.262111 +uz -9.264735 +za -9.297545 +gu -9.304824 +ep -9.326147 +ji -9.392615 +dr -9.405719 +oj -9.407185 +ic -9.408921 +gi -9.430337 +tv -9.465079 +od -9.469351 +eg -9.484746 +rb -9.485765 +uk -9.523803 +rm -9.525925 +lo -9.543793 +ul -9.564232 +ld -9.611704 +ez -9.619114 +ag -9.621732 +po -9.642698 +pu -9.674078 +kr -9.674624 +v_ -9.716850 +gr -9.719565 +jo -9.725054 +vu -9.787880 +br -9.810040 +sm -9.815962 +c_ -9.882858 +op -9.895784 +l_ -9.929945 +ok -10.014826 +_f -10.049345 +lv -10.052965 +sv -10.071718 +vo -10.079599 +sl -10.083482 +nd -10.112436 +_o -10.130888 +zs -10.157721 +ug -10.213423 +ca -10.223801 +lt -10.349376 +rd -10.407546 +rn -10.435242 +sn -10.437262 +mo -10.473830 +zn -10.509399 +zv -10.516238 +rg -10.525028 +cu -10.564466 +nk -10.567299 +og -10.568730 +ob -10.618319 +gs -10.645167 +rk -10.653483 +tn -10.684691 +je -10.689044 +uv -10.692898 +pl -10.696137 +fo -10.707302 +zu -10.711780 +eb -10.732779 +ia -10.738450 +ds -10.758922 +ip -10.787092 +ss -10.792343 +oz -10.838436 +io -10.840069 +zd -10.855488 +rp -10.868635 +tk -10.869982 +fi -10.890112 +tb -10.921823 +ps -10.934518 +up -10.955745 +zo -10.956563 +gl -10.978327 +ov -11.001830 +ea -11.004646 +tt -11.103713 +js -11.134641 +bo -11.193813 +ln -11.243660 +nf -11.250285 +zm -11.250327 +mn -11.261227 +oc -11.273398 +go -11.292021 +bl -11.305106 +rv -11.362814 +mt -11.371509 +pt -11.397082 +lg -11.407849 +mp -11.446036 +rz -11.490136 +nc -11.504958 +uc -11.525280 +fe -11.537256 +kc -11.664822 +zt -11.676503 +ua -11.685510 +td -11.715293 +ub -11.762760 +lk -11.768550 +pm -11.790003 +mb -11.797979 +gt -11.799506 +zr -11.804464 +lp -11.844507 +tl -11.877284 +ke -11.905091 +zg -11.950986 +tg -12.034399 +ng -12.106281 +aa -12.152834 +nv -12.190804 +j_ -12.224949 +tz -12.242455 +ee -12.264302 +p_ -12.291118 +zl -12.309015 +uj -12.315555 +vs -12.323003 +zp -12.328636 +ge -12.332610 +bs -12.361821 +lm -12.389979 +mm -12.420858 +lj -12.485655 +kv -12.533439 +km -12.544663 +rl -12.600747 +b_ -12.613583 +af -12.614336 +pg -12.642671 +of -12.644755 +fa -12.673700 +sd -12.718987 +kn -12.755984 +_h -12.781830 +tp -12.822731 +ll -12.845138 +ef -12.869180 +bn -12.878446 +pk -12.937339 +ho -12.980335 +ae -12.981859 +jn -12.992998 +zc -12.995376 +g_ -13.009020 +vn -13.025937 +zb -13.035406 +mj -13.101677 +eh -13.124921 +pj -13.135219 +vj -13.137689 +sr -13.201988 +lb -13.205709 +co -13.220688 +pn -13.221834 +fr -13.274035 +tm -13.309276 +dm -13.358486 +pv -13.379526 +zk -13.432752 +rc -13.450491 +hn -13.539346 +bk -13.552469 +fu -13.562805 +lc -13.569668 +nn -13.579079 +dn -13.582230 +nm -13.619937 +tc -13.653345 +vp -13.660431 +lz -13.672916 +gn -13.735612 +if -13.758239 +ha -13.778092 +eo -13.891023 +ft -13.896503 +md -13.902266 +ui -13.906473 +hi -13.949222 +fl -13.951665 +bj -14.001127 +vd -14.007325 +dv -14.008172 +zz -14.010151 +sg -14.072290 +mz -14.108475 +pd -14.207167 +pz -14.234667 +gv -14.290603 +tj -14.292666 +he -14.293698 +nz -14.306842 +kp -14.359927 +nl -14.380623 +pc -14.410180 +kd -14.526746 +oh -14.632510 +cs -14.639495 +lr -14.643877 +kk -14.683027 +mg -14.729311 +vl -14.741940 +oo -14.758952 +zj -14.792616 +sh -14.854758 +cp -14.885045 +bv -14.906999 +pb -14.956290 +ii -14.974941 +rh -15.014684 +f_ -15.028943 +mk -15.030664 +sf -15.059646 +eu -15.067278 +mr -15.092214 +bp -15.107870 +rr -15.152131 +sc -15.212691 +ih -15.233015 +pp -15.249617 +vv -15.284774 +oa -15.296455 +x_ -15.301293 +nb -15.315905 +gz -15.398655 +db -15.403105 +nr -15.438451 +lf -15.480159 +sb -15.480943 +dd -15.531205 +ct -15.592570 +jd -15.618211 +hu -15.623394 +ml -15.668221 +uu -15.669114 +ex -15.671797 +oe -15.719102 +rj -15.781468 +jb -15.865977 +dg -15.871106 +np -15.909635 +vf -16.026080 +sz -16.026080 +gm -16.095212 +oi -16.142797 +rf -16.159015 +nj -16.176686 +vr -16.180501 +fs -16.194576 +ah -16.208790 +bt -16.242953 +mv -16.244284 +bd -16.286143 +jv -16.309624 +sw -16.378066 +th -16.379526 +gd -16.434649 +h_ -16.471564 +y_ -16.498296 +w_ -16.525533 +cb -16.548356 +cr -16.638182 +ue -16.670008 +mf -16.711723 +ou -16.741470 +we -16.752786 +vk -16.804832 +ax -16.883489 +cd -16.895980 +xi -16.914922 +tf -17.011283 +bm -17.066689 +fg -17.080880 +_w -17.104846 +kg -17.146524 +zf -17.174148 +ch -17.192006 +ry -17.241624 +dl -17.244284 +ck -17.263038 +vz -17.315205 +dk -17.320808 +lh -17.357766 +cl -17.378066 +cm -17.407570 +kh -17.452989 +mw -17.499884 +dp -17.515861 +bc -17.650476 +iu -17.671797 +ht -17.682577 +cv -17.715408 +ww -17.756577 +bb -17.838593 +vm -17.867001 +ph -17.912805 +vt -17.964489 +vg -17.986583 +yo -18.013550 +nh -18.013550 +ya -18.045662 +ao -18.050308 +mx -18.064338 +hs -18.083259 +hl -18.121862 +df -18.121862 +cg -18.146524 +wi -18.254970 +wh -18.265737 +ow -18.276584 +oy -18.282039 +sj -18.293010 +vc -18.419542 +q_ -18.449916 +gg -18.564882 +ny -18.571545 +ty -18.578240 +ew -18.646953 +fk -18.661097 +wa -18.795051 +aq -18.802871 +ws -18.834580 +_x -18.867001 +fp -18.908580 +wo -18.925553 +xp -19.097614 +hr -19.126760 +gj -19.217908 +ky -19.337748 +ix -19.372236 +gp -19.443790 +ys -19.506254 +ff -19.519078 +ay -19.519078 +vb -19.519078 +jt -19.532017 +_y -19.545073 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lim b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lim new file mode 100644 index 00000000000..ad65f7711f4 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lim @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 6159798 +#UNIQUE_BIGRAMS 586 +#UNSEEN_LOG_PROB -17.402344 +#EXPECTED_SCORE -7.494543 +e_ -4.636264 +er -5.484425 +ie -5.673816 +n_ -5.689347 +t_ -5.710739 +_v -5.724902 +an -5.753021 +r_ -6.079130 +ge -6.194839 +_d -6.232981 +de -6.334716 +in -6.344484 +ao -6.347228 +te -6.356832 +s_ -6.427989 +aa -6.474946 +va -6.480206 +en -6.482507 +_w -6.499084 +_g -6.545213 +et -6.589637 +ee -6.633820 +or -6.640580 +ve -6.691863 +el -6.706636 +_m -6.787974 +_b -6.793290 +ei -6.899033 +_h -6.967289 +me -6.971692 +oe -6.976020 +_o -6.998095 +d_ -7.012206 +oo -7.044522 +le -7.045976 +eu -7.049136 +re -7.056506 +_z -7.061065 +ne -7.070699 +nd -7.081063 +_n -7.142279 +es -7.143305 +st -7.147516 +on -7.161832 +ch -7.162201 +_s -7.194874 +_a -7.198791 +l_ -7.207730 +g_ -7.268616 +_t -7.289789 +be -7.357005 +ur -7.375553 +k_ -7.378668 +_k -7.417140 +la -7.503243 +_l -7.551856 +ng -7.571369 +ke -7.572171 +ek -7.591285 +h_ -7.640394 +we -7.645496 +sj -7.667850 +_e -7.692395 +al -7.716627 +rt -7.733425 +nt -7.739118 +ri -7.747701 +li -7.750018 +at -7.756942 +ro -7.817416 +to -7.862327 +wa -7.897971 +he -7.914037 +it -7.920265 +zi -7.920435 +ti -7.926462 +di -7.936296 +ol -7.945793 +na -7.962929 +_i -7.971898 +da -8.028197 +ra -8.068685 +_p -8.084746 +eg -8.090480 +j_ -8.099253 +ed -8.128449 +ta -8.190180 +ae -8.198137 +wi -8.224165 +is -8.232030 +ev -8.236262 +ot -8.242420 +mi -8.244407 +ou -8.247607 +se -8.265081 +do -8.355393 +ha -8.360464 +om -8.386877 +rd -8.393241 +wo -8.402563 +rs -8.413302 +_r -8.446989 +je -8.469643 +ar -8.518965 +em -8.586235 +ze -8.597350 +ns -8.604716 +m_ -8.615148 +o_ -8.646622 +ma -8.674486 +pe -8.684472 +il -8.698513 +gr -8.708668 +bi -8.733475 +uu -8.742074 +ll -8.748103 +um -8.794459 +f_ -8.805315 +ew -8.899256 +vo -8.912174 +no -8.912287 +eb -8.929542 +og -8.959427 +p_ -8.969020 +ho -8.972074 +ec -9.013113 +uc -9.013718 +tr -9.038259 +am -9.045171 +bo -9.045171 +op -9.053610 +ni -9.094892 +nn -9.106206 +lo -9.107756 +ki -9.124260 +ts -9.140691 +ig -9.152239 +ij -9.183083 +ka -9.196487 +un -9.204203 +_c -9.204479 +_j -9.208908 +os -9.259831 +ld -9.271074 +ja -9.295297 +ik -9.304005 +rk -9.306969 +zo -9.324882 +jt -9.347437 +pr -9.348964 +ad -9.372523 +us -9.378434 +as -9.409953 +br -9.420667 +i_ -9.424203 +_f -9.425974 +ak -9.468315 +pa -9.469643 +a_ -9.472968 +ez -9.499339 +rl -9.499678 +nk -9.514847 +sc -9.518278 +rg -9.533472 +id -9.544448 +ef -9.575205 +ko -9.582369 +ru -9.655661 +w_ -9.663427 +mp -9.667421 +si -9.683702 +mo -9.720969 +dj -9.721759 +tu -9.724333 +ic -9.731283 +of -9.745286 +ok -9.810879 +vi -9.814249 +ss -9.824618 +ui -9.824831 +go -9.847308 +ga -9.860311 +ds -9.860311 +hi -9.871018 +pl -9.882247 +ba -9.893119 +tw -9.910370 +ac -9.914659 +od -9.916695 +rm -9.958495 +uw -9.972310 +gi -9.983647 +ks -9.994358 +oc -10.002263 +sp -10.023070 +ag -10.054108 +eh -10.064854 +mm -10.112804 +lu -10.113582 +ia -10.123476 +ls -10.125568 +so -10.129760 +dr -10.138974 +ob -10.150641 +ht -10.162403 +_u -10.168589 +kl -10.176428 +co -10.187310 +ej -10.212655 +ug -10.233369 +du -10.247820 +oa -10.250671 +kr -10.285325 +im -10.300309 +uv -10.312766 +lt -10.316644 +sa -10.346827 +gk -10.358772 +mb -10.365554 +tt -10.371126 +ov -10.394580 +ea -10.397105 +ce -10.406929 +bl -10.409475 +nu -10.421309 +hu -10.422916 +ep -10.428070 +fi -10.442664 +uk -10.468647 +po -10.472968 +af -10.492743 +tj -10.493418 +ul -10.515876 +ir -10.541827 +ms -10.549529 +uj -10.563284 +ut -10.564702 +vl -10.565412 +sl -10.567188 +lf -10.581472 +gs -10.581831 +ap -10.597350 +za -10.612670 +lg -10.634099 +rn -10.667612 +ci -10.669518 +vr -10.674103 +ji -10.681007 +rv -10.696859 +kt -10.700362 +ue -10.712101 +nw -10.714461 +fr -10.716824 +zu -10.718796 +pp -10.725522 +bu -10.761255 +lk -10.777196 +jo -10.789580 +bb -10.790409 +io -10.805839 +kk -10.884238 +au -10.898027 +rb -10.923275 +fe -10.978912 +ct -10.989826 +kd -11.047648 +ku -11.119823 +rp -11.120345 +th -11.126615 +nc -11.149310 +ca -11.166434 +dd -11.166973 +ud -11.230834 +jr -11.241569 +ai -11.332864 +fs -11.364010 +pi -11.365863 +nj -11.372678 +rw -11.443316 +ab -11.446581 +jl -11.489035 +aw -11.512792 +ju -11.535556 +eo -11.568610 +aj -11.609740 +iv -11.618550 +md -11.634843 +gd -11.637826 +jd -11.643060 +sd -11.683316 +wu -11.684087 +pu -11.715248 +gu -11.769817 +mt -11.847956 +av -11.884681 +rc -11.933315 +cu -11.936066 +tg -11.942505 +rr -11.964800 +mu -11.966674 +jp -11.976079 +fo -11.987446 +rz -12.033833 +ww -12.059596 +fd -12.063601 +su -12.070636 +zj -12.076693 +ub -12.097071 +pg -12.101181 +zw -12.120866 +ip -12.120866 +ff -12.155708 +ex -12.167511 +sk -12.209046 +rh -12.222415 +ow -12.260980 +ft -12.278327 +wd -12.279491 +oj -12.285325 +ua -12.302969 +up -12.340132 +pt -12.362159 +fa -12.380774 +jn -12.382024 +if -12.426457 +sy -12.430330 +u_ -12.443316 +yp -12.476301 +sm -12.492405 +nv -12.493756 +iz -12.516905 +nl -12.575741 +hr -12.619287 +lm -12.637080 +vu -12.644559 +dw -12.715248 +kw -12.737468 +cl -12.747097 +ib -12.766549 +nz -12.802908 +fg -12.812985 +lv -12.841925 +cr -12.887340 +fu -12.905195 +ym -12.908793 +nh -12.956399 +ty -12.958262 +y_ -12.969489 +hs -13.038752 +lj -13.040724 +kh -13.084810 +az -13.095020 +cc -13.162134 +iu -13.166434 +oz -13.168589 +js -13.168589 +td -13.214602 +rf -13.219061 +mg -13.266739 +tb -13.269049 +sn -13.276002 +ck -13.276002 +ws -13.328039 +gt -13.335283 +ii -13.335283 +tz -13.342563 +ys -13.349880 +b_ -13.354779 +nm -13.362159 +lb -13.389545 +mv -13.417460 +hh -13.422595 +gl -13.440709 +vv -13.445927 +fl -13.461694 +x_ -13.480310 +uf -13.496460 +nr -13.548827 +uz -13.551637 +oh -13.577172 +ps -13.585785 +fh -13.614872 +dh -13.617814 +tl -13.718401 +pv -13.756790 +jw -13.763289 +oi -13.792900 +bs -13.799564 +lp -13.833352 +sg -13.836775 +sv -13.854012 +zz -13.860965 +ly -13.860965 +nb -13.867951 +sh -13.892674 +tn -13.910595 +yn -13.914207 +sb -13.928743 +hn -13.965737 +cy -13.988398 +c_ -13.999863 +nf -14.042699 +jk -14.042699 +sw -14.062599 +xt -14.128187 +xi -14.162134 +sf -14.179412 +qu -14.188129 +jj -14.228022 +jf -14.237039 +xe -14.241569 +lh -14.246113 +hd -14.269049 +ax -14.282989 +wg -14.297064 +kn -14.301786 +gh -14.320832 +ph -14.344998 +xp -14.349880 +z_ -14.359695 +dt -14.389545 +fk -14.399634 +sr -14.445927 +gn -14.445927 +pd -14.461694 +lz -14.477636 +xa -14.510057 +pk -14.521029 +ry -14.526546 +dg -14.537643 +fp -14.554452 +mw -14.623714 +wl -14.629639 +kz -14.635588 +kb -14.641562 +ml -14.766549 +gg -14.766549 +dy -14.779665 +tv -14.812985 +mf -14.860965 +tm -14.860965 +gy -14.867951 +lw -14.896240 +ay -14.903400 +yo -14.917827 +jg -14.947121 +ah -14.947121 +hb -15.015293 +pz -15.023070 +jm -15.046657 +dn -15.054606 +bj -15.070636 +fw -15.070636 +ey -15.103240 +pm -15.103240 +ix -15.111508 +v_ -15.119823 +ux -15.136599 +lc -15.145061 +pn -15.188129 +_q -15.214602 +tk -15.214602 +gm -15.214602 +rj -15.278327 +vd -15.325633 +dz -15.335283 +hz -15.354779 +wt -15.354779 +bd -15.364627 +tc -15.394580 +dm -15.414900 +fy -15.435511 +yr -15.435511 +iw -15.510057 +db -15.565767 +ox -15.577172 +uy -15.577172 +pb -15.623714 +jh -15.647561 +_y -15.671809 +yl -15.721562 +yc -15.747097 +ny -15.773092 +jv -15.786267 +tf -15.799564 +cd -15.840206 +yt -15.840206 +by -15.840206 +wk -15.867951 +ln -15.867951 +vj -15.882026 +kg -15.896240 +xo -15.910595 +gb -15.925095 +jb -15.999863 +bt -16.046657 +mh -16.095020 +_x -16.111508 +dk -16.162134 +gv -16.179412 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lin b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lin new file mode 100644 index 00000000000..4f898f9dafd --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lin @@ -0,0 +1,424 @@ +#TOTAL_BIGRAMS 5147143 +#UNIQUE_BIGRAMS 420 +#UNSEEN_LOG_PROB -18.973412 +#EXPECTED_SCORE -6.626963 +a_ -4.492092 +o_ -4.579787 +i_ -4.678763 +_m -4.834733 +ko -5.239285 +_b -5.253318 +ba -5.497907 +ng -5.505590 +e_ -5.608922 +ak -5.725959 +al -5.747256 +an -5.786462 +li -5.794343 +_n -5.890299 +ka -5.981058 +_e -6.010173 +_k -6.024680 +la -6.035138 +ok -6.116656 +ki -6.203347 +mo -6.236250 +bo -6.299617 +ma -6.361742 +za -6.409620 +sa -6.473791 +go -6.623495 +is -6.624657 +ol -6.632060 +on -6.650724 +mb -6.653994 +mp -6.667921 +_a -6.735904 +lo -6.760095 +el -6.802329 +am -6.824872 +pe -6.885485 +oy -6.897098 +yo -6.919045 +to -6.953995 +bi -6.982528 +nd -6.982812 +_l -7.021945 +in -7.037805 +na -7.043932 +so -7.131769 +_o -7.167910 +at -7.171340 +si -7.196856 +ta -7.207836 +en -7.282934 +os -7.290725 +mi -7.342872 +ya -7.373639 +_y -7.375686 +u_ -7.386666 +ob -7.464132 +ga -7.473666 +ni -7.494087 +le -7.494845 +ye -7.506418 +as -7.520039 +ot -7.617015 +te -7.632450 +su -7.635790 +be -7.644457 +nz -7.675636 +om -7.713669 +wa -7.728097 +ek -7.739493 +ik -7.761828 +po -7.864235 +az -7.889202 +es -7.895462 +_t -7.896797 +et -7.904903 +de -7.979625 +_s -8.007412 +ti -8.125885 +ez -8.131141 +im -8.295428 +em -8.339872 +ai -8.351361 +gi -8.364695 +da -8.404411 +se -8.422858 +ab -8.429607 +ge -8.447892 +ib -8.449948 +ns -8.474264 +eb -8.480658 +oz -8.519627 +_p -8.593926 +ay -8.598916 +pa -8.672717 +il -8.752792 +no -8.806120 +nt -8.816824 +us -9.034568 +ne -9.048748 +tu -9.140681 +mu -9.186979 +ku -9.191233 +di -9.238703 +ke -9.256421 +ny -9.357415 +it -9.407739 +mw -9.450831 +_w -9.509071 +bu -9.517058 +ep -9.544424 +me -9.548826 +ut -9.574669 +uk -9.614102 +ia -9.622473 +un -9.711788 +ze -9.738116 +um -9.759336 +zi -9.797988 +ap -9.801485 +zo -9.864104 +va -10.010516 +lu -10.050382 +nk -10.067724 +ov -10.123851 +eh -10.150364 +ho -10.151957 +op -10.161556 +ey -10.161877 +yi -10.201923 +s_ -10.265018 +ri -10.282018 +ul -10.334614 +zw -10.338964 +er -10.359439 +oi -10.469984 +st -10.524677 +aw -10.527156 +bl -10.619824 +re -10.627342 +ar -10.642495 +fa -10.754728 +_c -10.767375 +n_ -10.770799 +gu -10.843099 +ra -10.849808 +_z -10.932301 +io -10.936140 +bw -10.978493 +_f -11.064720 +ro -11.135469 +pi -11.139257 +ie -11.157069 +co -11.173160 +do -11.233294 +t_ -11.305236 +ei -11.397495 +_d -11.404316 +wi -11.426518 +vi -11.449067 +we -11.465618 +iy -11.469587 +uf -11.501737 +_v -11.513981 +kr -11.591437 +fe -11.669632 +fi -11.698219 +_i -11.703818 +zu -11.706626 +_r -11.706626 +or -11.710378 +r_ -11.719801 +af -11.745556 +uy -11.767863 +ip -11.799485 +of -11.818594 +ao -11.826716 +ou -11.855510 +kw -11.889135 +ad -11.907323 +ly -11.908400 +kl -11.984728 +tr -11.985864 +iz -12.050977 +au -12.055742 +ua -12.067724 +ue -12.068928 +qu -12.076172 +id -12.089547 +ha -12.093217 +ty -12.167346 +oa -12.199943 +eu -12.214523 +fu -12.225219 +ud -12.313773 +_g -12.320926 +ae -12.389953 +ve -12.394474 +_j -12.397495 +pr -12.409644 +my -12.456137 +av -12.459290 +gw -12.520553 +ru -12.579378 +wu -12.598373 +yu -12.637129 +sh -12.640704 +ur -12.644289 +br -12.655095 +iw -12.669632 +ag -12.682472 +_u -12.689861 +rt -12.754244 +ef -12.779641 +he -12.795495 +eo -12.815560 +rn -12.827735 +ir -12.850326 +rd -12.866980 +l_ -12.881713 +m_ -12.900878 +od -12.915962 +ce -12.920301 +ll -12.924653 +ii -12.955490 +c_ -13.005322 +ic -13.030898 +fo -13.098124 +_q -13.112946 +pu -13.112946 +sr -13.137993 +nc -13.143056 +ch -13.210532 +ky -13.267434 +fr -13.298161 +rs -13.303819 +ss -13.344056 +ts -13.370528 +d_ -13.379461 +ca -13.391459 +ci -13.409644 +ah -13.415757 +ja -13.421896 +ee -13.459290 +iq -13.462450 +ea -13.487986 +pl -13.550507 +h_ -13.564021 +hi -13.591437 +rr -13.626455 +je -13.630005 +jo -13.647882 +uz -13.669632 +th -13.673289 +du -13.688010 +gr -13.725485 +uv -13.760065 +aa -13.767863 +nu -13.767863 +ed -13.767863 +if -13.779641 +ev -13.787546 +uw -13.823665 +og -13.856549 +_h -13.885949 +ew -13.898736 +ui -13.903023 +dc -13.915962 +oo -13.915962 +ow -13.942194 +rm -14.019216 +vr -14.061721 +dr -14.076172 +ec -14.090769 +x_ -14.095668 +iv -14.135469 +up -14.272973 +ph -14.278532 +oc -14.278532 +ct -14.295340 +by -14.335339 +y_ -14.346973 +ac -14.400523 +rw -14.513981 +rc -14.553873 +k_ -14.553873 +lm -14.581095 +p_ -14.615860 +ub -14.651484 +vo -14.748446 +rk -14.787546 +sy -14.811525 +gn -14.844129 +ux -14.860712 +eg -14.929018 +ig -15.019216 +vu -15.105516 +sp -15.115431 +ej -15.261917 +lb -15.306656 +yr -15.318061 +cs -15.318061 +cr -15.318061 +ff -15.400523 +cc -15.437359 +gl -15.437359 +wo -15.449850 +nj -15.449850 +lg -15.462450 +sk -15.462450 +pp -15.462450 +rg -15.513981 +tw -15.513981 +sl -15.527156 +ix -15.527156 +oh -15.594901 +sw -15.594901 +ps -15.608840 +aj -15.608840 +b_ -15.622915 +uc -15.637129 +ex -15.665984 +ys -15.680631 +tt -15.680631 +ju -15.756182 +ji -15.852397 +w_ -15.885949 +mm -15.885949 +tc -15.937788 +zy -16.028554 +g_ -16.028554 +rl -16.066522 +sm -16.186816 +mv -16.207878 +cl -16.207878 +nf -16.207878 +hn -16.229251 +dp -16.295340 +hw -16.295340 +ld -16.318061 +xe -16.341144 +cu -16.388450 +hm -16.388450 +mf -16.412697 +ms -16.487986 +oe -16.487986 +eq -16.513981 +f_ -16.540453 +hr -16.540453 +uj -16.594901 +hu -16.594901 +rq -16.651484 +uo -16.651484 +yy -16.710378 +zz -16.740752 +oj -16.740752 +ry -16.740752 +nv -16.740752 +yp -16.803487 +kk -16.835909 +fl -16.903023 +pt -16.937788 +oq -16.973412 +gb -16.973412 +dw -16.973412 +nn -17.047413 +v_ -17.166057 +lt -17.166057 +z_ -17.166057 +dj -17.207878 +kh -17.207878 +ks -17.295340 +dv -17.295340 +ug -17.341144 +kt -17.341144 +wh -17.341144 +hy -17.388450 +sc -17.388450 +zk -17.437359 +hh -17.487986 +cn -17.594901 +xo -17.651484 +gh -17.651484 +kp -17.771778 +rf -17.771778 +cy -17.835909 +yc -17.835909 +rz -17.973412 +qa -17.973412 +mr -18.047413 +ih -18.047413 +lp -18.125415 +rv -18.125415 +lj -18.125415 +vd -18.207878 +lw -18.207878 +sq -18.207878 +rh -18.295340 +yn -18.295340 +gy -18.388450 +lf -18.388450 +nr -18.487986 +rj -18.487986 +rb -18.487986 +dg -18.487986 +nq -18.594901 +ck -18.594901 +tl -18.594901 +jj -18.710378 +jh -18.710378 +kb -18.710378 +uh -18.710378 +pd -18.710378 +df -18.710378 +ws -18.710378 +py -18.835909 +sv -18.835909 +vp -18.835909 +tb -18.835909 +tp -18.973412 +pw -18.973412 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lit b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lit new file mode 100644 index 00000000000..966909e429d --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lit @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 326212535 +#UNIQUE_BIGRAMS 557 +#UNSEEN_LOG_PROB -19.965333 +#EXPECTED_SCORE -7.468986 +s_ -4.830383 +i_ -5.575775 +a_ -5.811887 +u_ -5.856043 +_p -5.925346 +o_ -5.975084 +ai -5.983521 +_k -6.132206 +_s -6.204497 +ti -6.218086 +in -6.248526 +as -6.287550 +e_ -6.346648 +is -6.352585 +ta -6.402008 +_t -6.443050 +au -6.531879 +ie -6.536979 +os -6.558543 +ka -6.568693 +ia -6.599745 +si -6.661189 +ri -6.665215 +pa -6.741411 +li -6.742491 +es -6.771972 +us -6.783328 +_v -6.830351 +ra -6.847006 +al -6.906440 +en -6.952813 +ar -6.986210 +iu -6.988614 +ma -7.012997 +ik -7.024872 +ni -7.046707 +tu -7.057328 +_a -7.062937 +_d -7.065597 +_m -7.081080 +_n -7.106419 +st -7.124969 +ne -7.168463 +ki -7.170917 +et -7.217987 +vi -7.242897 +im -7.296341 +da -7.318536 +at -7.367781 +te -7.368110 +mo -7.369083 +to -7.394586 +ei -7.395255 +_b -7.404106 +na -7.405528 +me -7.408323 +an -7.413498 +va -7.453606 +sa -7.491354 +_g -7.511503 +ci -7.560841 +pr -7.566905 +ga -7.580085 +ur -7.582804 +re -7.591855 +_i -7.623058 +am -7.634018 +la -7.634955 +ja -7.664637 +ko -7.690306 +io -7.702575 +uo -7.707232 +er -7.711928 +_l -7.739206 +nt -7.767895 +ku -7.794776 +jo -7.834090 +ve -7.846776 +_j -7.888419 +ij -7.898824 +di -7.926439 +ad -7.934958 +oj -7.968234 +_r -7.973124 +it -7.992389 +mi -7.993994 +ro -7.999244 +el -8.031507 +su -8.032866 +zi -8.042224 +vo -8.095847 +de -8.099408 +ir -8.103192 +sk -8.107409 +je -8.113477 +ek -8.150698 +bu -8.156673 +be -8.232247 +no -8.233409 +le -8.308751 +nk -8.345908 +on -8.348440 +av -8.354569 +or -8.368845 +om -8.379028 +nu -8.397872 +se -8.401781 +ok -8.408496 +gi -8.417937 +ba -8.444433 +ju -8.452536 +rt -8.461306 +ej -8.525459 +t_ -8.529547 +ke -8.554319 +ut -8.554954 +mu -8.571822 +pi -8.579792 +uv -8.647408 +ap -8.648045 +tr -8.653019 +ip -8.686657 +em -8.691780 +id -8.722562 +ru -8.726367 +_z -8.744093 +ak -8.763627 +kt -8.764415 +il -8.809101 +do -8.836313 +ot -8.863395 +ng -8.874444 +ug -8.905175 +ks -8.908929 +nd -8.909490 +kl -8.910928 +uk -8.944199 +ty -8.951414 +pe -8.982821 +yt -9.019555 +yr -9.095740 +_e -9.104708 +yb -9.121994 +d_ -9.124179 +az -9.143470 +p_ -9.147269 +ji -9.182290 +ud -9.205552 +ac -9.206139 +od -9.224123 +un -9.231864 +ge -9.262143 +lo -9.278315 +rb -9.278447 +ig -9.294239 +ol -9.316445 +po -9.318547 +du -9.349256 +l_ -9.349914 +kr -9.359413 +k_ -9.379424 +r_ -9.397225 +gu -9.425620 +um -9.446633 +ky -9.464853 +bi -9.476451 +ul -9.476822 +gr -9.488974 +ab -9.505505 +ed -9.509944 +og -9.513909 +dz -9.515271 +_y -9.519796 +ms -9.522936 +ly -9.534090 +yv -9.535569 +sv -9.538359 +rs -9.577283 +sp -9.594135 +vy -9.611207 +so -9.619953 +_u -9.703642 +yk -9.748653 +iv -9.759916 +uj -9.763679 +sl -9.763829 +go -9.778071 +gy -9.795467 +eg -9.801745 +ov -9.815894 +ys -9.823363 +ag -9.851683 +rm -9.938967 +ym -9.943541 +uz -9.947684 +ry -9.965910 +lu -9.990633 +bo -10.023016 +dy -10.023522 +iz -10.028710 +sm -10.039003 +dr -10.084433 +za -10.127477 +ze -10.138462 +ts -10.140053 +_f -10.141021 +pu -10.159344 +lt -10.165203 +ec -10.213383 +ui -10.223155 +ic -10.257342 +ny -10.271354 +ez -10.284217 +_c -10.288918 +zm -10.293018 +pl -10.293412 +_o -10.363603 +n_ -10.380871 +tv -10.425979 +ns -10.465735 +nc -10.529405 +rg -10.556024 +ld -10.576374 +up -10.583767 +rd -10.623549 +g_ -10.626979 +lb -10.639199 +ce -10.647688 +fo -10.662988 +ev -10.687087 +m_ -10.697794 +ep -10.701531 +sn -10.728200 +aj -10.737880 +ps -10.776573 +rk -10.804958 +yg -10.808316 +ln -10.843014 +fi -10.859813 +zu -10.866692 +mp -10.873372 +eb -10.895653 +sy -10.903103 +yj -10.907650 +sc -10.940413 +ob -10.960525 +oc -11.040800 +rn -11.084654 +op -11.086355 +vu -11.087490 +pt -11.091412 +zd -11.099367 +mt -11.101211 +lg -11.127527 +my -11.154633 +kv -11.164345 +ls -11.188944 +bl -11.220130 +gt -11.271244 +zo -11.285061 +rp -11.313054 +ch -11.320918 +dv -11.331843 +yn -11.472989 +yd -11.491219 +zn -11.616470 +ub -11.632276 +nf -11.635550 +fe -11.651852 +zy -11.756450 +tl -11.779742 +eu -11.814507 +lv -11.829961 +uc -11.836497 +kc -11.855021 +ua -11.898816 +oz -11.914522 +sr -11.929521 +ss -12.003278 +ea -12.019105 +yp -12.043019 +br -12.053978 +c_ -12.086480 +ib -12.142465 +rz -12.160371 +yl -12.180020 +km -12.191166 +gs -12.229582 +of -12.257656 +zs -12.271955 +yc -12.341061 +kn -12.358258 +zt -12.372117 +ii -12.458069 +gl -12.467857 +mb -12.472248 +zv -12.522597 +lk -12.582587 +yz -12.615485 +lm -12.633329 +rc -12.665607 +zk -12.687904 +kd -12.708862 +ef -12.738658 +hn -12.740292 +py -12.750313 +fa -12.798177 +bt -12.820941 +sd -12.888551 +tm -12.925851 +af -12.929056 +mz -12.938954 +rv -12.976494 +ho -12.998112 +dm -13.104635 +b_ -13.106350 +ae -13.107989 +lp -13.133038 +eo -13.158894 +gv -13.172835 +gn -13.258913 +v_ -13.264255 +tn -13.265691 +tg -13.320556 +he -13.348007 +fu -13.454986 +gz -13.455185 +_h -13.459363 +sg -13.512329 +cu -13.598242 +pk -13.606548 +hi -13.616123 +nv -13.691001 +ha -13.760618 +tk -13.780271 +if -13.812040 +pd -13.832604 +aa -13.862594 +rl -13.884432 +sb -13.922723 +ml -13.930298 +gd -13.942779 +_x -13.968709 +bj -14.041191 +pv -14.105843 +fr -14.132284 +mn -14.219106 +pj -14.225107 +by -14.241290 +j_ -14.274736 +ca -14.363119 +z_ -14.383675 +bd -14.417825 +oo -14.424325 +rj -14.488650 +gp -14.516987 +oe -14.570753 +gm -14.645179 +xi -14.654615 +oi -14.670327 +nz -14.689832 +th -14.695218 +co -14.699213 +y_ -14.794402 +vz -14.857202 +x_ -14.887310 +zr -14.919979 +tb -14.963118 +ou -14.976600 +md -15.050316 +pz -15.110686 +ft -15.138332 +oh -15.165868 +xv -15.174838 +f_ -15.202586 +oa -15.208937 +zp -15.226294 +_w -15.260951 +pc -15.285823 +bs -15.293618 +lc -15.299313 +ue -15.324498 +h_ -15.336159 +sf -15.347362 +pn -15.347731 +vm -15.424422 +mj -15.448545 +pm -15.461457 +ix -15.519894 +fl -15.535983 +tz -15.571292 +vs -15.573015 +nr -15.595831 +pg -15.629961 +ct -15.666067 +lf -15.688079 +cl -15.769731 +lr -15.850262 +ds -15.851569 +lz -15.891606 +hu -15.897533 +wi -15.943894 +w_ -15.964673 +jy -15.967220 +hy -16.078195 +uf -16.108185 +tp -16.198755 +zl -16.224938 +ow -16.232410 +ck -16.254022 +vc -16.265822 +dn -16.281237 +zg -16.286884 +mk -16.385283 +vd -16.419375 +we -16.421314 +cd -16.421702 +sh -16.429488 +mf -16.475896 +nn -16.542722 +ex -16.573015 +zb -16.603077 +vl -16.623472 +yo -16.624812 +ax -16.666527 +ee -16.692991 +vg -16.702393 +uh -16.818224 +vn -16.826424 +dk -16.870786 +zf -16.931403 +gb -16.941944 +tf -16.944173 +tt -16.974036 +cr -16.974606 +vp -16.981457 +ll -16.982029 +ew -17.106311 +mc -17.151954 +ph -17.159056 +bm -17.196429 +nb -17.221893 +pp -17.251950 +dp -17.263037 +ht -17.266519 +ws -17.270010 +kg -17.310412 +ww -17.369845 +mm -17.371344 +wo -17.391733 +nj -17.438887 +xp -17.455483 +xx -17.473077 +mw -17.474687 +nl -17.503160 +sw -17.508098 +vr -17.523014 +fg -17.576469 +ay -17.615013 +vv -17.632879 +bn -17.754738 +oy -17.767509 +bv -17.786381 +ux -17.852877 +ah -17.866552 +hm -17.876096 +wa -17.991218 +zz -18.010942 +sj -18.063279 +fs -18.096362 +df -18.098843 +lj -18.110060 +ya -18.113819 +tw -18.131490 +ff -18.140407 +bz -18.203086 +bc -18.263037 +fp -18.274210 +ao -18.281237 +bb -18.309693 +cc -18.311131 +mv -18.395541 +rr -18.410872 +gc -18.424811 +db -18.446766 +dg -18.457870 +kp -18.461058 +np -18.462655 +wh -18.481955 +hp -18.511399 +gh -18.516365 +vt -18.621241 +lh -18.749856 +hr -18.769484 +nh -18.795408 +tc -18.813631 +ox -18.827966 +hs -19.028572 +kh -19.033309 +dl -19.074223 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ltz b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ltz new file mode 100644 index 00000000000..4be8ae69b19 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ltz @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 7004437 +#UNIQUE_BIGRAMS 579 +#UNSEEN_LOG_PROB -17.641139 +#EXPECTED_SCORE -7.357176 +er -4.880897 +en -5.050201 +n_ -5.086881 +t_ -5.173130 +r_ -5.333608 +_d -5.402721 +de -5.522997 +ch -5.645296 +ei -6.040089 +ge -6.098760 +_g -6.226018 +s_ -6.388743 +_s -6.402826 +_a -6.439431 +_v -6.443083 +e_ -6.464168 +un -6.590329 +_e -6.596514 +_h -6.597411 +m_ -6.614021 +es -6.630150 +ng -6.721398 +ie -6.723769 +nn -6.726820 +te -6.745551 +re -6.795240 +et -6.824822 +sc -6.854284 +vu -6.866249 +ue -6.879502 +an -6.886138 +h_ -6.971550 +ne -7.001112 +ou -7.002511 +i_ -7.004652 +le -7.047522 +_m -7.053966 +se -7.063991 +ss -7.071368 +at -7.100412 +_b -7.111962 +st -7.124467 +em -7.133866 +g_ -7.215082 +ee -7.233902 +_w -7.259584 +el -7.264326 +we -7.273697 +in -7.362627 +ar -7.394640 +_f -7.462042 +ll -7.534923 +he -7.553027 +me -7.558802 +is -7.565770 +_k -7.568622 +nt -7.586523 +as -7.588781 +be -7.591043 +ec -7.594144 +um -7.611803 +l_ -7.639832 +al -7.643287 +ma -7.694889 +ht -7.797598 +on -7.801774 +_n -7.855047 +hi -7.871690 +ra -7.878654 +ir -7.879430 +it -7.879915 +_l -7.917964 +_o -7.927259 +ai -7.954334 +hu -7.956788 +rt -7.963250 +au -8.003171 +us -8.018204 +_p -8.035123 +ro -8.133143 +ha -8.175569 +wa -8.189632 +la -8.195598 +_z -8.301306 +f_ -8.378511 +fi -8.408151 +da -8.416150 +ri -8.424830 +_r -8.443352 +_j -8.461751 +tt -8.482229 +ze -8.493691 +go -8.507567 +uf -8.522562 +ta -8.530384 +d_ -8.542467 +li -8.638026 +ve -8.650222 +ti -8.650801 +aa -8.653121 +ic -8.665863 +eg -8.671563 +na -8.709084 +si -8.732197 +nd -8.732548 +il -8.774324 +ol -8.777845 +or -8.904379 +fe -8.941467 +ac -8.946742 +ke -8.964022 +gr -8.968142 +mm -8.971550 +_t -9.002485 +lt -9.013939 +ur -9.028278 +sa -9.030431 +tr -9.032047 +io -9.037340 +ef -9.037557 +ru -9.050932 +so -9.067191 +gi -9.077948 +_i -9.079842 +eb -9.094404 +ka -9.099254 +jo -9.103893 +rg -9.153584 +no -9.168322 +du -9.186448 +oc -9.213950 +ko -9.219465 +rs -9.231053 +ls -9.252249 +ns -9.264611 +kt -9.274526 +ba -9.285153 +bi -9.295729 +oe -9.338425 +om -9.356808 +am -9.364527 +do -9.374335 +tu -9.377757 +k_ -9.400544 +ni -9.403192 +sp -9.404866 +op -9.414111 +bu -9.416924 +_c -9.418473 +lo -9.418896 +ut -9.422425 +id -9.423556 +ck -9.427522 +pe -9.430646 +u_ -9.446222 +pr -9.464586 +to -9.465896 +_u -9.483187 +tz -9.483482 +ho -9.485547 +ed -9.528101 +mi -9.530384 +z_ -9.541239 +iw -9.553104 +vi -9.571636 +rc -9.576031 +pa -9.579336 +ek -9.624957 +af -9.631477 +ft -9.632457 +ew -9.635567 +fr -9.644276 +br -9.686591 +rd -9.708653 +os -9.716950 +ga -9.734213 +ts -9.743717 +od -9.748670 +ww -9.758449 +o_ -9.758627 +kr -9.858915 +di -9.883996 +dr -9.888674 +fa -9.888870 +nz -9.891215 +mo -9.938533 +nk -10.022804 +wi -10.039832 +a_ -10.042653 +lu -10.045045 +mb -10.057721 +nu -10.062558 +wo -10.076057 +im -10.099819 +ff -10.179744 +ug -10.189812 +ag -10.195839 +fo -10.208700 +of -10.215051 +su -10.269942 +rk -10.286567 +ap -10.296635 +ob -10.297672 +pl -10.305209 +rn -10.306252 +zu -10.319614 +ot -10.331773 +ik -10.346984 +za -10.347789 +pi -10.375703 +po -10.383661 +zw -10.387518 +ae -10.419601 +kl -10.428941 +ud -10.434631 +ad -10.443208 +ul -10.447804 +co -10.463422 +th -10.524911 +p_ -10.557443 +eh -10.563976 +pu -10.566161 +ii -10.585652 +oo -10.595817 +eu -10.598369 +zi -10.614424 +uc -10.624469 +aw -10.626421 +dd -10.646420 +ep -10.654697 +rl -10.665362 +hr -10.669717 +rm -10.669717 +iv -10.692714 +ki -10.699205 +pp -10.716776 +hl -10.723378 +pt -10.740895 +bl -10.744424 +ig -10.746545 +je -10.826200 +ez -10.835203 +ku -10.840858 +ju -10.845778 +tl -10.898273 +ea -10.920457 +gl -10.928463 +ce -10.937321 +bo -10.954385 +rz -10.956021 +mu -10.969587 +fl -11.003013 +rh -11.034637 +ab -11.039398 +ia -11.044174 +oi -11.086545 +lm -11.096432 +gu -11.098689 +rb -11.100045 +gt -11.128352 +rf -11.136675 +ph -11.149718 +og -11.162409 +rr -11.169033 +ak -11.204076 +rw -11.273761 +ca -11.300526 +mp -11.309908 +va -11.321985 +ua -11.338958 +if -11.347520 +iz -11.387794 +zo -11.389451 +pg -11.399988 +y_ -11.409481 +ov -11.415657 +qu -11.421860 +eo -11.430930 +wu -11.432068 +ub -11.440057 +hw -11.460227 +az -11.469542 +ld -11.478330 +nc -11.571792 +fu -11.619600 +ja -11.687270 +up -11.689309 +ok -11.694078 +ks -11.719552 +ex -11.741953 +ci -11.744777 +lw -11.756131 +hn -11.761842 +sg -11.769732 +zt -11.781285 +rv -11.828446 +gs -11.837463 +ux -11.884190 +ui -11.910908 +v_ -11.926056 +bs -11.948675 +xe -11.949489 +sk -12.011917 +td -12.022161 +ip -12.037665 +tw -12.039398 +mt -12.135284 +ev -12.140855 +vo -12.144580 +hs -12.166190 +c_ -12.216276 +dl -12.230063 +fg -12.258038 +ah -12.270196 +xi -12.314622 +ib -12.390004 +ys -12.393324 +oz -12.410041 +ps -12.542621 +oh -12.553723 +tg -12.593906 +gn -12.601566 +lg -12.610555 +sb -12.610555 +hm -12.644441 +nf -12.657689 +x_ -12.677792 +sy -12.694078 +zr -12.728610 +lb -12.738429 +ax -12.741247 +nl -12.741247 +ms -12.765423 +ty -12.766858 +sl -12.851094 +_q -12.867932 +ml -12.930873 +b_ -12.950304 +lz -12.968348 +av -13.003436 +rp -13.025592 +tk -13.029031 +iu -13.048094 +ly -13.076280 +dw -13.094179 +w_ -13.097786 +sw -13.114129 +nh -13.138067 +cl -13.181417 +uk -13.226110 +sh -13.228085 +sm -13.232043 +lk -13.234026 +ow -13.249990 +uw -13.286567 +ym -13.309385 +sd -13.313573 +ry -13.319877 +bj -13.336826 +nr -13.364798 +ct -13.386691 +eq -13.440630 +lp -13.480094 +yp -13.487172 +uz -13.515836 +mg -13.530384 +tj -13.532823 +ds -13.559929 +iq -13.605411 +fs -13.692714 +gc -13.698179 +uv -13.698179 +ay -13.711932 +lf -13.714698 +oy -13.731409 +ey -13.737023 +oj -13.748316 +xt -13.771171 +ij -13.785641 +sr -13.803200 +dm -13.841992 +gk -13.875652 +cc -13.888089 +ix -13.948675 +bt -13.984950 +ny -14.035934 +hy -14.106842 +yo -14.125128 +xp -14.151123 +fh -14.166190 +sz -14.181417 +_y -14.196806 +tn -14.220201 +ji -14.236012 +my -14.243983 +ye -14.243983 +pf -14.247985 +sn -14.321985 +xa -14.364798 +kg -14.382286 +cu -14.449819 +tm -14.454435 +ya -14.468375 +zl -14.473051 +tc -14.491910 +lh -14.501433 +hg -14.506218 +mf -14.525519 +hz -14.550013 +lv -14.569913 +yc -14.590091 +kn -14.626095 +dh -14.663022 +_x -14.762558 +hk -14.774053 +bb -14.785641 +pz -14.785641 +nv -14.809100 +cr -14.815025 +fb -14.845020 +sf -14.851094 +ln -14.875652 +tf -14.932483 +nb -15.005128 +cq -15.032479 +mr -15.053337 +mn -15.067412 +ox -15.132507 +fn -15.132507 +yt -15.139925 +tb -15.162409 +oa -15.162409 +dg -15.169982 +yn -15.169982 +uh -15.208456 +aj -15.264104 +wj -15.288627 +bw -15.296894 +dy -15.296894 +uu -15.321985 +dt -15.382286 +yr -15.399988 +gy -15.417910 +mk -15.491910 +kz -15.530384 +uy -15.550013 +dn -15.590091 +zd -15.600286 +yl -15.620897 +gh -15.641806 +kh -15.652375 +pb -15.673748 +xo -15.684555 +zb -15.706415 +hh -15.717470 +ao -15.717470 +tp -15.728610 +hb -15.751153 +zy -15.751153 +fk -15.762558 +lr -15.809100 +fz -15.820974 +yk -15.857195 +lj -15.857195 +gm -15.881857 +bz -15.894348 +cs -15.906948 +dk -15.919659 +pm -15.919659 +bn -15.945422 +pk -15.945422 +fw -16.039398 +nw -16.053337 +cy -16.053337 +db -16.154875 +lc -16.185249 +dc -16.200679 +rj -16.216276 +gg -16.232043 +mw -16.247985 +gb -16.247985 +vr -16.247985 +hp -16.280406 +fm -16.280406 +kk -16.280406 +cd -16.313573 +zz -16.330447 +by -16.330447 +yd -16.347520 +xz -16.347520 +ih -16.399988 +tv -16.417910 +xv -16.436057 +sv -16.436057 +pd -16.436057 +cf -16.491910 +dp -16.511019 +pw -16.530384 +nq -16.550013 +uj -16.590091 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lug b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lug new file mode 100644 index 00000000000..2e97c5fda3c --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lug @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 32815035 +#UNIQUE_BIGRAMS 548 +#UNSEEN_LOG_PROB -20.642923 +#EXPECTED_SCORE -7.188775 +a_ -4.068771 +e_ -5.373806 +ba -5.427553 +i_ -5.637738 +_o -5.755493 +_b -5.775688 +o_ -5.997576 +_e -6.030784 +an -6.069066 +_a -6.082691 +_n -6.110790 +_k -6.143597 +mu -6.159578 +wa -6.185566 +ga -6.205543 +ka -6.234669 +ku -6.269125 +ab -6.311010 +al -6.337427 +ya -6.389772 +ng -6.457974 +la -6.558119 +_m -6.693762 +na -6.708547 +am -6.735255 +ul -6.737669 +ok -6.789310 +li -6.823101 +u_ -6.842814 +ak -6.885658 +bu -6.886100 +er -6.895501 +en -6.942682 +dd -6.954995 +ki -6.980612 +aa -6.990099 +ir -7.067618 +de -7.105822 +ra -7.117795 +in -7.118552 +nt -7.192391 +mb -7.202464 +ma -7.229596 +uk -7.252106 +bi -7.254904 +om -7.267204 +we -7.298360 +ti -7.332546 +at -7.338846 +sa -7.347247 +si -7.411333 +ol -7.462337 +ag -7.466600 +nd -7.479229 +zi -7.512559 +ye -7.522160 +ny -7.532021 +te -7.553994 +ta -7.608293 +un -7.608336 +ee -7.637444 +on -7.637961 +nn -7.647582 +ko -7.649073 +as -7.664492 +tu -7.731456 +lu -7.751828 +ek -7.759676 +bw -7.774434 +ge -7.800396 +da -7.818176 +ub -7.835294 +_y -7.849024 +us -7.870398 +ri -7.889046 +za -7.892019 +is -7.902700 +yi -7.909471 +no -7.910528 +em -7.937477 +yo -7.969329 +se -7.977185 +_s -7.995053 +ob -8.006494 +eb -8.014123 +be -8.029388 +um -8.040828 +_g -8.054053 +ky -8.087876 +_l -8.123626 +ug -8.130300 +ad -8.149371 +ik -8.157571 +ay -8.185367 +ut -8.210542 +ii -8.262343 +im -8.269922 +it -8.329305 +by -8.397212 +ss -8.405060 +le -8.417287 +aw -8.455190 +so -8.475421 +wo -8.494481 +ez -8.496909 +_t -8.503101 +_w -8.533763 +gi -8.537417 +ze -8.555598 +uu -8.558363 +ja -8.562928 +kw -8.582177 +oo -8.599279 +ib -8.600524 +mi -8.604677 +gu -8.663734 +bo -8.668377 +az -8.725926 +ed -8.741536 +iz -8.744531 +re -8.760496 +go -8.776486 +kk -8.803634 +lo -8.891435 +ni -8.915838 +es -8.934323 +lw -8.954313 +to -8.962426 +di -8.982923 +ne -9.001979 +jj -9.005184 +uz -9.018095 +fu -9.022593 +me -9.027517 +ke -9.072491 +eg -9.100189 +mw -9.115861 +_p -9.123099 +_d -9.163773 +ro -9.165968 +ey -9.178218 +id -9.178855 +gg -9.251865 +nz -9.281517 +tt -9.281900 +ns -9.289281 +vu -9.310117 +gw -9.333411 +ig -9.338611 +ud -9.353288 +zz -9.461484 +gy -9.462414 +uw -9.482213 +su -9.517544 +aj -9.519189 +et -9.522774 +mp -9.525169 +af -9.541687 +pa -9.578189 +yu -9.582497 +va -9.593901 +nk -9.621375 +nj -9.703338 +av -9.730980 +bb -9.777527 +po -9.799338 +uv -9.805108 +r_ -9.888077 +oz -9.904122 +uy -9.906440 +mm -9.907284 +ot -9.938092 +_z -10.004732 +uf -10.021816 +dw -10.052815 +og -10.116934 +my -10.161203 +ew -10.193375 +n_ -10.214794 +du -10.253608 +ar -10.277255 +_f -10.279658 +d_ -10.297197 +ce -10.311149 +_c -10.327891 +wu -10.332589 +ly -10.376624 +_j -10.378788 +tw -10.407700 +ru -10.456410 +s_ -10.460121 +iw -10.513362 +do -10.531532 +y_ -10.555879 +fa -10.602420 +ec -10.609821 +ju -10.615534 +ow -10.630232 +mo -10.734309 +pu -10.736858 +ha -10.764812 +nl -10.789733 +pi -10.800279 +iy -10.814856 +fe -10.863091 +os -10.903785 +if -10.908002 +_u -10.910370 +h_ -10.916645 +od -10.959600 +iv -10.987982 +ia -11.056181 +zo -11.093737 +ff -11.093929 +l_ -11.094121 +ev -11.144586 +_r -11.171003 +ve -11.173641 +ch -11.175267 +uj -11.221864 +jo -11.232298 +rw -11.239827 +ap -11.241210 +oy -11.269692 +ca -11.278838 +fi -11.288264 +t_ -11.333383 +ty -11.342601 +vi -11.368522 +or -11.479135 +_h -11.498085 +ry -11.506886 +il -11.531793 +pe -11.563910 +ic -11.589423 +ll -11.598938 +ip -11.617605 +st -11.626057 +fo -11.647617 +up -11.648605 +je -11.790901 +m_ -11.810349 +_i -11.811138 +wi -11.816203 +nu -11.857370 +th -11.858676 +el -11.898572 +he -11.911893 +fr -12.046757 +sh -12.089611 +ah -12.103087 +ij -12.159493 +cc -12.218403 +ac -12.228284 +_v -12.246541 +zu -12.259416 +nf -12.264166 +of -12.274367 +ji -12.289694 +co -12.292558 +ur -12.296534 +ai -12.299633 +k_ -12.304740 +ef -12.319047 +c_ -12.319946 +dy -12.364459 +hi -12.371897 +ej -12.395627 +io -12.402514 +ea -12.417347 +nv -12.466016 +ho -12.529322 +ou -12.555019 +pr -12.580375 +kc -12.647899 +ep -12.735133 +ua -12.742044 +op -12.760229 +ie -12.775561 +tr -12.781120 +cr -12.809875 +au -12.810823 +py -12.819059 +ci -12.846644 +nc -12.881054 +rt -12.893378 +nr -12.920389 +eo -12.923801 +ov -12.951742 +ck -12.973147 +f_ -13.080252 +sw -13.105991 +rd -13.108319 +br -13.135755 +rm -13.141305 +oc -13.187314 +yw -13.194714 +p_ -13.198429 +hu -13.245900 +vv -13.251034 +g_ -13.301630 +rs -13.354064 +ph -13.354985 +ct -13.410390 +bl -13.424338 +ld -13.521287 +dc -13.532183 +rc -13.541589 +uh -13.660653 +ue -13.662932 +ei -13.677835 +cu -13.722895 +pt -13.752927 +oa -13.783598 +oj -13.819377 +gr -13.854111 +uc -13.873776 +oh -13.874436 +fd -13.882381 +w_ -13.889036 +pp -13.902438 +b_ -13.909185 +rl -13.925510 +df -13.965742 +bd -14.010752 +rk -14.037116 +sc -14.051974 +ex -14.066232 +hn -14.088270 +rr -14.150071 +vt -14.198016 +kh -14.228073 +rg -14.266547 +cl -14.339408 +ih -14.354064 +ao -14.424822 +dr -14.450184 +nw -14.471999 +ts -14.508422 +pl -14.511499 +qu -14.559524 +sp -14.584149 +kl -14.684765 +gh -14.684765 +fm -14.804204 +kr -14.814302 +hr -14.818107 +dh -14.892374 +sd -14.916645 +x_ -14.937186 +xp -14.951045 +rn -15.004958 +fl -15.031216 +lt -15.038595 +pd -15.122364 +wn -15.241635 +ae -15.277856 +sy -15.288374 +ls -15.333043 +iu -15.336677 +cs -15.347634 +gl -15.397998 +pc -15.490095 +yn -15.506374 +ui -15.533225 +sl -15.535312 +rb -15.537401 +v_ -15.547893 +sq -15.564842 +hh -15.588475 +j_ -15.599347 +ps -15.664073 +ks -15.673233 +ds -15.787945 +ms -15.899075 +oi -15.962229 +_q -15.970674 +ηη -15.979169 +z_ -15.987714 +rv -16.031216 +sm -16.054964 +lv -16.082157 +eh -16.103667 +lb -16.112985 +ym -16.131803 +wr -16.147675 +ys -16.160499 +cp -16.267414 +bs -16.334858 +td -16.338497 +oe -16.353144 +cy -16.390425 +dp -16.413265 +hw -16.436472 +dn -16.452154 +gn -16.471999 +vo -16.537401 +hm -16.562712 +yp -16.597166 +ox -16.664073 +ht -16.682451 +yl -16.719926 +tn -16.753535 +sr -16.782978 +_η -16.807982 +lg -16.828302 +gt -16.843732 +nh -16.848913 +yc -16.864566 +tm -16.885705 +mz -16.896391 +ix -16.945486 +ηe -16.973500 +mt -16.979169 +xi -17.002069 +ft -17.037116 +sg -17.066987 +jw -17.085211 +lf -17.085211 +vs -17.097489 +fc -17.122364 +rp -17.141305 +yd -17.226387 +yf -17.253608 +wh -17.260495 +gs -17.267414 +hy -17.281353 +uo -17.302518 +eu -17.345802 +bc -17.353144 +ηa -17.420959 +lk -17.436472 +yr -17.476001 +pm -17.516643 +dv -17.550001 +sf -17.558463 +dl -17.575536 +q_ -17.575536 +cb -17.584149 +ws -17.636937 +bm -17.645926 +tl -17.729449 +nb -17.748685 +ax -17.838571 +mn -17.912571 +rf -17.979169 +aq -17.990574 +gb -18.147675 +dg -18.186494 +tb -18.199669 +iq -18.226387 +sk -18.253608 +nm -18.267414 +eq -18.309642 +dm -18.309642 +cw -18.323997 +lc -18.338497 +vn -18.444292 +aη -18.444292 +mg -18.476001 +zr -18.524910 +xe -18.610302 +gm -18.628004 +tc -18.682451 +uη -18.758400 +tv -18.797929 +yt -18.818107 +xt -18.818107 +yy -18.880391 +ηo -18.923460 +xa -18.945486 +mk -19.013657 +tz -19.060963 +xy -19.109873 +yv -19.134964 +eη -19.160499 +lm -19.186494 +dj -19.267414 +wy -19.295428 +mc -19.323997 +_x -19.413265 +cd -19.413265 +xx -19.444292 +sj -19.476001 +vy -19.476001 +mr -19.476001 +fw -19.541589 +hl -19.541589 +tg -19.575536 +yg -19.575536 +hk -19.610302 +cq -19.682451 +wb -19.758400 +bf -19.758400 +zd -19.797929 +fs -19.797929 +sb -19.838571 +sn -19.880391 +qa -19.880391 +lh -19.923460 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lvs b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lvs new file mode 100644 index 00000000000..f5750eb5bb5 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/lvs @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 21814455 +#UNIQUE_BIGRAMS 637 +#UNSEEN_LOG_PROB -18.978712 +#EXPECTED_SCORE -7.425708 +s_ -4.512953 +a_ -4.864604 +as -5.251089 +ie -5.589343 +_p -6.002406 +u_ -6.031164 +ta -6.168662 +ja -6.207487 +ka -6.215996 +_s -6.258888 +ar -6.311604 +i_ -6.380653 +ra -6.472384 +_v -6.473053 +_k -6.517494 +_a -6.531687 +la -6.587885 +sa -6.595968 +ti -6.663697 +st -6.704272 +na -6.705513 +ri -6.735841 +at -6.810142 +ij -6.812742 +vi -6.816807 +es -6.829944 +pa -6.843612 +_t -6.867971 +is -6.871745 +ma -6.909433 +al -6.920174 +li -6.943298 +da -6.946084 +ga -6.968512 +va -6.984503 +_d -7.081963 +_l -7.087098 +ai -7.111235 +en -7.139378 +_i -7.149333 +in -7.215171 +et -7.271390 +em -7.279259 +ad -7.279937 +an -7.313596 +er -7.317136 +m_ -7.354139 +el -7.354789 +_g -7.355449 +au -7.386573 +ni -7.389431 +_b -7.395643 +ak -7.455047 +ve -7.502636 +_m -7.560948 +zi -7.580373 +pi -7.598459 +tu -7.627119 +am -7.639739 +ik -7.681230 +ek -7.692568 +me -7.702177 +_n -7.708042 +um -7.721058 +dz -7.735545 +ci -7.747490 +te -7.747704 +iz -7.754685 +us -7.806051 +ba -7.823368 +di -7.837216 +ku -7.871079 +e_ -7.883816 +tr -7.889106 +pe -7.892842 +it -7.894808 +ur -7.905868 +re -7.906788 +si -7.924803 +ot -7.925446 +os -7.966509 +id -7.991353 +_r -8.012561 +ne -8.027859 +no -8.042867 +aj -8.069269 +t_ -8.086154 +ko -8.108705 +to -8.108979 +ju -8.140078 +ru -8.141683 +de -8.141870 +ir -8.171881 +il -8.197376 +ro -8.202998 +le -8.206119 +sk -8.209952 +ej -8.222659 +bi -8.267054 +mi -8.270134 +ts -8.291774 +on -8.297485 +av -8.304828 +ei -8.312482 +se -8.388566 +ib -8.455291 +lu -8.461386 +ap -8.465424 +r_ -8.496950 +mu -8.506226 +ed -8.520484 +im -8.521480 +ce -8.524448 +ks -8.535547 +or -8.535572 +_c -8.599883 +ze -8.626845 +ig -8.640742 +ec -8.642828 +rs -8.649957 +nu -8.690695 +o_ -8.695156 +iv -8.744679 +_z -8.753358 +ls -8.786412 +ns -8.815287 +sp -8.821676 +za -8.852435 +do -8.855801 +nt -8.883270 +_u -8.895658 +ac -8.928375 +ul -8.930471 +_j -8.970352 +lo -8.974771 +su -8.977268 +pr -8.980104 +kl -9.004489 +ol -9.011639 +az -9.015741 +ez -9.044648 +om -9.049090 +ms -9.052527 +oj -9.058968 +be -9.086495 +ki -9.090059 +ev -9.127299 +rt -9.139332 +bu -9.140638 +ut -9.144038 +un -9.172377 +kt -9.185371 +_f -9.186296 +gu -9.188764 +k_ -9.196658 +rm -9.208034 +ug -9.214424 +od -9.243553 +kr -9.245278 +du -9.249216 +_o -9.264063 +pu -9.298796 +br -9.308786 +_e -9.343295 +ud -9.362713 +uz -9.367947 +ab -9.378605 +gr -9.399847 +op -9.409574 +z_ -9.444307 +ji -9.485148 +sm -9.516386 +uk -9.541350 +c_ -9.549257 +gi -9.563198 +nd -9.612407 +eg -9.626035 +ic -9.635840 +mo -9.636788 +po -9.702273 +dr -9.740798 +lt -9.743233 +uv -9.751076 +zv -9.782766 +ok -9.803656 +vo -9.839138 +jo -9.868944 +tv -9.889937 +io -9.890439 +sl -9.910157 +ag -9.923711 +pl -9.942525 +ca -10.001164 +tn -10.023086 +vu -10.069376 +so -10.070087 +fi -10.072221 +gs -10.086603 +zs -10.109435 +lv -10.135086 +mp -10.145461 +rb -10.215918 +rd -10.232212 +zu -10.256034 +zo -10.267238 +up -10.274346 +og -10.329613 +je -10.330379 +ep -10.381690 +ds -10.381778 +rn -10.416608 +ov -10.470107 +ps -10.490513 +mt -10.534664 +sn -10.535547 +ip -10.540661 +gl -10.542632 +bo -10.544903 +d_ -10.570420 +rp -10.571426 +ld -10.580613 +n_ -10.582234 +zn -10.590471 +ob -10.606260 +sv -10.619517 +ke -10.663141 +cu -10.665932 +ln -10.685730 +ia -10.716669 +rg -10.728066 +zm -10.753186 +l_ -10.756844 +nk -10.787609 +nc -10.800644 +_h -10.803006 +rk -10.846791 +v_ -10.855097 +eb -10.855464 +mb -10.855464 +ng -10.942200 +fo -10.949244 +oz -10.975769 +pt -10.994672 +ss -11.023981 +tt -11.058686 +nv -11.112873 +zd -11.116833 +fe -11.125820 +rv -11.125968 +ub -11.139779 +js -11.149211 +ea -11.200583 +uc -11.201829 +tk -11.235079 +lm -11.273855 +ge -11.283054 +fr -11.302132 +oc -11.308325 +zc -11.344326 +ha -11.353815 +go -11.426950 +p_ -11.459732 +he -11.466079 +lp -11.475652 +bl -11.477726 +rz -11.504953 +uj -11.609356 +lg -11.630588 +sd -11.632897 +tb -11.659392 +lb -11.659606 +af -11.660676 +fa -11.677908 +lk -11.780031 +zl -11.803715 +eo -11.872474 +pm -11.898748 +sr -11.904568 +hi -11.936356 +zp -11.971513 +of -12.056008 +kn -12.074715 +tl -12.107610 +ui -12.134417 +ho -12.134715 +rl -12.145461 +vs -12.167501 +rj -12.171462 +zb -12.222066 +zt -12.227447 +pg -12.245318 +km -12.277790 +ua -12.311011 +gz -12.311011 +ah -12.340891 +kc -12.349149 +pv -12.363366 +zr -12.365109 +zg -12.371403 +co -12.379838 +ll -12.383367 +fu -12.428590 +rc -12.473770 +j_ -12.486618 +b_ -12.502648 +kv -12.503032 +tg -12.510344 +pk -12.532899 +dn -12.560199 +ef -12.577881 +pd -12.586804 +if -12.593737 +lz -12.628912 +mn -12.632267 +eh -12.646190 +th -12.652137 +dm -12.657682 +gt -12.659820 +lj -12.701061 +mj -12.707240 +bs -12.777475 +rh -12.782591 +vn -12.821317 +nf -12.854729 +vj -13.021229 +mm -13.044508 +ii -13.168110 +ee -13.202608 +y_ -13.239230 +sb -13.254014 +g_ -13.272873 +bj -13.273527 +tm -13.276805 +nn -13.297964 +pj -13.312692 +pp -13.325534 +tz -13.357107 +nz -13.377373 +tp -13.380191 +pn -13.471890 +zk -13.476406 +pb -13.482449 +hn -13.486238 +_w -13.495374 +lc -13.528594 +nb -13.534075 +nr -13.540365 +lf -13.561797 +sf -13.573037 +gn -13.577073 +td -13.580309 +tj -13.601526 +fl -13.608117 +bn -13.613910 +tc -13.626400 +ou -13.644918 +ae -13.674878 +sc -13.676608 +sg -13.755814 +pz -13.788194 +ih -13.825152 +aa -13.869996 +ch -13.925510 +oa -13.960929 +vd -13.981106 +ck -14.005916 +vp -14.067033 +rr -14.094535 +nl -14.104985 +lr -14.109654 +dv -14.135607 +jn -14.212618 +sz -14.220171 +hu -14.246924 +ft -14.252077 +jb -14.261138 +cs -14.335754 +ue -14.346735 +gv -14.356413 +fs -14.424585 +h_ -14.433337 +nh -14.451003 +hr -14.455454 +oo -14.488517 +mg -14.560199 +mv -14.560199 +cp -14.581120 +gm -14.599062 +x_ -14.608943 +nm -14.615569 +sh -14.632267 +bk -14.633947 +oi -14.642379 +lh -14.645766 +hl -14.659392 +vv -14.676608 +wi -14.676608 +md -14.690531 +oh -14.702824 +_x -14.702824 +ct -14.727729 +hs -14.771451 +tf -14.884926 +oe -14.892952 +wa -14.909139 +vr -14.937912 +ht -14.960929 +ow -15.056853 +ay -15.126116 +kk -15.164462 +jr -15.184024 +vl -15.193906 +bp -15.278119 +rf -15.293973 +zz -15.318085 +vg -15.436267 +yo -15.436267 +np -15.436267 +pc -15.436267 +sj -15.459918 +ys -15.477914 +_y -15.499198 +xi -15.508416 +kp -15.527032 +f_ -15.530158 +ix -15.545891 +gh -15.568209 +we -15.571426 +wo -15.594146 +cl -15.613910 +ty -15.620558 +w_ -15.654267 +eu -15.749424 +iu -15.753072 +ry -15.767756 +my -15.831887 +vk -15.839622 +cr -15.859145 +kh -15.915257 +jv -15.982176 +mk -16.021229 +oy -16.052352 +ws -16.135607 +vm -16.140376 +tw -16.154779 +qu -16.169328 +cn -16.179109 +dk -16.188957 +mz -16.203855 +zj -16.203855 +dl -16.208856 +ex -16.223963 +jl -16.265039 +bv -16.280749 +ny -16.296632 +ao -16.301965 +xv -16.312692 +dd -16.318085 +ml -16.345358 +ew -16.384428 +kg -16.390096 +sy -16.401501 +mf -16.424585 +ly -16.448044 +bc -16.471890 +fg -16.477914 +dt -16.483963 +aw -16.502264 +gd -16.514595 +jp -16.539577 +nj -16.545891 +cb -16.577881 +ey -16.610597 +jt -16.623894 +ph -16.678341 +vz -16.699301 +uf -16.713445 +dp -16.742156 +uh -16.764071 +ye -16.771451 +sq -16.847400 +bt -16.863081 +ox -16.870986 +uu -16.903048 +dg -16.927570 +wh -16.944153 +ql -16.952516 +bd -17.038931 +ya -17.038931 +gg -17.047864 +wn -17.111995 +kd -17.149962 +_q -17.270257 +mc -17.270257 +hw -17.291318 +kb -17.312692 +hm -17.345358 +db -17.367554 +dc -17.412997 +ff -17.436267 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mal b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mal new file mode 100644 index 00000000000..a2c8432ebb6 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mal @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 442672 +#UNIQUE_BIGRAMS 803 +#UNSEEN_LOG_PROB -13.495848 +#EXPECTED_SCORE -7.869713 +ര_ -4.824864 +യ_ -4.990489 +_വ -5.362220 +_പ -5.398327 +ട_ -5.498933 +_ക -5.563278 +ന_ -5.732471 +ല_ -5.744651 +ത_ -5.936498 +_അ -5.958014 +_മ -5.972880 +_സ -5.980474 +_ഒ -5.981709 +ഒര -6.001826 +_ന -6.365172 +ണ_ -6.518370 +ള_ -6.674728 +_ഇ -6.855012 +കട -6.895956 +_ര -6.908430 +വര -6.922593 +പര -6.974929 +കര -6.978623 +നയ -7.064571 +മയ -7.227424 +വ_ -7.233297 +ക_ -7.240672 +ed -7.257029 +തയ -7.290312 +_p -7.295423 +te -7.295423 +st -7.315528 +os -7.330138 +po -7.344897 +d_ -7.344897 +വല -7.359809 +_ത -7.371634 +_ആ -7.448678 +മത -7.530069 +യട -7.550085 +മന -7.562969 +റ_ -7.575970 +പത -7.578459 +_ജ -7.586580 +രണ -7.587206 +സമ -7.628529 +തന -7.629819 +_ശ -7.639535 +രത -7.665766 +അവ -7.719705 +_ഭ -7.733511 +ലയ -7.738374 +രപ -7.755879 +മ_ -7.773598 +സ_ -7.798051 +_ച -7.814097 +തര -7.823664 +രവ -7.829583 +പല -7.835526 +_ദ -7.878594 +രള -7.887056 +വധ -7.889372 +ധ_ -7.951748 +ഗ_ -7.991007 +കള -8.064135 +വട -8.070254 +അത -8.114730 +വസ -8.166227 +മല -8.177506 +ഇത -8.181285 +നട -8.245114 +റയ -8.272063 +പ_ -8.277109 +സന -8.277109 +വവ -8.279132 +പട -8.342251 +തമ -8.408257 +ഴ_ -8.435078 +_യ -8.438466 +ദവ -8.455526 +ഷ_ -8.503213 +_ഓ -8.554980 +_ഏ -8.561122 +വത -8.582201 +ബ_ -8.626596 +രമ -8.629174 +ഇവ -8.672399 +നമ -8.699241 +ടത -8.704670 +പണ -8.719705 +_ബ -8.734899 +യമ -8.777168 +_f -8.778599 +fe -8.792983 +മര -8.792983 +രട -8.797326 +ഭമ -8.823664 +ഭഗ -8.826620 +സഭ -8.829583 +ടയ -8.832551 +തവ -8.844487 +_ഗ -8.853503 +നക -8.865614 +ജ_ -8.880897 +മറ -8.908821 +രയ -8.908821 +പയ -8.913528 +കല -8.922989 +ദശ -8.956597 +ശ_ -8.959839 +യള -8.986041 +_ല -8.989350 +eb -9.011045 +വന -9.022863 +ദ_ -9.034779 +ചല -9.036490 +b_ -9.060650 +ജന -9.062392 +അന -9.099454 +നര -9.112022 +വള -9.135659 +ധന -9.137493 +യര -9.143010 +_എ -9.159689 +ടപ -9.169039 +_j -9.176563 +വണ -9.178450 +സഘ -9.180339 +കത -9.191729 +ഴയ -9.199373 +വഴ -9.212847 +സവ -9.216720 +നത -9.216720 +വദ -9.226448 +t_ -9.244126 +ഇന -9.262023 +ശത -9.268039 +ശയ -9.278120 +മസ -9.292354 +വക -9.310864 +ഹ_ -9.331712 +തട -9.348611 +നവ -9.365710 +ഗത -9.365710 +ശഷ -9.398327 +മഖ -9.422723 +യഗ -9.429449 +ഭവ -9.436206 +ജല -9.449817 +സഹ -9.468166 +ടക -9.477429 +ദന -9.482083 +ആര -9.484416 +ധക -9.486752 +യല -9.493784 +പഴ -9.498491 +ചറ -9.505580 +n_ -9.524657 +പക -9.546425 +ളട -9.548864 +ഓര -9.558662 +ടന -9.575970 +തക -9.590972 +കസ -9.590972 +യയ -9.618887 +നല -9.629174 +ലവ -9.644743 +പറ -9.665766 +_ഉ -9.684416 +കണ -9.730739 +ഗമ -9.736288 +_ട -9.739070 +ളര -9.744651 +_ഹ -9.744651 +ഭ_ -9.747450 +ചന -9.755879 +ജയ -9.778599 +an -9.795877 +ഭര -9.795877 +ഗര -9.810435 +കയ -9.813364 +വശ -9.816299 +ja -9.825141 +അഗ -9.837015 +വഹ -9.837015 +രശ -9.842989 +ധര -9.858033 +_റ -9.858033 +ഘട -9.870182 +ശര -9.873236 +ജവ -9.888600 +ഹര -9.891692 +തല -9.894792 +ഏറ -9.926156 +ഘ_ -9.954979 +മപ -9.961463 +ri -9.981092 +ma -10.007686 +സല -10.021169 +ലക -10.021169 +ണക -10.038202 +ബബ -10.045072 +കവ -10.055439 +ഏത -10.062392 +ഹക -10.079922 +അധ -10.090543 +ഖ_ -10.090543 +_ഫ -10.090543 +രജ -10.104827 +വറ -10.108420 +s_ -10.112022 +ct -10.115634 +വയ -10.141169 +ta -10.144854 +ഖല -10.159689 +ar -10.163422 +_m -10.205132 +_ഷ -10.205132 +യത -10.252053 +ശല -10.256033 +ലസ -10.276098 +യവ -10.276098 +വമ -10.308795 +തഴ -10.329614 +തഭ -10.333814 +മണ -10.333814 +കഴ -10.338026 +മഅ -10.338026 +അദ -10.338026 +രധ -10.372174 +റണ -10.376500 +_d -10.380839 +_t -10.380839 +പന -10.393935 +ആദ -10.398327 +ദഹ -10.411583 +th -10.416029 +മഹ -10.416029 +നഗ -10.420488 +സഗ -10.429449 +രക -10.433951 +ലന -10.452098 +ഗവ -10.456671 +ളയ -10.461258 +യറ -10.475108 +ബര -10.475108 +റമ -10.475108 +ദര -10.475108 +കമ -10.475108 +ലള -10.489092 +വഭ -10.493784 +സര -10.498491 +co -10.507951 +എറ -10.536710 +ഡയ -10.536710 +ആയ -10.541559 +സധ -10.585954 +രസ -10.585954 +യക -10.606131 +i_ -10.621452 +രഗ -10.621452 +ee -10.626596 +ac -10.626596 +ഷണ -10.626596 +ലമ -10.626596 +ju -10.642136 +ഹയ -10.647354 +വഷ -10.652591 +തദ -10.657847 +ec -10.679063 +മബ -10.679063 +_a -10.684416 +റപ -10.700596 +ലര -10.716960 +അഴ -10.739070 +ഴമ -10.739070 +de -10.744651 +c_ -10.744651 +_n -10.744651 +_o -10.755879 +ടബ -10.755879 +കറ -10.772885 +ളക -10.772885 +ലത -10.795877 +സബ -10.807511 +er -10.848988 +ഭഷ -10.855012 +വജ -10.867135 +ഇര -10.873236 +ഹന -10.897898 +കശ -10.897898 +മള -10.904130 +മക -10.904130 +ഗല -10.910389 +നധ -10.910389 +ഫല -10.922989 +ഗക -10.935700 +ru -10.942097 +ണറ -10.948524 +p_ -10.948524 +ഠന -10.954979 +ശദ -10.954979 +ലട -10.954979 +സക -10.974519 +മഗ -10.994327 +ഉപ -11.000991 +ub -11.014412 +_ധ -11.041633 +പഠ -11.048519 +മധ -11.055439 +ഴല -11.062392 +hu -11.069378 +അപ -11.069378 +ആല -11.083453 +oc -11.090543 +ശബ -11.090543 +തസ -11.097667 +_h -11.104827 +ul -11.112022 +ആറ -11.112022 +ദയ -11.112022 +ണയ -11.133827 +ചര -11.133827 +at -11.141169 +hr -11.141169 +bh -11.141169 +um -11.141169 +mi -11.141169 +വഗ -11.141169 +സജ -11.148548 +നള -11.163422 +റസ -11.201290 +നറ -11.208984 +ലപ -11.216720 +ഗള -11.216720 +യന -11.216720 +l_ -11.232317 +റഡ -11.232317 +ഫസ -11.232317 +ജത -11.240179 +രദ -11.248084 +അല -11.296447 +ഫ_ -11.296447 +യണ -11.304667 +on -11.338026 +നശ -11.338026 +സച -11.346488 +ദമ -11.354999 +എഴ -11.354999 +ഴത -11.354999 +ഹദ -11.354999 +ഷജ -11.372174 +_c -11.398327 +op -11.398327 +py -11.398327 +yr -11.398327 +ig -11.398327 +gh -11.398327 +ht -11.398327 +ശധ -11.398327 +ടറ -11.407150 +ep -11.407150 +ഗണ -11.407150 +ണന -11.407150 +മഴ -11.433951 +കന -11.442996 +ba -11.452098 +തപ -11.452098 +ho -11.452098 +രഖ -11.452098 +y_ -11.461258 +e_ -11.461258 +re -11.470476 +ബജ -11.479754 +വഡ -11.489092 +_s -11.498491 +se -11.498491 +മവ -11.507951 +ശശ -11.517474 +nt -11.527060 +_ഡ -11.527060 +no -11.546425 +ov -11.546425 +v_ -11.546425 +ഓഹ -11.546425 +ഹത -11.546425 +ഹജ -11.556206 +നസ -11.556206 +au -11.566054 +ug -11.566054 +g_ -11.566054 +ഓഫ -11.585954 +ഏക -11.606131 +ഉട -11.616327 +db -11.626596 +ck -11.626596 +kc -11.626596 +if -11.626596 +ff -11.626596 +fc -11.626596 +ca -11.626596 +rs -11.626596 +sa -11.626596 +ab -11.626596 +bo -11.626596 +ou -11.626596 +ut -11.626596 +കഷ -11.626596 +om -11.626596 +ay -11.636938 +ആക -11.636938 +അമ -11.657847 +വപ -11.657847 +പദ -11.668416 +ളല -11.668416 +ളജ -11.679063 +ലല -11.679063 +ടമ -11.689789 +_ഞ -11.711484 +ഞന -11.711484 +ണവ -11.711484 +es -11.733511 +me -11.733511 +നദ -11.733511 +ആന -11.744651 +സശ -11.825141 +കദ -11.825141 +ഭയ -11.825141 +ne -11.837015 +ബന -11.837015 +അഭ -11.837015 +ലച -11.848988 +ധവ -11.848988 +is -11.861061 +തത -11.873236 +ഇട -11.873236 +തറ -11.873236 +അറ -11.897898 +മഘ -11.922989 +സദ -11.935700 +ഏഴ -11.948524 +അട -11.961463 +ew -11.974519 +ws -11.974519 +എത -11.974519 +റത -11.974519 +ശഭ -11.974519 +സയ -11.974519 +ആധ -11.987694 +it -11.987694 +ഷയ -11.987694 +r_ -11.987694 +നഭ -12.000991 +രര -12.000991 +ic -12.000991 +ബഗ -12.014412 +ടല -12.041633 +രഹ -12.055439 +ഡ_ -12.055439 +മഷ -12.069378 +ടവ -12.069378 +ധയ -12.069378 +ap -12.083453 +ണത -12.097667 +ag -12.112022 +gs -12.112022 +ചമ -12.112022 +ഡല -12.112022 +അസ -12.126522 +അബ -12.126522 +മട -12.155966 +റക -12.155966 +ആള -12.155966 +റല -12.155966 +ആഗ -12.170916 +ബദ -12.186023 +രഷ -12.186023 +ജമ -12.186023 +ളത -12.201290 +un -12.216720 +ra -12.216720 +എവ -12.232317 +ഭദ -12.232317 +ഓട -12.232317 +ot -12.232317 +ബല -12.248084 +ദപ -12.248084 +ഗഢ -12.264026 +ഢല -12.264026 +പവ -12.264026 +ഗന -12.280145 +ഖന -12.280145 +ബസ -12.280145 +_ഐ -12.280145 +അഥ -12.296447 +ഥവ -12.296447 +ഫറ -12.296447 +ജബ -12.296447 +തഗ -12.296447 +a_ -12.312935 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mar b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mar new file mode 100644 index 00000000000..261daa1a7d3 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mar @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 13323346 +#UNIQUE_BIGRAMS 1474 +#UNSEEN_LOG_PROB -14.471014 +#EXPECTED_SCORE -8.014749 +ल_ -4.666485 +त_ -4.789881 +_आ -5.215011 +न_ -5.240480 +र_ -5.277221 +_क -5.373330 +_स -5.832235 +_अ -5.860822 +ह_ -5.970363 +_म -6.012971 +च_ -6.032933 +_प -6.262544 +ण_ -6.355652 +_व -6.392529 +_य -6.414788 +आह -6.416609 +_न -6.449221 +हत -6.494157 +_ह -6.555726 +कर -6.631988 +_द -6.702695 +क_ -6.720376 +_ज -6.906655 +अस -6.984925 +स_ -6.990449 +व_ -7.129173 +_त -7.134397 +_ब -7.147694 +कल -7.237948 +रत -7.396136 +_र -7.437470 +_श -7.464029 +ळ_ -7.487291 +_ग -7.495475 +_ल -7.537667 +य_ -7.570182 +_भ -7.595656 +आण -7.621949 +तल -7.695146 +वर -7.701083 +पर -7.770340 +ड_ -7.781328 +श_ -7.868475 +द_ -7.882946 +सर -8.001479 +सत -8.024527 +_घ -8.028621 +_झ -8.048036 +म_ -8.084605 +रव -8.113044 +यन -8.139671 +नह -8.172598 +झल -8.223441 +रण -8.229246 +सन -8.269946 +नव -8.282467 +ज_ -8.321944 +आल -8.347640 +तन -8.351702 +तत -8.367065 +यच -8.368567 +सम -8.369426 +_च -8.378047 +ग_ -8.389294 +तर -8.394532 +णर -8.420304 +लल -8.428228 +गत -8.444922 +ट_ -8.458723 +_ख -8.465138 +कम -8.483779 +गल -8.527063 +ठ_ -8.557622 +मन -8.561504 +दर -8.579742 +मर -8.587843 +रन -8.589845 +मह -8.596871 +जत -8.614418 +गर -8.626393 +पल -8.679834 +सल -8.702571 +वत -8.702751 +_उ -8.704241 +नक -8.727828 +ध_ -8.737085 +लक -8.759999 +भर -8.761549 +मळ -8.773583 +मल -8.775622 +_ए -8.781518 +दल -8.792040 +रज -8.795885 +वळ -8.798052 +वन -8.809326 +हर -8.814972 +कत -8.816778 +जन -8.858891 +_ठ -8.866048 +वल -8.870146 +यत -8.893902 +दव -8.964252 +रक -8.966201 +सच -8.973258 +लग -8.977946 +जर -8.994254 +नत -9.005397 +सग -9.012985 +वड -9.034458 +दन -9.055391 +अश -9.062205 +भग -9.094043 +एक -9.102351 +वस -9.104257 +_फ -9.170724 +रप -9.185401 +कह -9.198511 +थ_ -9.208342 +सह -9.219853 +अन -9.220370 +कव -9.241975 +मत -9.244469 +घत -9.287736 +_इ -9.295200 +पण -9.304208 +ख_ -9.306537 +यथ -9.307223 +चर -9.358616 +कळ -9.358972 +रच -9.363530 +कण -9.366386 +कस -9.369176 +मज -9.370609 +रल -9.381617 +तच -9.403304 +कन -9.404698 +हल -9.414049 +धक -9.428451 +मद -9.431289 +गव -9.436082 +दश -9.440063 +शक -9.450631 +प_ -9.491982 +जव -9.494791 +रह -9.495026 +पह -9.504983 +वच -9.523193 +तस -9.524708 +हन -9.526543 +सव -9.530943 +ळत -9.534632 +रस -9.564165 +बर -9.569503 +वक -9.577506 +नग -9.587718 +तक -9.603216 +डल -9.618626 +कय -9.631623 +टक -9.632740 +चल -9.653345 +जग -9.653781 +पस -9.654829 +पड -9.665781 +ब_ -9.685351 +_ध -9.685618 +कच -9.709898 +गण -9.713348 +मग -9.725305 +सठ -9.734316 +धर -9.743754 +आप -9.748031 +लन -9.758591 +ऊन -9.766964 +तय -9.770553 +कड -9.771499 +बद -9.772730 +वज -9.780518 +ई_ -9.784524 +लस -9.787870 +नद -9.793817 +हज -9.796896 +लव -9.801816 +धन -9.820004 +लख -9.826479 +नस -9.827954 +नच -9.845779 +खल -9.854574 +शह -9.861810 +अध -9.867969 +पय -9.885786 +वध -9.891226 +वट -9.906837 +ठव -9.909751 +हण -9.925986 +वव -9.926197 +शत -9.931263 +बल -9.939958 +पच -9.967013 +ळव -9.977237 +टल -9.980734 +पट -9.982814 +वढ -9.984678 +दत -9.991055 +_ट -9.991496 +यव -9.997904 +ढ_ -9.999676 +बध -10.001895 +मठ -10.023372 +ष_ -10.031622 +दस -10.037641 +रग -10.043000 +लत -10.048838 +वश -10.053203 +मड -10.077568 +कट -10.092623 +सक -10.098666 +गळ -10.112864 +ळल -10.138633 +वद -10.147203 +यज -10.162262 +खर -10.173473 +शव -10.180367 +नल -10.183260 +वह -10.188052 +बई -10.204312 +नय -10.214182 +जक -10.228272 +बब -10.231522 +थल -10.232955 +ठक -10.233737 +हव -10.255677 +मब -10.260318 +सद -10.267508 +सभ -10.280244 +रम -10.281860 +ठर -10.300721 +ईल -10.304824 +वण -10.305372 +डन -10.309764 +यल -10.318310 +भव -10.327047 +दख -10.334018 +आद -10.341586 +मक -10.342288 +_ड -10.345807 +पव -10.363244 +णक -10.363387 +पढ -10.364957 +आत -10.367530 +पन -10.374413 +टन -10.382195 +डत -10.390892 +लच -10.391765 +घड -10.400374 +इत -10.407857 +जण -10.418933 +नम -10.426066 +यर -10.436532 +डय -10.438184 +पत -10.452527 +चन -10.463188 +बज -10.469008 +सघ -10.469929 +कद -10.481648 +वग -10.481648 +तह -10.487544 +जल -10.493776 +घट -10.502232 +ऊ_ -10.502703 +रठ -10.511212 +हस -10.511686 +णत -10.515961 +सध -10.529822 +जय -10.529982 +फर -10.532546 +गड -10.552735 +कश -10.578334 +सब -10.581317 +रय -10.587302 +उप -10.587635 +चक -10.600355 +घर -10.610477 +यम -10.615055 +यक -10.618966 +शर -10.637131 +आय -10.644219 +भ_ -10.652910 +कप -10.676995 +नर -10.681611 +डक -10.688207 +_थ -10.690531 +शन -10.702210 +मध -10.702210 +धव -10.705280 +सय -10.706003 +तव -10.706003 +वभ -10.716168 +टत -10.725122 +यण -10.729895 +लह -10.732104 +सप -10.736162 +बन -10.742641 +बह -10.753442 +बक -10.779090 +डण -10.781566 +जम -10.783473 +नन -10.783855 +मव -10.793432 +पद -10.795355 +भष -10.802880 +अम -10.826675 +खड -10.827659 +रख -10.832390 +तथ -10.839317 +डव -10.842892 +पक -10.844882 +वप -10.856279 +रद -10.856681 +बस -10.861306 +तप -10.885069 +वष -10.893902 +हम -10.897822 +वय -10.900717 +मच -10.934438 +यद -10.937408 +खळ -10.938682 +हच -10.973531 +नट -10.977892 +गद -10.989733 +डळ -10.992599 +ढल -10.993923 +बत -11.004117 +_ओ -11.007234 +घऊ -11.011252 +पठ -11.012593 +अत -11.038324 +हऊ -11.042886 +बळ -11.052743 +रब -11.064986 +आव -11.065682 +आज -11.080144 +णस -11.084841 +दह -11.095227 +मट -11.096649 +धत -11.100923 +भत -11.105688 +षय -11.105926 +ळन -11.106165 +यग -11.109272 +ळच -11.112145 +टर -11.118390 +तम -11.123213 +गन -11.137779 +तळ -11.166362 +कठ -11.174098 +यस -11.181372 +रश -11.185654 +कक -11.207765 +खन -11.208790 +धल -11.211355 +रध -11.217015 +डच -11.233607 +सड -11.234911 +शल -11.237001 +जह -11.253561 +गच -11.253561 +णच -11.254354 +शस -11.255677 +जब -11.271116 +रड -11.272455 +तश -11.274599 +शध -11.281591 +सण -11.282130 +रष -11.295132 +गय -11.297038 +शळ -11.301951 +मण -11.312377 +चत -11.322880 +शष -11.324823 +आम -11.330110 +नश -11.330947 +आर -11.335696 +अड -11.347499 +दच -11.360822 +लब -11.371397 +जप -11.379741 +तद -11.380895 +आध -11.382918 +वम -11.396866 +चव -11.398619 +कध -11.409771 +अभ -11.421900 +हद -11.442246 +दण -11.444055 +सस -11.446772 +ठय -11.452223 +भम -11.454349 +रर -11.458609 +_छ -11.460134 +भज -11.460134 +अग -11.461660 +शय -11.466861 +सट -11.467474 +कढ -11.469622 +बड -11.475160 +सज -11.480410 +मस -11.480410 +लम -11.487855 +णव -11.492839 +लप -11.496276 +ळक -11.504747 +तड -11.525665 +हक -11.533027 +अर -11.546891 +घ_ -11.558929 +जच -11.562527 +घल -11.567448 +टय -11.576679 +रभ -11.578665 +फट -11.578996 +अव -11.581981 +णन -11.584306 +वई -11.596326 +खस -11.608785 +शच -11.614206 +चग -11.629906 +झ_ -11.630936 +उभ -11.631623 +डस -11.635407 +लय -11.653084 +सश -11.661127 +गप -11.661478 +हप -11.661478 +लद -11.667453 +अप -11.675577 +टब -11.705642 +खप -11.708900 +णल -11.711803 +अज -11.723839 +खत -11.724939 +ढत -11.729711 +अक -11.739305 +दक -11.741528 +गम -11.745984 +यई -11.763571 +फल -11.783283 +हई -11.783664 +हय -11.784046 +खव -11.791320 +गध -11.796703 +नध -11.802880 +षद -11.816485 +दऊ -11.821571 +णम -11.837334 +षण -11.841302 +दड -11.874663 +टच -11.877919 +बग -11.882409 +हब -11.884046 +सळ -11.884046 +दप -11.884864 +ळण -11.897202 +रळ -11.901338 +लट -11.908397 +गट -11.915072 +शख -11.923881 +भट -11.942939 +उम -11.943366 +पळ -11.951920 +षत -11.958800 +ळख -11.962685 +ठण -11.972225 +गह -11.977455 +घन -11.979640 +_औ -11.989733 +आठ -11.991937 +दम -11.996354 +_ऐ -12.005675 +अट -12.011922 +आढ -12.022245 +मझ -12.027660 +नज -12.044944 +हभ -12.061048 +शम -12.066611 +मख -12.079207 +आन -12.082960 +डग -12.090969 +जऊ -12.091914 +बठ -12.100923 +नप -12.103780 +आक -12.112385 +टळ -12.114304 +लभ -12.118631 +फ_ -12.122006 +गज -12.125389 +नभ -12.132664 +यब -12.146834 +लढ -12.147817 +ढव -12.150768 +जध -12.155700 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mhr b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mhr new file mode 100644 index 00000000000..a821a0f413f --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mhr @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 2172238 +#UNIQUE_BIGRAMS 697 +#UNSEEN_LOG_PROB -15.923771 +#EXPECTED_SCORE -7.700303 +е_ -5.056375 +н_ -5.156620 +_к -5.613031 +ын -5.867309 +ла -5.901599 +_т -6.158445 +ым -6.198514 +ен -6.276375 +_п -6.356175 +де -6.381312 +ии -6.417699 +_в -6.426355 +_и -6.426641 +ан -6.474089 +ал -6.550220 +о_ -6.554833 +те -6.563036 +ыш -6.583145 +м_ -6.586843 +ш_ -6.590231 +ар -6.689287 +_ш -6.728541 +а_ -6.741132 +_м -6.789905 +ты -6.804604 +шт -6.822684 +к_ -6.922031 +аш -6.942390 +_с -7.006613 +лы -7.023017 +ма -7.027516 +и_ -7.033768 +ку -7.062776 +т_ -7.064020 +ер -7.100651 +р_ -7.139452 +ак -7.203987 +ту -7.214700 +_о -7.227981 +ве -7.253596 +не -7.255523 +_л -7.257554 +ко -7.296281 +ны -7.338224 +ле -7.340160 +иы -7.375235 +_д -7.468609 +шо -7.504580 +ил -7.506873 +ме -7.511713 +ол -7.523274 +ор -7.528782 +ч_ -7.529518 +ка -7.534066 +мы -7.558146 +ра -7.565550 +ыл -7.605089 +ык -7.641758 +на -7.648805 +_а -7.655083 +ке -7.676798 +гы -7.691825 +ем -7.698845 +_у -7.704930 +та -7.717176 +чы -7.729245 +ды -7.801193 +кы -7.804159 +_г -7.804307 +ли -7.833853 +еш -7.868667 +ел -7.869754 +уд -7.876449 +ше -7.882392 +се -7.883176 +же -7.886787 +ам -7.889934 +ры -7.893878 +еч -7.907527 +ур -7.939778 +ри -7.976108 +нд -7.982140 +вл -7.993605 +по -7.996486 +са -8.006357 +ат -8.007211 +ча -8.011146 +то -8.017328 +га -8.022327 +ас -8.025957 +ва -8.026823 +па -8.043373 +ог -8.141420 +л_ -8.143672 +ст -8.149884 +ыт -8.177307 +он -8.188307 +ыр -8.219839 +ре -8.228778 +ик -8.248234 +мо -8.271852 +шк -8.314349 +ун -8.316253 +ыж -8.317100 +ша -8.329652 +_н -8.335003 +кт -8.372371 +ни -8.399251 +рт -8.402393 +шы -8.403517 +аи -8.411184 +че -8.416620 +ге -8.491374 +оч -8.497842 +ло -8.516934 +_ч -8.537023 +ет -8.550905 +да -8.559649 +ид -8.584419 +уг -8.591063 +_р -8.624224 +ум -8.635537 +_э -8.643483 +ек -8.650672 +му -8.661657 +ти -8.684975 +ио -8.709232 +рл -8.767952 +лт -8.767952 +ом -8.793952 +ро -8.830977 +нч -8.864946 +ов -8.922434 +уш -8.940594 +од -8.965610 +ыч -8.976275 +ав -8.982308 +ви -8.991068 +мл -9.000222 +уч -9.002604 +гу -9.018705 +ут -9.018705 +ин -9.028730 +_я -9.044775 +лу -9.051455 +ач -9.060292 +у_ -9.086410 +ки -9.092561 +пе -9.168872 +ок -9.188113 +пы -9.191216 +ул -9.195104 +ук -9.219048 +до -9.228977 +во -9.228977 +кш -9.245007 +шу -9.254305 +жа -9.264481 +со -9.266116 +го -9.271031 +от -9.275552 +тр -9.278436 +пу -9.311392 +ич -9.328370 +рн -9.377883 +ои -9.401045 +ес -9.443420 +эр -9.462973 +но -9.465319 +ош -9.483270 +ль -9.486601 +лн -9.493287 +ыг -9.506269 +ян -9.548914 +зы -9.556395 +оз -9.564921 +ис -9.566935 +лм -9.590807 +чо -9.591319 +ев -9.599540 +ед -9.604184 +жы -9.616123 +_ы -9.631317 +рг -9.633426 +ми -9.638181 +зе -9.653076 +ос -9.716477 +ап -9.740138 +иж -9.747541 +с_ -9.747541 +кс -9.775208 +кл -9.797494 +уж -9.841297 +мд -9.843126 +тн -9.865875 +ит -9.899100 +аж -9.923401 +ня -9.926629 +ҥа -9.943534 +си -9.965278 +ыс -9.978618 +жо -10.008407 +в_ -10.017328 +рд -10.019394 +рж -10.060647 +ыз -10.102384 +ск -10.112641 +тл -10.122232 +сы -10.125938 +за -10.170402 +_ж -10.190440 +уҥ -10.215490 +ир -10.262848 +_ф -10.265298 +вы -10.274318 +ад -10.278436 +мш -10.281739 +ь_ -10.296698 +ял -10.297534 +я_ -10.301719 +ну -10.307599 +ж_ -10.345983 +пр -10.352914 +ру -10.364250 +иг -10.369512 +рв -10.375676 +еҥ -10.379210 +ив -10.382752 +ву -10.404192 +из -10.455493 +иу -10.499042 +пи -10.530132 +сс -10.538010 +су -10.547919 +ыи -10.580092 +ii -10.621344 +_е -10.680063 +зи -10.693199 +ци -10.694299 +аз -10.718714 +ус -10.732208 +уи -10.811152 +ди -10.817131 +еп -10.823135 +гр -10.827956 +ҥ_ -10.840079 +ши -10.853534 +лш -10.854763 +ыв -10.863399 +сп -10.863399 +аг -10.897199 +ьы -10.927923 +ие -10.933108 +ҥе -10.965942 +иа -10.977948 +уа -10.986008 +им -11.013204 +тy -11.020084 +п_ -11.022845 +рш -11.036730 +нж -11.040922 +дв -11.045126 +нт -11.050751 +_ю -11.077771 +ае -11.086410 +фе -11.092198 +тш -11.106771 +оэ -11.114113 +км -11.128910 +сн -11.178846 +i_ -11.181928 +оҥ -11.222614 +др -11.230572 +лв -11.241787 +св -11.259588 +лл -11.261217 +ыд -11.292528 +чм -11.300881 +лк -11.316041 +ия -11.322830 +мб -11.322830 +дм -11.348578 +ба -11.353783 +кр -11.364250 +тв -11.366002 +эл -11.369512 +ям -11.392539 +уз -11.396115 +лз -11.410506 +рс -11.410506 +ҥг -11.417756 +мм -11.419574 +фи -11.423217 +нс -11.428699 +иш -11.460164 +вт -11.480895 +кн -11.484697 +рк -11.486601 +ег -11.501929 +ть -11.515475 +яр -11.525230 +еж -11.535051 +д_ -11.546925 +ез -11.556895 +нн -11.562911 +жн -11.583145 +уэ -11.609882 +ҥы -11.630791 +бл -11.637123 +аҥ -11.641360 +эҥ -11.658433 +рм -11.664888 +яш -11.667046 +еи -11.673540 +эш -11.680063 +ье -11.680063 +чк -11.695400 +еа -11.697604 +ят -11.728823 +ги -11.731079 +уб -11.751543 +з_ -11.779288 +их -11.783964 +рь -11.793363 +эт -11.795722 +лг -11.809959 +дл -11.824339 +ля -11.829164 +нь -11.831582 +хо -11.838862 +чу -11.870842 +ха -11.875825 +yр -11.880826 +мг -11.903546 +_x -11.903546 +ув -11.950088 +_б -11.976609 +ая -11.995468 +ее -11.998183 +рч -12.003627 +пл -12.009092 +чи -12.014577 +мж -12.031160 +ац -12.033942 +шл -12.047936 +лд -12.050751 +жу -12.082084 +o_ -12.111172 +тс -12.125938 +нл -12.125938 +эн -12.155933 +_з -12.165054 +дш -12.174234 +шн -12.177307 +мп -12.199002 +бо -12.208400 +об -12.217861 +тк -12.272674 +зо -12.285879 +yҥ -12.302558 +зв -12.309284 +бр -12.312658 +yм -12.316041 +oр -12.326237 +мн -12.346847 +оп -12.350311 +оя -12.410506 +vi -12.425042 +юж -12.439726 +жл -12.439726 +ья -12.447124 +xi -12.469550 +уы -12.511592 +_y -12.531114 +нг -12.531114 +ип -12.535051 +ож -12.554896 +_х -12.575017 +оф -12.579076 +ео -12.697604 +ьн -12.697604 +фа -12.697604 +аф -12.724321 +гл -12.728823 +як -12.733338 +_i -12.760732 +лю -12.760732 +нк -12.783964 +шм -12.793363 +фр -12.812346 +ау -12.821932 +эм -12.821932 +юм -12.826749 +сл -12.826749 +фо -12.846180 +иo -12.860926 +це -12.880826 +yд -12.906093 +уп -12.911199 +нц -12.931810 +ъя -12.957994 +иц -12.995468 +лж -13.000902 +yл -13.033942 +эс -13.056397 +лo -13.073471 +лх -13.079207 +лy -13.084966 +ръ -13.102384 +вр -13.120013 +кy -13.137861 +см -13.162008 +ищ -13.174234 +г_ -13.186565 +ех -13.199002 +ср -13.224202 +ҥж -13.236970 +зн -13.236970 +пк -13.236970 +мс -13.249851 +ое -13.275964 +щы -13.302558 +тм -13.302558 +уя -13.329652 +_v -13.329652 +бу -13.329652 +ьо -13.350311 +пь -13.357264 +юц -13.364250 +бе -13.364250 +эз -13.392539 +кч -13.428699 +юл -13.428699 +сy -13.436041 +шy -13.458294 +лс -13.473322 +юр -13.503856 +xv -13.527189 +тч -13.535051 +хн -13.575017 +вн -13.575017 +яб -13.583145 +жг -13.616123 +дс -13.632898 +хи -13.667046 +пy -13.684429 +бы -13.684429 +аб -13.693199 +оц -13.702023 +ҥд -13.719834 +yн -13.719834 +еф -13.746970 +пш -13.774626 +кв -13.793363 +ьт -13.793363 +рз -13.812346 +ьм -13.812346 +кц -13.821932 +иy -13.860926 +иб -13.860926 +дд -13.921468 +би -13.921468 +сб -13.942226 +вг -13.973935 +тя -13.973935 +yш -13.973935 +зд -14.050751 +ец -14.050751 +уе -14.084966 +вя -14.084966 +зь -14.096554 +пп -14.096554 +пo -14.096554 +эк -14.108236 +жм -14.108236 +жи -14.120013 +ду -14.120013 +нм -14.143860 +yс -14.143860 +яҥ -14.143860 +э_ -14.155933 +ъе -14.155933 +нф -14.155933 +ьк -14.180386 +кп -14.217861 +ыҥ -14.217861 +б_ -14.243396 +кo -14.309284 +iv -14.322830 +v_ -14.322830 +дж -14.336505 +чл -14.378325 +ух -14.392539 +ьж -14.392539 +пс -14.421394 +нр -14.436041 +чш -14.465788 +нв -14.465788 +ию -14.480895 +кк -14.496162 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/min b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/min new file mode 100644 index 00000000000..9051b6460ea --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/min @@ -0,0 +1,477 @@ +#TOTAL_BIGRAMS 5839717 +#UNIQUE_BIGRAMS 473 +#UNSEEN_LOG_PROB -19.155539 +#EXPECTED_SCORE -7.062884 +la -5.050089 +da -5.078523 +i_ -5.181752 +an -5.328147 +_a -5.607030 +am -5.659984 +_d -5.689768 +h_ -5.746533 +ah -5.749427 +do -5.774550 +li -5.783776 +ad -5.836996 +ol -5.847285 +ri -5.885609 +ar -5.908962 +id -6.004507 +il -6.070232 +ta -6.073956 +ae -6.135653 +e_ -6.200089 +mi -6.238074 +ng -6.261536 +_f -6.313876 +al -6.344887 +fa -6.351932 +_s -6.357432 +k_ -6.405188 +a_ -6.470462 +s_ -6.474169 +g_ -6.581182 +_t -6.666168 +_l -6.704225 +ek -6.814826 +er -6.857161 +mb -6.912603 +ra -6.966332 +o_ -7.050220 +_p -7.051104 +le -7.116723 +sa -7.119468 +si -7.164868 +ba -7.181268 +es -7.207500 +on -7.239137 +_m -7.249438 +_c -7.285984 +_k -7.336877 +ua -7.365231 +it -7.422227 +ro -7.448784 +ci -7.448870 +n_ -7.481347 +ma -7.493583 +as -7.501530 +ia -7.509430 +ku -7.534907 +ni -7.535182 +ce -7.566871 +um -7.619923 +at -7.634871 +ak -7.644429 +pa -7.656939 +ag -7.657039 +us -7.723624 +pe -7.912305 +sp -7.935100 +ph -7.945476 +is -7.946756 +yc -8.016499 +te -8.046884 +st -8.088037 +nd -8.098698 +in -8.107802 +na -8.117031 +ab -8.135253 +en -8.160478 +by -8.163592 +nt -8.169270 +op -8.176257 +hi -8.188457 +om -8.263072 +du -8.266263 +ja -8.266491 +nj -8.358202 +ga -8.369514 +or -8.384462 +ac -8.399233 +_o -8.413314 +_n -8.439234 +po -8.458571 +di -8.467464 +mo -8.469564 +ti -8.490203 +tu -8.503680 +ca -8.523725 +lo -8.525090 +t_ -8.537612 +ch -8.577827 +ie -8.584924 +ny -8.614152 +my -8.620555 +mu -8.625913 +ai -8.649430 +se -8.649926 +go -8.689666 +os -8.743498 +ap -8.782456 +tr -8.834513 +ic -8.847769 +to -8.850731 +_b -8.853128 +de -8.860459 +bu -8.889924 +_e -8.967445 +el -8.981113 +ii -8.999709 +bi -9.009479 +ks -9.057770 +oi -9.100121 +m_ -9.121153 +ho -9.152583 +ik -9.157795 +yo -9.165009 +_i -9.170266 +u_ -9.214051 +d_ -9.226133 +sc -9.242650 +ul -9.257089 +ne -9.260721 +ka -9.423372 +ut -9.429661 +im -9.446455 +no -9.455099 +ud -9.456314 +au -9.459963 +gi -9.466065 +yi -9.471492 +lu -9.522725 +_h -9.530926 +ha -9.547654 +ip -9.581324 +gn -9.584924 +rb -9.634724 +kn -9.644577 +ns -9.644577 +tl -9.684270 +yl -9.701857 +_j -9.712181 +ge -9.714047 +wa -9.726341 +ep -9.763866 +co -9.796229 +so -9.832709 +et -9.875928 +pu -9.904530 +un -9.912841 +em -9.936855 +pr -9.966950 +aw -9.969672 +ju -9.998950 +ui -10.021626 +th -10.022654 +ig -10.038675 +hr -10.045708 +nu -10.055139 +ec -10.097277 +ok -10.105690 +hy -10.112238 +ru -10.127633 +_g -10.130399 +oc -10.137339 +uo -10.145990 +ei -10.152443 +ea -10.197566 +cu -10.200470 +og -10.201052 +_r -10.204546 +ll -10.241453 +ke -10.326133 +me -10.334403 +if -10.380423 +od -10.414409 +io -10.418799 +fu -10.420152 +ot -10.427618 +he -10.452674 +re -10.475355 +bo -10.487363 +sy -10.505923 +ye -10.507002 +p_ -10.545252 +pt -10.564204 +ps -10.628062 +ur -10.683864 +r_ -10.708869 +yt -10.724668 +eu -10.730533 +gr -10.734316 +pi -10.777027 +dr -10.847655 +gg -10.925278 +be -10.952436 +l_ -10.955867 +dy -10.967692 +lt -10.978119 +uk -11.006808 +ya -11.008334 +ko -11.051727 +yz -11.072858 +rt -11.110052 +ir -11.112238 +su -11.166286 +ob -11.195537 +gu -11.240855 +cr -11.330898 +iu -11.357229 +zi -11.418123 +ex -11.456487 +pl -11.463447 +cl -11.534220 +lm -11.536419 +_u -11.557114 +ct -11.570576 +eo -11.572832 +br -11.604023 +up -11.656490 +yr -11.658885 +vi -11.710110 +x_ -11.711767 +za -11.747846 +ry -11.805926 +ys -11.830908 +yp -11.830908 +mp -11.909511 +rc -11.912365 +hu -11.913318 +rh -11.920961 +rn -11.939278 +eg -11.951946 +_v -11.966703 +ed -11.968682 +nn -12.062782 +ki -12.069137 +rp -12.091605 +ty -12.126528 +ss -12.196696 +tt -12.201343 +oe -12.251055 +ly -12.286408 +mn -12.325182 +nc -12.353346 +cy -12.419475 +va -12.449561 +rg -12.500187 +rm -12.520365 +rs -12.524726 +ym -12.560095 +ox -12.569074 +ax -12.584165 +ve -12.599416 +gl -12.636689 +rd -12.638263 +uc -12.641417 +ub -12.654100 +hl -12.655693 +rr -12.701034 +xi -12.712595 +xo -12.722580 +fo -12.754659 +yd -12.797987 +fi -12.806811 +eb -12.877554 +ez -12.938308 +gy -12.949990 +yn -12.971655 +_w -12.999709 +sm -13.015988 +ay -13.028318 +yb -13.036598 +xy -13.059615 +lc -13.093763 +sn -13.146550 +fr -13.217724 +oh -13.229540 +je -13.241453 +ev -13.258299 +av -13.263148 +ov -13.302541 +mm -13.332809 +cc -13.358526 +bj -13.368943 +ib -13.427618 +lp -13.433073 +xa -13.441293 +fe -13.455099 +ew -13.483114 +aj -13.491625 +ao -13.529100 +_y -13.540829 +af -13.558604 +oa -13.573585 +ji -13.622599 +tc -13.625718 +eh -13.647744 +zo -13.666895 +fl -13.670112 +wi -13.673336 +ih -13.683051 +iw -13.686304 +yx -13.709283 +bs -13.739375 +gm -13.742757 +ix -13.756368 +zz -13.790966 +bl -13.794472 +xe -13.830009 +sh -13.833611 +iv -13.870137 +ml -13.938308 +qu -13.949990 +_z -14.026256 +iz -14.051202 +jo -14.128739 +gk -14.137617 +bd -14.142077 +yg -14.142077 +uf -14.169128 +ht -14.196696 +_x -14.224802 +kt -14.248648 +hn -14.272896 +rv -14.292592 +pp -14.307542 +ug -14.322649 +lb -14.363725 +ou -14.368943 +yh -14.411378 +y_ -14.427618 +yu -14.449561 +ue -14.455099 +ck -14.466240 +ff -14.488782 +py -14.517465 +aa -14.607102 +tz -14.631977 +we -14.657288 +sl -14.683051 +ef -14.736000 +ze -14.777027 +hm -14.870137 +ee -14.900038 +b_ -14.907611 +lv -14.915225 +kl -15.009861 +tm -15.042839 +zu -15.093763 +rl -15.093763 +ow -15.119915 +vo -15.128739 +nb -15.164584 +dt -15.220079 +vu -15.229540 +az -15.368943 +sk -15.400651 +zy -15.477467 +nf -15.488782 +cn -15.558604 +nk -15.570576 +_q -15.594824 +ey -15.619486 +ld -15.631977 +vr -15.644577 +uh -15.805042 +ls -15.819256 +ts -15.862757 +oo -15.969672 +c_ -15.985614 +sb -15.985614 +sw -16.034524 +kh -16.068076 +ij -16.102428 +oz -16.102428 +wy -16.210680 +cd -16.248648 +rk -16.287642 +wn -16.368943 +nz -16.411378 +eq -16.433073 +rw -16.477467 +gh -16.546730 +v_ -16.546730 +bn -16.546730 +oq -16.570576 +dz -16.570576 +tn -16.570576 +of -16.594824 +ky -16.594824 +dg -16.644577 +uv -16.644577 +ds -16.670112 +uj -16.670112 +cs -16.696107 +uu -16.749547 +sz -16.777027 +zh -16.805042 +wo -16.805042 +ej -16.833611 +sr -16.833611 +vt -16.862757 +sv -16.892505 +lg -16.922878 +w_ -16.922878 +ux -16.922878 +kr -16.922878 +ws -16.985614 +xt -17.051202 +rf -17.051202 +cq -17.051202 +dl -17.119915 +bt -17.229540 +hs -17.229540 +uz -17.307542 +nm -17.348184 +nl -17.348184 +xh -17.390004 +kk -17.390004 +dj -17.433073 +dw -17.433073 +hk -17.570576 +nh -17.619486 +nq -17.619486 +nv -17.619486 +lh -17.670112 +xu -17.670112 +ms -17.670112 +uy -17.777027 +dh -17.777027 +db -17.777027 +gs -17.777027 +f_ -17.833611 +iq -17.833611 +lk -18.018035 +yv -18.018035 +ft -18.018035 +nr -18.018035 +lr -18.155539 +sj -18.155539 +lz -18.155539 +lj -18.307542 +oy -18.390004 +bb -18.477467 +gq -18.477467 +zm -18.477467 +wd -18.477467 +cz -18.477467 +dd -18.477467 +gt -18.670112 +jd -18.670112 +kv -18.670112 +bf -18.670112 +km -18.670112 +bk -18.670112 +dn -18.777027 +kw -18.892505 +dp -18.892505 +rq -18.892505 +wu -18.892505 +bw -18.892505 +hw -19.018035 +hb -19.018035 +uw -19.018035 +dm -19.155539 +xc -19.155539 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mkd b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mkd new file mode 100644 index 00000000000..34fc71e7a81 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mkd @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 77358958 +#UNIQUE_BIGRAMS 979 +#UNSEEN_LOG_PROB -17.311741 +#EXPECTED_SCORE -7.670007 +а_ -4.703242 +и_ -5.499676 +_п -5.781281 +е_ -5.913383 +ат -5.991708 +на -6.064997 +та -6.116752 +т_ -6.125520 +о_ -6.162254 +ни -6.235098 +_с -6.267662 +от -6.268564 +ра -6.375400 +те -6.379428 +то -6.438848 +ка -6.447839 +ст -6.491450 +ко -6.536590 +ен -6.679470 +_к -6.680864 +ва -6.720828 +_д -6.737199 +ре -6.758285 +по -6.760214 +ит -6.787307 +_н -6.802113 +ја -6.813824 +пр -6.829404 +_о -7.021943 +ан -7.022445 +ри -7.030395 +де -7.038745 +_м -7.090779 +но -7.104577 +ов -7.114288 +ти -7.134150 +ед -7.144680 +ин -7.169423 +_и -7.257391 +од -7.258598 +_т -7.275255 +ек -7.308180 +ет -7.343436 +ор -7.352155 +_в -7.359792 +ро -7.363691 +_б -7.388791 +иј -7.400940 +ак -7.437973 +во -7.445573 +ск -7.452750 +ма -7.471672 +ли -7.478877 +ле -7.486000 +до -7.549723 +ки -7.565657 +ви -7.581802 +ал -7.602856 +ме -7.607351 +ве -7.610569 +не -7.630719 +ос -7.632339 +ар -7.633660 +ди -7.663904 +ел -7.682924 +ав -7.686612 +_г -7.705289 +н_ -7.721804 +ла -7.726927 +го -7.751626 +ув -7.756872 +ер -7.760176 +он -7.761672 +ци -7.770608 +ис -7.771826 +ол -7.795852 +_р -7.854208 +_з -7.898389 +ој -7.911378 +ил -7.963194 +ад -8.002587 +ем -8.033365 +ио -8.046598 +би -8.072088 +тр -8.128818 +ми -8.179201 +се -8.182080 +ес -8.214424 +да -8.221069 +за -8.225377 +ло -8.266008 +л_ -8.313264 +па -8.345039 +мо -8.361598 +вр -8.367948 +шт -8.371817 +об -8.377809 +из -8.384531 +га -8.441347 +ам -8.461559 +со -8.483839 +_а -8.487134 +ик -8.536166 +ј_ -8.549464 +ас -8.557698 +ог -8.582977 +_е -8.583507 +дн -8.606133 +ај -8.607500 +аа -8.609662 +ба -8.618027 +ом -8.656815 +ив -8.669623 +гр -8.674514 +д_ -8.681266 +им -8.682116 +ок -8.729514 +си -8.732477 +ње -8.743146 +ег -8.748919 +бе -8.749714 +нс -8.756457 +тв -8.776828 +р_ -8.778833 +_у -8.791173 +бо -8.802428 +ир -8.807624 +ањ -8.809732 +у_ -8.822770 +иц -8.873160 +ку -8.877442 +_ш -8.907638 +ру -8.945772 +че -8.955137 +ев -8.976331 +др -8.984111 +ке -9.006591 +оп -9.014247 +лу -9.021565 +сп -9.028600 +нт -9.041091 +к_ -9.043412 +ид -9.067453 +ие -9.093002 +сл -9.104865 +_л -9.134923 +_ф -9.150440 +бр -9.189116 +са -9.220424 +вн -9.244158 +зи -9.255216 +ту -9.264201 +лн -9.279974 +ои -9.296731 +чи -9.311325 +ич -9.329292 +еш -9.331981 +ап -9.350951 +аз -9.360249 +кр -9.408024 +пе -9.409786 +м_ -9.424628 +аш -9.427219 +зн -9.456269 +аб -9.486532 +ше -9.495389 +оз -9.504978 +св -9.553004 +це -9.553593 +с_ -9.602062 +гу -9.602366 +_ч -9.612185 +оа -9.628108 +в_ -9.659241 +ез -9.682315 +оч -9.685030 +кт -9.695738 +сн -9.696466 +ац -9.696558 +чн -9.729807 +_ц -9.770583 +тс -9.792743 +ож -9.795591 +ач -9.805620 +же -9.807407 +ус -9.817014 +пи -9.824097 +уг -9.831265 +жа -9.842899 +аг -9.843824 +ги -9.844681 +жи -9.847238 +ну -9.847943 +уч -9.862296 +_ј -9.862400 +зе -9.902819 +рв -9.902872 +чк -9.935317 +дв -9.945929 +рт -9.949483 +ур -9.969107 +иг -9.975833 +су -9.980347 +еј -9.980592 +рн -10.001870 +еб -10.026653 +ду -10.026750 +гл -10.033888 +фи -10.054981 +тн -10.058258 +ца -10.064095 +ии -10.065793 +вл -10.096541 +ум -10.101777 +нд -10.121149 +аж -10.167411 +уд -10.183434 +рс -10.233566 +иш -10.235409 +уп -10.255740 +ул -10.269532 +зв -10.279626 +_ж -10.279766 +еч -10.281784 +ун -10.294282 +пл -10.297799 +вс -10.321205 +ча -10.332294 +лк -10.345579 +_х -10.356442 +нц -10.356809 +ср -10.359452 +шн -10.364754 +зб -10.396729 +мн -10.399951 +ук -10.403559 +ге -10.407555 +кл -10.409304 +еп -10.421939 +рж -10.424064 +ша -10.425653 +бл -10.441645 +тк -10.442787 +см -10.470302 +рм -10.472580 +ое -10.488192 +оц -10.490846 +јо -10.510952 +еа -10.549199 +уш -10.574997 +уб -10.577332 +јк -10.613192 +ош -10.616643 +рк -10.641422 +з_ -10.643002 +ае -10.674354 +нк -10.689858 +кс -10.699067 +пс -10.700743 +фо -10.707525 +ши -10.711835 +му -10.724780 +кв -10.839768 +бу -10.850093 +рг -10.876355 +ву -10.878952 +зо -10.881518 +вт -10.886663 +рд -10.889986 +дб -10.894807 +ут -10.899573 +ш_ -10.903497 +ец -10.906680 +фе -10.910660 +ња -10.911414 +рш -10.947198 +јн -10.949704 +фа -10.950369 +жу -10.955915 +мј -10.963045 +ип -10.984649 +ењ -10.990139 +зд -11.000226 +је -11.002673 +ју -11.016360 +мп -11.037294 +жн -11.042005 +пу -11.050168 +зг -11.070117 +рз -11.082803 +еж -11.120464 +шк -11.126831 +зл -11.127623 +лб -11.166146 +чу -11.166275 +г_ -11.180228 +дс -11.211242 +тп -11.246784 +дл -11.268151 +хо -11.292316 +гн -11.317654 +лт -11.327877 +лс -11.353121 +нг -11.433627 +бј -11.449604 +јд -11.477357 +уж -11.499649 +вк -11.548417 +кц -11.550205 +уа -11.557944 +хр -11.567591 +цр -11.601033 +еф -11.610565 +рц -11.634854 +рб -11.664150 +п_ -11.682913 +фр -11.689735 +оф -11.735868 +оо -11.739881 +мс -11.740136 +кн -11.791437 +пш -11.803785 +мв -11.836354 +фу -11.847582 +аф -11.848544 +ео -11.855369 +ха -11.878636 +шу -11.878987 +јс -11.881025 +рл -11.898861 +уз -11.911809 +чо -11.926543 +зм -11.933602 +дг -11.986956 +мр -11.994470 +зј -11.997593 +пј -12.037020 +зу -12.043697 +ц_ -12.067513 +лж -12.088315 +нф -12.120921 +иф -12.143188 +бн -12.192964 +_џ -12.202074 +јз -12.238740 +дм -12.249052 +ау -12.252961 +уе -12.267323 +иу -12.313661 +лг -12.317844 +мб -12.331140 +чл -12.336628 +аи -12.383191 +џе -12.384786 +жб -12.394996 +зр -12.429249 +бв -12.445385 +ао -12.457292 +ји -12.470355 +лм -12.478528 +рп -12.483432 +хи -12.493828 +ее -12.513431 +јц -12.534519 +дд -12.535073 +иа -12.568667 +пт -12.578786 +иж -12.616233 +јм -12.617288 +иб -12.620103 +јв -12.648679 +c_ -12.675634 +хе -12.677099 +мл -12.688626 +ех -12.690351 +нз -12.709585 +gc -12.712586 +уц -12.741668 +рч -12.757853 +ињ -12.759403 +јг -12.767443 +шо -12.768874 +оу -12.889491 +ох -12.921832 +пн -12.924584 +ч_ -12.947383 +вм -12.955952 +вч -12.961147 +их -12.971296 +тт -13.022050 +јб -13.028113 +б_ -13.038274 +пк -13.042831 +тл -13.050563 +лј -13.057860 +цн -13.075782 +мк -13.097358 +х_ -13.113464 +љу -13.152667 +рх -13.207004 +јт -13.219579 +e_ -13.306275 +дп -13.395699 +тх -13.453939 +нв -13.461703 +нч -13.462756 +_љ -13.465495 +_ѕ -13.478208 +уј -13.499433 +шп -13.504625 +ах -13.508098 +лц -13.509184 +вг -13.515722 +цо -13.532198 +ф_ -13.569120 +еи -13.581641 +вј -13.603062 +_n -13.606780 +сф -13.654799 +уи -13.659618 +пц -13.661308 +џо -13.689858 +ж_ -13.711710 +ng -13.724275 +cg -13.739244 +еу -13.773306 +јч -13.779849 +ух -13.783000 +сц -13.824063 +вд -13.846963 +цу -13.869675 +ѕв -13.889632 +сд -13.899858 +шв -13.911019 +жј -13.944439 +лд -13.960404 +g_ -13.969649 +хн -13.972644 +тм -13.993177 +s_ -14.132932 +_m -14.148427 +јп -14.174743 +фл -14.195936 +џи -14.217446 +_p -14.226355 +жо -14.315941 +жр -14.317464 +пм -14.323951 +нб -14.332775 +нр -14.357616 +_t -14.363108 +км -14.372175 +џа -14.386083 +шл -14.443098 +вп -14.476719 +_c -14.488246 +јш -14.490820 +уџ -14.494259 +фт -14.500297 +аџ -14.500729 +th -14.529108 +вц -14.533966 +pg -14.561209 +јл -14.584845 +ху -14.607478 +дз -14.613543 +_s -14.624335 +_њ -14.628581 +њу -14.646644 +en -14.662517 +he -14.666876 +ma -14.669304 +_i -14.671735 +мф -14.674171 +цв -14.684937 +_u -14.693312 +тф -14.705717 +нл -14.730345 +as -14.746658 +t_ -14.756949 +be -14.774090 +ur -14.797797 +јр -14.826770 +_e -14.842025 +on -14.853022 +l_ -14.859105 +иџ -14.874708 +n_ -14.883701 +a_ -14.892749 +сч -14.897295 +уф -14.920819 +хм -14.934186 +пч -14.949446 +ii -14.950627 +nd -14.967258 +r_ -14.971445 +d_ -15.005393 +нџ -15.018951 +_b -15.019570 +mc -15.032638 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mlg b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mlg new file mode 100644 index 00000000000..b1548121b82 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mlg @@ -0,0 +1,413 @@ +#TOTAL_BIGRAMS 3928853 +#UNIQUE_BIGRAMS 409 +#UNSEEN_LOG_PROB -18.583749 +#EXPECTED_SCORE -6.563911 +a_ -3.712589 +y_ -4.656415 +na -4.678522 +an -4.697738 +ra -5.057726 +in -5.346643 +ia -5.500203 +_a -5.505781 +ar -5.727693 +_m -5.737318 +ka -5.853045 +ao -5.886651 +_d -5.902752 +di -5.928867 +_i -6.007808 +o_ -6.088768 +_t -6.112683 +on -6.264698 +ta -6.321331 +ni -6.382176 +am -6.387577 +ma -6.436639 +ny -6.472124 +ha -6.497314 +_f -6.535909 +tr -6.603966 +sa -6.634980 +at -6.688628 +mi -6.715582 +_k -6.782202 +ah -6.813456 +mp -6.850437 +la -6.965684 +si -6.991198 +is -7.001183 +fi -7.010859 +om -7.059746 +ry -7.161999 +it -7.185433 +nd -7.191217 +dr -7.195893 +za -7.199936 +fa -7.215733 +ts -7.216989 +ak -7.322241 +_h -7.384199 +sy -7.385426 +_n -7.455497 +to -7.460985 +oa -7.526501 +al -7.530094 +ay -7.551152 +hi -7.609477 +ik -7.617748 +nt -7.630862 +ro -7.633922 +iz -7.648289 +as -7.649542 +m_ -7.681826 +va -7.694473 +pi -7.766205 +_s -7.802553 +ba -7.834047 +eo -7.883569 +e_ -7.885825 +re -7.888085 +_p -7.893401 +mb -7.911323 +av -7.941336 +eh -8.010291 +so -8.049251 +en -8.066178 +im -8.096512 +af -8.156122 +ri -8.178394 +ko -8.204154 +il -8.207732 +te -8.235795 +_j -8.244789 +or -8.273931 +_r -8.287177 +vo -8.291772 +po -8.293845 +pa -8.294652 +no -8.348931 +n_ -8.354329 +od -8.370037 +et -8.382727 +lo -8.395531 +es -8.408573 +az -8.426907 +ir -8.434129 +ho -8.448937 +je -8.476008 +da -8.595490 +iv -8.605181 +ai -8.607614 +ol -8.608474 +_v -8.618542 +os -8.683486 +ly -8.690144 +_l -8.694853 +pe -8.802717 +er -8.815068 +de -8.840766 +se -8.855488 +og -8.881576 +gr -8.883656 +zy -8.925537 +ad -8.998223 +_e -9.104373 +ot -9.121655 +ee -9.129244 +ns -9.137906 +he -9.154133 +me -9.186502 +ef -9.189715 +ok -9.210448 +ga -9.215242 +id -9.323535 +ty -9.372590 +ne -9.392443 +oh -9.463770 +_z -9.465586 +ih -9.484927 +do -9.589395 +ng -9.589961 +fo -9.592510 +oe -9.610482 +_o -9.653899 +bo -9.730127 +s_ -9.756247 +ja -9.760381 +el -9.827192 +ab -9.867444 +mo -9.896898 +ti -9.906734 +vy -10.144126 +ov -10.164210 +vi -10.190144 +_b -10.197886 +ki -10.211754 +dy -10.236349 +le -10.275410 +hy -10.285457 +_c -10.301124 +nj -10.468965 +ag -10.470527 +ve -10.494166 +li -10.544281 +zo -10.684090 +be -10.717706 +nk -10.836227 +ke -10.875010 +co -10.895149 +oy -10.903565 +t_ -10.998035 +mm -11.033772 +un -11.082310 +ky -11.144957 +mu -11.159162 +ev -11.196593 +io -11.202638 +ie -11.317899 +au -11.357818 +_g -11.475224 +r_ -11.587134 +ap -11.607614 +if -11.622588 +st -11.626066 +ib -11.680469 +go -11.750859 +ut -11.761019 +ys -11.782849 +jo -11.974939 +fe -12.044590 +em -12.047696 +kr -12.069626 +my -12.090293 +ij -12.095105 +ez -12.117774 +fy -12.160843 +ur -12.194870 +oi -12.279968 +ou -12.427918 +ca -12.454466 +l_ -12.539355 +fr -12.565827 +ip -12.681675 +ll -12.686508 +of -12.703553 +i_ -12.725768 +su -12.799768 +h_ -12.839588 +bi -12.858553 +br -12.888868 +ea -12.897248 +py -12.957310 +ob -12.957310 +aj -12.983836 +ch -12.995784 +ge -13.007831 +gy -13.016934 +rt -13.041491 +oz -13.082310 +by -13.085498 +d_ -13.215679 +ek -13.219176 +ss -13.290967 +jy -13.294652 +rd -13.320714 +c_ -13.328248 +nn -13.366518 +ce -13.370401 +zi -13.454466 +gi -13.479412 +ze -13.597338 +ji -13.611056 +eu -13.629552 +pr -13.672057 +aa -13.681675 +ep -13.720801 +op -13.750859 +eg -13.791935 +gn -13.866758 +rn -13.872254 +v_ -13.916992 +us -14.023034 +x_ -14.047696 +vr -14.066473 +eb -14.104777 +ic -14.150789 +u_ -14.177756 +rc -14.205237 +sn -14.290967 +bl -14.313220 +rr -14.366518 +ux -14.397882 +rs -14.471049 +nc -14.530637 +ex -14.539355 +th -14.565827 +pl -14.574760 +ac -14.715852 +rg -14.715852 +p_ -14.776394 +g_ -14.776394 +xi -14.786736 +ii -14.797152 +ul -14.807645 +ud -14.894450 +fp -14.916992 +nv -14.916992 +gc -14.939892 +_w -14.951480 +ae -14.963162 +oc -14.963162 +gl -15.010859 +ey -15.047696 +pm -15.150789 +ue -15.177756 +oj -15.191431 +wi -15.233251 +qu -15.276320 +cr -15.290967 +tg -15.305764 +gv -15.305764 +ed -15.305764 +km -15.397882 +sc -15.397882 +ci -15.462733 +rm -15.462733 +ei -15.638890 +oo -15.676858 +_q -15.676858 +ec -15.696223 +ui -15.715852 +rl -15.755930 +k_ -15.755930 +ig -15.797152 +ju -15.818214 +cy -15.818214 +fj -15.839588 +jk -15.839588 +uv -15.839588 +ps -15.883309 +um -15.905677 +lu -15.928397 +pt -15.928397 +gm -15.974939 +mt -15.974939 +_u -15.974939 +lm -16.023034 +ej -16.047696 +ua -16.177756 +yl -16.205237 +kt -16.233251 +rk -16.261821 +ix -16.261821 +cl -16.261821 +kn -16.290967 +sp -16.290967 +dc -16.351088 +uc -16.479412 +_y -16.479412 +sk -16.513359 +tt -16.513359 +lh -16.548125 +up -16.620275 +ye -16.620275 +ru -16.657749 +z_ -16.696223 +_て -16.696223 +す_ -16.735752 +lb -16.735752 +wo -16.776394 +yv -16.776394 +ub -16.776394 +pu -16.776394 +ct -16.818214 +hr -16.818214 +tv -16.818214 +rh -16.861283 +yp -16.861283 +rv -16.861283 +ph -16.861283 +vm -16.861283 +rj -16.905677 +uy -16.951480 +_い -16.998786 +sm -17.047696 +yo -17.047696 +yn -17.098322 +sr -17.150789 +tu -17.150789 +ew -17.205237 +w_ -17.205237 +xp -17.205237 +mg -17.205237 +てす -17.261821 +ug -17.261821 +f_ -17.261821 +ls -17.320714 +ff -17.320714 +fk -17.320714 +い_ -17.320714 +tw -17.382115 +uj -17.446245 +ml -17.446245 +pp -17.446245 +fl -17.446245 +ds -17.513359 +rb -17.513359 +cc -17.583749 +gu -17.583749 +いて -17.583749 +て_ -17.583749 +vh -17.583749 +hv -17.583749 +gt -17.657749 +cs -17.657749 +pv -17.657749 +ms -17.818214 +sb -17.818214 +bu -17.998786 +kl -17.998786 +md -17.998786 +dm -17.998786 +tp -18.098322 +nl -18.098322 +fm -18.205237 +sl -18.205237 +rp -18.205237 +lp -18.320714 +tl -18.320714 +iq -18.320714 +ks -18.320714 +cq -18.320714 +pc -18.320714 +_さ -18.320714 +さい -18.320714 +てい -18.320714 +yr -18.446245 +du -18.446245 +yc -18.446245 +ck -18.446245 +_x -18.446245 +xa -18.446245 +_く -18.446245 +くた -18.446245 +た_ -18.446245 +_た -18.446245 +たさ -18.446245 +さ_ -18.446245 +_ま -18.446245 +ます -18.446245 +jl -18.446245 +lf -18.446245 +dd -18.583749 +gs -18.583749 +hm -18.583749 +mf -18.583749 +hl -18.583749 +nu -18.583749 +b_ -18.583749 +pd -18.583749 +we -18.583749 +いま -18.583749 +ま_ -18.583749 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mlt b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mlt new file mode 100644 index 00000000000..e80d1aa30be --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mlt @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 37128072 +#UNIQUE_BIGRAMS 699 +#UNSEEN_LOG_PROB -16.578061 +#EXPECTED_SCORE -7.792132 +a_ -4.972606 +ta -5.891885 +i_ -5.925220 +l_ -5.951671 +ha -5.963974 +n_ -6.096174 +gh -6.099669 +_t -6.111735 +ie -6.116528 +al -6.150157 +_m -6.251521 +t_ -6.313574 +u_ -6.330855 +_k -6.395634 +in -6.463750 +_j -6.472443 +ti -6.533995 +_g -6.545292 +ar -6.546748 +_i -6.661275 +en -6.691571 +li -6.832628 +ll -6.851041 +_f -6.875018 +_d -6.884266 +mi -6.896880 +ni -6.935055 +_s -6.938121 +is -6.938359 +ra -6.969225 +il -6.979741 +ji -7.043544 +an -7.068618 +r_ -7.071202 +_b -7.124414 +na -7.138996 +ja -7.140147 +et -7.189863 +_h -7.225922 +er -7.228891 +_p -7.251077 +st -7.269394 +ma -7.298827 +da -7.303203 +on -7.334839 +m_ -7.344918 +ri -7.345366 +at -7.377874 +nt -7.433735 +_a -7.494598 +d_ -7.510055 +di -7.583653 +_n -7.610929 +x_ -7.625097 +it -7.627246 +ko -7.646924 +bi -7.671160 +s_ -7.705308 +em -7.730901 +el -7.792464 +me -7.795724 +fi -7.827147 +ku -7.872476 +ol -7.892944 +ki -7.893037 +_l -7.937839 +tt -7.947495 +un -8.015728 +ss -8.020331 +la -8.048860 +ag -8.051414 +hu -8.078541 +id -8.105707 +or -8.107399 +jo -8.110263 +si -8.128350 +lu -8.138410 +ik -8.142884 +wa -8.148861 +ej -8.152947 +sa -8.178330 +mm -8.213504 +zz -8.217732 +ka -8.224991 +_q -8.226795 +es -8.227132 +se -8.228600 +re -8.238436 +te -8.250515 +ed -8.275570 +ho -8.283007 +iz -8.300725 +im -8.302785 +aw -8.308712 +ij -8.309624 +zj -8.318131 +he -8.337282 +ur -8.345271 +je -8.348320 +k_ -8.350690 +ad -8.352543 +ah -8.380112 +om -8.412859 +ba -8.425562 +qa -8.425602 +ir -8.425616 +de -8.466115 +eg -8.482032 +pa -8.483631 +_r -8.498015 +le -8.539616 +nn -8.546530 +ew -8.567177 +tu -8.575025 +az -8.583168 +am -8.608910 +as -8.621894 +_e -8.629137 +_w -8.646379 +nd -8.651308 +pr -8.670907 +ru -8.706663 +rt -8.707069 +ro -8.714134 +ab -8.735920 +aj -8.746445 +tr -8.779480 +ex -8.782779 +w_ -8.784406 +aq -8.818297 +hi -8.823251 +if -8.836726 +q_ -8.853074 +du -8.864910 +po -8.889338 +gi -8.923344 +um -8.929868 +ul -8.938821 +jn -8.957051 +ne -8.957302 +ke -8.963768 +pe -8.970243 +ek -8.975789 +dd -9.045180 +jj -9.046721 +fu -9.050631 +kk -9.060887 +zi -9.075110 +b_ -9.127242 +nu -9.129786 +ig -9.138870 +za -9.142555 +eh -9.149489 +fe -9.150439 +ga -9.157345 +ak -9.157411 +lt -9.168459 +ib -9.179569 +f_ -9.271338 +fa -9.272082 +rr -9.293136 +ge -9.294966 +_o -9.302920 +ju -9.309390 +h_ -9.315962 +_z -9.345916 +uq -9.354946 +af -9.361423 +ci -9.376350 +hh -9.419044 +os -9.421413 +ve -9.427688 +wi -9.450942 +pp -9.464906 +be -9.482393 +su -9.488186 +ic -9.499702 +_c -9.505790 +sp -9.523526 +eb -9.523555 +_x -9.533139 +ns -9.546036 +lh -9.559431 +mu -9.563453 +nz -9.574077 +ng -9.607636 +_u -9.607879 +mp -9.626371 +bb -9.635181 +og -9.674459 +vi -9.679102 +iv -9.679580 +to -9.686543 +ax -9.710272 +gu -9.729968 +ce -9.737677 +z_ -9.749436 +g_ -9.767204 +lo -9.767983 +ez -9.806261 +rs -9.821053 +ze -9.831778 +hr -9.840088 +qe -9.840836 +e_ -9.877502 +op -9.886668 +rj -9.891015 +nk -9.901941 +kt -9.913361 +bu -9.914111 +_v -9.917714 +bh -9.933701 +va -9.947946 +ca -9.967965 +rn -9.972330 +hd -9.984324 +mh -9.994317 +ux -10.005697 +fo -10.017650 +jk -10.020594 +oh -10.037156 +rm -10.037401 +gr -10.052961 +ff -10.075299 +dh -10.077775 +wn -10.082738 +so -10.103193 +ap -10.107731 +ih -10.120348 +nh -10.144687 +mo -10.146668 +we -10.164886 +cc -10.200243 +bl -10.211533 +ut -10.228821 +sk -10.242596 +od -10.251473 +uz -10.268004 +pl -10.300468 +pi -10.314057 +gg -10.325280 +hm -10.343945 +ix -10.369009 +ud -10.381446 +nj -10.381550 +ip -10.387524 +wk -10.387940 +rg -10.410874 +uk -10.431225 +no -10.455300 +ot -10.469995 +ht -10.485733 +lm -10.496077 +ac -10.496975 +lj -10.507797 +us -10.513296 +go -10.546152 +dw -10.550983 +ww -10.551216 +hn -10.576448 +fn -10.586869 +tk -10.589321 +pu -10.600077 +mb -10.601948 +xi -10.629199 +ok -10.636356 +cj -10.669514 +o_ -10.684016 +th -10.685871 +jg -10.687088 +sj -10.690616 +kb -10.695118 +bo -10.712030 +ec -10.713596 +qu -10.724534 +fr -10.726508 +ft -10.734827 +kl -10.741863 +rb -10.745528 +tg -10.751076 +xx -10.753824 +bd -10.797071 +rd -10.811943 +br -10.816281 +pj -10.816912 +tl -10.836460 +tw -10.866832 +xe -10.901792 +zd -10.916437 +ea -10.926914 +nf -10.927974 +jt -10.944955 +uw -10.979609 +hl -10.981179 +ub -10.981886 +ob -11.002464 +kr -11.004219 +qi -11.005896 +ef -11.007175 +fl -11.012945 +zm -11.014793 +of -11.024231 +oc -11.067856 +do -11.070779 +xo -11.081096 +qs -11.090216 +ml -11.099224 +gv -11.119138 +zo -11.124593 +uh -11.209921 +p_ -11.217766 +hs -11.224910 +jr -11.273140 +nq -11.286472 +dn -11.293770 +kw -11.305623 +tn -11.335536 +nc -11.354946 +dr -11.365365 +qt -11.374827 +oz -11.377306 +oq -11.388617 +wr -11.413946 +qg -11.415962 +sh -11.422026 +ov -11.424908 +ch -11.447194 +zg -11.455682 +qq -11.464769 +dj -11.488800 +tf -11.489359 +ep -11.495853 +rv -11.518359 +io -11.525329 +kn -11.527507 +zu -11.533369 +up -11.537752 +xa -11.572715 +lb -11.580310 +bn -11.587108 +sn -11.589980 +eq -11.598028 +vo -11.605637 +ug -11.605758 +mg -11.638212 +ln -11.654655 +c_ -11.678402 +jh -11.705008 +j_ -11.706825 +bq -11.742995 +uf -11.753287 +wg -11.772462 +gw -11.779685 +av -11.781872 +ox -11.806435 +ev -11.809222 +qo -11.818876 +js -11.828312 +rk -11.830858 +dm -11.838949 +tm -11.930019 +y_ -11.935640 +ia -11.949866 +hb -11.971081 +tb -11.997530 +rc -12.005816 +rl -12.030801 +bz -12.047810 +mr -12.067022 +xt -12.080254 +tq -12.083792 +xu -12.084636 +nv -12.086663 +wl -12.097520 +fs -12.115340 +sl -12.116892 +dl -12.132510 +co -12.134780 +wt -12.144423 +gd -12.146535 +sf -12.170159 +lf -12.232557 +sm -12.244386 +iq -12.252326 +nx -12.255553 +bj -12.258215 +sq -12.265276 +rz -12.266041 +nb -12.313908 +lk -12.318664 +gl -12.334632 +wd -12.360963 +uc -12.402225 +lq -12.423413 +lg -12.435846 +mk -12.439943 +qr -12.456664 +qb -12.462793 +bt -12.472256 +v_ -12.476901 +vu -12.482227 +hj -12.494507 +tx -12.525558 +tj -12.533830 +sb -12.567163 +jb -12.612921 +mx -12.617553 +qd -12.686832 +sc -12.705397 +ld -12.731850 +vv -12.770968 +mn -12.820702 +ms -12.826617 +cl -12.853398 +rh -12.863208 +ou -12.864657 +ai -12.913886 +gj -12.924420 +zv -12.959583 +xh -12.962992 +kj -12.968276 +jm -12.975769 +fh -12.976708 +bs -12.981414 +zl -12.985820 +lw -13.038463 +gb -13.076557 +zn -13.100589 +sw -13.114995 +ls -13.182027 +jd -13.212316 +lv -13.213423 +jl -13.219711 +lx -13.227516 +rf -13.242502 +pt -13.244009 +ks -13.265276 +dx -13.268339 +oe -13.289582 +bg -13.301301 +rp -13.309168 +jf -13.322242 +hz -13.371632 +gn -13.381965 +ql -13.393208 +eo -13.400333 +vj -13.405805 +sr -13.424481 +dt -13.472256 +wh -13.513466 +rx -13.527163 +nw -13.535905 +ow -13.536367 +au -13.593338 +oa -13.629815 +xj -13.639699 +md -13.648654 +wo -13.662191 +ry -13.665722 +ph -13.739271 +ct -13.776410 +ck -13.829160 +cu -13.830291 +qf -13.833124 +rq -13.853110 +jw -13.871047 +ee -13.876880 +hw -13.897486 +zb -13.902238 +qw -13.912387 +qh -13.914186 +uv -13.922609 +ay -13.973580 +km -14.007735 +ws -14.059207 +ue -14.060535 +iw -14.071197 +oo -14.079245 +mq -14.083286 +ts -14.095478 +hq -14.109147 +mt -14.113961 +mw -14.146712 +fx -14.153069 +ds -14.158743 +zw -14.165867 +wu -14.173745 +gz -14.227889 +wz -14.230875 +jz -14.258786 +jq -14.302871 +cr -14.312326 +ei -14.323436 +ae -14.356473 +lp -14.357289 +hk -14.386119 +ui -14.399493 +ua -14.423200 +rw -14.423200 +xr -14.462135 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mon b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mon new file mode 100644 index 00000000000..088fd8984e1 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mon @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 8425755 +#UNIQUE_BIGRAMS 1222 +#UNSEEN_LOG_PROB -15.579622 +#EXPECTED_SCORE -7.939047 +н_ -4.718126 +_б -5.542962 +_х -5.723313 +ии -5.852338 +ин -5.972210 +_т -6.256793 +г_ -6.272022 +ар -6.280822 +ан -6.292491 +д_ -6.303296 +аи -6.414969 +аа -6.544927 +ол -6.558968 +р_ -6.589148 +ба -6.625964 +ээ -6.652694 +и_ -6.658269 +он -6.668004 +га -6.689404 +эр -6.838114 +_а -6.858057 +да -6.901653 +эн -6.911802 +аг -6.952809 +ал -6.969750 +_о -7.051247 +л_ -7.065189 +_д -7.066818 +уу -7.079036 +_г -7.094187 +эг -7.096060 +са -7.195753 +ил -7.234344 +_н -7.250574 +х_ -7.260464 +ла -7.285703 +ын -7.289635 +ха -7.292478 +_с -7.300661 +гэ -7.329922 +үү -7.358045 +ж_ -7.370968 +дэ -7.391435 +бо -7.402256 +иг -7.402487 +а_ -7.404169 +ул -7.416255 +та -7.434385 +эл -7.455087 +_з -7.471828 +_э -7.537401 +нэ -7.546719 +хэ -7.567615 +ор -7.593871 +нд -7.594102 +_м -7.605628 +_у -7.616978 +э_ -7.623919 +ло -7.680647 +тэ -7.700633 +т_ -7.723793 +то -7.732579 +_ү -7.780339 +с_ -7.781205 +на -7.793954 +ны -7.809504 +ам -7.843708 +ги -7.844416 +ы_ -7.891819 +лэ -7.904645 +ни -7.913576 +ра -7.960700 +өр -7.964673 +оо -7.999785 +ри -8.003120 +хо -8.019955 +хи -8.035864 +эд -8.074483 +өө -8.096294 +үн -8.101787 +сэ -8.103481 +хү -8.115208 +рэ -8.129954 +ур -8.138324 +_ө -8.143640 +ог -8.149560 +рг -8.151506 +ах -8.171755 +лд -8.173583 +нг -8.204767 +ад -8.208207 +го -8.247319 +лс -8.258077 +жи -8.259388 +өн -8.287360 +ли -8.339651 +эх -8.362349 +лт -8.367430 +ас -8.436222 +эс -8.437172 +өг -8.439785 +_ш -8.457372 +ир -8.505844 +ид -8.511332 +за -8.533874 +ши -8.545023 +уд -8.548865 +лг -8.570249 +до -8.571225 +ма -8.598244 +үр -8.605562 +үи -8.609703 +_и -8.621591 +гд -8.680086 +чи -8.682686 +ис -8.711323 +ро -8.726256 +со -8.788493 +тө -8.794106 +ту -8.799437 +ав -8.812772 +_ц -8.834572 +лж -8.850845 +ч_ -8.855913 +рт -8.873232 +бү -8.900630 +гө -8.904317 +аж -8.906534 +гт -8.917835 +ху -8.924142 +ер -8.934074 +гл -8.939024 +ох -8.944497 +ои -8.946354 +эм -8.955421 +м_ -8.958995 +о_ -8.961639 +үл -8.963091 +гү -8.969600 +мэ -8.992005 +гу -8.994274 +но -9.006727 +ва -9.014321 +өл -9.020177 +од -9.041222 +жэ -9.054361 +им -9.101176 +ыг -9.103434 +лы -9.117536 +рд -9.122395 +су -9.139385 +эж -9.155211 +тү -9.161963 +ом -9.165597 +өд -9.166974 +ры -9.194498 +у_ -9.194498 +рх -9.246174 +лб -9.251278 +би -9.253785 +ду -9.258287 +бу -9.271771 +ун -9.277285 +мж -9.277285 +_я -9.279518 +эи -9.303661 +ди -9.317141 +ус -9.319874 +ца -9.338819 +нх -9.344485 +рл -9.344485 +их -9.364323 +хө -9.374288 +ат -9.375539 +мо -9.414969 +бө -9.457793 +рс -9.464915 +_ж -9.467337 +ст -9.477188 +ру -9.504164 +цэ -9.553490 +гч -9.553619 +ос -9.556839 +_ч -9.569143 +в_ -9.583259 +зэ -9.598974 +тг -9.602430 +дл -9.609904 +ь_ -9.630113 +от -9.630928 +дө -9.641146 +сг -9.644568 +өв -9.650060 +ну -9.655988 +дс -9.685997 +рч -9.686702 +лл -9.687267 +лө -9.706880 +аш -9.770809 +уг -9.781619 +нө -9.808390 +эв -9.809311 +_к -9.811002 +иж -9.832697 +мө -9.847765 +үх -9.871948 +сы -9.876931 +ит -9.878058 +ша -9.887433 +ды -9.894098 +те -9.897034 +рө -9.927056 +ут -9.931899 +зү -9.931899 +ө_ -9.959933 +мг -9.993750 +ув -10.021956 +өм -10.023916 +үс -10.053815 +ми -10.066979 +аз -10.088443 +зо -10.099672 +ти -10.121632 +үе -10.126791 +вч -10.176256 +эт -10.186595 +ов -10.217045 +ух -10.221535 +нт -10.235298 +_е -10.237363 +ү_ -10.266383 +цо -10.283994 +вэ -10.299663 +үд -10.303120 +сн -10.308755 +ш_ -10.312452 +си -10.329095 +мт -10.329316 +ты -10.330858 +_п -10.339485 +дү -10.352186 +ое -10.354874 +ед -10.372471 +ио -10.377246 +ив -10.396042 +сө -10.429890 +ик -10.437943 +үг -10.454186 +өх -10.478165 +зу -10.531908 +үз -10.535970 +ль -10.536224 +ьд -10.538769 +рн -10.542850 +лх -10.548994 +лц -10.556710 +уи -10.558000 +мы -10.563431 +хн -10.570444 +ме -10.588258 +ча -10.590369 +мн -10.593805 +вс -10.602564 +бэ -10.608433 +ая -10.613789 +лн -10.622130 +ко -10.628622 +ви -10.632694 +рв -10.640326 +үт -10.656264 +сү -10.675178 +ки -10.678541 +ож -10.682194 +юу -10.701168 +иш -10.714916 +яв -10.721550 +ес -10.732288 +рь -10.748103 +вь -10.754300 +мд -10.759337 +ич -10.803638 +лу -10.803945 +тр -10.833010 +дг -10.840839 +лч -10.859488 +сл -10.870665 +_р -10.910318 +ц_ -10.917255 +дн -10.923228 +ем -10.925890 +хя -10.966427 +рш -10.967456 +к_ -10.980543 +нц -10.984354 +нж -11.019466 +чл -11.023737 +мь -11.027307 +_л -11.036628 +вд -11.037348 +жл -11.037348 +иа -11.043117 +гс -11.045649 +ен -11.047822 +үч -11.048547 +дв -11.053633 +чу -11.080079 +зө -11.081191 +_ф -11.083419 +мп -11.096481 +ял -11.100611 +дд -11.114211 +рж -11.119916 +сс -11.119916 +лү -11.131777 +гж -11.136395 +пр -11.138710 +др -11.152285 +оц -11.169535 +шу -11.170324 +ян -11.174276 +тн -11.178238 +вл -11.204666 +еи -11.245655 +ую -11.260280 +гы -11.262382 +по -11.281434 +тл -11.287414 +уч -11.288698 +му -11.303769 +үм -11.308973 +эз -11.322064 +ср -11.341483 +өс -11.365678 +рү -11.366130 +ет -11.375652 +вт -11.378385 +рм -11.398120 +ел -11.398582 +_в -11.405997 +хл -11.438894 +ум -11.443178 +ие -11.454186 +хь -11.461893 +шэ -11.466731 +эц -11.474019 +хд -11.514521 +ле -11.542339 +жу -11.567583 +из -11.579062 +гш -11.586941 +яа -11.641693 +уж -11.651574 +гр -11.664300 +па -11.673778 +шү -11.679383 +иу -11.692925 +вр -11.694059 +мб -11.695194 +вө -11.699743 +цу -11.724445 +ац -11.725604 +ка -11.753118 +нү -11.762606 +св -11.764391 +жү -11.766179 +иц -11.779360 +вш -11.801192 +ят -11.839584 +чэ -11.878380 +ек -11.891331 +ге -11.903743 +гн -11.913617 +ев -11.917586 +зи -11.919575 +жа -11.928224 +цө -11.935583 +ву -11.936254 +вх -11.947030 +не -11.952449 +зр -11.954486 +нь -11.988870 +мү -12.049998 +ьс -12.082304 +жт -12.103999 +сд -12.114591 +хы -12.122968 +e_ -12.134469 +ци -12.143737 +ям -12.153845 +ех -12.156969 +з_ -12.219288 +тт -12.229119 +ач -12.242332 +кт -12.251487 +хт -12.266594 +оп -12.268282 +лм -12.273359 +өт -12.298153 +ре -12.299879 +яг -12.304202 +хг -12.305068 +се -12.313759 +чд -12.349056 +мц -12.350844 +фи -12.360716 +я_ -12.368844 +уц -12.395350 +ью -12.412983 +ош -12.419535 +нс -12.423292 +чн -12.428002 +це -12.441272 +ак -12.453705 +фр -12.491660 +во -12.540808 +тс -12.551047 +е_ -12.560325 +цс -12.561360 +нз -12.595923 +яр -12.595923 +яп -12.602297 +ьж -12.606563 +ьц -12.623751 +еэ -12.628080 +цл -12.679945 +гц -12.686702 +дж -12.700313 +ок -12.719817 +ют -12.730250 +ве -12.732579 +лз -12.748987 +шө -12.804251 +_ю -12.821499 +гв -12.845243 +пе -12.845243 +ьт -12.849028 +вг -12.864267 +өж -12.874518 +нч -12.884841 +үж -12.890031 +кс -12.891331 +бр -12.924225 +вц -12.928224 +де -12.929559 +_s -12.938940 +_t -12.947030 +t_ -12.959251 +мс -12.971576 +s_ -12.978469 +шн -12.984007 +сп -13.000750 +пь -13.016271 +чө -13.019111 +рк -13.042034 +зл -13.066795 +үв -13.078597 +зг -13.081562 +зд -13.084534 +мя -13.086022 +пи -13.091989 +чт -13.097982 +оч -13.120678 +ео -13.146840 +er -13.153065 +шт -13.157752 +цы -13.157752 +аю -13.165597 +ск -13.187792 +уя -13.199020 +яз -13.211959 +аф -13.216841 +нк -13.228297 +б_ -13.233235 +рб -13.241503 +мш -13.244823 +жм -13.246486 +in -13.248151 +нб -13.254830 +нш -13.263223 +n_ -13.281861 +бе -13.288698 +_m -13.297291 +d_ -13.314631 +мч -13.344596 +сч -13.355323 +өц -13.375197 +тм -13.389826 +_a -13.389826 +дм -13.421412 +пы -13.423292 +тд -13.423292 +ii -13.423292 +чр -13.434622 +l_ -13.442225 +дт -13.451786 +гм -13.453705 +_c -13.465278 +нн -13.467216 +фа -13.476944 +шо -13.480854 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mri b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mri new file mode 100644 index 00000000000..23ac8cd7067 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mri @@ -0,0 +1,412 @@ +#TOTAL_BIGRAMS 7189758 +#UNIQUE_BIGRAMS 408 +#UNSEEN_LOG_PROB -19.455584 +#EXPECTED_SCORE -6.315113 +a_ -3.720832 +i_ -4.706421 +_t -5.037454 +ng -5.250317 +ta -5.346765 +u_ -5.397440 +_k -5.437418 +ga -5.464283 +an -5.492697 +ha -5.513142 +ka -5.622555 +at -5.723745 +_a -5.747276 +ra -5.767792 +_m -5.830914 +_w -5.969264 +e_ -5.974617 +ma -6.002686 +_n -6.008035 +ak -6.027066 +na -6.031772 +ah -6.047267 +_h -6.114621 +_r -6.153559 +to -6.187700 +o_ -6.196988 +wh -6.240460 +hi -6.276063 +ar -6.301050 +au -6.358309 +ki -6.538809 +re -6.556170 +ai -6.589908 +ua -6.626019 +tu -6.712853 +ko -6.725984 +ho -6.734100 +ei -6.789559 +ri -6.791270 +ia -6.810826 +ti -6.825430 +_p -6.916595 +ro -6.961603 +te -6.969075 +on -7.037151 +wa -7.038760 +or -7.065710 +ne -7.193430 +in -7.209437 +no -7.258889 +ou -7.261289 +er -7.272972 +pa -7.307012 +he -7.310543 +en -7.387007 +ok -7.459181 +ea -7.532813 +ku -7.545691 +ir -7.552116 +oa -7.594303 +it -7.617404 +_i -7.640280 +ke -7.662712 +nu -7.801664 +ih -7.811727 +hu -7.817193 +pu -7.828916 +oh -7.877872 +ur -7.895394 +am -7.988131 +me -7.999229 +ot -8.006332 +ap -8.006900 +ru -8.015285 +ae -8.021737 +aw -8.107464 +ut -8.152260 +un -8.186282 +et -8.275923 +ik -8.340735 +ao -8.533743 +ui -8.571490 +mo -8.611898 +_o -8.662468 +po -8.794162 +_e -8.830606 +gi -8.987061 +iw -8.993286 +pe -9.010155 +ek -9.093092 +uk -9.103099 +eh -9.133656 +aa -9.136024 +up -9.145653 +oe -9.156032 +wi -9.243574 +mu -9.328492 +go -9.435020 +pi -9.435298 +ow -9.561221 +ni -9.638440 +mi -9.701698 +uh -9.720535 +oi -9.732947 +_u -9.906954 +om -9.915649 +op -9.918365 +we -9.990630 +im -10.091887 +io -10.132754 +ip -10.132754 +em -10.235931 +eo -10.366266 +um -10.540302 +ep -10.695696 +ew -10.866869 +d_ -10.940278 +th -11.046193 +nd -11.242236 +ue -11.320644 +y_ -11.491532 +ie -11.495582 +t_ -11.696694 +n_ -11.732435 +ii -11.754451 +s_ -11.756532 +_c -11.804532 +al -11.861632 +uw -11.879666 +la -11.894105 +r_ -11.949375 +iu -12.000256 +oo -12.160963 +_b -12.202918 +_f -12.270708 +ge -12.323212 +l_ -12.387343 +_s -12.484040 +w_ -12.520124 +ee -12.635405 +ll -12.661168 +tr -12.674224 +is -12.695363 +gu -12.696694 +h_ -12.783158 +ic -12.834997 +st -12.848253 +_d -12.881179 +le -12.910233 +_z -12.916425 +ze -12.916425 +ry -12.922643 +ch -13.036045 +el -13.066705 +ct -13.080544 +m_ -13.115734 +es -13.131853 +li -13.137267 +ag -13.160963 +_l -13.194429 +co -13.228690 +fo -13.234480 +il -13.291682 +nt -13.320131 +_j -13.419960 +uo -13.428784 +di -13.494423 +ed -13.496741 +ve -13.503716 +so -13.508385 +ce -13.531959 +_g -13.568058 +eu -13.595117 +fr -13.698027 +ol -13.703370 +se -13.708734 +de -13.814616 +rt -13.820410 +ns -13.840874 +bo -13.846774 +gr -13.928889 +k_ -13.992877 +ay -14.073608 +bi -14.101555 +og -14.148155 +as -14.148155 +hy -14.170181 +ba -14.222923 +ci -14.253950 +ph -14.257876 +ck -14.273686 +c_ -14.293696 +ac -14.293696 +be -14.297732 +sh -14.305837 +lt -14.334568 +vi -14.347059 +rs -14.355447 +jo -14.389494 +us -14.393807 +lo -14.402472 +ul -14.446595 +ex -14.482891 +nc -14.506049 +si -14.510725 +_v -14.534338 +ec -14.568058 +iv -14.568058 +tt -14.592636 +ov -14.597603 +sl -14.612605 +uc -14.632854 +ca -14.632854 +da -14.637960 +ty -14.663770 +_y -14.695363 +cu -14.744089 +xt -14.760703 +rd -14.823315 +kl -14.882694 +sa -14.907147 +ss -14.907147 +ey -14.925763 +gh -14.932022 +hn -14.983096 +av -14.989609 +mm -15.015961 +yo -15.029319 +ld -15.029319 +ig -15.091011 +id -15.098032 +lu -15.126460 +pt -15.155460 +oc -15.155460 +ad -15.155460 +bu -15.200083 +rn -15.246130 +rc -15.253950 +cl -15.269717 +do -15.309906 +fi -15.326301 +rg -15.368121 +ts -15.368121 +br -15.376632 +ls -15.393807 +pl -15.393807 +wo -15.411190 +g_ -15.419960 +va -15.428784 +uu -15.437662 +os -15.437662 +gs -15.501387 +gt -15.510725 +wn -15.539107 +ja -15.548693 +je -15.690049 +pd -15.690049 +ms -15.722229 +tc -15.788827 +sc -15.858649 +fa -15.858649 +ds -15.870621 +rl -15.882694 +f_ -15.907147 +mp -15.944622 +ly -15.957333 +ob -15.983096 +bl -16.049591 +ff -16.063266 +nl -16.091011 +eg -16.133656 +gl -16.133656 +sp -16.133656 +ks -16.133656 +df -16.162802 +ib -16.177599 +ny -16.192549 +cr -16.207656 +ab -16.238353 +ws -16.385194 +vo -16.385194 +du -16.473731 +rk -16.473731 +ya -16.492110 +rr -16.510725 +tw -16.548693 +pr -16.548693 +of -16.548693 +ug -16.548693 +rw -16.568058 +hr -16.568058 +oy -16.587687 +tm -16.607587 +yc -16.648229 +mc -16.668987 +od -16.690049 +ub -16.690049 +b_ -16.690049 +cy -16.711423 +ht -16.711423 +rb -16.711423 +sb -16.755144 +nr -16.777512 +ef -16.800232 +ev -16.823315 +iz -16.944622 +sm -17.022624 +gy -17.022624 +af -17.049591 +lk -17.105086 +rv -17.105086 +yn -17.133656 +su -17.222923 +ju -17.253950 +yt -17.285659 +lb -17.351247 +nn -17.385194 +mb -17.419960 +ys -17.455584 +ye -17.492110 +kn -17.492110 +tl -17.529584 +sy -17.529584 +fe -17.529584 +p_ -17.568058 +if -17.568058 +fl -17.648229 +zy -17.648229 +ft -17.690049 +by -17.733118 +za -17.733118 +dy -17.777512 +sr -17.823315 +xa -17.823315 +mr -17.823315 +yp -17.870621 +dr -17.870621 +ה_ -17.870621 +dw -17.919531 +lf -17.919531 +wc -17.970157 +sd -18.022624 +nk -18.077072 +wu -18.077072 +ps -18.133656 +my -18.192549 +dd -18.192549 +fu -18.192549 +rp -18.253950 +nb -18.318080 +gb -18.318080 +dh -18.385194 +hb -18.385194 +nh -18.385194 +yd -18.455584 +dn -18.455584 +tb -18.529584 +lw -18.529584 +wy -18.529584 +ud -18.529584 +py -18.607587 +eb -18.690049 +_ס -18.777512 +סל -18.777512 +לה -18.777512 +bn -18.870621 +yi -18.870621 +yl -18.870621 +bb -18.870621 +xe -18.970157 +_י -18.970157 +יה -18.970157 +הו -18.970157 +וה -18.970157 +nm -18.970157 +cs -18.970157 +td -18.970157 +pb -19.077072 +tv -19.077072 +vn -19.077072 +nz -19.077072 +z_ -19.077072 +dm -19.077072 +vy -19.192549 +cm -19.192549 +nw -19.192549 +lg -19.192549 +bs -19.192549 +lm -19.192549 +ym -19.192549 +_q -19.192549 +qu -19.192549 +ox -19.192549 +x_ -19.192549 +sf -19.318080 +wr -19.318080 +az -19.318080 +nj -19.318080 +ij -19.318080 +ji -19.318080 +ln -19.455584 +xc -19.455584 +tz -19.455584 +zr -19.455584 +tf -19.455584 +wl -19.455584 +dg -19.455584 +pm -19.455584 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/msa b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/msa new file mode 100644 index 00000000000..82336b08600 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/msa @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 57402439 +#UNIQUE_BIGRAMS 662 +#UNSEEN_LOG_PROB -19.109887 +#EXPECTED_SCORE -7.159094 +an -4.379266 +n_ -4.938777 +a_ -5.158675 +ng -5.544099 +er -5.715478 +da -5.832693 +_m -5.887019 +ka -5.955128 +i_ -5.961408 +_d -5.994569 +la -6.028874 +_p -6.054379 +en -6.068012 +_s -6.118212 +me -6.288937 +ra -6.332098 +_b -6.351306 +_k -6.364083 +ta -6.418516 +ar -6.434070 +ah -6.510787 +ya -6.537028 +ga -6.619422 +se -6.620757 +at -6.621814 +_t -6.634790 +h_ -6.662574 +ma -6.672869 +pe -6.673402 +g_ -6.675587 +ak -6.675970 +pa -6.678856 +in -6.782771 +al -6.787267 +ba -6.816690 +am -6.970610 +ke -6.990795 +be -7.001283 +_a -7.006011 +di -7.019063 +sa -7.025872 +em -7.027527 +k_ -7.040961 +un -7.041587 +na -7.107456 +te -7.133319 +el -7.141880 +as -7.144484 +ad -7.148593 +u_ -7.163818 +tu -7.179078 +ri -7.212899 +t_ -7.288355 +ha -7.316130 +ia -7.380661 +ti -7.392069 +_i -7.566250 +ai -7.603232 +ny -7.656945 +it -7.688628 +r_ -7.689131 +ik -7.700138 +nt -7.711214 +si -7.719692 +li -7.733837 +ja -7.748150 +ua -7.769840 +uk -7.779885 +_y -7.853796 +m_ -7.906431 +_l -7.963968 +s_ -7.970490 +le -7.975239 +eb -7.987146 +ni -7.994011 +ap -8.016548 +ru -8.117067 +wa -8.128113 +nd -8.151442 +bu -8.200496 +ek -8.252748 +mb -8.254880 +il -8.259302 +ag -8.322858 +_j -8.342532 +is -8.350680 +pu -8.385935 +_u -8.387660 +l_ -8.400834 +mp -8.420687 +de -8.440702 +et -8.448267 +ol -8.483429 +ur -8.502353 +ut -8.513790 +ki -8.535766 +mu -8.536671 +lu -8.564955 +_h -8.598611 +gi -8.599088 +_r -8.611549 +es -8.638248 +ep -8.640443 +ay -8.649104 +ul -8.650487 +us -8.650699 +ir -8.653106 +du -8.676875 +bi -8.680222 +su -8.695791 +um -8.703251 +id -8.730631 +ku -8.738414 +au -8.753791 +ju -8.784073 +rt -8.805560 +_n -8.816441 +on -8.818823 +or -8.847224 +aw -8.863287 +re -8.864762 +mi -8.914019 +aa -8.968072 +ih -8.981043 +ne -8.997970 +ko -9.000259 +gg -9.008572 +gu -9.008921 +ca -9.035039 +pi -9.040945 +_o -9.081271 +rk -9.100016 +im -9.114487 +hu -9.130259 +hi -9.146049 +eh -9.155592 +ab -9.171142 +ge -9.196767 +aj -9.242070 +nj -9.275059 +d_ -9.288355 +gk -9.294655 +ud -9.299699 +_c -9.330904 +eg -9.392221 +st -9.432656 +_g -9.513579 +rs -9.536055 +ed -9.546429 +ip -9.583588 +uh -9.601986 +rl -9.697355 +ib -9.697835 +p_ -9.715011 +up -9.738392 +nu -9.742305 +ro -9.770125 +nc -9.792997 +_w -9.870845 +_e -9.918889 +po -9.932430 +rb -9.983955 +e_ -10.145622 +bo -10.152271 +ug -10.179818 +ej -10.217325 +lo -10.245453 +_f -10.255126 +rm -10.314473 +to -10.366080 +om -10.380783 +ji -10.392322 +ig -10.396653 +je -10.421048 +no -10.428961 +ci -10.433576 +ub -10.443587 +os -10.470186 +sk -10.474506 +o_ -10.479844 +pr -10.496631 +kh -10.499394 +ok -10.511780 +rj -10.529279 +rn -10.564813 +ye -10.570840 +gs -10.592486 +sy -10.602806 +rd -10.605232 +ec -10.612808 +tr -10.630747 +kt -10.744761 +rg -10.765436 +mo -10.771706 +ks -10.783663 +fi -10.831499 +nn -10.856444 +ot -10.870397 +oh -10.883490 +ce -10.907475 +ns -10.922616 +go -10.935405 +tk -10.960477 +b_ -10.982683 +he -11.001830 +yu -11.001985 +rh -11.003222 +gh -11.024949 +hk -11.033879 +so -11.049349 +od -11.060310 +ac -11.079380 +uj -11.094194 +ea -11.120980 +eo -11.129007 +cu -11.135494 +ew -11.150556 +ch -11.235571 +fa -11.235814 +rp -11.258601 +za -11.262609 +ys -11.269790 +af -11.301601 +og -11.315113 +ui -11.333286 +y_ -11.367275 +ho -11.385851 +op -11.394486 +io -11.419671 +f_ -11.422772 +if -11.431423 +nk -11.441523 +do -11.451061 +vi -11.503292 +lk -11.523126 +sh -11.554684 +co -11.581006 +ve -11.587334 +th -11.604762 +ic -11.626370 +kk -11.730300 +sn -11.732864 +kn -11.745667 +ij -11.781582 +ll -11.805762 +sl -11.806662 +gr -11.830720 +iv -11.862281 +wi -11.863779 +rc -11.886627 +br -11.914686 +kr -11.919059 +_v -12.002191 +sp -12.011604 +kl -12.019721 +az -12.031037 +mn -12.031983 +ie -12.055327 +_z -12.090518 +ky -12.106278 +jo -12.106943 +sm -12.130076 +hl -12.154045 +zi -12.188590 +yi -12.193997 +lb -12.231456 +ml -12.261375 +iu -12.330759 +ei -12.336857 +ob -12.339199 +ae -12.343242 +tn -12.357943 +uc -12.429758 +pt -12.461159 +c_ -12.476260 +hn -12.483150 +fe -12.499357 +yo -12.515160 +fo -12.602181 +q_ -12.624862 +ee -12.635857 +aq -12.636177 +bk -12.694457 +sr -12.743942 +pl -12.776018 +ss -12.782733 +iz -12.790012 +pk -12.812978 +lt -12.820412 +ov -12.858356 +hd -12.888912 +dr -12.929903 +ry -12.938361 +mk -12.938756 +va -12.940335 +np -12.950644 +gn -12.982225 +ou -12.984668 +dm -12.994479 +iw -13.002913 +ez -13.009115 +gl -13.042017 +bd -13.064233 +hm -13.102625 +bl -13.154618 +oo -13.161971 +w_ -13.184490 +jl -13.212366 +ev -13.273767 +dd -13.275759 +eu -13.356493 +ii -13.364687 +tl -13.365218 +z_ -13.393877 +oa -13.396043 +fr -13.398755 +of -13.411295 +we -13.423117 +oj -13.475973 +lm -13.485454 +ey -13.486319 +uz -13.492389 +ld -13.554835 +sw -13.563329 +ck -13.574323 +dn -13.585402 +kw -13.596878 +ln -13.610016 +wu -13.614737 +ow -13.617262 +mm -13.628358 +ef -13.695791 +j_ -13.721701 +rw -13.772145 +sc -13.875252 +av -13.901934 +iy -13.914298 +km -13.923641 +hw -13.927160 +tt -13.933044 +ue -13.938164 +ps -13.957225 +hr -13.972092 +fu -13.974922 +sj -13.982225 +ts -13.983853 +vo -14.001057 +nf -14.016802 +rr -14.046263 +ff -14.065956 +db -14.067250 +wo -14.103510 +ty -14.112831 +oc -14.140251 +bh -14.201908 +hb -14.220020 +oi -14.224824 +ft -14.258909 +ms -14.260388 +fl -14.275759 +bb -14.282255 +ls -14.296345 +gt -14.313129 +_q -14.344156 +ct -14.380146 +dk -14.381218 +ze -14.393607 +gj -14.435315 +ht -14.436429 +hy -14.452117 +oy -14.477692 +tm -14.481712 +pp -14.486319 +ex -14.494998 +uf -14.526088 +sd -14.534414 +qu -14.538595 +lf -14.576164 +pb -14.585402 +ph -14.600932 +nl -14.605310 +rf -14.617262 +zu -14.688472 +cr -14.695124 +x_ -14.728849 +ly -14.739122 +hs -14.809546 +my -14.882066 +bj -14.892730 +cl -14.898859 +ds -14.900396 +bn -14.915851 +nw -14.936981 +bs -14.969671 +sb -14.975327 +pd -14.998176 +xi -15.035672 +nr -15.065525 +pn -15.181218 +yr -15.182152 +zo -15.257924 +rv -15.272772 +dh -15.281754 +nm -15.285764 +v_ -15.295839 +mr -15.301918 +fg -15.314153 +ao -15.373729 +lg -15.389825 +tc -15.419258 +nv -15.431423 +zh -15.447056 +yd -15.449303 +bt -15.475401 +oe -15.514865 +sf -15.517221 +hp -15.577392 +dy -15.631225 +by -15.660867 +qa -15.681852 +mc -15.691129 +pm -15.712563 +jk -15.753629 +yt -15.780255 +lh -15.793041 +wn -15.827702 +_x -15.864716 +wt -15.915074 +gd -15.962431 +cy -16.028094 +nz -16.029775 +ws -16.036516 +dw -16.081122 +cc -16.111051 +yl -16.116397 +tp -16.136173 +nb -16.170982 +tw -16.184022 +ix -16.206653 +fs -16.216188 +jr -16.218103 +kp -16.251047 +lv -16.256939 +kd -16.260881 +nh -16.268797 +ox -16.286769 +lp -16.292809 +fk -16.339980 +gm -16.342067 +lc -16.388746 +dl -16.397398 +ax -16.423670 +zm -16.436987 +xa -16.441453 +tz -16.448179 +kb -16.459459 +uy -16.470828 +dv -16.529056 +uw -16.538595 +dg -16.562720 +iq -16.592214 +wh -16.640182 +zl -16.647904 +uo -16.663473 +bc -16.668700 +mf -16.738435 +oz -16.752241 +tf -16.811713 +bm -16.817507 +pw -16.832094 +dz -16.835029 +mt -16.840918 +yy -16.843871 +cs -16.944886 +jp -16.957625 +yh -16.967254 +tv -17.088108 +yn -17.119973 +zn -17.163584 +ym -17.163584 +qi -17.182152 +md -17.189646 +dp -17.231577 +xc -17.251047 +tb -17.251047 +vd -17.262856 +pg -17.262856 +rz -17.311084 +xx -17.335817 +gy -17.352544 +bp -17.373729 +hj -17.390904 +uv -17.390904 +yb -17.408286 +mz -17.412665 +yp -17.417057 +fc -17.425881 +cd -17.425881 +lj -17.434759 +jm -17.448179 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mya b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mya new file mode 100644 index 00000000000..c9d40c59b37 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mya @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 20069076 +#UNIQUE_BIGRAMS 1887 +#UNSEEN_LOG_PROB -14.740942 +#EXPECTED_SCORE -8.642807 +က_ -5.565328 +_အ -5.756859 +e_ -5.801651 +မ_ -6.157049 +_မ -6.491036 +_က -6.514925 +_s -6.639605 +_c -6.641994 +s_ -6.693972 +on -6.766649 +t_ -6.804027 +ပ_ -6.823745 +er -6.825040 +ရ_ -6.918760 +n_ -6.922522 +in -6.952667 +_a -7.019311 +r_ -7.061293 +an -7.073731 +re -7.091435 +d_ -7.132724 +_m -7.145339 +ar -7.222898 +_p -7.234283 +te -7.270097 +တ_ -7.283899 +at -7.316472 +_သ -7.323363 +ti -7.328681 +_ပ -7.375673 +co -7.428265 +_တ -7.460999 +_t -7.518519 +y_ -7.600259 +or -7.615587 +al -7.624766 +st -7.632733 +_ရ -7.644581 +သ_ -7.662470 +လ_ -7.679037 +ri -7.693458 +l_ -7.694679 +io -7.717374 +ma -7.720221 +en -7.722178 +_f -7.736364 +_d -7.799039 +_ဘ -7.805731 +le -7.819224 +ng -7.855525 +es -7.890323 +_b -7.892661 +a_ -7.892746 +ne -7.962756 +ro -7.996578 +nt -8.013661 +li -8.021634 +ra -8.021821 +_ဆ -8.022307 +စ_ -8.041081 +ic -8.060697 +de -8.080759 +me -8.083467 +_လ -8.084989 +_r -8.103890 +g_ -8.118440 +it -8.128262 +_i -8.149354 +se -8.160048 +_n -8.165095 +ea -8.166478 +ခ_ -8.168565 +nd -8.172604 +la -8.175553 +ve -8.222254 +_ဒ -8.241380 +_l -8.242838 +အခ -8.270807 +_g -8.289872 +th -8.296999 +_w -8.310446 +ac -8.313233 +_h -8.326948 +အမ -8.349750 +he -8.359799 +lo -8.369086 +ce -8.370822 +om -8.380875 +န_ -8.381067 +ta -8.386133 +ca -8.388419 +ဘစ -8.403067 +မက -8.405137 +ဘဘ -8.417939 +တက -8.441812 +_စ -8.446895 +si -8.449205 +un -8.451493 +to -8.459822 +ch -8.469116 +သမ -8.479983 +op -8.485821 +_e -8.531082 +el -8.560282 +k_ -8.569510 +o_ -8.577754 +ha -8.581412 +ဒ_ -8.618961 +ed -8.624823 +ig -8.637134 +na -8.651776 +is -8.652991 +p_ -8.671572 +_န -8.675388 +ho -8.675447 +tr -8.676211 +ge -8.684439 +am -8.695066 +ns -8.697183 +as -8.733531 +_o -8.740863 +mo -8.752008 +ss -8.762616 +ad -8.764115 +m_ -8.768717 +_u -8.807872 +et -8.813036 +ap -8.821857 +oo -8.832960 +di -8.840454 +ou -8.840618 +sh -8.882617 +ni -8.885878 +pr -8.891602 +ဆ_ -8.892695 +mi -8.917265 +ur -8.926539 +il -8.930287 +pl -8.931831 +ut -8.936648 +ol -8.938622 +gh -8.940564 +c_ -8.944525 +ot -8.962594 +အစ -8.973322 +pa -8.981547 +id -8.981765 +po -8.990806 +h_ -8.992124 +ထ_ -9.029427 +ယ_ -9.036317 +ht -9.037261 +so -9.039983 +မသ -9.040665 +ia -9.070848 +rt -9.090034 +vi -9.092936 +ll -9.099703 +ec -9.103415 +us -9.105513 +စရ -9.137180 +_ထ -9.137990 +pe -9.157480 +i_ -9.171547 +ag -9.179736 +_v -9.184329 +cr -9.200564 +ct -9.207984 +hi -9.226813 +fo -9.245148 +ဆပ -9.245759 +fi -9.257679 +စမ -9.269653 +em -9.271518 +ke -9.275433 +ow -9.282533 +ရက -9.297835 +bu -9.298876 +အရ -9.307869 +ya -9.309282 +နရ -9.320959 +iv -9.321051 +da -9.321603 +ck -9.332777 +tt -9.347032 +pp -9.347641 +_k -9.355625 +ဟ_ -9.357746 +os -9.366735 +my -9.382434 +ရမ -9.382865 +py -9.383201 +sa -9.390420 +nm -9.418234 +nc -9.421829 +ဝဒ -9.423358 +အက -9.432071 +ay -9.432121 +အဆ -9.436498 +od -9.456965 +ie -9.458430 +yr -9.465986 +မရ -9.473836 +မဝ -9.476701 +no -9.481113 +_ခ -9.483890 +do -9.485023 +bo -9.509701 +ba -9.512849 +ဘ_ -9.515688 +အ_ -9.542723 +rs -9.567491 +ai -9.571806 +we -9.577837 +ee -9.581082 +ရသ -9.581908 +_j -9.606915 +sp -9.621110 +bl -9.632363 +wi -9.633504 +mm -9.642611 +tu -9.644106 +အသ -9.655830 +ရရ -9.660302 +ry -9.662048 +og -9.668469 +ထမ -9.673919 +wa -9.690515 +be -9.693547 +ga -9.698855 +ip -9.705802 +im -9.708746 +ရပ -9.709830 +nl -9.711094 +eb -9.713869 +ph -9.760868 +သက -9.775096 +go -9.780460 +dr -9.788448 +oc -9.791821 +မမ -9.804364 +ir -9.805715 +ts -9.808871 +အလ -9.816240 +အဒ -9.820589 +အထ -9.823647 +rd -9.826125 +သပ -9.828803 +ပပ -9.838050 +su -9.860663 +ပမ -9.863271 +fa -9.866422 +ki -9.882685 +_ယ -9.903395 +ld -9.910711 +ab -9.937036 +ty -9.963419 +စပ -9.967084 +w_ -9.973502 +x_ -9.995275 +gr -10.002188 +ကလ -10.006693 +oa -10.007950 +ev -10.014627 +ဆရ -10.020663 +_ဥ -10.023354 +u_ -10.031908 +_y -10.057955 +စက -10.064561 +ci -10.072357 +တရ -10.072976 +mp -10.073131 +သတ -10.073208 +ကန -10.083701 +ex -10.094743 +မတ -10.103099 +pt -10.106028 +ib -10.112061 +ဥပ -10.115008 +rn -10.121640 +အန -10.126855 +bi -10.135319 +လမ -10.138800 +ok -10.166705 +oi -10.172003 +လက -10.172335 +ဒသ -10.178819 +မလ -10.195497 +ul -10.197184 +စသ -10.206837 +up -10.208707 +fe -10.208707 +of -10.218352 +ပတ -10.220495 +ep -10.220495 +cl -10.222126 +ပက -10.224704 +_ဦ -10.235670 +lu -10.236103 +ew -10.241663 +ob -10.252145 +gi -10.257855 +b_ -10.268544 +gl -10.276636 +ua -10.279135 +sc -10.282981 +ik -10.284146 +rm -10.303003 +rc -10.321741 +ပရ -10.324780 +_ဧ -10.347828 +ကရ -10.348953 +wo -10.354024 +rk -10.355531 +ws -10.362233 +yo -10.368112 +ပထ -10.375350 +လတ -10.379654 +ဆန -10.385219 +သဘ -10.388202 +ဆတ -10.413765 +lt -10.425779 +au -10.434606 +ခလ -10.436697 +နပ -10.438192 +ပဒ -10.439290 +ms -10.464766 +fr -10.481524 +မဆ -10.489770 +ကမ -10.539510 +ဒမ -10.550896 +pi -10.554027 +ရယ -10.556623 +br -10.561503 +ru -10.571205 +မပ -10.589807 +bs -10.594135 +_ဗ -10.613826 +ဒလ -10.619356 +သသ -10.621506 +va -10.632077 +ည_ -10.633561 +ကပ -10.637220 +um -10.644912 +ရစ -10.648138 +ခအ -10.679862 +သရ -10.699811 +pu -10.703283 +တခ -10.707724 +ov -10.707965 +သခ -10.725628 +ရတ -10.741909 +aw -10.756385 +cu -10.757131 +ys -10.758251 +တတ -10.760370 +av -10.769250 +ကက -10.777176 +ခရ -10.781851 +nf -10.782737 +du -10.785526 +rl -10.792904 +နက -10.805072 +f_ -10.815527 +eo -10.816046 +ဝ_ -10.819679 +gu -10.820199 +tw -10.827757 +nu -10.829326 +ub -10.837195 +tc -10.857725 +အတ -10.858792 +ly -10.862669 +ဒတ -10.863606 +သလ -10.866690 +တမ -10.868705 +နမ -10.868974 +eg -10.873013 +ထက -10.876658 +wn -10.877198 +ft -10.886150 +ခခ -10.886422 +ds -10.890374 +သစ -10.890647 +မစ -10.920988 +if -10.932744 +ဒက -10.943604 +လခ -10.943746 +မအ -10.946723 +ယခ -10.949279 +nn -10.957118 +ခမ -10.958977 +rv -10.963707 +ud -10.971624 +ဖ_ -10.984967 +အပ -10.991977 +ey -10.994762 +ui -10.995789 +_ည -10.996523 +ps -10.996817 +_ဂ -11.029652 +တပ -11.034771 +ja -11.046278 +mb -11.051457 +uc -11.063406 +စတ -11.064484 +ue -11.067103 +ပလ -11.075456 +_z -11.081207 +sl -11.084794 +ff -11.103020 +ပည -11.104127 +ကစ -11.108565 +hu -11.120359 +mu -11.126292 +rg -11.134350 +ဓက -11.134996 +_ဟ -11.135481 +ls -11.138233 +ကသ -11.140179 +sm -11.142289 +cc -11.144078 +တယ -11.147661 +အဓ -11.151744 +ပသ -11.158794 +_င -11.167201 +ဘရ -11.169351 +sy -11.178153 +qu -11.199972 +_ဖ -11.207772 +ei -11.209473 +ax -11.221610 +ဧပ -11.222297 +ပဘ -11.224360 +cs -11.229701 +ဂ_ -11.231429 +iz -11.242359 +_x -11.248292 +သဆ -11.248992 +pd -11.268722 +ဌန -11.283877 +_ဝ -11.291425 +ဒယ -11.308097 +ရဘ -11.321097 +jo -11.321833 +nk -11.325702 +ဏ_ -11.352896 +လပ -11.374491 +ju -11.385988 +rr -11.395059 +je -11.395834 +နတ -11.411805 +ef -11.414157 +xy -11.416121 +dd -11.448904 +ak -11.451116 +ခက -11.465884 +sk -11.493392 +လသ -11.494221 +ပစ -11.500664 +fl -11.519534 +ခပ -11.525032 +ka -11.526515 +တဟ -11.526728 +ဦက -11.553703 +ကအ -11.575915 +vo -11.577672 +ဘသ -11.602717 +ဆသ -11.607195 +ox -11.614840 +ny -11.625249 +ဘတ -11.630937 +hy -11.638251 +ဆက -11.644681 +င_ -11.654845 +ဘဝ -11.673039 +ဧရ -11.681042 +zi -11.685297 +မန -11.686718 +လရ -11.692893 +ဘမ -11.698138 +yp -11.702683 +ဒဟ -11.705802 +ရထ -11.705802 +ks -11.713989 +fu -11.716648 +ဦစ -11.720039 +kn -11.727333 +ye -11.729528 +gy -11.760120 +ze -11.761617 +gn -11.771385 +za -11.774907 +rp -11.775663 +dm -11.781978 +ကတ -11.785526 +ရန -11.790611 +gs -11.810613 +hr -11.817861 +ဘလ -11.819939 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mya-zaw b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mya-zaw new file mode 100644 index 00000000000..f90ec7d621c --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/mya-zaw @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 22320851 +#UNIQUE_BIGRAMS 2002 +#UNSEEN_LOG_PROB -14.424209 +#EXPECTED_SCORE -8.737291 +က_ -5.593906 +_အ -5.745762 +တ_ -5.844549 +မ_ -6.090687 +e_ -6.181183 +_မ -6.300030 +_က -6.355881 +_သ -6.400869 +_တ -6.477340 +လ_ -6.562074 +ပ_ -6.634955 +ရ_ -6.676518 +န_ -6.749846 +_s -6.893528 +_ပ -7.017225 +s_ -7.034729 +r_ -7.057751 +er -7.072108 +n_ -7.105106 +in -7.121447 +_m -7.134510 +_a -7.172812 +ar -7.209746 +t_ -7.211316 +_လ -7.240624 +an -7.285567 +on -7.296764 +d_ -7.306348 +_ရ -7.311432 +သ_ -7.317089 +_ဒ -7.355187 +_c -7.359183 +re -7.367078 +_p -7.446905 +y_ -7.481671 +ma -7.548731 +te -7.583083 +ဖ_ -7.642748 +_ဆ -7.710244 +_t -7.724404 +ti -7.745082 +_f -7.784084 +or -7.798632 +l_ -7.833988 +_r -7.844824 +at -7.849378 +g_ -7.867181 +en -7.867226 +ne -7.884870 +a_ -7.922684 +ng -7.940563 +me -7.943758 +_d -7.973665 +st -7.981926 +al -7.993806 +le -8.045430 +_n -8.054302 +_b -8.109660 +သတ -8.118004 +ro -8.126812 +io -8.203731 +nt -8.204512 +_h -8.207203 +ho -8.215191 +k_ -8.302956 +es -8.311637 +ta -8.362508 +ra -8.377133 +ခ_ -8.391929 +_l -8.395951 +ic -8.426692 +ၿပ -8.426936 +it -8.432173 +_i -8.434475 +ed -8.453721 +ၿမ -8.454764 +_န -8.464115 +ဆ_ -8.464891 +nd -8.471392 +om -8.478498 +_ဘ -8.504482 +ce -8.507136 +co -8.508266 +to -8.510574 +ri -8.511494 +_စ -8.511871 +o_ -8.526002 +li -8.533982 +de -8.543307 +un -8.567085 +se -8.569956 +th -8.587353 +ၾက -8.593831 +အဖ -8.602095 +_ၿ -8.630939 +_v -8.632015 +la -8.632272 +သမ -8.639059 +vi -8.676412 +ac -8.676624 +_w -8.685830 +အခ -8.686735 +တက -8.689401 +_g -8.715084 +ve -8.715410 +ou -8.726072 +_ထ -8.732079 +lo -8.754179 +_e -8.775576 +he -8.779888 +na -8.788293 +m_ -8.789551 +el -8.792586 +il -8.794192 +ဘ_ -8.797725 +ch -8.809306 +ထ_ -8.824551 +ew -8.836261 +oo -8.874549 +et -8.880172 +စ_ -8.886184 +_ဖ -8.907815 +ot -8.912323 +su -8.916971 +အ_ -8.927349 +ge -8.951049 +ad -8.958489 +si -8.961612 +ea -8.961741 +di -8.966066 +ag -9.000245 +ca -9.028556 +ha -9.034271 +နရ -9.038140 +is -9.041031 +ry -9.046728 +ur -9.063610 +da -9.064717 +p_ -9.079992 +ow -9.082618 +_ခ -9.090137 +ie -9.099183 +um -9.101169 +mm -9.107217 +ap -9.124104 +_o -9.125511 +ya -9.132750 +as -9.135837 +ws -9.148913 +pa -9.156934 +ai -9.162812 +ss -9.171172 +po -9.171470 +mo -9.179955 +am -9.196849 +us -9.229068 +im -9.229921 +မရ -9.233029 +ဟ_ -9.235248 +_k -9.245765 +တရ -9.257941 +id -9.261626 +ia -9.261903 +hi -9.275458 +အရ -9.281479 +mi -9.283492 +we -9.288818 +bo -9.289870 +ႀက -9.319090 +my -9.334823 +_ယ -9.347525 +rt -9.352798 +ll -9.355378 +ol -9.357624 +_u -9.371770 +ay -9.379326 +ကလ -9.382127 +ကမ -9.388221 +ကန -9.397084 +pr -9.398871 +em -9.403985 +အန -9.407229 +fa -9.419614 +အလ -9.426269 +nm -9.440255 +i_ -9.444347 +ct -9.444572 +ec -9.448586 +c_ -9.450529 +ရသ -9.476493 +မတ -9.479305 +do -9.482122 +_ဗ -9.487123 +so -9.500169 +sh -9.503402 +ut -9.503590 +ထမ -9.510503 +rs -9.513004 +fo -9.519298 +sa -9.522813 +ယ_ -9.555317 +မမ -9.556534 +_ဥ -9.569342 +အဒ -9.578603 +tr -9.579246 +စရ -9.585688 +_j -9.588870 +fi -9.590065 +ph -9.598909 +os -9.600363 +ရမ -9.601016 +w_ -9.604534 +အစ -9.605591 +pl -9.610787 +be -9.629658 +h_ -9.630836 +ပပ -9.630888 +_ဦ -9.631605 +eb -9.637719 +ev -9.650286 +ဒန -9.650909 +လက -9.652105 +ni -9.668264 +ir -9.676121 +ee -9.689935 +ဆတ -9.706202 +ဒက -9.720309 +pe -9.722710 +ns -9.737752 +nl -9.740348 +ဆရ -9.762015 +အထ -9.766061 +မလ -9.767638 +wa -9.771587 +လတ -9.781622 +ဒ_ -9.782020 +ck -9.782817 +အတ -9.792815 +ရက -9.803923 +no -9.813778 +op -9.831218 +မသ -9.832926 +go -9.842983 +au -9.860481 +wi -9.872063 +ပတ -9.885023 +nc -9.889001 +_y -9.903537 +ld -9.907504 +အပ -9.913476 +aw -9.916159 +oc -9.917908 +rd -9.934510 +pp -9.937549 +မက -9.942565 +နန -9.954764 +_ဟ -9.955663 +ရပ -9.962805 +dr -9.980914 +အသ -9.984903 +ba -10.012210 +og -10.017292 +တန -10.017828 +_ဧ -10.022930 +ပၚ -10.024948 +_ၾ -10.025689 +သပ -10.042769 +iv -10.054199 +မန -10.064960 +oi -10.084687 +ke -10.088764 +ig -10.089890 +ty -10.097235 +bl -10.109393 +သလ -10.117483 +ရရ -10.121870 +yo -10.127354 +ဆပ -10.128294 +ga -10.130031 +သဘ -10.132061 +tu -10.132641 +ရတ -10.137001 +သက -10.140499 +mg -10.144809 +ဒလ -10.153323 +tt -10.159371 +va -10.161516 +bu -10.169459 +u_ -10.174007 +စက -10.176922 +ယန -10.193174 +ok -10.199240 +မပ -10.212753 +ပမ -10.214442 +တပ -10.216132 +gr -10.217363 +oa -10.217517 +အဆ -10.223377 +_င -10.224150 +ဒသ -10.228874 +ဘလ -10.237665 +မၼ -10.252175 +br -10.253673 +ru -10.256438 +cl -10.256754 +rm -10.258178 +od -10.269143 +သရ -10.287848 +rn -10.289627 +up -10.298391 +ကတ -10.298879 +ly -10.299775 +ကစ -10.309421 +ci -10.309421 +cr -10.309995 +အမ -10.325752 +ပထ -10.326831 +sp -10.331071 +သစ -10.344370 +ab -10.352037 +ဗဒ -10.357454 +ts -10.373055 +wo -10.381222 +ပလ -10.388134 +ip -10.407142 +fr -10.408107 +fe -10.412241 +ki -10.416210 +ခလ -10.434609 +yi -10.440884 +mp -10.457511 +ရလ -10.471392 +တတ -10.475987 +ul -10.484203 +_ည -10.495169 +လပ -10.498345 +ep -10.508007 +ov -10.509043 +ဥပ -10.511116 +ua -10.517639 +ei -10.523907 +vo -10.523907 +wn -10.539887 +ကက -10.540657 +of -10.542488 +သသ -10.546542 +ကရ -10.548283 +ဥက -10.549154 +av -10.555171 +b_ -10.567968 +ပရ -10.578010 +ထက -10.599411 +စၥ -10.599611 +tw -10.606346 +ex -10.608767 +rc -10.608868 +ဒမ -10.609978 +သန -10.666109 +ဒတ -10.671370 +ကပ -10.682905 +ၚ_ -10.683224 +_ႀ -10.684606 +စပ -10.687162 +x_ -10.701836 +f_ -10.721127 +ၼတ -10.724951 +မဆ -10.730760 +lu -10.738138 +nu -10.760052 +bi -10.763419 +ပဘ -10.778893 +ဟသ -10.788579 +sc -10.791898 +အက -10.805599 +gi -10.815466 +တခ -10.833870 +rk -10.834460 +ရစ -10.838005 +မၻ -10.839781 +gh -10.842033 +lt -10.852870 +nn -10.855742 +ht -10.859100 +ဒၶ -10.864994 +ရန -10.868978 +ပက -10.870792 +ကသ -10.879046 +ပအ -10.897298 +ob -10.897668 +mb -10.906077 +ပသ -10.907566 +ပဒ -10.919159 +ဦက -10.919535 +စတ -10.928955 +လသ -10.932361 +ဂ_ -10.939452 +ပၿ -10.970241 +လန -10.983790 +ik -10.986017 +တမ -10.988641 +ja -10.991797 +du -10.994695 +gl -11.000643 +ည_ -11.000907 +ဒၚ -11.002365 +ယခ -11.005019 +hu -11.005950 +cu -11.008743 +ခရ -11.009543 +eg -11.019571 +eo -11.023871 +ဘသ -11.024006 +သခ -11.034000 +ဧရ -11.035628 +gt -11.041610 +တလ -11.055299 +ka -11.059431 +ဏ_ -11.063022 +ak -11.063852 +လမ -11.065375 +ခအ -11.070648 +င_ -11.072874 +nk -11.077336 +pi -11.105542 +ခမ -11.105827 +ရယ -11.106824 +တယ -11.109678 +ရအ -11.131118 +_z -11.135183 +ub -11.151557 +ဗဟ -11.156860 +pu -11.161886 +သဆ -11.164555 +ey -11.170054 +ft -11.178269 +ရၿ -11.204569 +za -11.215440 +ခၚ -11.232446 +pt -11.241338 +နတ -11.241964 +ff -11.243530 +cy -11.248711 +pd -11.251072 +rr -11.253910 +ox -11.278425 +ထသ -11.287283 +မၸ -11.294570 +ဒယ -11.296032 +ဆၿ -11.296520 +ms -11.317316 +လၿ -11.323100 +ၚလ -11.326748 +ပည -11.327246 +ကၡ -11.329074 +ၸဏ -11.336744 +ဘရ -11.338417 +bs -11.340762 +စန -11.341768 +if -11.343782 +ue -11.354743 +rl -11.355590 +ပစ -11.362380 +ရဖ -11.377262 +ရဘ -11.388481 +ပန -11.390735 +အဓ -11.390909 +ဖဖ -11.400487 +ဧၿ -11.404161 +gu -11.420544 +cc -11.423914 +ju -11.426936 +နက -11.437474 +ye -11.444843 +ခက -11.446826 +_ဂ -11.456420 +နပ -11.467360 +လရ -11.469008 +မစ -11.477829 +gy -11.482260 +ညန -11.520675 +ဓက -11.524097 +ဘမ -11.535947 +လလ -11.536907 +ဘတ -11.549445 +rv -11.561311 +ဒပ -11.566203 +ys -11.568753 +ၻ_ -11.577615 +dd -11.578208 +ခတ -11.588521 +ef -11.590513 +ရထ -11.605138 +ny -11.606145 +ဆက -11.607556 +ၚမ -11.613012 +v_ -11.627049 +pk -11.628686 +ze -11.635661 +rf -11.635867 +ဝ_ -11.644532 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nan b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nan new file mode 100644 index 00000000000..5517e1679cf --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nan @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 5150634 +#UNIQUE_BIGRAMS 660 +#UNSEEN_LOG_PROB -17.970339 +#EXPECTED_SCORE -6.828484 +_c -4.620734 +n_ -4.620747 +_k -4.705585 +ch -4.771088 +g_ -4.888088 +ng -4.900375 +u_ -5.074137 +_t -5.162243 +an -5.304719 +_s -5.724122 +hi -5.736449 +ia -5.832826 +ko -5.899430 +en -6.001734 +un -6.054056 +i_ -6.066129 +e_ -6.184959 +th -6.195677 +ha -6.199974 +a_ -6.201519 +au -6.202777 +ho -6.223998 +oa -6.330129 +on -6.373943 +_h -6.487983 +k_ -6.497644 +iu -6.523128 +ki -6.591631 +_p -6.708688 +ok -6.775761 +_l -6.816570 +he -6.879983 +hu -6.882261 +ku -6.913188 +la -6.977705 +in -6.994572 +se -7.011494 +hn -7.024090 +o_ -7.029422 +_b -7.164663 +ai -7.209602 +ti -7.282778 +si -7.307190 +io -7.314083 +ka -7.364981 +t_ -7.569781 +_m -7.681781 +ar -7.780619 +ta -7.785802 +sk -7.795539 +to -7.804967 +p_ -7.807537 +kh -7.821155 +hh -7.852598 +ie -7.898978 +ph -8.073675 +it -8.160850 +at -8.175838 +bu -8.250303 +or -8.250559 +ni -8.275339 +so -8.341395 +ci -8.385676 +li -8.390086 +_i -8.449849 +ma -8.492793 +y_ -8.513320 +ip -8.521840 +h_ -8.547182 +_o -8.565106 +s_ -8.582932 +oe -8.589715 +as -8.594146 +lo -8.594254 +am -8.647062 +ui -8.665824 +m_ -8.692229 +ow -8.718536 +ra -8.740771 +te -8.746654 +_g -8.761773 +al -8.763111 +_e -8.784319 +_a -8.800588 +ol -8.812629 +pa -8.862342 +ap -8.863516 +er -8.896774 +sa -8.909648 +na -8.977917 +ri -8.988122 +ic -9.001411 +_j -9.002847 +pi -9.039225 +ne -9.062699 +_n -9.094960 +po -9.134714 +mi -9.141342 +ty -9.158207 +ns -9.168969 +_w -9.215834 +va -9.231407 +is -9.239342 +om -9.281426 +ro -9.283869 +le -9.286315 +ck -9.386238 +be -9.394886 +bi -9.413294 +ak -9.422490 +ca -9.423836 +su -9.442229 +no -9.448283 +mu -9.453575 +ah -9.454361 +ji -9.471161 +il -9.518652 +wi -9.525655 +ba -9.529997 +_v -9.532899 +ur -9.533729 +ll -9.535183 +el -9.556960 +ek -9.564363 +es -9.570739 +ge -9.574365 +ec -9.588959 +ag -9.595879 +st -9.609380 +r_ -9.615080 +tu -9.616179 +co -9.625884 +go -9.629207 +pe -9.671295 +nd -9.715353 +rk -9.747737 +nt -9.771042 +bo -9.826677 +ls -9.839964 +vi -9.844592 +d_ -9.846912 +_d -9.865866 +rt -9.887723 +re -9.898377 +rs -9.902660 +ut -9.983152 +ke -9.990257 +da -9.991682 +wa -9.991967 +ou -9.997397 +mo -10.026023 +os -10.055527 +lu -10.073222 +od -10.074429 +ov -10.100946 +op -10.155489 +nn -10.171228 +ya -10.174461 +sl -10.194343 +de -10.203892 +ws -10.276728 +dp -10.304088 +cd -10.304797 +wn -10.307989 +oh -10.314395 +az -10.373734 +gi -10.375966 +_r -10.402638 +zo -10.438726 +eg -10.439504 +pu -10.464220 +me -10.468182 +ot -10.479735 +ad -10.507193 +ye -10.510457 +rg -10.517010 +tr -10.572658 +ss -10.581644 +w_ -10.636323 +ob -10.640341 +ew -10.650660 +ga -10.664687 +im -10.675641 +yl -10.691302 +iy -10.697336 +rm -10.713706 +do -10.718418 +ac -10.743169 +rn -10.779634 +gy -10.802463 +_f -10.802963 +ir -10.860648 +l_ -10.877412 +aw -10.889582 +av -10.899179 +br -10.904001 +za -10.913154 +mb -10.921279 +oz -10.924542 +gr -10.932731 +c_ -10.939867 +di -10.953688 +_z -11.023106 +sy -11.034224 +yo -11.034811 +ce -11.045428 +em -11.090525 +sz -11.128273 +us -11.141500 +ve -11.147842 +pr -11.153573 +dz -11.156767 +ex -11.166392 +zs -11.171551 +id -11.187141 +sh -11.198945 +ub -11.219503 +rz -11.230229 +ab -11.235623 +rd -11.249195 +ul -11.258085 +fo -11.285790 +eh -11.300551 +ze -11.314751 +ry -11.325494 +ig -11.326213 +lv -11.327652 +ay -11.335593 +lk -11.350875 +nc -11.357474 +oi -11.358945 +uc -11.367800 +ja -11.401501 +zy -11.427496 +ys -11.441450 +et -11.497847 +xa -11.511684 +sc -11.518241 +oc -11.524004 +tt -11.554009 +zi -11.571805 +_y -11.587235 +gh -11.619480 +nk -11.654267 +zk -11.670610 +cz -11.695476 +ug -11.701061 +wo -11.715118 +eb -11.759100 +dl -11.817549 +ln -11.822613 +if -11.823628 +ru -11.856488 +je -11.861690 +ny -11.879521 +ea -11.886928 +ee -11.921279 +ed -11.946485 +zu -11.987980 +ky -12.031876 +we -12.033049 +_u -12.044836 +ao -12.055527 +lg -12.055527 +kr -12.056720 +fl -12.062699 +kl -12.072317 +rr -12.182576 +og -12.185183 +yn -12.243751 +uj -12.247832 +gt -12.250559 +um -12.250559 +eo -12.311900 +ey -12.313325 +cl -12.380439 +wy -12.383429 +ik -12.386426 +ts -12.409098 +sn -12.427496 +pl -12.427496 +dn -12.430585 +jo -12.461848 +mp -12.471360 +ly -12.513320 +v_ -12.523179 +ue -12.526481 +ev -12.526481 +rc -12.566698 +by -12.641683 +du -12.707604 +sw -12.720779 +rl -12.720779 +ld -12.734076 +gu -12.735986 +ud -12.737898 +_อ -12.780619 +อา -12.780619 +าเ -12.780619 +เภ -12.780619 +ภอ -12.780619 +อ_ -12.780619 +lm -12.804465 +cu -12.812503 +fa -12.816538 +ez -12.855449 +oo -12.855449 +yd -12.867958 +ks -12.886928 +ms -12.912614 +aj -12.923454 +lt -12.945379 +bl -12.954244 +gl -12.965402 +pp -12.985706 +uz -12.994822 +cr -13.003997 +rb -13.029532 +dr -13.053145 +rp -13.069906 +fe -13.079573 +tn -13.113924 +iz -13.128900 +wu -13.154212 +fr -13.167036 +tl -13.167036 +ps -13.200922 +yc -13.208856 +sp -13.216834 +ih -13.230229 +x_ -13.241036 +ib -13.249195 +ua -13.262896 +dk -13.319039 +my -13.327652 +ei -13.365581 +ef -13.383429 +iv -13.383429 +af -13.398473 +up -13.404535 +oy -13.432132 +lc -13.438338 +ff -13.488964 +ep -13.498657 +fi -13.505156 +uk -13.595879 +zc -13.606321 +z_ -13.609818 +j_ -13.681609 +ej -13.700129 +ct -13.741730 +fy -13.788524 +qu -13.849235 +kn -13.853375 +zn -13.865866 +lb -13.904001 +ux -13.908301 +gs -13.925631 +nu -13.947590 +zz -13.960928 +oj -13.960928 +xi -13.965402 +tv -13.978906 +cs -14.010916 +mm -14.043653 +xe -14.053145 +tm -14.091747 +yu -14.121393 +wr -14.136447 +uh -14.187794 +dw -14.198287 +nr -14.214170 +市_ -14.224856 +vo -14.224856 +ii -14.262896 +ml -14.353804 +js -14.365581 +b_ -14.371506 +ox -14.377455 +ht -14.395452 +dd -14.419802 +nh -14.450829 +dg -14.450829 +wk -14.476140 +zt -14.501903 +ds -14.508416 +dy -14.528134 +ju -14.534767 +zl -14.548126 +zd -14.561609 +tc -14.595879 +tg -14.623893 +uo -14.638107 +hr -14.638107 +jn -14.674267 +hl -14.718890 +wl -14.741730 +rf -14.741730 +ae -14.749424 +rw -14.772757 +yt -14.796473 +vs -14.836887 +rh -14.853375 +oq -14.921279 +eu -14.938767 +wh -14.956469 +yk -14.956469 +mc -14.965402 +lp -14.992538 +gn -14.992538 +zp -15.010916 +ym -15.038931 +zb -15.057914 +uf -15.057914 +sr -15.057914 +lh -15.086865 +nl -15.086865 +gd -15.096646 +sv -15.096646 +_高 -15.126394 +tw -15.146571 +uy -15.146571 +sb -15.146571 +pk -15.219503 +vc -15.230229 +yr -15.307634 +pt -15.353804 +jd -15.389428 +bc -15.425954 +vl -15.463429 +nm -15.476140 +bs -15.488964 +wf -15.554852 +mr -15.638107 +sm -15.681609 +bn -15.681609 +中_ -15.696406 +學_ -15.726463 +mt -15.772757 +zg -15.772757 +dh -15.772757 +kw -15.772757 +lw -15.772757 +vu -15.804465 +tf -15.804465 +uw -15.820585 +ax -15.820585 +f_ -15.836887 +pn -15.904001 +_臺 -15.921279 +nw -15.921279 +md -15.938767 +bb -15.938767 +gw -15.974391 +yp -15.992538 +yz -15.992538 +of -16.010916 +yf -16.029532 +mn -16.067500 +ft -16.067500 +jh -16.086865 +hy -16.086865 +北_ -16.106494 +vr -16.126394 +vg -16.146571 +高級 -16.146571 +級_ -16.146571 +_中 -16.167036 +中學 -16.167036 +_市 -16.187794 +立_ -16.187794 +nb -16.208856 +ij -16.296319 +iw -16.296319 +lf -16.319039 +mz -16.342122 +_級 -16.365581 +級中 -16.365581 +臺北 -16.365581 +yi -16.365581 +df -16.365581 +_北 -16.413676 +北市 -16.413676 +wt -16.413676 +lj -16.438338 +yw -16.488964 +nv -16.488964 +fu -16.514959 +aq -16.541431 +aa -16.568398 +gm -16.568398 +pc -16.568398 +zw -16.568398 +mk -16.623893 +vn -16.623893 +zm -16.623893 +sd -16.623893 +mf -16.681609 +灣_ -16.681609 +cc -16.711356 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nds b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nds new file mode 100644 index 00000000000..6d193131a4d --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nds @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 6129486 +#UNIQUE_BIGRAMS 583 +#UNSEEN_LOG_PROB -17.656759 +#EXPECTED_SCORE -7.412294 +n_ -4.402089 +t_ -4.719446 +en -4.867861 +er -5.399464 +_d -5.606459 +r_ -5.713592 +_s -5.982039 +un -6.054245 +_v -6.178675 +ee -6.234806 +_w -6.269303 +ch -6.283974 +or -6.295224 +at -6.383941 +te -6.507602 +de -6.540570 +an -6.584303 +st -6.621039 +_h -6.672810 +e_ -6.699030 +he -6.764976 +_b -6.769437 +_m -6.807633 +da -6.827251 +ie -6.841648 +vu -6.907344 +nn -6.907711 +ne -6.981489 +re -7.045124 +et -7.058804 +_a -7.109160 +ge -7.136983 +sc -7.231831 +ar -7.257820 +it -7.267905 +s_ -7.280621 +ve -7.318854 +we -7.319042 +el -7.319155 +d_ -7.332181 +oo -7.333167 +_k -7.363661 +tt -7.382467 +_g -7.421316 +_f -7.451855 +_n -7.522758 +be -7.549273 +le -7.549273 +l_ -7.571755 +nd -7.605737 +rr -7.611709 +_e -7.643547 +si -7.644159 +h_ -7.653986 +me -7.664071 +rt -7.666316 +_t -7.669188 +in -7.692807 +_l -7.707491 +rn -7.753630 +do -7.764132 +ll -7.765719 +rs -7.773835 +wa -7.780961 +mi -7.860342 +g_ -7.888008 +k_ -7.902127 +on -7.919630 +_o -7.971027 +se -7.980864 +ri -8.015527 +hr -8.025673 +ma -8.050481 +la -8.053854 +li -8.075469 +al -8.081194 +ho -8.098573 +ke -8.129021 +aa -8.132848 +su -8.191227 +es -8.195981 +ra -8.206859 +to -8.208598 +eg -8.217398 +ic -8.219501 +ut -8.222240 +_r -8.224773 +is -8.226182 +ei -8.227239 +ro -8.247554 +na -8.275945 +ol -8.281280 +ha -8.308034 +_j -8.310797 +ns -8.325521 +us -8.329377 +ah -8.329907 +ni -8.341389 +m_ -8.358900 +ru -8.388567 +nt -8.405068 +ta -8.419340 +_u -8.423860 +em -8.425235 +oh -8.426368 +_p -8.439954 +ng -8.449220 +ed -8.503282 +uu -8.556965 +fo -8.576240 +ht -8.585342 +am -8.618631 +ti -8.627354 +hn -8.648828 +vo -8.658877 +ka -8.662115 +ts -8.684213 +lt -8.691006 +ur -8.716918 +pe -8.721382 +ig -8.746132 +ot -8.758005 +ss -8.761167 +eh -8.761371 +gr -8.763927 +_i -8.793387 +lo -8.846137 +ek -8.851998 +wi -8.858101 +kt -8.869065 +il -8.894938 +wo -8.947654 +ik -8.959323 +op -8.972268 +no -8.976767 +bi -9.013032 +wu -9.040407 +od -9.054355 +sp -9.057361 +os -9.063519 +ev -9.092907 +ak -9.136883 +um -9.153944 +ck -9.184980 +bo -9.186625 +di -9.244553 +du -9.248127 +eb -9.256596 +as -9.279670 +om -9.293783 +rd -9.310125 +br -9.318215 +mm -9.338643 +oc -9.381957 +ul -9.402039 +so -9.405068 +jo -9.405387 +pr -9.406665 +hu -9.448974 +ba -9.457884 +tr -9.458546 +lu -9.458878 +a_ -9.462028 +ud -9.474197 +p_ -9.484445 +fi -9.496126 +tu -9.497826 +ov -9.497997 +ko -9.498678 +mo -9.510818 +ag -9.514600 +ff -9.522714 +sa -9.537681 +kr -9.539782 +rg -9.608123 +fr -9.610513 +fa -9.631642 +rk -9.634258 +ac -9.649300 +af -9.673122 +dr -9.682955 +hi -9.710890 +ft -9.721382 +dd -9.724366 +ad -9.737366 +rb -9.753935 +ku -9.755561 +gt -9.759024 +ls -9.761678 +bl -9.787863 +ze -9.806289 +tw -9.818139 +mt -9.821117 +og -9.861491 +ga -9.874688 +nw -9.879558 +id -9.926886 +i_ -9.943940 +uc -9.951145 +av -10.013518 +bu -10.044503 +bb -10.053229 +th -10.054981 +_c -10.074644 +rm -10.098960 +pa -10.099218 +gi -10.128955 +o_ -10.144056 +au -10.145922 +rl -10.177465 +ju -10.204149 +gg -10.247268 +bt -10.251566 +pl -10.253576 +ap -10.256452 +po -10.265405 +mu -10.297332 +z_ -10.316714 +va -10.320621 +sl -10.331195 +hl -10.359055 +of -10.364009 +nk -10.388725 +nz -10.434895 +ok -10.478556 +mb -10.482255 +rh -10.537856 +go -10.571129 +if -10.605920 +io -10.608858 +dt -10.612538 +pp -10.618446 +ef -10.632950 +lm -10.634820 +f_ -10.662401 +hm -10.671201 +_z -10.678898 +je -10.681988 +ec -10.690909 +ob -10.697539 +v_ -10.702237 +fe -10.705377 +up -10.710495 +ld -10.725162 +ih -10.771313 +ki -10.824954 +gl -10.825381 +fu -10.892251 +ep -10.907994 +tz -10.915703 +nf -10.922539 +rw -10.960964 +fl -10.996588 +uk -11.027207 +ai -11.046493 +ds -11.075659 +ja -11.087391 +gu -11.095608 +ia -11.097670 +tl -11.100251 +ab -11.103873 +kl -11.131593 +sn -11.142193 +im -11.148591 +y_ -11.157166 +pt -11.181012 +ir -11.184843 +gs -11.295852 +iv -11.298221 +sw -11.305947 +ms -11.340320 +ue -11.351347 +nu -11.384943 +lv -11.409703 +rv -11.436199 +ks -11.440772 +rf -11.450620 +ua -11.503624 +zi -11.509102 +eu -11.513224 +u_ -11.515289 +ea -11.528439 +nh -11.539607 +ph -11.553689 +rp -11.562204 +lk -11.566481 +mp -11.632202 +lg -11.632950 +vi -11.638192 +eo -11.648733 +nl -11.680056 +ew -11.681601 +nb -11.689354 +pi -11.701845 +fs -11.708131 +oe -11.740785 +co -11.751295 +ug -11.769258 +ln -11.792447 +ca -11.837389 +iu -11.986047 +td -12.006238 +ou -12.047489 +dw -12.090980 +ps -12.100251 +ez -12.102320 +zt -12.111664 +vt -12.150730 +tm -12.167956 +ip -12.215298 +ex -12.216418 +nr -12.220905 +tb -12.263089 +rz -12.274705 +pu -12.297036 +dl -12.369915 +lf -12.391252 +tk -12.395050 +sm -12.398858 +hs -12.414192 +nc -12.436199 +vs -12.441426 +x_ -12.477214 +ow -12.497486 +pd -12.504307 +tg -12.543114 +ub -12.554396 +sd -12.557231 +tf -12.560071 +iz -12.567195 +lp -12.574355 +lb -12.585885 +sb -12.644960 +ty -12.647978 +ce -12.652517 +sh -12.663164 +ml -12.720786 +dm -12.722376 +w_ -12.736763 +uh -12.746435 +sy -12.784122 +ys -12.824527 +zo -12.905283 +sk -12.917978 +dk -13.019858 +ix -13.023773 +ci -13.098186 +rc -13.137944 +pb -13.144323 +ym -13.167956 +hw -13.234452 +cu -13.241273 +ly -13.245838 +ay -13.252714 +pn -13.259622 +hd -13.282892 +pf -13.282892 +uz -13.316113 +ae -13.333016 +ry -13.372409 +tn -13.410344 +ii -13.420630 +nm -13.444047 +tj -13.505676 +kn -13.513912 +aw -13.522195 +gn -13.524967 +fh -13.555813 +ey -13.587333 +sr -13.610697 +ux -13.628471 +db -13.631455 +ib -13.655551 +qu -13.670818 +kh -13.673891 +yo -13.686248 +tc -13.686248 +gd -13.686248 +xi -13.686248 +b_ -13.695586 +oa -13.711284 +oi -13.714445 +az -13.717612 +gk -13.762700 +_y -13.765975 +xt -13.765975 +sg -13.765975 +uf -13.785783 +ui -13.789111 +pg -13.812625 +lr -13.816016 +yp -13.826235 +cl -13.864340 +oz -13.881999 +c_ -13.947422 +sz -14.071601 +lw -14.100251 +hy -14.104391 +lh -14.108543 +_q -14.112706 +ye -14.125270 +xp -14.167956 +fb -14.181012 +hk -14.189783 +my -14.194188 +zz -14.207485 +ax -14.211944 +mf -14.234452 +ny -14.285240 +ox -14.304161 +df -14.328166 +yt -14.333016 +fg -14.337881 +bs -14.347662 +dh -14.377410 +mg -14.387463 +ya -14.402676 +ct -14.402676 +pw -14.444047 +gh -14.470519 +oy -14.502941 +nv -14.508416 +xa -14.524967 +dg -14.541710 +zl -14.547335 +bn -14.581550 +gy -14.581550 +xh -14.610697 +mw -14.646468 +wh -14.664692 +fk -14.670818 +mn -14.701845 +yn -14.727156 +tp -14.746435 +dn -14.759432 +cd -14.765975 +km -14.805868 +hc -14.812625 +yy -14.833089 +hf -14.839976 +wj -14.903478 +hb -14.910710 +fd -14.954878 +ky -14.962372 +lz -14.962372 +oj -15.023773 +md -15.047489 +fw -15.063519 +xe -15.063519 +pm -15.087903 +sf -15.104391 +zu -15.121070 +za -15.146455 +sv -15.207485 +dp -15.216418 +uv -15.216418 +kf -15.243554 +bj -15.280548 +yl -15.280548 +zb -15.289947 +gm -15.289947 +ao -15.289947 +mr -15.470519 +zy -15.604820 +dy -15.664692 +tv -15.676970 +zh -15.714445 +zw -15.752919 +pk -15.752919 +vn -15.765975 +np -15.765975 +by -15.765975 +gb -15.792447 +wn -15.805868 +lj -15.819414 +mc -15.833089 +cc -15.833089 +uo -15.874909 +fm -15.874909 +dv -15.889123 +vd -15.917978 +pz -15.977479 +yw -16.008176 +zd -16.008176 +cs -16.055482 +vw -16.071601 +bz -16.087903 +cr -16.087903 +mk -16.104391 +hp -16.104391 +yr -16.104391 +kg -16.121070 +hh -16.121070 +kw -16.172295 +wr -16.172295 +rj -16.243554 +gj -16.243554 +ji -16.261932 +yd -16.261932 +hg -16.397588 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nep b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nep new file mode 100644 index 00000000000..69fb557f7ab --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nep @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 11410184 +#UNIQUE_BIGRAMS 1391 +#UNSEEN_LOG_PROB -14.811460 +#EXPECTED_SCORE -7.907723 +क_ -4.281196 +न_ -4.845072 +र_ -5.300725 +ल_ -5.305642 +_स -5.358341 +म_ -5.704559 +_प -5.946060 +_भ -6.077129 +_ग -6.119506 +_क -6.166413 +_ब -6.206581 +_न -6.214897 +एक -6.237854 +गर -6.425499 +रक -6.457266 +_म -6.462731 +_द -6.543282 +त_ -6.561833 +_र -6.614926 +_ह -6.717627 +य_ -6.853963 +_अ -6.859458 +_ल -6.881442 +_ज -6.938100 +उन -7.010070 +कर -7.042873 +_आ -7.171280 +_उ -7.192281 +पन -7.209245 +ई_ -7.249466 +नक -7.270122 +_व -7.310697 +जन -7.313711 +सम -7.320849 +भए -7.354389 +_थ -7.372335 +ग_ -7.471759 +_त -7.530438 +पल -7.536177 +हर -7.564283 +नप -7.603385 +_य -7.615484 +लई -7.629336 +लक -7.631767 +ट_ -7.634453 +हन -7.647678 +सर -7.647804 +लग -7.715765 +रह -7.805157 +द_ -7.924551 +हल -7.930523 +सक -7.932529 +हक -7.945530 +स_ -7.946092 +भन -7.949401 +मल -8.007530 +ह_ -8.021327 +पर -8.051199 +ए_ -8.109126 +_च -8.110663 +_ख -8.121609 +रम -8.169113 +रल -8.211061 +तक -8.213686 +दन -8.233033 +कम -8.262472 +नल -8.266905 +नत -8.280680 +थ_ -8.309874 +मत -8.320627 +कल -8.333784 +_ए -8.390317 +व_ -8.415783 +नम -8.417901 +छ_ -8.427707 +_छ -8.445978 +रप -8.456510 +ड_ -8.461583 +सह -8.463947 +रज -8.464260 +थय -8.556979 +वद -8.557265 +लम -8.611720 +रण -8.624188 +धन -8.643576 +मन -8.655508 +मह -8.678740 +सव -8.686898 +ख_ -8.706096 +बर -8.725286 +ण_ -8.731077 +नस -8.748754 +सल -8.797316 +यस -8.810199 +कन -8.839613 +हम -8.863148 +वध -8.863383 +कक -8.873548 +कस -8.908118 +पछ -8.919338 +धर -8.946403 +थए -8.952529 +रत -8.959688 +ज_ -8.971636 +अन -8.976213 +बल -8.999710 +सन -9.004183 +दल -9.007693 +_श -9.019849 +लल -9.028605 +बन -9.029794 +दख -9.034030 +जर -9.046010 +बट -9.053851 +तथ -9.080026 +भर -9.092603 +_फ -9.098760 +बत -9.099870 +ब_ -9.109126 +_घ -9.115284 +वक -9.116406 +तल -9.143180 +वर -9.144038 +चर -9.174254 +खल -9.187684 +मर -9.195743 +मओ -9.225861 +ओव -9.273737 +बह -9.287736 +रर -9.290663 +गत -9.303149 +कह -9.316792 +सब -9.325526 +नभ -9.328775 +एर -9.330321 +दश -9.334396 +श_ -9.336929 +रए -9.347680 +नय -9.364417 +नह -9.380929 +_ध -9.411859 +फल -9.411859 +हद -9.436791 +नर -9.456644 +भय -9.481284 +नग -9.504791 +पह -9.508838 +पम -9.513913 +यम -9.525421 +_ठ -9.525887 +तम -9.531865 +सग -9.536740 +मक -9.538808 +प_ -9.547013 +कत -9.583314 +तय -9.593632 +रब -9.598035 +लख -9.607472 +वन -9.614790 +धक -9.619755 +जत -9.620053 +बढ -9.630438 +ध_ -9.631140 +_ट -9.650317 +मय -9.653776 +गक -9.656528 +छन -9.663074 +गल -9.666152 +टक -9.666769 +दए -9.691543 +तन -9.759508 +बक -9.772057 +भग -9.777262 +नए -9.778483 +तर -9.790862 +आए -9.798498 +लन -9.811505 +दर -9.819823 +यज -9.832333 +चन -9.834755 +हस -9.858856 +जक -9.885757 +तए -9.890549 +यत -9.890549 +वस -9.891750 +दव -9.905994 +यर -9.915975 +इन -9.924551 +यग -9.949463 +नव -9.977487 +कठ -9.979273 +टल -9.991320 +अध -9.991577 +च_ -10.007237 +मड -10.029398 +रन -10.047080 +हज -10.061735 +कट -10.064170 +सच -10.064711 +यन -10.074494 +डक -10.083657 +ढ_ -10.087092 +गए -10.089707 +वल -10.089707 +खर -10.099939 +रव -10.101744 +तह -10.105361 +गउ -10.111922 +दई -10.113882 +सभ -10.115424 +ठक -10.131645 +ष_ -10.139182 +इर -10.143609 +आय -10.150778 +हत -10.153224 +बज -10.164208 +रस -10.164498 +लए -10.171626 +नउ -10.176739 +ठम -10.193668 +यक -10.203475 +मज -10.204220 +घ_ -10.218159 +यह -10.219516 +चल -10.235889 +रख -10.237110 +उट -10.239553 +आफ -10.276714 +उद -10.289159 +मग -10.306187 +एउ -10.328125 +बस -10.335621 +वश -10.337092 +खन -10.339220 +वत -10.357517 +गम -10.359010 +पय -10.369008 +एप -10.370347 +सथ -10.371016 +अह -10.387012 +पट -10.391590 +जव -10.392440 +उह -10.416085 +शक -10.421798 +पच -10.422839 +कप -10.439247 +पस -10.449112 +तप -10.450526 +सय -10.473174 +एम -10.483636 +दक -10.506261 +खए -10.542763 +आध -10.547486 +उप -10.548054 +रय -10.552415 +चत -10.556217 +आउ -10.569798 +टन -10.587588 +घट -10.588366 +फर -10.594023 +अव -10.595783 +लत -10.596761 +सत -10.607571 +जल -10.614096 +ठउ -10.618264 +उ_ -10.625636 +बच -10.643121 +नद -10.647373 +ठल -10.650215 +वष -10.654081 +खक -10.656732 +षय -10.662869 +रध -10.675014 +इ_ -10.675634 +लड -10.681021 +जस -10.687262 +मद -10.687262 +भव -10.688305 +वट -10.693949 +_ड -10.709321 +बठ -10.730217 +मध -10.731722 +टम -10.734089 +गड -10.737754 +पत -10.742296 +पग -10.744897 +तव -10.761921 +तउ -10.762800 +सद -10.764999 +सध -10.768964 +इए -10.774491 +गन -10.780261 +लह -10.782709 +हय -10.783377 +लब -10.787394 +मस -10.790750 +इत -10.792991 +फ_ -10.793889 +भइ -10.802670 +नज -10.805835 +अभ -10.815145 +सस -10.816285 +घर -10.819480 +दय -10.821081 +अघ -10.823828 +शर -10.825433 +मट -10.845999 +वव -10.846232 +सफ -10.861206 +थल -10.863089 +भष -10.865446 +सघ -10.868989 +_इ -10.883486 +भम -10.887552 +उम -10.891870 +ठन -10.927873 +एन -10.931078 +मम -10.932066 +नब -10.939993 +पए -10.943973 +गव -10.945717 +लय -10.972906 +दब -10.993896 +नन -10.996994 +हइ -11.002171 +ङ_ -11.007888 +दह -11.013889 +डल -11.019390 +कय -11.023859 +अम -11.025175 +गठ -11.026757 +शल -11.030191 +_झ -11.040008 +दम -11.041606 +कद -11.050696 +टर -11.056340 +उस -11.084344 +अस -11.103969 +झ_ -11.111502 +यद -11.115424 +जग -11.121046 +जम -11.142608 +वह -11.151497 +लर -11.152360 +लस -11.156106 +ठ_ -11.157839 +रड -11.165660 +रच -11.178790 +अग -11.184958 +नइ -11.197078 +तब -11.205414 +अर -11.210499 +पउ -11.216504 +छर -11.225861 +रग -11.226467 +नट -11.243533 +कइ -11.247831 +बब -11.248447 +लफ -11.251834 +यल -11.253377 +शम -11.268893 +पठ -11.274833 +आर -11.282687 +रउ -11.291217 +पक -11.298523 +ढक -11.307148 +गय -11.325851 +छल -11.343813 +नछ -11.349411 +अप -11.354037 +भट -11.356356 +खत -11.359342 +पई -11.365001 +मब -11.368005 +चह -11.370681 +_ढ -11.377057 +चक -11.384136 +शह -11.391590 +थप -11.391930 +कश -11.392610 +सप -11.392950 +पड -11.393970 +हभ -11.397377 +शष -11.398400 +सझ -11.404900 +वज -11.422492 +चव -11.423881 +बध -11.444876 +पख -11.448758 +लप -11.449465 +जड -11.455134 +पइ -11.463679 +मच -11.467613 +आद -11.467613 +यव -11.470121 +षण -11.484179 +रद -11.484179 +दइ -11.491806 +कव -11.511235 +मथ -11.522722 +जद -11.531678 +सड -11.536552 +शख -11.551276 +आइ -11.551655 +धव -11.559648 +इल -11.560794 +भ_ -11.570375 +भक -11.571913 +पद -11.586615 +डम -11.589730 +एस -11.594023 +शन -11.597937 +भज -11.600683 +फट -11.614492 +लउ -11.626036 +हट -11.637269 +इस -11.655100 +हड -11.659184 +सज -11.665331 +घष -11.665742 +भल -11.673981 +बद -11.703616 +आम -11.716324 +थह -11.718026 +धम -11.719731 +_ओ -11.720158 +छड -11.774048 +वम -11.804930 +कए -11.809008 +षक -11.818566 +वभ -11.824516 +वच -11.834640 +जय -11.843441 +आग -11.866390 +जट -11.880146 +लज -11.888271 +ईक -11.895960 +रई -11.907086 +खज -11.910489 +कथ -11.923691 +गण -11.935529 +मव -11.937015 +खस -11.955979 +पज -11.961515 +लइ -12.008669 +कष -12.014412 +डर -12.018078 +णक -12.030720 +_औ -12.036020 +हब -12.039209 +शव -12.042406 +शस -12.045609 +बझ -12.047749 +जध -12.060114 +थव -12.069866 +लव -12.070954 +उठ -12.072587 +रष -12.073676 +अथ -12.079137 +बग -12.114583 +णम -12.127537 +एव -12.163048 +चम -12.174692 +टब -12.175277 +जह -12.176447 +दत -12.183487 +रङ -12.187610 +गह -12.191153 +इक -12.206011 +दद -12.221024 +घक -12.223440 +मघ -12.230107 +जठ -12.256467 +अल -12.268893 +रट -12.289001 +खम -12.292168 +टउ -12.299160 +आव -12.323581 +फइ -12.324878 +घइ -12.333988 +थर -12.333988 +लद -12.344471 +सए -12.345129 +डत -12.349741 +ढउ -12.351062 +भत -12.359675 +छट -12.361004 +डय -12.382448 +डन -12.382448 +गद -12.392610 +उछ -12.393290 +भड -12.401475 +सइ -12.402160 +पश -12.405586 +हव -12.420757 +उक -12.441707 +सट -12.448052 +फन -12.462251 +आश -12.463679 +कड -12.464394 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/new b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/new new file mode 100644 index 00000000000..5c1eb2468b5 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/new @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 1081556 +#UNIQUE_BIGRAMS 602 +#UNSEEN_LOG_PROB -16.278310 +#EXPECTED_SCORE -6.734327 +ग_ -3.453418 +_भ -4.765211 +_छ -4.858795 +छग -5.009233 +य_ -5.083680 +यग -5.117316 +_स -5.264496 +ल_ -5.474406 +भष -5.753794 +_ज -5.781408 +न_ -5.825055 +तय -5.829902 +_क -5.934765 +भर -6.073582 +रत -6.079524 +_न -6.093210 +_म -6.098045 +स_ -6.207049 +षय -6.245598 +_ग -6.302999 +प_ -6.446159 +सक -6.483866 +र_ -6.528608 +जन -6.696364 +_त -6.804035 +_य -6.872093 +त_ -6.909128 +कथ -6.936479 +कप -6.937460 +जग -6.962195 +नग -6.963859 +रक -6.968195 +_अ -6.994999 +कल -7.001308 +थ_ -7.165094 +गण -7.166051 +णन -7.179330 +मर -7.192733 +मल -7.209219 +नप -7.230495 +_ख -7.244181 +नज -7.263112 +क_ -7.281465 +अम -7.298583 +_ध -7.353151 +लग -7.467248 +_व -7.473161 +लय -7.496337 +मन -7.540603 +ष_ -7.635816 +इल -7.714881 +धस -7.797343 +म_ -7.807467 +_द -7.822184 +कय -7.856088 +रय -7.865079 +तह -7.884806 +_थ -7.901613 +खन -7.904486 +_प -7.946974 +गढ -7.950929 +हर -7.964192 +_ल -7.974221 +हस -8.028913 +मइ -8.084313 +पर -8.116529 +टर -8.125069 +जत -8.132162 +मट -8.140042 +दग -8.140419 +_इ -8.185142 +रण -8.198403 +_र -8.221310 +तम -8.253514 +छ_ -8.257591 +वल -8.330861 +सर -8.394972 +तग -8.410320 +ण_ -8.425833 +सम -8.430427 +जम -8.444299 +वक -8.459245 +मत -8.493930 +नत -8.503096 +कस -8.528484 +कम -8.529963 +पय -8.545329 +ट_ -8.563887 +उ_ -8.596044 +मउ -8.602252 +तल -8.621561 +रग -8.628407 +लम -8.648607 +थस -8.698163 +_श -8.736338 +_ब -8.771464 +ढव -8.784345 +यल -8.826114 +गल -8.862283 +जज -8.888594 +सह -8.899382 +कर -8.903847 +दय -8.932889 +इत -8.952581 +_आ -9.036949 +भय -9.041159 +पल -9.046792 +वर -9.091936 +सल -9.099965 +च_ -9.100697 +थल -9.134784 +ढ_ -9.139290 +सत -9.141548 +पद -9.169696 +तप -9.175083 +_ह -9.195272 +ज_ -9.274839 +गर -9.319311 +खत -9.350319 +शह -9.351190 +सट -9.378453 +पथ -9.423541 +डप -9.435498 +सच -9.439197 +द_ -9.442906 +वव -9.456899 +लत -9.458775 +आद -9.686026 +सड -9.737476 +नस -9.740896 +धर -9.782582 +रज -9.794379 +इ_ -9.805078 +यन -9.834006 +ह_ -9.856088 +ड_ -9.887330 +भग -9.978588 +णक -10.077451 +हम -10.080336 +ख_ -10.090481 +भम -10.099233 +सध -10.116899 +पभ -10.163563 +बह -10.211787 +नय -10.216540 +हक -10.226095 +ई_ -10.269890 +थग -10.291460 +लक -10.296484 +रम -10.400821 +_ट -10.415320 +धग -10.420795 +उन -10.422625 +रप -10.435498 +गज -10.437347 +मज -10.446624 +टउ -10.486256 +शस -10.495855 +व_ -10.505518 +थय -10.521115 +लस -10.532924 +लप -10.556837 +थन -10.575035 +सप -10.599662 +_फ -10.601733 +सग -10.616317 +_च -10.620511 +फट -10.626824 +अन -10.700381 +नई -10.750056 +लब -10.763906 +बग -10.775550 +थर -10.796749 +मस -10.813456 +नद -10.830358 +दल -10.862283 +लज -10.879770 +यत -10.884806 +भल -10.936152 +गव -10.941389 +दन -10.946645 +मग -10.965192 +धक -10.994828 +नक -11.055992 +_उ -11.081781 +थक -11.102162 +जल -11.105098 +जइ -11.143810 +रव -11.155934 +यय -11.214954 +जप -11.250261 +पह -11.276493 +पच -11.279805 +श_ -11.340773 +वन -11.358176 +कड -11.372252 +अल -11.471030 +चम -11.490088 +रन -11.536882 +षक -11.585245 +बम -11.597594 +दर -11.605885 +जय -11.614224 +_ड -11.665299 +धन -11.669638 +दक -11.678355 +गत -11.709287 +बद -11.731794 +ङ_ -11.731794 +नड -11.745469 +मथ -11.777890 +रद -11.787289 +वग -11.801503 +रस -11.830358 +उच -11.845005 +चइ -11.845005 +गम -11.854852 +यण -11.859802 +सद -11.874752 +कश -11.874752 +इर -11.889859 +हव -11.925736 +नर -11.930935 +चय -11.936152 +कट -11.936152 +धइ -11.946645 +दह -11.973215 +यइ -11.994828 +हड -12.000283 +रल -12.067397 +षत -12.078893 +सय -12.084675 +मच -12.113940 +जर -12.113940 +_घ -12.113940 +बल -12.174312 +लन -12.192928 +मय -12.199187 +जव -12.199187 +खर -12.205473 +नव -12.224498 +सन -12.230896 +आध -12.289789 +हल -12.309967 +सफ -12.323578 +डह -12.358176 +_झ -12.358176 +इग -12.429967 +यक -12.429967 +रड -12.437347 +_ए -12.452220 +वद -12.482435 +हट -12.497782 +मह -12.513295 +वह -12.536882 +कग -12.536882 +चन -12.544831 +बर -12.593466 +लर -12.601733 +फ_ -12.678355 +एर -12.713760 +तध -12.713760 +मम -12.722749 +कत -12.731794 +नह -12.740896 +खस -12.759275 +गक -12.759275 +दव -12.759275 +मद -12.759275 +घर -12.796749 +वश -12.796749 +चल -12.815858 +लभ -12.825508 +अर -12.845005 +ब_ -12.864768 +रघ -12.894930 +घड -12.894930 +गय -12.905126 +हन -12.905126 +आइ -12.925736 +इड -12.925736 +झक -12.936152 +हत -12.936152 +आस -12.967861 +रब -13.000283 +बय -13.011254 +डग -13.022309 +लल -13.033450 +रङ -13.044677 +_ऒ -13.102162 +ऒर -13.102162 +जध -13.102162 +छई -13.125814 +ईख -13.125814 +खद -13.125814 +तख -13.137786 +नल -13.149859 +आप -13.162034 +नम -13.174312 +भस -13.199187 +पन -13.199187 +थथ -13.211787 +बज -13.224498 +रह -13.237322 +दस -13.276493 +वभ -13.289789 +लव -13.303210 +टन -13.303210 +पख -13.330431 +धव -13.386465 +वय -13.400821 +दथ -13.429967 +लख -13.429967 +डल -13.474821 +वस -13.474821 +कन -13.490088 +भज -13.505518 +थइ -13.521115 +नष -13.536882 +तक -13.536882 +यर -13.552824 +वई -13.568944 +अध -13.618412 +अथ -13.722749 +जस -13.740896 +इम -13.740896 +डड -13.759275 +_ई -13.777890 +ईल -13.777890 +गग -13.777890 +बच -13.796749 +_ऐ -13.796749 +तथ -13.796749 +भत -13.835224 +घट -13.854852 +लट -13.894930 +मक -13.894930 +सघ -13.915394 +सण -13.936152 +भन -13.936152 +यप -13.936152 +भव -13.978588 +ऐत -14.000283 +सभ -14.022309 +फय -14.044677 +पण -14.044677 +तस -14.044677 +तन -14.044677 +तर -14.044677 +पक -14.067397 +दश -14.137786 +उद -14.162034 +तव -14.162034 +उड -14.186696 +डस -14.186696 +झप -14.186696 +खट -14.211787 +टङ -14.211787 +दङ -14.237322 +ए_ -14.237322 +रच -14.263317 +नख -14.263317 +अछ -14.289789 +छम -14.289789 +शष -14.316756 +r_ -14.344237 +फल -14.372252 +बभ -14.429967 +शय -14.429967 +यस -14.459714 +बत -14.490088 +तड -14.490088 +यद -14.490088 +रभ -14.490088 +पत -14.490088 +मख -14.521115 +_ഒ -14.521115 +ഒര -14.521115 +ര_ -14.521115 +लई -14.521115 +चध -14.521115 +धय -14.521115 +जङ -14.552824 +आय -14.552824 +चर -14.552824 +घ_ -14.585245 +वत -14.585245 +सव -14.618412 +आख -14.618412 +उर -14.618412 +लह -14.652360 +छत -14.652360 +बक -14.652360 +गप -14.687125 +यछ -14.687125 +चक -14.722749 +फत -14.722749 +कफ -14.759275 +आक -14.759275 +ar -14.759275 +डट -14.796749 +गद -14.796749 +शक -14.796749 +कव -14.835224 +यम -14.835224 +कष -14.835224 +नन -14.835224 +चथ -14.874752 +इफ -14.874752 +शर -14.874752 +_ऋ -14.915394 +ऋष -14.915394 +लए -14.915394 +अत -15.000283 +वड -15.000283 +गड -15.000283 +_ओ -15.044677 +ओख -15.044677 +खल -15.044677 +लढ -15.044677 +ढग -15.044677 +नब -15.044677 +बझ -15.044677 +झङ -15.044677 +मण -15.044677 +बख -15.044677 +हभ -15.044677 +गथ -15.044677 +धद -15.090481 +_a -15.090481 +an -15.090481 +nd -15.090481 +d_ -15.090481 +अप -15.090481 +_औ -15.090481 +उस -15.090481 +मछ -15.137786 +छप -15.137786 +नउ -15.137786 +उक -15.137786 +भ_ -15.137786 +णच -15.186696 +_t -15.186696 +th -15.186696 +he -15.186696 +e_ -15.186696 +पइ -15.186696 +कइ -15.186696 +घष -15.186696 +लद -15.186696 +चत -15.237322 +सउ -15.237322 +उथ -15.237322 +वच -15.237322 +_ढ -15.237322 +ढक -15.237322 +मप -15.237322 +अव -15.237322 +गभ -15.289789 +जक -15.289789 +सथ -15.289789 +उप -15.289789 +खम -15.289789 +रथ -15.344237 +थम -15.344237 +तत -15.344237 +घय -15.344237 +कई -15.344237 +वज -15.344237 +शख -15.400821 +_ठ -15.400821 +शश -15.459714 +शद -15.459714 +अग -15.459714 +सस -15.459714 +यह -15.459714 +_n -15.521115 +na -15.521115 +ag -15.521115 +ga -15.521115 +धई -15.521115 +वष -15.521115 +थज -15.521115 +खग -15.521115 +णत -15.521115 +बआ -15.585245 +णप -15.585245 +मव -15.585245 +इह -15.585245 +हप -15.585245 +कछ -15.585245 +छट -15.585245 +बस -15.585245 +छउ -15.652360 +हए -15.652360 +वध -15.652360 +पम -15.652360 +औस -15.722749 +लछ -15.722749 +छन -15.722749 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nld b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nld new file mode 100644 index 00000000000..4d9d8fb9079 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nld @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 304918631 +#UNIQUE_BIGRAMS 648 +#UNSEEN_LOG_PROB -17.505112 +#EXPECTED_SCORE -7.416122 +n_ -4.622767 +en -4.873287 +t_ -5.240832 +er -5.351950 +e_ -5.547488 +an -5.785947 +_v -5.837478 +ee -5.996077 +de -6.090828 +r_ -6.150710 +_h -6.191809 +ge -6.198203 +aa -6.322138 +te -6.328740 +et -6.328873 +ij -6.477979 +_d -6.499625 +el -6.547493 +he -6.566403 +ie -6.577532 +or -6.626398 +_e -6.647668 +s_ -6.665146 +_w -6.715126 +oo -6.720074 +d_ -6.760597 +va -6.781643 +st -6.804689 +_b -6.885817 +in -6.887156 +_m -6.913066 +_g -6.925392 +nd -6.966355 +ar -7.011541 +ve -7.053799 +_a -7.062211 +me -7.107274 +re -7.116716 +ch -7.123090 +rd -7.202607 +on -7.213515 +_s -7.216953 +_o -7.243006 +le -7.266772 +li -7.288514 +_t -7.317383 +at -7.332185 +we -7.351900 +_z -7.384627 +al -7.407269 +g_ -7.429993 +be -7.493657 +ng -7.568148 +_n -7.582102 +it -7.601476 +ke -7.638167 +vo -7.647464 +ri -7.674401 +_k -7.711128 +di -7.713811 +ti -7.754726 +es -7.755106 +da -7.765194 +l_ -7.769992 +k_ -7.777881 +la -7.820750 +oe -7.842915 +ro -7.866274 +ne -7.934750 +zi -7.938829 +ui -7.973045 +nt -7.980857 +_p -8.024843 +ed -8.069414 +ra -8.071224 +wa -8.071485 +rs -8.091062 +_l -8.091569 +na -8.106984 +ni -8.156995 +se -8.163363 +ma -8.228818 +j_ -8.239919 +ns -8.254042 +sc -8.255957 +ei -8.274803 +hi -8.330809 +to -8.334415 +am -8.334656 +ig -8.352571 +ta -8.373263 +_r -8.400504 +is -8.400956 +eg -8.403739 +pe -8.431201 +ha -8.457441 +do -8.476894 +ht -8.501394 +ev -8.525461 +ze -8.544464 +jk -8.605204 +rt -8.605869 +jn -8.614286 +ek -8.621709 +em -8.624849 +om -8.629969 +wo -8.652953 +as -8.693216 +id -8.709523 +m_ -8.742234 +_u -8.787984 +ot -8.818035 +bi -8.833130 +ld -8.838654 +op -8.846850 +il -8.848801 +ic -8.863302 +_c -8.892152 +ol -8.906314 +ll -8.929418 +_i -8.952694 +tr -8.953880 +mi -8.958475 +ls -8.995238 +ou -8.995676 +_f -8.996282 +ho -9.013599 +ur -9.112581 +no -9.120644 +ts -9.124803 +gr -9.161183 +ec -9.178759 +lo -9.186055 +eb -9.196219 +og -9.214845 +eu -9.215416 +ez -9.220293 +_j -9.236597 +rk -9.245315 +ak -9.266596 +ac -9.272132 +ad -9.276328 +zo -9.276659 +ag -9.284471 +mo -9.291929 +ko -9.307461 +kt -9.321336 +pr -9.329799 +wi -9.355880 +ok -9.381910 +ef -9.402057 +a_ -9.405233 +ru -9.406485 +un -9.408477 +uw -9.413101 +ov -9.415930 +vi -9.429477 +ka -9.462636 +br -9.465199 +us -9.479273 +pa -9.522475 +sp -9.554194 +bo -9.563379 +si -9.563640 +ss -9.585895 +rl -9.589419 +ep -9.603395 +nn -9.633320 +tu -9.639930 +co -9.666964 +f_ -9.673452 +so -9.674043 +gi -9.769147 +pl -9.810844 +p_ -9.814009 +ce -9.828378 +af -9.842999 +jd -9.861076 +dr -9.861639 +ik -9.868080 +ap -9.869478 +ct -9.875639 +ft -9.885733 +ja -9.889915 +ds -9.899969 +ga -9.910677 +sl -9.911885 +rg -9.913590 +rm -9.917031 +dt -9.927604 +bl -9.935416 +nk -9.976596 +od -9.976921 +h_ -9.998344 +ba -10.027820 +tw -10.031168 +io -10.105010 +hu -10.132035 +du -10.141297 +ud -10.203018 +rv -10.256997 +po -10.290962 +rn -10.308274 +lt -10.346991 +os -10.370174 +fi -10.373955 +tt -10.377885 +ew -10.381591 +ia -10.385149 +th -10.412489 +je -10.414515 +oc -10.431246 +o_ -10.435951 +pp -10.436187 +ki -10.441777 +lg -10.464741 +go -10.471651 +vr -10.479013 +gd -10.484067 +lu -10.493428 +fa -10.501561 +rb -10.505579 +ul -10.512280 +vl -10.520555 +kr -10.549137 +i_ -10.549705 +ci -10.578652 +kl -10.613052 +za -10.613986 +w_ -10.628833 +sa -10.632914 +ca -10.641203 +kk -10.689556 +pu -10.726845 +uu -10.728040 +gt -10.747552 +bu -10.755669 +fe -10.764811 +lf -10.795647 +ug -10.798172 +nw -10.802678 +ub -10.812736 +eh -10.833134 +lk -10.848982 +ks -10.851629 +au -10.851690 +ut -10.853991 +pi -10.863498 +um -10.870479 +of -10.871214 +mm -10.872926 +md -10.873849 +im -10.888690 +mp -10.946284 +ms -10.957437 +ea -10.957616 +dd -10.993832 +ku -11.009391 +ai -11.016941 +nc -11.022255 +ir -11.075274 +hr -11.085889 +rz -11.116541 +rw -11.122436 +pt -11.129202 +fr -11.136139 +uc -11.153990 +iv -11.171585 +jo -11.229653 +tg -11.237674 +y_ -11.249456 +gs -11.291781 +mb -11.314508 +u_ -11.332454 +nu -11.338984 +bb -11.360134 +rh -11.364892 +ae -11.365142 +fd -11.370431 +kw -11.401567 +rr -11.403207 +fo -11.433103 +mu -11.472559 +rp -11.506597 +mt -11.515809 +ab -11.540697 +su -11.542178 +nl -11.543604 +ob -11.557199 +nv -11.570664 +eo -11.617587 +uk -11.659598 +jf -11.662755 +av -11.699089 +oi -11.718315 +pg -11.727029 +ju -11.783637 +ex -11.814303 +cl -11.816059 +ff -11.819937 +jv -11.838738 +ip -11.872385 +iz -11.887775 +zu -11.928175 +cu -11.999613 +js -12.006430 +sn -12.012105 +nz -12.018334 +fs -12.031564 +gg -12.047218 +lm -12.054284 +oa -12.080766 +sd -12.116940 +jg -12.125963 +wd -12.131854 +b_ -12.135171 +zw -12.152600 +gu -12.166845 +rc -12.204357 +ua -12.244890 +ue -12.262473 +cr -12.315316 +fg -12.318527 +tv -12.342850 +jz -12.355043 +tb -12.384567 +ow -12.408033 +sh -12.486556 +nm -12.487235 +tj -12.488974 +if -12.497349 +lv -12.503215 +nb -12.514521 +nh -12.529213 +ps -12.537712 +dw -12.554521 +sm -12.556743 +uz -12.558997 +lb -12.594169 +sy -12.678534 +lp -12.690025 +gh -12.695224 +ck -12.776880 +fl -12.778242 +kh -12.788583 +ys -12.874124 +jl -12.899314 +nf -12.931923 +ry -12.935773 +rf -12.953866 +dh -12.955707 +sv -12.970479 +gl -12.986556 +ph -12.997928 +tz -13.002386 +c_ -13.003862 +sk -13.022481 +fu -13.031842 +tm -13.050787 +ty -13.078350 +mg -13.144673 +cc -13.145273 +ib -13.154260 +pn -13.155857 +ii -13.165953 +x_ -13.174589 +sb -13.215452 +hn -13.226112 +bs -13.243490 +tl -13.277193 +oz -13.289031 +az -13.294583 +vu -13.301397 +iu -13.322471 +pv -13.347207 +td -13.347306 +kn -13.353235 +ly -13.357251 +oh -13.363171 +up -13.457897 +nr -13.478649 +ym -13.500580 +lj -13.527201 +yp -13.528766 +ay -13.533975 +ws -13.538923 +kb -13.552388 +qu -13.560996 +dv -13.615299 +xi -13.667657 +gn -13.702617 +lh -13.712174 +dg -13.744083 +sj -13.817800 +ey -13.842122 +lw -13.856226 +ax -13.888080 +xt -13.898447 +dk -13.911073 +xp -13.926167 +z_ -13.941122 +nj -13.976835 +_y -13.992944 +uv -14.019021 +xe -14.020750 +fh -14.061992 +sg -14.078350 +pd -14.092166 +sw -14.125857 +tk -14.134426 +dz -14.165997 +ln -14.168434 +yo -14.228381 +dy -14.268343 +sr -14.277428 +db -14.284492 +oj -14.290074 +fw -14.299488 +dj -14.359984 +yn -14.375790 +tp -14.381535 +my -14.404437 +np -14.413082 +kz -14.420325 +jw -14.423857 +yc -14.440803 +fk -14.462109 +dp -14.472397 +bt -14.529325 +tn -14.550173 +ny -14.558825 +yl -14.577907 +aj -14.590458 +v_ -14.591976 +kj -14.603119 +ah -14.643358 +pz -14.680893 +fb -14.732638 +gk -14.759027 +oy -14.760339 +dm -14.763494 +tc -14.771809 +kg -14.780171 +gb -14.789118 +xa -14.816844 +jt -14.822316 +hy -14.827397 +mh -14.836366 +sf -14.845670 +hz -14.852932 +ya -14.859106 +wt -14.862767 +ix -14.872952 +pb -14.879355 +aw -14.883211 +gm -14.900905 +jp -14.917355 +yr -14.930888 +pm -14.932663 +by -14.944251 +mv -14.962262 +dl -14.972873 +gv -14.978514 +ao -14.989708 +_q -15.017843 +lc -15.082694 +cy -15.136214 +sz -15.214463 +rj -15.231108 +ye -15.241335 +ux -15.252004 +vd -15.264241 +gy -15.273019 +pj -15.276395 +ji -15.293205 +wn -15.325285 +ox -15.336400 +mr -15.360283 +ml -15.368466 +jb -15.371471 +lr -15.400237 +yt -15.426042 +uf -15.512308 +wu -15.529884 +mf -15.540218 +pk -15.553127 +wr -15.607129 +tf -15.623277 +jh -15.629980 +uo -15.654907 +wb -15.658328 +lz -15.664213 +kp -15.697265 +gz -15.698523 +bj -15.733153 +fv -15.734958 +vv -15.765997 +fz -15.799875 +mw -15.817800 +fm -15.818073 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nno b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nno new file mode 100644 index 00000000000..28ae475104a --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nno @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 41923838 +#UNIQUE_BIGRAMS 632 +#UNSEEN_LOG_PROB -19.026090 +#EXPECTED_SCORE -7.403722 +e_ -5.068556 +r_ -5.375939 +t_ -5.412953 +n_ -5.522592 +_s -5.531147 +de -5.713123 +ar -5.725455 +er -5.824188 +en -5.872939 +an -6.126876 +_d -6.135042 +_f -6.155097 +a_ -6.228632 +et -6.250057 +ei -6.281813 +in -6.327464 +or -6.339137 +te -6.373268 +re -6.426621 +st -6.492809 +_m -6.503900 +le -6.546644 +_v -6.548579 +_t -6.594175 +_h -6.675859 +_e -6.685670 +ti -6.809295 +me -6.846389 +ra -6.857915 +_k -6.898652 +d_ -6.905175 +om -6.921831 +ne -6.957851 +l_ -6.981039 +il -7.032041 +g_ -7.070599 +el -7.095119 +ve -7.135744 +va -7.146459 +_b -7.149587 +m_ -7.183711 +ng -7.187734 +fo -7.202955 +ta -7.256135 +la -7.265561 +ha -7.285674 +sk -7.298732 +al -7.308681 +_a -7.325191 +nd -7.335270 +so -7.335320 +se -7.341161 +tt -7.363706 +je -7.369129 +rt -7.379509 +nn -7.485211 +ed -7.532033 +_l -7.535323 +_o -7.588981 +_n -7.619156 +_g -7.622909 +na -7.646117 +ka -7.665059 +is -7.704783 +li -7.710077 +ll -7.753163 +eg -7.755318 +ge -7.773288 +on -7.778296 +ri -7.805098 +k_ -7.827654 +ke -7.827951 +ik -7.855406 +ga -7.859892 +da -7.875541 +es -7.883085 +it -7.887984 +ma -7.925424 +_i -7.974330 +s_ -8.014965 +i_ -8.078131 +sa -8.080430 +un -8.149788 +_r -8.172870 +am -8.182496 +_p -8.182586 +fr -8.227149 +kk -8.244765 +kj -8.252048 +at -8.254274 +ko -8.272153 +si -8.285781 +ns -8.353962 +id -8.369186 +vi -8.381367 +as -8.384354 +to -8.391431 +jo -8.410859 +ek -8.421049 +ad -8.442079 +rs -8.452445 +_u -8.453373 +no -8.459841 +nt -8.461757 +ag -8.508150 +tr -8.531174 +he -8.573914 +rd -8.602668 +ol -8.661996 +ro -8.674287 +gj -8.684799 +pe -8.711497 +kt -8.719453 +ig -8.730081 +be -8.745064 +op -8.779687 +sj -8.794279 +ru -8.878810 +ok -8.888465 +lo -8.912357 +ak -8.924027 +ss -8.934395 +pp -8.969034 +ut -8.974944 +us -8.994662 +dr -9.010495 +ov -9.021219 +em -9.043399 +lt -9.058622 +ir -9.084020 +ld -9.116849 +kr -9.172692 +ul -9.199410 +di -9.200483 +ni -9.230080 +pr -9.259348 +og -9.270271 +ba -9.283099 +mo -9.289179 +gr -9.290902 +rk -9.343227 +bl -9.349971 +ot -9.364438 +mi -9.387415 +pa -9.395018 +fa -9.400519 +sl -9.414588 +iv -9.419010 +ho -9.442307 +ku -9.451722 +od -9.457347 +ur -9.464696 +fe -9.469396 +sp -9.481547 +br -9.481744 +av -9.504809 +gg -9.519584 +ja -9.541189 +dd -9.551610 +ae -9.567528 +ie -9.616661 +mm -9.629769 +ev -9.648234 +tu -9.671141 +v_ -9.673303 +yr -9.674540 +ia -9.679470 +os -9.694446 +by -9.697557 +fi -9.699817 +_j -9.744517 +au -9.822730 +uk -9.833773 +ks -9.835910 +p_ -9.853948 +gs -9.879912 +o_ -9.891468 +kv -9.913104 +rn -9.919255 +fl -9.932846 +bo -9.949355 +kl -9.956551 +oy -10.017451 +mu -10.020379 +pl -10.057962 +ap -10.073006 +do -10.081371 +ds -10.107290 +go -10.125011 +rr -10.144627 +rl -10.152204 +ls -10.168903 +ki -10.176928 +ny -10.199532 +po -10.230121 +lv -10.233887 +ts -10.249805 +ty -10.268275 +gi -10.289179 +yt -10.354131 +dt -10.384768 +io -10.388407 +vo -10.392194 +kn -10.452445 +ud -10.463675 +ft -10.464356 +im -10.465377 +bu -10.484379 +rg -10.488575 +gd -10.495911 +ys -10.496309 +yg -10.502785 +nk -10.514718 +dl -10.516380 +sv -10.519458 +um -10.523302 +lk -10.538679 +_c -10.548180 +rm -10.567633 +rb -10.570717 +mn -10.605627 +bi -10.630506 +lu -10.643273 +yn -10.697100 +yk -10.711050 +y_ -10.745728 +ye -10.768118 +nl -10.772385 +tn -10.790434 +ep -10.807108 +fj -10.817628 +sy -10.844585 +of -10.930225 +su -10.934799 +if -10.939995 +gl -10.963784 +ms -10.971502 +tl -10.982740 +sm -10.987483 +ju -10.998002 +ml -10.999974 +ly -11.018200 +th -11.034277 +gt -11.053237 +lb -11.106872 +ry -11.107708 +fy -11.125588 +hi -11.141047 +ea -11.146264 +mp -11.148918 +ob -11.184837 +hu -11.257451 +gn -11.258209 +mb -11.280122 +tv -11.280721 +af -11.286898 +fu -11.315029 +lg -11.316257 +gu -11.327887 +yd -11.352511 +mt -11.369072 +mr -11.402870 +ky -11.413813 +my -11.445038 +rv -11.445614 +pt -11.458824 +co -11.464064 +yl -11.495613 +eb -11.513812 +sn -11.517944 +up -11.529392 +pu -11.531123 +ug -11.538576 +ef -11.564086 +du -11.571398 +ff -11.582331 +ya -11.582964 +nf -11.620611 +rf -11.680006 +pi -11.686797 +nu -11.692366 +eh -11.750819 +ab -11.771362 +ch -11.786114 +ua -11.786600 +lj -11.818560 +mf -11.842877 +h_ -11.848069 +vu -11.884036 +ee -11.890292 +ic -11.913203 +_w -11.941618 +vs -11.955219 +lm -12.000467 +eo -12.004562 +ip -12.005127 +ai -12.049367 +_y -12.073933 +oe -12.076755 +ue -12.089296 +aa -12.096663 +hj -12.179959 +ub -12.220934 +u_ -12.242783 +ou -12.246959 +ca -12.251147 +ps -12.280293 +nb -12.291808 +tj -12.303415 +vd -12.315117 +ce -12.370894 +lp -12.406509 +eu -12.411000 +dv -12.421155 +sd -12.439389 +dy -12.447439 +sb -12.449362 +bb -12.469518 +tf -12.491347 +sh -12.493329 +tg -12.502685 +ac -12.522998 +oh -12.557848 +nh -12.586346 +f_ -12.601237 +vt -12.615419 +lf -12.620178 +rj -12.623865 +ct -12.624517 +pn -12.645090 +ph -12.682379 +tb -12.755451 +vl -12.759980 +nv -12.781382 +yp -12.798441 +wi -12.813721 +rh -12.814216 +sc -12.855446 +dn -12.857488 +ib -12.894217 +c_ -12.905790 +ym -12.917723 +sf -12.923058 +oa -12.942160 +ec -12.944057 +ck -12.954399 +db -13.018629 +rp -13.033267 +dj -13.073340 +df -13.078689 +rc -13.087348 +md -13.131443 +dm -13.141358 +sr -13.178841 +ii -13.179480 +ji -13.199734 +uf -13.204924 +gv -13.206874 +pd -13.215359 +wa -13.227190 +cl -13.233805 +vg -13.238121 +td -13.310390 +ay -13.331518 +oo -13.389422 +ow -13.394601 +nc -13.411000 +gh -13.446670 +b_ -13.458243 +lh -13.468738 +gm -13.471471 +ey -13.484428 +mg -13.504684 +bj -13.514718 +w_ -13.529291 +we -13.580643 +vn -13.585288 +yo -13.645751 +pg -13.654156 +oc -13.675159 +ci -13.711627 +bs -13.718569 +nr -13.740538 +nj -13.766679 +ew -13.768118 +tm -13.783079 +uv -13.800649 +hy -13.811987 +nm -13.914531 +dh -13.931099 +vr -13.948402 +pf -13.948402 +ui -13.950035 +hn -14.015776 +vj -14.034132 +gy -14.042237 +dk -14.070377 +hr -14.094855 +ah -14.117308 +iu -14.163921 +ex -14.182996 +cu -14.212743 +oi -14.234468 +_z -14.263276 +mj -14.265985 +mk -14.287844 +ln -14.305852 +x_ -14.375092 +dg -14.376555 +vh -14.378020 +tk -14.378020 +km -14.434047 +sg -14.434047 +yv -14.441684 +ze -14.452445 +kd -14.540728 +lr -14.554739 +wo -14.589948 +tp -14.605305 +kg -14.640908 +za -14.727876 +yf -14.736305 +gf -14.753311 +kh -14.811492 +z_ -14.920388 +ht -14.975862 +mv -15.014067 +az -15.038179 +qu -15.163921 +mh -15.176609 +pm -15.199734 +vv -15.203624 +cr -15.210132 +uc -15.231155 +j_ -15.279608 +aj -15.332583 +vk -15.358371 +yb -15.384630 +tw -15.403895 +aw -15.405388 +wh -15.441684 +xp -15.450903 +_q -15.466399 +ao -15.513913 +zi -15.541548 +vy -15.546480 +vp -15.563044 +pv -15.574753 +gb -15.638273 +dp -15.693734 +vf -15.693734 +vb -15.701048 +ix -15.760935 +wn -15.778236 +np -15.797706 +iz -15.855701 +xi -15.857743 +py -15.920388 +nz -15.926805 +kf -15.926805 +cc -15.957133 +tz -16.049804 +hl -16.070969 +pb -16.080476 +hs -16.082863 +xa -16.124051 +mc -16.133915 +zz -16.148840 +fv -16.168983 +hm -16.171520 +gk -16.181716 +yh -16.199734 +bn -16.204924 +kb -16.247126 +tc -16.260572 +ez -16.304459 +uh -16.367071 +vm -16.384630 +dw -16.399427 +ax -16.417386 +sw -16.441684 +pc -16.517136 +uo -16.543190 +rw -16.670216 +hv -16.688272 +ws -16.691911 +ox -16.695559 +ih -16.710243 +fs -16.732553 +ij -16.903415 +ux -16.946228 +pk -16.954945 +_x -16.981417 +bc -17.092449 +kp -17.126511 +rx -17.131443 +zo -17.141358 +oj -17.156361 +yi -17.202326 +ej -17.202326 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nob b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nob new file mode 100644 index 00000000000..152f4006937 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nob @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 220751723 +#UNIQUE_BIGRAMS 615 +#UNSEEN_LOG_PROB -18.846654 +#EXPECTED_SCORE -7.366934 +e_ -4.947828 +er -5.138526 +r_ -5.180846 +en -5.361760 +t_ -5.397380 +_s -5.572669 +n_ -5.638384 +de -5.781144 +et -5.908642 +te -5.961367 +_f -6.090645 +or -6.123190 +re -6.129727 +an -6.377987 +_d -6.396863 +_m -6.450972 +le -6.489623 +ar -6.524644 +ne -6.530372 +_h -6.552287 +st -6.640350 +_t -6.650883 +_b -6.693435 +me -6.701526 +ti -6.751909 +fo -6.786871 +in -6.810554 +se -6.905830 +il -6.922431 +el -6.932308 +_v -6.978543 +ke -6.988115 +ge -6.992106 +g_ -7.003602 +l_ -7.014117 +li -7.047404 +om -7.059051 +_k -7.073888 +tt -7.106187 +d_ -7.130012 +ve -7.232871 +ng -7.233907 +ed -7.257328 +sk -7.307243 +_a -7.330056 +m_ -7.345538 +s_ -7.359551 +ha -7.390787 +nn -7.393999 +_o -7.410206 +es -7.415356 +nd -7.417423 +so -7.429863 +al -7.450756 +ra -7.541802 +ll -7.585143 +is -7.619605 +_e -7.648296 +ik -7.664346 +a_ -7.670073 +_n -7.683138 +va -7.684254 +_l -7.713510 +ig -7.722000 +ri -7.728319 +rt -7.753138 +_i -7.764919 +la -7.820276 +ta -7.847867 +_g -7.860086 +on -7.882496 +_p -7.905976 +ns -7.951141 +k_ -7.999808 +ka -8.014598 +eg -8.024089 +sa -8.024715 +nt -8.039521 +si -8.046918 +at -8.048720 +je -8.049406 +un -8.069671 +vi -8.074183 +be -8.102996 +kk -8.120277 +bl -8.165734 +_r -8.169854 +ma -8.174589 +rs -8.195299 +_u -8.252070 +ol -8.263341 +id -8.282460 +to -8.393680 +jo -8.393959 +ko -8.418671 +pe -8.425013 +tr -8.431019 +it -8.449481 +ro -8.455966 +he -8.458364 +no -8.480478 +kt -8.505382 +fr -8.555615 +am -8.563190 +ag -8.607588 +as -8.648827 +na -8.676730 +ut -8.685570 +op -8.706035 +ak -8.773200 +mm -8.783707 +em -8.789051 +ie -8.808873 +di -8.813744 +ni -8.820988 +da -8.831418 +pp -8.844973 +dr -8.857869 +ad -8.864028 +ek -8.866076 +ae -8.872182 +lt -8.878981 +ss -8.924621 +rd -8.935859 +ru -8.979774 +lo -8.980325 +ld -8.991059 +ei -8.993412 +ba -9.005561 +ga -9.068390 +mi -9.078466 +ot -9.103114 +sj -9.103434 +os -9.104391 +ho -9.105517 +pr -9.111253 +ov -9.158954 +ok -9.180172 +ls -9.224166 +sl -9.248561 +gj -9.273061 +fa -9.276182 +_j -9.281118 +mo -9.284810 +av -9.302813 +kr -9.314858 +rk -9.318893 +od -9.365974 +ir -9.378943 +sp -9.399042 +gs -9.409257 +fi -9.417376 +gr -9.425433 +ul -9.441870 +br -9.482770 +i_ -9.490375 +ev -9.493970 +us -9.500133 +ts -9.511097 +iv -9.513503 +og -9.531977 +ku -9.544241 +rn -9.579673 +kj -9.636959 +po -9.664406 +pa -9.668203 +gi -9.669240 +ks -9.678598 +ap -9.697922 +hv -9.719386 +p_ -9.730508 +fe -9.751119 +gg -9.789539 +kl -9.802723 +rg -9.804113 +o_ -9.808443 +ur -9.819124 +dd -9.884644 +uk -9.906688 +dt -9.928883 +v_ -9.976376 +lg -10.038933 +fl -10.062899 +oe -10.066952 +pl -10.104247 +tu -10.104751 +nk -10.107531 +dl -10.124473 +bo -10.136333 +hu -10.159137 +sv -10.179676 +ye -10.183683 +rb -10.239451 +ny -10.250554 +ty -10.251199 +pi -10.255379 +do -10.264185 +vo -10.267798 +mu -10.292806 +bi -10.303875 +ia -10.346439 +rm -10.348448 +rl -10.366953 +ft -10.379592 +ds -10.400817 +lv -10.403930 +lu -10.432095 +yt -10.460277 +ki -10.470274 +by -10.525817 +rr -10.535552 +sy -10.556078 +ep -10.561707 +oy -10.586887 +yr -10.606734 +mp -10.606856 +im -10.638198 +gt -10.653138 +gn -10.679434 +nl -10.702304 +yk -10.705902 +yn -10.732430 +io -10.738569 +ly -10.757203 +if -10.805720 +ob -10.810372 +ys -10.823944 +lk -10.856484 +ef -10.867907 +go -10.870890 +mt -10.878104 +ud -10.883872 +y_ -10.918783 +of -10.927425 +sn -10.931771 +kn -10.986504 +rf -11.008995 +sm -11.035047 +_c -11.053652 +af -11.053722 +um -11.060098 +yg -11.080899 +bu -11.086871 +tn -11.089874 +fu -11.096255 +su -11.133004 +kv -11.135546 +tl -11.145845 +ry -11.145949 +tv -11.156069 +eh -11.156308 +ja -11.166306 +ff -11.168065 +pt -11.174515 +nf -11.189182 +vn -11.213155 +gh -11.213434 +lb -11.243399 +ms -11.287854 +ea -11.331128 +ue -11.365030 +gu -11.387510 +th -11.401108 +rh -11.438529 +du -11.439309 +gl -11.441198 +rv -11.444819 +eb -11.461143 +hj -11.493113 +pu -11.516453 +hi -11.531000 +my -11.550079 +nu -11.573291 +ju -11.587319 +ab -11.588748 +au -11.594617 +mb -11.624762 +yl -11.660958 +bb -11.677367 +ub -11.715188 +yd -11.722811 +ml -11.762313 +ch -11.769756 +ua -11.782339 +ip -11.825591 +nr -11.837716 +up -11.859650 +mf -11.879926 +ps -11.922114 +dn -11.986186 +tf -12.005618 +mr -12.012109 +nh -12.024036 +lm -12.034279 +ca -12.053736 +vs -12.086587 +fj -12.112428 +fy -12.131919 +f_ -12.147253 +vt -12.172251 +vd -12.178448 +nv -12.200119 +_w -12.200426 +sd -12.214086 +co -12.224182 +tb -12.238101 +sc -12.245254 +ai -12.254708 +dv -12.259699 +tg -12.273158 +gd -12.280292 +gy -12.312176 +ee -12.318204 +ug -12.364599 +lp -12.373900 +ic -12.376122 +ky -12.376504 +nb -12.387493 +lf -12.425564 +eo -12.441362 +ce -12.473895 +eu -12.494715 +ou -12.502468 +tj -12.506265 +sb -12.513355 +b_ -12.535416 +lj -12.540897 +_y -12.560542 +yp -12.570327 +pn -12.601912 +sh -12.613742 +h_ -12.614849 +rp -12.622082 +dy -12.658040 +oh -12.759025 +ib -12.786974 +dm -12.864004 +yh -12.903718 +ac -12.913769 +vu -12.936592 +pd -12.957545 +gv -13.008496 +pg -13.016597 +u_ -13.092711 +vg -13.106767 +sr -13.118343 +pf -13.187321 +ck -13.210426 +sf -13.230386 +uf -13.231077 +np -13.255667 +wi -13.279707 +wa -13.284460 +bj -13.315437 +nc -13.329496 +x_ -13.338742 +oo -13.354262 +vh -13.387423 +nm -13.414283 +mg -13.426463 +lh -13.429344 +mh -13.463264 +ph -13.499211 +td -13.516798 +ya -13.547064 +ym -13.585992 +ci -13.622370 +df -13.630884 +vr -13.636033 +db -13.667066 +nj -13.676276 +c_ -13.702086 +dk -13.711523 +we -13.719435 +hr -13.720935 +ay -13.723673 +bs -13.801503 +dh -13.872653 +mk -13.878646 +w_ -13.898768 +ow -13.898968 +vl -13.905271 +hy -13.931274 +aa -13.981553 +ey -13.988229 +ix -13.998675 +dg -14.007043 +oa -14.020665 +ew -14.025888 +hn -14.032553 +sg -14.054625 +mn -14.058522 +oc -14.059638 +yo -14.097057 +yv -14.101415 +dj -14.127145 +cu -14.132535 +uv -14.141601 +ah -14.144202 +kh -14.201165 +rc -14.222869 +tm -14.266767 +gb -14.272964 +ex -14.294602 +md -14.339013 +ii -14.393247 +vv -14.415639 +kg -14.424809 +iu -14.426391 +oi -14.428983 +kd -14.477654 +ui -14.497622 +ec -14.552628 +vj -14.556718 +pm -14.602806 +tp -14.605736 +tk -14.663076 +z_ -14.703829 +wo -14.827775 +lr -14.877662 +rj -14.933624 +ct -14.947392 +gf -14.954430 +vf -15.000816 +gk -15.062095 +mv -15.097859 +cl -15.137591 +_z -15.139477 +gm -15.148231 +ao -15.201903 +cr -15.218004 +ih -15.219000 +hl -15.225746 +ln -15.268185 +ht -15.290275 +yb -15.299733 +tc -15.322048 +ji -15.372167 +ez -15.389736 +vk -15.391701 +tw -15.426679 +uc -15.443763 +az -15.461346 +pk -15.470219 +za -15.493848 +vb -15.552943 +pb -15.576381 +vp -15.634038 +uh -15.652433 +aj -15.675164 +rw -15.717849 +j_ -15.726328 +fs -15.788591 +dp -15.831010 +xa -15.842868 +xi -15.851730 +ax -15.859868 +zi -15.902867 +bn -15.933624 +_q -15.942239 +aw -15.985682 +pv -15.987804 +ze -15.995469 +fg -16.007473 +ws -16.023927 +cc -16.044540 +qu -16.125392 +fp -16.138062 +hm -16.146097 +km -16.288443 +wh -16.311644 +bc -16.316970 +vm -16.354262 +wn -16.399872 +uo -16.421507 +nz -16.442889 +fn -16.466959 +ox -16.499286 +dw -16.516951 +tz -16.572554 +iz -16.581499 +bd -16.629724 +zu -16.661212 +kb -16.668682 +hs -16.669363 +kf -16.696175 +fk -16.707321 +sw -16.781948 +gp -16.813215 +bm -16.836735 +py -16.862981 +mj -16.909690 +kp -16.920188 +mc -16.948012 +fb -16.952978 +ij -16.989079 +lc -16.991631 +_x -17.031349 +zo -17.071291 +wd -17.105903 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nso b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nso new file mode 100644 index 00000000000..f71c8c5f8b4 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/nso @@ -0,0 +1,414 @@ +#TOTAL_BIGRAMS 3039254 +#UNIQUE_BIGRAMS 410 +#UNSEEN_LOG_PROB -18.213358 +#EXPECTED_SCORE -6.862546 +a_ -4.100812 +o_ -4.639640 +e_ -4.876405 +ts -5.209998 +le -5.405983 +_b -5.490636 +_t -5.507315 +go -5.622022 +_m -5.719152 +se -5.722657 +la -5.763977 +el -5.952497 +mo -6.061136 +lo -6.148278 +di -6.228797 +na -6.233718 +ng -6.292595 +sa -6.339337 +ba -6.400900 +an -6.457802 +et -6.459556 +bo -6.486246 +_l -6.547755 +_g -6.550333 +on -6.579954 +wa -6.734689 +en -6.769793 +ag -6.770052 +_d -6.784684 +ho -6.856851 +_s -6.862584 +re -6.887490 +th -6.889290 +ga -6.897020 +g_ -6.917531 +at -6.945576 +ol -6.996309 +_k -7.015527 +ka -7.034693 +ma -7.052037 +_n -7.082401 +or -7.095130 +tl -7.153608 +ha -7.167052 +hl -7.242030 +so -7.260835 +eg -7.290030 +i_ -7.296806 +al -7.336841 +we -7.427251 +me -7.443603 +gw -7.481614 +_h -7.484332 +il -7.488929 +be -7.538989 +is -7.619686 +ok -7.670713 +ne -7.673714 +eo -7.703187 +ak -7.709333 +he -7.723309 +_p -7.729441 +te -7.850866 +si -7.869395 +kg -7.872061 +ra -7.875401 +sw -7.892106 +ta -7.892444 +ot -7.932935 +om -7.937233 +eb -7.945284 +it -7.966617 +nt -8.011601 +_f -8.048325 +fe -8.078418 +_i -8.088332 +_e -8.100397 +ab -8.163645 +ko -8.175262 +pe -8.204649 +fa -8.247285 +_j -8.265575 +pa -8.300319 +ar -8.335767 +eh -8.343764 +bj -8.360204 +es -8.381575 +ir -8.408581 +ph -8.420730 +ro -8.457636 +oh -8.474759 +ke -8.484417 +_r -8.514132 +ja -8.524059 +ap -8.526332 +od -8.526507 +to -8.537577 +_y -8.555146 +ny -8.574922 +li -8.576552 +sh -8.632157 +u_ -8.640279 +ao -8.659920 +im -8.675945 +mm -8.695295 +of -8.697658 +os -8.707348 +in -8.710526 +ae -8.724714 +ye -8.740870 +je -8.754336 +ob -8.796771 +ah -8.803330 +op -8.812052 +ya -8.825340 +ik -8.862639 +ri -8.874621 +as -8.876182 +og -8.885581 +ek -8.899794 +ut -8.904336 +ed -8.906157 +_a -8.957621 +er -8.965193 +hu -9.006099 +ef -9.007809 +po -9.033449 +ge -9.050966 +em -9.082530 +du -9.092861 +oa -9.094936 +am -9.131475 +oo -9.226094 +no -9.253066 +us -9.269671 +bi -9.373209 +ti -9.376992 +ee -9.430032 +ib -9.430360 +lw -9.435938 +jo -9.455801 +ku -9.472902 +su -9.495339 +ih -9.607508 +ip -9.621274 +ei -9.717503 +um -9.762971 +kh -9.784579 +af -9.804391 +kw -9.939562 +ep -9.940028 +oe -9.986464 +hw -10.002443 +ad -10.064119 +ud -10.064627 +fi -10.065135 +yo -10.074318 +ea -10.077905 +hi -10.079445 +ll -10.085106 +ul -10.124305 +up -10.194323 +ki -10.252777 +nn -10.262073 +_w -10.289733 +mi -10.306467 +ru -10.307670 +ai -10.358489 +nk -10.395095 +ps -10.492259 +st -10.518478 +rw -10.535286 +au -10.570945 +mp -10.647304 +pi -10.648827 +_o -10.657235 +mu -10.720703 +fo -10.770414 +ia -10.772073 +eu -10.805665 +gi -10.809920 +oi -10.950323 +ni -11.089236 +da -11.101700 +pu -11.133340 +bu -11.145117 +fr -11.166779 +lh -11.201013 +kr -11.248728 +uk -11.260325 +ch -11.339299 +ua -11.362858 +tu -11.374154 +_c -11.442529 +fu -11.449150 +wi -11.465165 +aa -11.481360 +wo -11.623894 +oy -11.635929 +lu -11.648065 +yw -11.655703 +n_ -11.747383 +id -11.783742 +un -11.783742 +ie -11.840058 +r_ -11.843542 +s_ -11.861094 +ur -11.873508 +if -11.957857 +de -11.978780 +t_ -12.047446 +ew -12.053486 +gk -12.057528 +tw -12.079959 +io -12.094417 +ns -12.109021 +_u -12.130144 +tt -12.136542 +fs -12.153743 +pr -12.182139 +iw -12.294494 +ic -12.306467 +_q -12.393179 +ay -12.408581 +d_ -12.493627 +ca -12.589842 +ju -12.646543 +y_ -12.674199 +ou -12.721505 +nd -12.744123 +ue -12.767101 +ii -12.790452 +tr -12.821040 +qe -12.884234 +l_ -12.909577 +do -12.946571 +h_ -12.992254 +lm -13.067680 +ij -13.067680 +gs -13.142968 +ov -13.147269 +tj -13.147269 +qa -13.213358 +aj -13.254515 +k_ -13.287358 +bl -13.316117 +ow -13.360360 +m_ -13.385539 +oc -13.474590 +jh -13.524059 +ve -13.558006 +va -13.616423 +ey -13.677305 +ec -13.677305 +ej -13.683537 +ug -13.747383 +nc -13.780398 +nw -13.780398 +aw -13.827927 +ss -13.834846 +mb -13.834846 +co -13.855806 +cr -14.043433 +ry -14.051470 +bs -14.075854 +br -14.117433 +pt -14.160246 +c_ -14.177734 +ui -14.204369 +uo -14.204369 +vi -14.231505 +az -14.268499 +ac -14.277898 +ub -14.316117 +oj -14.345461 +kl -14.355377 +ly -14.395734 +ce -14.406003 +_v -14.416345 +ig -14.426761 +w_ -14.458470 +gr -14.524059 +iv -14.558006 +ci -14.569502 +rk -14.581090 +jw -14.592771 +ft -14.592771 +gl -14.664921 +uf -14.702396 +rg -14.740870 +rr -14.740870 +nu -14.753926 +p_ -14.780398 +gu -14.807365 +rd -14.821040 +rs -14.821040 +eq -14.848785 +b_ -14.862860 +yp -14.891430 +av -14.905929 +rn -14.905929 +yu -14.920576 +tc -14.935373 +ck -14.935373 +nl -15.043433 +ty -15.059552 +wn -15.059552 +oq -15.092342 +pl -15.177734 +yi -15.231505 +sp -15.268499 +rt -15.306467 +ct -15.426761 +aq -15.469197 +pp -15.535286 +zo -15.581090 +hn -15.604548 +cu -15.604548 +rl -15.628395 +qh -15.677305 +zi -15.727931 +rc -15.753926 +lt -15.753926 +gm -15.780398 +iu -15.780398 +qo -15.807365 +cy -15.807365 +za -15.834846 +yr -15.834846 +_z -15.891430 +v_ -15.920576 +f_ -16.011724 +mn -16.043433 +sm -16.043433 +my -16.142968 +yn -16.177734 +ji -16.177734 +ox -16.213358 +xf -16.213358 +gy -16.249884 +x_ -16.287358 +dp -16.287358 +ls -16.365361 +ws -16.406003 +ld -16.490892 +sl -16.535286 +kt -16.581090 +dr -16.581090 +kc -16.677305 +iy -16.677305 +wu -16.677305 +dm -16.677305 +zu -16.727931 +sx -16.727931 +rm -16.727931 +sk -16.780398 +kn -16.834846 +tp -16.834846 +ff -16.834846 +gn -16.834846 +ht -16.834846 +rb -16.834846 +cs -16.891430 +ze -16.891430 +ex -16.891430 +sy -16.950323 +gq -17.011724 +hr -17.011724 +qu -17.075854 +bw -17.142968 +wh -17.142968 +pd -17.287358 +df -17.287358 +ms -17.287358 +nz -17.287358 +lc -17.365361 +lf -17.447823 +hs -17.447823 +ky -17.447823 +yz -17.447823 +zs -17.447823 +rv -17.447823 +tm -17.535286 +nj -17.535286 +vd -17.535286 +ds -17.535286 +rh -17.628395 +yt -17.628395 +sb -17.727931 +ks -17.727931 +sc -17.727931 +dl -17.834846 +ml -17.834846 +dg -17.834846 +nm -17.834846 +xy -17.834846 +dc -17.950323 +uu -17.950323 +py -17.950323 +gh -17.950323 +yc -17.950323 +cl -17.950323 +xi -17.950323 +gd -18.075854 +ev -18.075854 +by -18.213358 +ax -18.213358 +rf -18.213358 +nq -18.213358 +vo -18.213358 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/oci b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/oci new file mode 100644 index 00000000000..0bdfc726182 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/oci @@ -0,0 +1,503 @@ +#TOTAL_BIGRAMS 6964180 +#UNIQUE_BIGRAMS 499 +#UNSEEN_LOG_PROB -19.409594 +#EXPECTED_SCORE -7.319309 +s_ -4.724303 +a_ -5.063891 +_p -5.732932 +on -5.870993 +t_ -5.877360 +_d -5.961930 +er -5.983788 +en -6.015681 +as -6.164457 +n_ -6.188883 +_c -6.198497 +re -6.220789 +an -6.284988 +ra -6.326946 +_s -6.386290 +es -6.388874 +ta -6.401393 +e_ -6.406814 +nt -6.426226 +_a -6.434319 +de -6.559524 +la -6.614447 +r_ -6.653163 +ca -6.695628 +ue -6.728443 +ar -6.736550 +_m -6.759462 +_l -6.763328 +na -6.774738 +at -6.789878 +ci -6.815923 +or -6.929965 +in -6.975148 +ma -6.976217 +co -6.985900 +ri -7.011946 +pe -7.025188 +me -7.049433 +ns -7.049570 +el -7.051932 +ia -7.056337 +_e -7.067992 +qu -7.082603 +al -7.097051 +_f -7.101340 +is -7.107755 +i_ -7.112678 +pa -7.169369 +da -7.212592 +et -7.225307 +_t -7.271450 +_r -7.278830 +st -7.279281 +ic -7.317564 +le -7.341353 +te -7.350993 +io -7.352584 +it -7.413615 +l_ -7.451186 +se -7.482372 +di -7.493304 +ti -7.493416 +tr -7.515155 +om -7.523097 +ei -7.534382 +si -7.606310 +ro -7.629587 +os -7.684100 +au -7.701588 +am -7.702149 +pr -7.713365 +li -7.762001 +un -7.762360 +ac -7.776780 +so -7.782379 +nc -7.788183 +sa -7.806849 +_q -7.825853 +lo -7.838315 +_v -7.841543 +gu -7.896607 +us -8.004346 +po -8.044693 +_g -8.047760 +ai -8.083221 +u_ -8.086089 +rt -8.107355 +ve -8.128707 +em -8.128939 +to -8.175974 +_n -8.185954 +fo -8.196125 +_u -8.244876 +nd -8.276580 +ni -8.278830 +ie -8.299763 +vi -8.306765 +ts -8.326913 +eg -8.341017 +ad -8.345390 +ec -8.355872 +fi -8.381757 +_b -8.390420 +ss -8.421193 +mi -8.453871 +_i -8.471926 +ne -8.480706 +ol -8.511597 +su -8.537920 +ge -8.542470 +il -8.582569 +og -8.583204 +mp -8.608451 +ir -8.611284 +no -8.643148 +ce -8.662912 +mb -8.691575 +mo -8.692089 +ur -8.716281 +tu -8.727828 +va -8.741773 +im -8.757018 +ba -8.777689 +gr -8.784707 +c_ -8.787359 +ap -8.791300 +ot -8.814523 +_o -8.837367 +rr -8.877628 +ut -8.906862 +eu -8.911942 +rs -8.926282 +ls -8.946478 +be -8.972674 +id -8.981024 +ga -9.035641 +ha -9.051163 +oc -9.057992 +do -9.059428 +d_ -9.137314 +ch -9.139532 +aq -9.150086 +ct -9.160125 +op -9.171189 +ua -9.178973 +gi -9.246951 +ul -9.247706 +h_ -9.253258 +pu -9.269148 +br -9.274526 +ng -9.300678 +bl -9.342563 +rm -9.356618 +_j -9.358929 +ab -9.368482 +rc -9.405654 +ig -9.412132 +iv -9.417647 +ui -9.418356 +mu -9.436327 +pl -9.473840 +av -9.475756 +lh -9.480780 +uc -9.500451 +fa -9.527868 +ep -9.616641 +b_ -9.633655 +ip -9.637609 +pi -9.690376 +ob -9.728356 +od -9.728707 +sc -9.741241 +ov -9.742305 +rn -9.748528 +cr -9.753169 +fr -9.814336 +fe -9.821067 +nh -9.872764 +cc -9.907757 +p_ -9.919746 +jo -9.928602 +af -9.971842 +ag -9.973507 +cu -9.977470 +rd -9.990687 +ud -9.991320 +go -9.999355 +sp -10.011919 +du -10.043272 +bi -10.048089 +m_ -10.067075 +gl -10.158585 +lu -10.163091 +ev -10.176214 +vo -10.177653 +eo -10.180294 +ed -10.185592 +ea -10.201360 +ex -10.225462 +iu -10.276709 +o_ -10.301592 +ll -10.344582 +iz -10.381964 +tg -10.399765 +if -10.419206 +cl -10.422899 +ja -10.428598 +he -10.460935 +ib -10.479744 +za -10.499401 +um -10.590373 +th -10.615503 +nb -10.646714 +ub -10.650705 +ho -10.676918 +bo -10.687128 +rg -10.691918 +sm -10.758183 +ru -10.760338 +cs -10.770072 +ef -10.786444 +dr -10.801154 +eb -10.853088 +lt -11.017277 +ou -11.084064 +nf -11.109013 +up -11.111302 +xi -11.116352 +ju -11.123267 +oa -11.130680 +ps -11.135798 +hi -11.165468 +ii -11.187041 +oi -11.206491 +uo -11.207960 +pt -11.244687 +_h -11.314197 +nn -11.341353 +fl -11.418639 +z_ -11.467666 +tz -11.478265 +g_ -11.497902 +rv -11.563477 +ug -11.637444 +bu -11.640748 +of -11.654039 +fu -11.662744 +rl -11.731522 +_x -11.784616 +nu -11.800785 +je -11.821629 +nq -11.832165 +gn -11.850408 +ej -11.860387 +hs -12.076886 +xe -12.097619 +aj -12.185592 +rq -12.216808 +tl -12.219769 +rb -12.228690 +lm -12.262898 +oe -12.285473 +sv -12.309457 +ds -12.390559 +sh -12.453073 +eq -12.456562 +x_ -12.513564 +sq -12.523288 +xt -12.567872 +ze -12.621691 +sf -12.624305 +y_ -12.664088 +iq -12.669476 +v_ -12.696723 +pc -12.696723 +ms -12.703616 +nv -12.704999 +rf -12.709154 +lq -12.745680 +tj -12.771520 +mt -12.817137 +xp -12.868885 +_z -12.978305 +ix -13.036294 +sl -13.091277 +xa -13.093086 +lg -13.126043 +nr -13.167373 +oj -13.178853 +xv -13.180775 +bs -13.207960 +mm -13.257816 +bj -13.263917 +mn -13.284439 +ay -13.290653 +bt -13.322131 +zo -13.324254 +dm -13.334917 +_w -13.393900 +cn -13.407342 +lc -13.427741 +f_ -13.450751 +hu -13.471779 +nj -13.566615 +bb -13.646714 +k_ -13.673530 +ax -13.681674 +_k -13.867336 +ph -13.873541 +ye -13.927391 +ao -13.946887 +nl -13.966650 +ee -13.986688 +lb -14.000203 +ux -14.000203 +ny -14.105813 +uv -14.120497 +uf -14.124192 +uq -14.176933 +vc -14.192363 +az -14.255789 +ae -14.259847 +nz -14.276195 +lv -14.305257 +ld -14.382794 +wi -14.382794 +ck -14.382794 +wa -14.450751 +ki -14.483595 +vr -14.493117 +_y -14.497902 +oh -14.512354 +lf -14.522069 +tm -14.551613 +rp -14.556596 +ox -14.571651 +sb -14.576704 +zi -14.649373 +ik -14.670826 +gs -14.670826 +ke -14.709154 +ah -14.830655 +gt -14.879773 +tt -14.943620 +oq -14.976635 +pp -14.983329 +bc -15.010423 +ya -15.010423 +ry -15.131609 +we -15.146560 +yo -15.154093 +tn -15.154093 +dv -15.169280 +xc -15.184628 +hn -15.192363 +ey -15.200141 +oo -15.255789 +ka -15.263917 +sn -15.305257 +gm -15.305257 +bn -15.365200 +ty -15.400605 +ak -15.455398 +ez -15.493117 +kh -15.551613 +ym -15.581775 +tf -15.591971 +sr -15.622998 +oy -15.676240 +tc -15.698099 +ly -15.709154 +rk -15.754242 +hr -15.754242 +uj -15.777326 +xu -15.873541 +uu -15.886032 +lp -15.937106 +cq -15.990055 +fs -16.003602 +ew -16.073311 +tb -16.116812 +ys -16.116812 +ok -16.239669 +tp -16.255789 +ln -16.272090 +xo -16.288579 +gh -16.288579 +nk -16.339205 +gj -16.356483 +yr -16.391672 +eh -16.409594 +qi -16.409594 +aw -16.409594 +dg -16.427741 +uy -16.464736 +ku -16.502703 +w_ -16.522069 +dw -16.522069 +gd -16.541698 +zh -16.581775 +q_ -16.622998 +uz -16.665433 +yv -16.687128 +uh -16.687128 +ht -16.687128 +vu -16.687128 +dn -16.709154 +ij -16.731522 +cb -16.754242 +hm -16.800785 +bd -16.824631 +xx -16.873541 +ji -16.950162 +sd -16.976635 +rh -17.031082 +gg -17.059097 +sy -17.059097 +cf -17.176933 +cy -17.176933 +ko -17.207960 +dy -17.239669 +np -17.305257 +j_ -17.373970 +ft -17.373970 +sw -17.409594 +wo -17.446120 +zz -17.483595 +yl -17.483595 +fg -17.522069 +sg -17.561597 +ff -17.561597 +cm -17.602239 +zu -17.602239 +nx -17.644059 +dj -17.644059 +ow -17.687128 +rj -17.687128 +fm -17.777326 +aa -17.777326 +rx -17.924167 +sk -17.924167 +wt -17.976635 +gv -17.976635 +kr -17.976635 +wh -18.031082 +zf -18.087666 +gb -18.087666 +km -18.146560 +hy -18.146560 +yi -18.146560 +mv -18.146560 +vd -18.207960 +yc -18.207960 +dd -18.207960 +fc -18.272090 +tx -18.272090 +lr -18.339205 +nw -18.339205 +my -18.339205 +wn -18.339205 +sz -18.339205 +zt -18.339205 +yu -18.339205 +rz -18.409594 +lk -18.409594 +gp -18.483595 +yp -18.483595 +yd -18.561597 +ml -18.561597 +uk -18.561597 +by -18.731522 +yt -18.731522 +vl -18.731522 +yf -18.731522 +gc -18.731522 +rw -18.924167 +kg -18.924167 +bw -18.924167 +gk -19.031082 +xf -19.031082 +pv -19.031082 +yw -19.146560 +hw -19.146560 +zm -19.146560 +ek -19.146560 +kl -19.146560 +ws -19.146560 +cd -19.146560 +fk -19.146560 +vb -19.146560 +nm -19.272090 +gy -19.272090 +cp -19.272090 +oz -19.272090 +zb -19.272090 +py -19.409594 +fb -19.409594 +wl -19.409594 +wu -19.409594 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ori b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ori new file mode 100644 index 00000000000..4305a08dcb8 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ori @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 1998546 +#UNIQUE_BIGRAMS 1313 +#UNSEEN_LOG_PROB -14.557341 +#EXPECTED_SCORE -7.798519 +ର_ -4.121455 +_କ -5.058590 +ଲ_ -5.513689 +କର -5.662855 +_ଏ -5.715517 +ବ_ -5.740926 +_ପ -5.751854 +ହ_ -5.880075 +_ସ -5.906202 +_ହ -5.916324 +କ_ -5.925509 +_ବ -5.971876 +ନ_ -6.165596 +ଏହ -6.242050 +ଏ_ -6.256217 +_ମ -6.330490 +_ଅ -6.338880 +ତ_ -6.341453 +ଥଲ -6.342507 +_ଦ -6.437727 +ପର -6.497977 +_ନ -6.516561 +_ର -6.593176 +_ଭ -6.692563 +_ଜ -6.718783 +ଣ_ -6.779899 +ଇ_ -6.825102 +ଳ_ -7.061311 +ଇଥ -7.185265 +_ଯ -7.195598 +ହର -7.211665 +ହଇ -7.222298 +_ତ -7.266405 +_ଆ -7.295368 +ଟ_ -7.305610 +_ଗ -7.350968 +ଛ_ -7.382540 +_ଥ -7.494459 +ସହ -7.497325 +ରଥ -7.558607 +ବର -7.588028 +ଗ_ -7.600723 +ଥବ -7.622749 +ସର -7.676524 +ଯଏ -7.706668 +ୟ_ -7.792568 +ଜ_ -7.871175 +ରବ -7.877442 +ସମ -7.909887 +ମ_ -7.916673 +ତର -7.931576 +_ଲ -7.931753 +ସ_ -7.957898 +ପଇ -7.965276 +ଥଏ -8.014267 +ମନ -8.019689 +ରଣ -8.066914 +ରଯ -8.097629 +ଶର -8.157174 +_ଶ -8.169799 +ଶ_ -8.176258 +ବଶ -8.204301 +ଜବ -8.227481 +ଯଇ -8.250380 +_ଉ -8.301390 +ମସ -8.304126 +ଅନ -8.332002 +ଠର -8.393301 +ରତ -8.415066 +ନସ -8.432418 +ହଏ -8.444439 +ରହ -8.459098 +ଭବ -8.502421 +ଗର -8.524846 +ନମ -8.539544 +ଅଟ -8.539813 +ନକ -8.547625 +ଜଣ -8.563378 +ରଗ -8.577924 +ଧର -8.580133 +ବହ -8.580409 +ଭର -8.581515 +ଦନ -8.590948 +ଏବ -8.618204 +_ଚ -8.633030 +ବକ -8.639637 +ଷ_ -8.692712 +ବଳ -8.702001 +ସବ -8.706216 +ଉପ -8.718631 +ରର -8.742858 +ହଲ -8.751854 +ଦ_ -8.773172 +ୟର -8.775385 +ମୟ -8.779820 +ହତ -8.813850 +କଲ -8.814175 +ରଜ -8.820036 +ମର -8.836772 +ଥ_ -8.877951 +ଦଇ -8.885784 +_ଡ -8.916150 +ଇନ -8.927001 +ରକ -8.931224 +ଅଧ -8.956105 +ମହ -8.958976 +କହ -8.973780 +ମଳ -8.995354 +ପ_ -9.011284 +ଦବ -9.022127 +_ଧ -9.042157 +କମ -9.042918 +ହକ -9.054770 +ଯଗ -9.077990 +ନର -9.082288 +ଜନ -9.084637 +ହବ -9.096443 +ଧକ -9.123567 +କଳ -9.126792 +ଥର -9.140986 +ରପ -9.145885 +ଡଇ -9.170215 +ଳର -9.190317 +ଦଶ -9.210275 +ଧ_ -9.224887 +ଆ_ -9.225319 +ତୟ -9.230947 +ତକ -9.239648 +ଲକ -9.258536 +ନହ -9.271862 +ଦର -9.273201 +ଖ_ -9.283511 +ବଲ -9.284411 +ନଜ -9.293442 +ଭଗ -9.294801 +ଜର -9.303442 +ବନ -9.305724 +ହଉ -9.330142 +ଦଖ -9.343680 +_ଫ -9.354035 +ଧନ -9.372577 +ନତ -9.422229 +_ଖ -9.426694 +କଟ -9.455293 +ଯହ -9.457321 +ତହ -9.463423 +_ଠ -9.470576 +ଆର -9.535521 +ଜତ -9.544657 +ଲଗ -9.585114 +କତ -9.586778 +ସନ -9.605776 +ଗଟ -9.610283 +ରଚ -9.615936 +ଉଥ -9.617069 +ଯଉ -9.638198 +ମତ -9.643962 +ଟକ -9.687941 +ବସ -9.699298 +ଆସ -9.713167 +ରମ -9.737610 +ହସ -9.753100 +ଅଛ -9.761847 +ତନ -9.774436 +_ଘ -9.776334 +ଇଛ -9.789051 +ସଧ -9.794169 +ରଖ -9.794169 +ହଯ -9.807045 +ଭତ -9.808986 +କବ -9.820036 +ଗଲ -9.822648 +ଶଷ -9.830514 +ଉଛ -9.847705 +_ଇ -9.848370 +_ଛ -9.851701 +ତଥ -9.854372 +ଟଏ -9.857717 +ପତ -9.885442 +ଳନ -9.889545 +କଶ -9.899852 +ଭଷ -9.903996 +ଦଆ -9.914408 +ବଷ -9.914408 +ବଧ -9.943966 +ବଦ -9.949666 +ଆଦ -9.951094 +ପଳ -9.954671 +ନବ -9.971967 +କଣ -9.982882 +ହଛ -9.992410 +ପଜ -10.015387 +ଗତ -10.017630 +ଟର -10.056306 +ଖର -10.058614 +ଲଖ -10.060155 +ବତ -10.067107 +ଚତ -10.075651 +ଚନ -10.086598 +ତବ -10.090528 +ଚର -10.094469 +ଠ_ -10.103177 +ଆଯ -10.108745 +କଛ -10.121555 +ଳକ -10.131238 +ବଜ -10.145885 +ଣସ -10.188210 +ବବ -10.209420 +ଖବ -10.219713 +ଏଠ -10.231815 +ନଇ -10.232683 +ଖଯ -10.261634 +ନୟ -10.277674 +ପଖ -10.282162 +ଡ_ -10.290274 +ଗଣ -10.314890 +ମକ -10.327820 +ରସ -10.328749 +ପଥ -10.330607 +ତଆ -10.338062 +ସଥ -10.338997 +ଭ_ -10.340868 +ମଟ -10.353091 +ମଣ -10.370187 +ସକ -10.370187 +ଅଭ -10.373056 +ଏକ -10.380735 +ସଲ -10.385555 +ହଥ -10.389423 +ନଆ -10.392330 +ଏଥ -10.407938 +ତଣ -10.419755 +ଣର -10.448720 +ଷର -10.496934 +ବଣ -10.507403 +ଣତ -10.521128 +ହଳ -10.541426 +ପନ -10.547895 +ହନ -10.586223 +ତଳ -10.591783 +ପଦ -10.594013 +_ଟ -10.595129 +ଯବ -10.629023 +ନଦ -10.639349 +ପଣ -10.643962 +ଭଳ -10.653232 +ଫଳ -10.656724 +ଳତ -10.660224 +_ଔ -10.671953 +ବୟ -10.677854 +ଷଧ -10.688536 +ଗବ -10.710141 +ସତ -10.727171 +ଉ_ -10.729621 +ଆମ -10.729621 +ଇବ -10.734532 +ଷୟ -10.753100 +ଅବ -10.753100 +ଔଷ -10.764356 +ପଟ -10.806398 +ଠକ -10.810281 +ଘର -10.833804 +ସଭ -10.836442 +ରଧ -10.837762 +ପଠ -10.839084 +ଅତ -10.849702 +ନଥ -10.855040 +ଇପ -10.880671 +ଷଣ -10.883395 +ସଠ -10.891600 +ଶଳ -10.930519 +ଆକ -10.940415 +ନଟ -10.943255 +ରଳ -10.947526 +ବପ -10.957540 +ଇଟ -10.970517 +ଇତ -10.970517 +ଆଗ -10.973417 +ମଜ -10.985076 +କଥ -10.989472 +_ଓ -10.990940 +ଇର -10.992410 +ଇଲ -11.028144 +ପକ -11.032674 +ନଗ -11.034187 +ଦହ -11.044823 +ବମ -11.054002 +ମଶ -11.085029 +ୟନ -11.103971 +ଛଟ -11.113536 +ହପ -11.119948 +ଜଗ -11.123164 +ଜକ -11.124775 +ଡକ -11.139356 +ଚଲ -11.145885 +ଲଭ -11.152442 +ଦଳ -11.167307 +ପଲ -11.170631 +ରଇ -11.182326 +ସଦ -11.189052 +ଶବ -11.194117 +ଗଠ -11.209420 +ଦଲ -11.212843 +ୟକ -11.216274 +ଚଳ -11.231815 +ଗଡ -11.245771 +ଶଣ -11.247525 +ଳଥ -11.249281 +ଣକ -11.277674 +ଗଜ -11.284861 +ଆଧ -11.286663 +ଇଯ -11.297524 +ଶସ -11.345557 +ଫଲ -11.349319 +ହଜ -11.368277 +ରଦ -11.375930 +ଖଳ -11.383625 +ଭନ -11.391361 +ହଦ -11.403042 +ଘଟ -11.404999 +ବଭ -11.406957 +ମଲ -11.414820 +ଅଶ -11.416792 +ଜମ -11.422725 +ଭୟ -11.430673 +ଜପ -11.432668 +ଫର -11.456814 +ଖଲ -11.464953 +ଲର -11.469040 +ଆଲ -11.473138 +କଦ -11.481371 +କଠ -11.481371 +ରଷ -11.500067 +ଲମ -11.510559 +ସଗ -11.514778 +ଭମ -11.519008 +ଖଇ -11.527507 +ସଫ -11.529640 +ୟସ -11.531776 +ଶକ -11.536057 +ଠନ -11.544657 +ମଦ -11.544657 +ଦୟ -11.544657 +ଗଳ -11.546815 +ଅ_ -11.577373 +ଚ_ -11.577373 +ବଠ -11.588445 +ଶଖ -11.610847 +ତତ -11.624458 +ଶତ -11.631311 +_ଝ -11.635899 +ତମ -11.635899 +ହଣ -11.647431 +ଣଯ -11.649749 +ୟତ -11.652070 +ଭଜ -11.652070 +ସପ -11.677854 +ସଇ -11.687345 +ଣଶ -11.696900 +ରଛ -11.701701 +ଦସ -11.704107 +କଷ -11.708932 +ଷକ -11.716200 +ଯଥ -11.721066 +ଏସ -11.728396 +ସଚ -11.730847 +ଉଭ -11.743167 +ଜଳ -11.765612 +ଗକ -11.773172 +ଯତ -11.778235 +ଜଲ -11.788412 +ରଟ -11.788412 +ବଟ -11.790968 +ରୟ -11.801236 +ଜୟ -11.806398 +ଶଶ -11.808986 +ନଳ -11.816777 +ଯଦ -11.821995 +ଫସ -11.827232 +ଉଠ -11.829857 +ଟଣ -11.843057 +ଆୟ -11.848370 +ସୟ -11.851035 +ଅପ -11.859057 +ଫଟ -11.861741 +ମଧ -11.861741 +ଚଉ -11.883395 +ଗମ -11.888860 +ଅଲ -11.891600 +ଭଟ -11.897096 +ନଭ -11.899852 +ବଢ -11.902613 +ତଲ -11.922091 +ଧବ -11.922091 +ରଲ -11.933340 +ପଶ -11.938998 +ଖକ -11.941835 +ଥପ -11.944677 +ନଯ -11.947526 +ନନ -11.950380 +ରଶ -11.953239 +କଇ -11.967623 +କକ -11.993881 +ଆଣ -12.002741 +ସଆ -12.011656 +ଅସ -12.032674 +ବଚ -12.032674 +ୟମ -12.035702 +ଲପ -12.038736 +d_ -12.047876 +ଓ_ -12.057075 +କନ -12.066333 +ଲବ -12.085029 +ଗଇ -12.085029 +e_ -12.088169 +ଗଆ -12.088169 +ଛର -12.088169 +ଆଜ -12.094469 +ପଡ -12.097629 +ଢ_ -12.097629 +ଦଗ -12.113536 +ଜଧ -12.116738 +ଳବ -12.116738 +ଲଚ -12.126388 +ଖଥ -12.136103 +ଲଇ -12.142617 +ଳଦ -12.142617 +ଇଗ -12.152442 +ଣନ -12.159030 +ମଗ -12.159030 +ଣବ -12.162335 +ଉଳ -12.165648 +ଲସ -12.185686 +ଦକ -12.192427 +ଯୟ -12.195810 +ଲଜ -12.209420 +କପ -12.212843 +ଗହ -12.212843 +ସଯ -12.240521 +ଲହ -12.240521 +ଣଷ -12.244019 +ଶନ -12.244019 +ପସ -12.244019 +ଯଜ -12.268741 +ଚକ -12.283061 +ଚପ -12.290274 +ଗଦ -12.290274 +ଉଦ -12.301163 +ବଝ -12.308468 +ଏପ -12.312134 +ନଲ -12.319495 +ଉର -12.334330 +ସସ -12.338062 +ଗଛ -12.341805 +ଆଖ -12.349319 +କଡ -12.353091 +ଲଲ -12.360664 +ପହ -12.372099 +ରଫ -12.375930 +ହମ -12.375930 +ମଆ -12.399138 +ଆଇ -12.399138 +ନଶ -12.410883 +ଛଡ -12.414820 +nd -12.418767 +ଅଣ -12.422725 +ଚଷ -12.434664 +ଆହ -12.438666 +ନପ -12.442679 +ଠତ -12.458844 +ଦତ -12.458844 +କସ -12.458844 +ଭଇ -12.462914 +ସଘ -12.462914 +ଯକ -12.466995 +ଘ_ -12.491727 +ଦଉ -12.504255 +ଶମ -12.516891 +_ଋ -12.521128 +he -12.533915 +th -12.538202 +ଏମ -12.538202 +ଶଇ -12.542502 +an -12.555480 +ସଛ -12.555480 +ଅମ -12.555480 +ଗପ -12.559832 +ଘଷ -12.568576 +ଳସ -12.572967 +ଟଳ -12.590669 +ହଭ -12.599602 +ଏତ -12.617636 +ମପ -12.622180 +ହୟ -12.626739 +_t -12.649749 +ଝଅ -12.654395 +ଲଳ -12.659056 +ଯପ -12.663733 +ଷତ -12.673132 +in -12.677854 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/orm b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/orm new file mode 100644 index 00000000000..96e0e7e1095 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/orm @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 37553504 +#UNIQUE_BIGRAMS 745 +#UNSEEN_LOG_PROB -20.512242 +#EXPECTED_SCORE -7.083730 +a_ -4.282907 +aa -4.354762 +an -5.264076 +n_ -5.386512 +i_ -5.601167 +ee -5.640148 +oo -6.002216 +ma -6.005499 +ii -6.145193 +ra -6.163229 +_k -6.226813 +sa -6.320306 +ka -6.333846 +uu -6.344517 +at -6.378276 +_a -6.408314 +u_ -6.421481 +e_ -6.449504 +ha -6.470019 +ar -6.484756 +_i -6.486680 +ba -6.537656 +ti -6.553818 +am -6.561337 +ga -6.563981 +_h -6.595424 +_d -6.632160 +al -6.637673 +ir -6.681449 +ta -6.690832 +na -6.691849 +o_ -6.747722 +_b -6.799028 +ab -6.805872 +_g -6.821229 +dh -6.822590 +in -6.837899 +tt -6.904624 +_j -7.011031 +is -7.054027 +la -7.126485 +_m -7.133060 +hi -7.146463 +ni -7.160662 +wa -7.176980 +da -7.218161 +ya -7.229324 +kk -7.311433 +_w -7.318161 +um -7.333849 +mm -7.386737 +hu -7.474453 +rr -7.479628 +_o -7.492141 +_s -7.510860 +_t -7.553504 +_q -7.569038 +ji -7.582495 +qa -7.590514 +ro -7.594763 +ak -7.606979 +fa -7.609100 +mo -7.616273 +un -7.666737 +or -7.692008 +es -7.732400 +ad -7.733544 +ch -7.735475 +en -7.749759 +nn -7.786760 +ke -7.852542 +ss -7.889605 +tu -7.896811 +bi -7.912349 +ot -7.929508 +om -7.951031 +it -8.058951 +af -8.073635 +yy -8.098217 +f_ -8.102086 +_f -8.131088 +_y -8.176391 +as -8.188321 +si -8.194745 +mi -8.246576 +_n -8.248234 +iy -8.253008 +ur -8.302485 +su -8.306007 +du -8.330123 +to -8.341343 +je -8.396096 +mu -8.425289 +er -8.427325 +le -8.437251 +dd -8.447488 +li -8.495236 +ne -8.499776 +il -8.515576 +on -8.528484 +bb -8.539120 +ag -8.541723 +ru -8.585576 +re -8.638255 +di -8.638591 +fi -8.663174 +ja -8.691499 +ay -8.728060 +ac -8.738655 +ed -8.761848 +uf -8.764135 +ku -8.765722 +ny -8.821134 +te -8.821760 +_u -8.835804 +gu -8.894944 +bo -8.934358 +_l -8.940404 +go -8.962542 +yo -8.987577 +ko -8.994242 +rg -8.999207 +he -9.008635 +de -9.033181 +bu -9.047441 +ri -9.062870 +me -9.076038 +se -9.101516 +s_ -9.140228 +so -9.143570 +ll -9.164912 +ho -9.171476 +ol -9.184449 +ah -9.256282 +be -9.259810 +ye -9.265166 +ud -9.267177 +id -9.323339 +if -9.365695 +fu -9.366227 +nd -9.379932 +ok -9.385934 +od -9.404143 +ub -9.423481 +im -9.434790 +bs -9.471846 +_e -9.472692 +nu -9.484917 +_r -9.498275 +ut -9.511280 +gg -9.622013 +of -9.629875 +ul -9.642992 +aq -9.653907 +aj -9.665466 +jj -9.673945 +_c -9.682885 +no -9.692612 +sh -9.800535 +rs -9.859877 +ff -9.882507 +aw -9.888357 +lu -9.934715 +qo -9.937124 +ge -9.960091 +rt -10.014365 +ib -10.026134 +nt -10.056249 +oj -10.060304 +rb -10.088428 +ec -10.117239 +ef -10.138300 +ki -10.159937 +mn -10.166412 +eg -10.220342 +ek -10.237539 +l_ -10.239581 +ng -10.320192 +et -10.332424 +do -10.350016 +op -10.385394 +lo -10.385600 +ca -10.401309 +ic -10.411685 +ph -10.425567 +rk -10.482689 +fe -10.506187 +ms -10.544001 +os -10.560267 +ns -10.564915 +rm -10.572910 +qu -10.583954 +ww -10.611277 +ij -10.627351 +yu -10.661851 +lm -10.704935 +em -10.736179 +nf -10.739788 +yi -10.796805 +ob -10.822733 +lc -10.865170 +t_ -10.882326 +gi -10.894926 +rn -10.907858 +us -10.952991 +eb -10.976717 +_x -11.062110 +st -11.078383 +oc -11.085044 +og -11.087550 +bn -11.093162 +ug -11.130054 +ci -11.154979 +xa -11.162004 +ik -11.166147 +lq -11.187044 +mt -11.190901 +_p -11.194678 +r_ -11.198555 +ig -11.256399 +uy -11.266017 +kn -11.304463 +tn -11.348563 +xi -11.367521 +qi -11.424669 +qe -11.448306 +lt -11.482744 +ft -11.526499 +bd -11.526839 +d_ -11.577599 +lf -11.640476 +yk -11.700581 +el -11.746306 +uk -11.817732 +lb -11.937237 +qq -11.967380 +cu -12.063096 +ld -12.065070 +ix -12.091149 +rd -12.092659 +mb -12.099218 +m_ -12.189823 +wo -12.191440 +br -12.199729 +xu -12.271990 +iq -12.299419 +pa -12.318327 +ih -12.364985 +po -12.383135 +jo -12.422453 +nj -12.433886 +nq -12.449917 +rq -12.486046 +pi -12.500221 +y_ -12.514761 +ax -12.530131 +ks -12.636189 +fn -12.665091 +sb -12.670341 +nk -12.696878 +ey -12.710203 +pl -12.792029 +g_ -12.806543 +fr -12.820647 +pd -12.825659 +ia -12.839389 +nc -12.870698 +we -12.952077 +fo -12.963692 +lk -13.001312 +tp -13.004781 +c_ -13.033805 +wb -13.052287 +ew -13.072332 +wu -13.089642 +io -13.096691 +sn -13.101411 +k_ -13.121127 +zi -13.125240 +rf -13.127645 +co -13.136267 +uq -13.141117 +ap -13.148424 +pr -13.164559 +wi -13.203891 +mp -13.212617 +th -13.223232 +kt -13.254052 +ea -13.311476 +iv -13.312648 +xx -13.347461 +h_ -13.405471 +ce -13.452068 +ow -13.454654 +bc -13.474194 +lg -13.474631 +kr -13.475069 +sl -13.515885 +lp -13.526273 +az -13.547734 +eq -13.580302 +gd -13.589270 +cc -13.627655 +ep -13.629114 +oy -13.668089 +_v -13.679132 +qs -13.680140 +ju -13.680140 +b_ -13.683675 +kf -13.687217 +ve -13.712264 +vi -13.761565 +fs -13.766375 +ex -13.866675 +fx -13.890981 +bl -13.907416 +oq -13.972002 +iw -14.002573 +oh -14.048051 +gr -14.054573 +ej -14.102424 +ip -14.121470 +va -14.124211 +uw -14.137305 +ou -14.178026 +sp -14.232446 +xe -14.236890 +dn -14.247312 +ty -14.266869 +df -14.279037 +_z -14.296711 +xg -14.313039 +uc -14.320094 +pp -14.320094 +qb -14.349465 +ze -14.352676 +ct -14.445625 +fd -14.452499 +ez -14.475069 +w_ -14.476819 +uj -14.482085 +yr -14.507808 +bx -14.571857 +gh -14.575604 +nb -14.575604 +ht -14.588797 +mr -14.619412 +pt -14.620380 +ts -14.621348 +gn -14.627169 +oa -14.694839 +vo -14.733037 +hm -14.740379 +dg -14.744592 +sg -14.760498 +nx -14.773351 +ys -14.790668 +fb -14.798310 +iz -14.812610 +rl -14.818148 +au -14.886320 +ua -14.912146 +tr -14.963999 +ry -14.970151 +p_ -14.970151 +ws -14.988767 +py -15.007626 +ux -15.038323 +za -15.122154 +kh -15.138690 +ai -15.175180 +ev -15.206794 +qn -15.230229 +rc -15.369654 +yt -15.400893 +pe -15.422664 +rj -15.451638 +ie -15.460272 +xo -15.488252 +uh -15.506019 +ue -15.525820 +lh -15.527633 +nv -15.536735 +ui -15.566254 +yx -15.590692 +mf -15.623285 +eo -15.678628 +z_ -15.698920 +sf -15.711233 +sm -15.780901 +gs -15.807093 +eh -15.827054 +yd -15.831527 +hd -15.838264 +dr -15.847295 +ls -15.888649 +yl -15.895658 +gl -15.928825 +cr -15.957873 +uz -15.995026 +zd -16.000053 +bj -16.017786 +uo -16.028018 +dw -16.056536 +ae -16.101748 +xt -16.118050 +pu -16.140076 +fk -16.185164 +gt -16.193777 +ck -16.199548 +sx -16.202442 +ov -16.240603 +ly -16.270660 +nl -16.285927 +gb -16.307576 +sw -16.307576 +up -16.329554 +sr -16.374542 +ml -16.384367 +zo -16.407557 +ds -16.500666 +wn -16.573730 +tl -16.588797 +hn -16.611697 +tw -16.642808 +jn -16.670591 +rp -16.765839 +ei -16.787405 +yf -16.796122 +sj -16.881673 +oi -16.886320 +lx -16.890981 +sk -16.919270 +v_ -16.924039 +bh -16.943276 +hr -16.948125 +wh -16.977569 +yn -17.053920 +sc -17.085629 +md -17.090982 +cy -17.101748 +gy -17.101748 +rh -17.112596 +hg -17.118050 +sy -17.173760 +nz -17.173760 +lj -17.185164 +tc -17.214077 +j_ -17.214077 +sq -17.231707 +q_ -17.279801 +mh -17.292079 +hb -17.323240 +hc -17.368028 +eu -17.414251 +rw -17.448199 +rx -17.468957 +qx -17.468957 +tb -17.475944 +bt -17.475944 +jr -17.525820 +fl -17.525820 +x_ -17.555114 +dm -17.569987 +dp -17.615550 +km -17.646744 +nh -17.654650 +rv -17.654650 +av -17.719501 +qt -17.753053 +wr -17.761565 +cl -17.778740 +ox -17.787405 +tv -17.796122 +nm -17.849561 +zu -17.849561 +kl -17.858663 +qw -17.933626 +dk -17.962772 +لي -18.033161 +kw -18.107162 +fc -18.196660 +hy -18.208248 +fq -18.243581 +dl -18.329554 +zz -18.342265 +pc -18.368028 +yp -18.368028 +by -18.368028 +ym -18.394260 +oe -18.434524 +xr -18.448199 +js -18.448199 +wf -18.490019 +dc -18.623285 +wm -18.638882 +ps -18.654650 +tm -18.804892 +hk -18.914517 +wz -18.972620 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pan b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pan new file mode 100644 index 00000000000..c7299f5d82c --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pan @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 3613843 +#UNIQUE_BIGRAMS 1275 +#UNSEEN_LOG_PROB -14.819169 +#EXPECTED_SCORE -7.687971 +ਤ_ -4.709843 +_ਵ -5.073328 +ਰ_ -5.130746 +ਨ_ -5.176775 +_ਸ -5.283654 +_ਕ -5.497570 +ਲ_ -5.560196 +ਦ_ -5.569930 +_ਅ -5.746998 +_ਨ -5.759898 +ਕ_ -5.773220 +ਚ_ -5.846625 +_ਪ -5.908681 +ਵਚ -5.941427 +ਆ_ -5.957735 +_ਜ -5.986782 +_ਮ -6.258695 +_ਹ -6.357233 +_ਇ -6.472893 +ਣ_ -6.517987 +_ਤ -6.523742 +_ਦ -6.545914 +_ਬ -6.547407 +ਅਤ -6.627913 +_ਰ -6.717668 +ਮ_ -6.861079 +ਕਰ -6.905423 +_ਲ -6.981173 +ਸ_ -7.115165 +_ਗ -7.120822 +ਜ_ -7.231053 +_ਉ -7.277184 +ਇਕ -7.300342 +ਹ_ -7.319281 +ਰਤ -7.509487 +ਵ_ -7.555683 +ਕਤ -7.559970 +_ਆ -7.566842 +ਬ_ -7.612519 +_ਭ -7.617371 +ਤਰ -7.642676 +ਡ_ -7.667946 +ਗ_ -7.696397 +ਹਦ -7.731177 +ਪਰ -7.777287 +ਜਦ -7.829362 +ਵਰ -7.847821 +ਸਰ -7.885179 +ਟ_ -7.979459 +ਵਲ -8.018988 +ਖ_ -8.020542 +ਦਰ -8.027296 +ਨਲ -8.047327 +_ਚ -8.071931 +ਰਨ -8.089113 +ਸਮ -8.113451 +ਕਲ -8.183100 +ਲਕ -8.193814 +ਦਆ -8.268787 +ਈ_ -8.270265 +ਗਰ -8.289122 +ਰਕ -8.377702 +ਰਜ -8.424941 +_ਖ -8.428238 +ਮਰ -8.430164 +ਥ_ -8.449154 +ਪਜ -8.471511 +ਸਤ -8.490338 +ਭਰ -8.492493 +ਗਆ -8.493212 +ਮਨ -8.494940 +ਹਰ -8.496957 +ਸਲ -8.509705 +ਪਣ -8.556584 +ਬਰ -8.573062 +ਕਸ -8.586657 +ਸਨ -8.594506 +ਰਹ -8.597750 +ਮਲ -8.605971 +ੜ_ -8.616117 +ਹਲ -8.660012 +ਰਦ -8.661304 +ਜਬ -8.662113 +ਇਆ -8.668434 +ਸਕ -8.685262 +ਸਦ -8.699298 +ਨਮ -8.724238 +ਆਪ -8.731177 +ਜਨ -8.737467 +ਪ_ -8.744984 +ਰਮ -8.749272 +_ਫ -8.789335 +ਹਤ -8.819318 +ਧ_ -8.823834 +ਨਵ -8.860661 +ਧਰ -8.895598 +ਉਤ -8.906477 +ਤਕ -8.914160 +ਘ_ -8.955578 +ਸਹ -8.976943 +ਪਹ -8.995976 +ਬਲ -9.009286 +ਮਹ -9.021060 +ਰਸ -9.043635 +ਸਘ -9.050393 +ਹਨ -9.080334 +ਹਣ -9.082064 +ਪਤ -9.101231 +ਪਡ -9.106942 +_ਯ -9.125329 +ਮਤ -9.132930 +ਸਥ -9.134275 +ਕਦ -9.141923 +ਹਸ -9.147345 +ਜਸ -9.158481 +ਬਣ -9.162135 +ਰਪ -9.179391 +ਤਨ -9.206730 +ਇਸ -9.209327 +ਜਵ -9.224770 +ਦਤ -9.249584 +ਰਵ -9.251286 +ਆਰ -9.270388 +ਨਹ -9.273350 +ਕਮ -9.314189 +ਅਦ -9.315715 +ਲਗ -9.322856 +ਬਹ -9.370153 +ਨਕ -9.376507 +ਕਹ -9.381291 +ਲਖ -9.407079 +ਚਰ -9.411422 +ਦਨ -9.415232 +ਉਸ -9.436651 +ਹਇ -9.464584 +ਲਆ -9.484464 +ਟਰ -9.506363 +ਸਟ -9.511890 +ਰਗ -9.515683 +ਬਦ -9.518316 +_ਧ -9.549686 +ਮਕ -9.555082 +ਗਲ -9.557486 +ਵਨ -9.560196 +ਅਨ -9.584204 +ਰਖ -9.628387 +ਸਬ -9.647791 +ਸਖ -9.679521 +ਵਖ -9.681815 +ਖਆ -9.718677 +ਰਆ -9.727449 +ਲਮ -9.743785 +ਰਬ -9.764817 +_ਘ -9.765859 +ਜਹ -9.775974 +ਪਨ -9.781936 +ਕਵ -9.806035 +ਵਧ -9.808538 +ਏ_ -9.818597 +ਗਤ -9.830543 +ਖਤ -9.867357 +ਵਡ -9.891043 +ਨਰ -9.905902 +ਜਣ -9.911658 +ਤਬ -9.913582 +ਫਰ -9.915509 +ਰਣ -9.917438 +_ਟ -9.935306 +ਝ_ -9.939220 +ਸਪ -9.942752 +ਸਵ -9.947080 +ਲਦ -9.953004 +ਮਦ -9.953399 +ਰਚ -9.957760 +ਰਥ -9.964525 +ਅਰ -9.972524 +ਅਕ -9.976138 +ਦਲ -9.976540 +_ਛ -9.976540 +ਪਦ -9.978150 +ਸਭ -9.983394 +ਨਸ -9.985011 +ਵਸ -9.987846 +ਨਤ -9.994754 +ਚਨ -10.001286 +ਦਸ -10.008258 +ਬਅ -10.019402 +ਵਦ -10.038168 +ਰਲ -10.051663 +ਪਸ -10.058458 +ਉਦ -10.073866 +ਦਵ -10.075588 +ਆਨ -10.092923 +ਤਆ -10.092923 +ਭਗ -10.096852 +ਖਣ -10.102108 +ਫਲ -10.109145 +ਧਨ -10.115774 +ਤਹ -10.125107 +ਅਜ -10.144406 +ਨਆ -10.156201 +ਵਕ -10.173155 +ਆਦ -10.173616 +ਹਈ -10.209091 +ਵਗ -10.209091 +ਬਧ -10.216671 +ਕਨ -10.242071 +ਸਗ -10.255672 +ਚਲ -10.258603 +ਇਤ -10.260561 +ਮਸ -10.269403 +ਸਸ -10.280283 +ਪਛ -10.282271 +_ਡ -10.295756 +ਹਮ -10.303303 +ਅਧ -10.310382 +ਉਣ -10.325671 +ਦਖ -10.332346 +ਮਟ -10.345791 +ਕਟ -10.361461 +ਪਲ -10.367250 +ਅਮ -10.368833 +ਮਜ -10.371474 +ਸਧ -10.384223 +ਅਗ -10.391712 +ਣਆ -10.396009 +ਜਰ -10.412237 +ਪਕ -10.416596 +ਕਝ -10.431404 +ਲਵ -10.453626 +ਇਹ -10.493932 +ਖਰ -10.507234 +ਚਕ -10.514223 +ਜਤ -10.521246 +ਠ_ -10.523008 +ਘਟ -10.533620 +ਭਵ -10.550285 +ਜਲ -10.550285 +ਗਣ -10.575649 +ਲਤ -10.575649 +_ਥ -10.576258 +ਵਜ -10.579309 +ਟਕ -10.597750 +ਤਮ -10.597750 +ਵਤ -10.625862 +ਕਈ -10.631550 +ਥਤ -10.635355 +ਹਬ -10.637897 +ਆਲ -10.649393 +ਵਆ -10.657108 +ਮਖ -10.659043 +ਰਟ -10.659043 +ਤਲ -10.659689 +ਕਆ -10.660981 +ਯਨ -10.672663 +ਬਕ -10.678539 +ਉਹ -10.696314 +ਇਲ -10.718340 +ਲਬ -10.719013 +ਭ_ -10.721707 +ਤਵ -10.732534 +ਹਏ -10.736616 +ਜਗ -10.754435 +ਦਦ -10.764817 +ਲਹ -10.765512 +ਕਬ -10.769687 +ਹਕ -10.775274 +ਇਥ -10.780180 +_ਐ -10.782991 +ਬਨ -10.798549 +ਲਪ -10.817876 +ਲੜ -10.822206 +ਕਪ -10.824376 +ਦਜ -10.838196 +ਹਵ -10.850674 +ਮਗ -10.852150 +ਖਡ -10.856584 +ਲਡ -10.869223 +ਡਰ -10.874460 +ਅਸ -10.887257 +ਧਆ -10.888770 +ਸਆ -10.889527 +ਉਪ -10.900169 +ਥਨ -10.977747 +ਧਤ -10.981778 +ਪਆ -10.990687 +ਟਮ -10.995569 +ਲਣ -11.010315 +ਰਡ -11.010315 +ਗਦ -11.036910 +ਈਆ -11.037749 +ਦਕ -11.052935 +ਜਕ -11.053783 +ਖਦ -11.064003 +ਨਗ -11.064858 +ਲਨ -11.079470 +ਬਚ -11.083796 +ਨਦ -11.104743 +ਸਚ -11.117104 +ਆਉ -11.117991 +ਜਆ -11.123324 +ਨਨ -11.128678 +ਹੜ -11.128678 +ਲਜ -11.136745 +ਵਪ -11.162135 +ਆਸ -11.175924 +ਪਟ -11.182404 +ਨਟ -11.185190 +ਚਤ -11.192645 +ਇਨ -11.194515 +ਲਸ -11.196388 +ਚਣ -11.210509 +ਦਹ -11.241104 +ਣਤ -11.257625 +ਮਡ -11.263502 +ਯਗ -11.264484 +ਟਆ -11.269403 +ਛਟ -11.270388 +ਟਨ -11.284261 +ਜੜ -11.299273 +ਤਤ -11.311397 +ਵਅ -11.317497 +ਭਆ -11.330803 +_ਔ -11.336986 +ਕੜ -11.359886 +ਖਕ -11.364090 +ਲਟ -11.365142 +ਲਈ -11.371474 +ਮਣ -11.381025 +ਰਧ -11.386359 +ਅਲ -11.419874 +ਬਬ -11.436374 +ਆਧ -11.447480 +ਜਥ -11.456427 +ਆਹ -11.457550 +ਆਚ -11.465430 +ਨਖ -11.466560 +ੜਆ -11.483606 +ਉਮ -11.489333 +ਘਰ -11.492781 +ਛਮ -11.504332 +ਤਜ -11.504332 +ਫ_ -11.506653 +ਬਤ -11.510142 +ਉਚ -11.540739 +ਦਮ -11.573214 +ਰਰ -11.606438 +ਬਸ -11.611425 +ਛ_ -11.629019 +ਭਈ -11.631550 +ਲਧ -11.659689 +ਗਇ -11.660981 +ਥਪ -11.688387 +ਡਆ -11.691025 +ਚਹ -11.702953 +_ਝ -11.717668 +ਥਰ -11.723056 +ਅਬ -11.776674 +ਮਬ -11.778075 +ਟਲ -11.779478 +ਅਖ -11.789335 +ਵਈ -11.790749 +ਣਕ -11.793581 +ਜਮ -11.800684 +ਅਪ -11.807822 +ਢ_ -11.809254 +ਅ_ -11.809254 +ਣਉ -11.833818 +ਖਲ -11.833818 +_ਢ -11.838196 +ਮਝ -11.845523 +ਭਤ -11.849937 +ਦਣ -11.866239 +ਢਲ -11.870717 +ਊਰ -11.884236 +ਆਖ -11.885745 +ਯਰ -11.888770 +ਮਢ -11.888770 +ਔਰ -11.897882 +ਣਇ -11.908585 +ਲਲ -11.919369 +ਹਠ -11.920916 +ਭਜ -11.924015 +ਜਪ -11.924015 +ਕਉ -11.934916 +ਮਚ -11.934916 +ਧਕ -11.941181 +ਬਜ -11.942752 +ਕਥ -11.947474 +ਮਆ -11.947474 +ਮਧ -11.952212 +ਣਨ -11.961735 +ਰਭ -11.979359 +ਨਭ -11.985821 +ਡਲ -12.003743 +ਬਗ -12.005383 +ਅਵ -12.013613 +ਯਕ -12.016918 +ਤਪ -12.023551 +ਗਵ -12.025214 +ਬੜ -12.025214 +ਉਥ -12.038588 +ਜਧ -12.047010 +ਕਣ -12.047010 +ਗਭ -12.048700 +ਵਬ -12.062295 +ਪੜ -12.064003 +ਲਰ -12.069140 +ਉਕ -12.098602 +_ਈ -12.102108 +ਰਫ -12.105622 +ਵਟ -12.110910 +ਚਦ -12.119766 +ਤਸ -12.130466 +ਆਇ -12.134051 +ਹਚ -12.144857 +ਚਆ -12.161221 +ਪਖ -12.166717 +ਗਸ -12.172234 +ਨੜ -12.177772 +ਲਚ -12.181476 +ਅਹ -12.187050 +ਥਵ -12.196388 +ੜਈ -12.203902 +ਸਫ -12.209563 +ਖਜ -12.211455 +ਨਜ -12.211455 +ਇਮ -12.238208 +_ਓ -12.240138 +ਨਪ -12.267433 +ਖਨ -12.269403 +ਓ_ -12.281277 +ਨਬ -12.291247 +ਕਢ -12.291247 +ਸਇ -12.293249 +ਵਵ -12.299273 +ਟਪ -12.309369 +ਆਟ -12.311397 +ਹਥ -12.311397 +ਵਹ -12.317497 +ਗਮ -12.342159 +ਹਆ -12.350474 +ਬਠ -12.356742 +ਨਅ -12.371474 +ਆਣ -12.386359 +ਛਡ -12.388498 +ਡਕ -12.390640 +ਸਜ -12.397085 +ਫਟ -12.399240 +ਯਤ -12.407892 +ਸਡ -12.410063 +ੜਕ -12.416596 +ਛਣ -12.423159 +ਕਡ -12.431956 +ਥਕ -12.440806 +ਬਈ -12.445252 +_ੲ -12.456427 +_ਏ -12.476763 +_ਠ -12.479041 +ਈਸ -12.485894 +ਪਠ -12.490482 +ਪਧ -12.499700 +ਲਅ -12.499700 +ਬਟ -12.504332 +ਭਮ -12.508978 +ਭਨ -12.520660 +ਊਟ -12.523008 +ਕਗ -12.530074 +ਥਊ -12.539550 +ਯਮ -12.549088 +ਟਵ -12.551483 +ਡਗ -12.561101 +ਵਇ -12.563515 +ਆਗ -12.565934 +ਗਗ -12.568357 +ਗਜ -12.570783 +ਤਦ -12.573214 +ਦਪ -12.585430 +ਆਕ -12.597750 +ਉਰ -12.600227 +ਸਯ -12.600227 +ਮਮ -12.605193 +ਇਰ -12.612675 +ਯਦ -12.625231 +ਫਜ -12.625231 +ਗਧ -12.630284 +ਖੜ -12.630284 +ਗਈ -12.632818 +ਰਅ -12.635355 +ਕਠ -12.635355 +ਣਈ -12.642995 +ਨਯ -12.650676 +ਸਣ -12.663569 +ਗਨ -12.668758 +ਖਸ -12.673967 +ਆਤ -12.681815 +ਫਸ -12.692345 +ਛਪ -12.700294 +ਬਆ -12.708287 +ਪਉ -12.710961 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pes b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pes new file mode 100644 index 00000000000..b751c941799 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pes @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 77876177 +#UNIQUE_BIGRAMS 1344 +#UNSEEN_LOG_PROB -15.524239 +#EXPECTED_SCORE -7.870762 +_ا -4.950792 +ي_ -5.027082 +ن_ -5.376995 +ه_ -5.498819 +ان -5.548883 +ت_ -5.737591 +د_ -5.756236 +_م -5.777093 +اي -5.820098 +_ب -6.023098 +ر_ -6.035307 +ار -6.093857 +ست -6.169238 +را -6.470635 +اس -6.470844 +_د -6.491223 +ين -6.645474 +_ن -6.795325 +_س -6.795817 +_ك -6.804782 +_ت -6.817460 +ل_ -6.847098 +ري -6.876872 +دا -6.892533 +ال -6.977193 +_ش -6.993066 +ام -7.035362 +ما -7.055864 +ند -7.150741 +ده -7.188353 +تا -7.211578 +م_ -7.218652 +رد -7.221552 +ا_ -7.300840 +ني -7.322676 +اد -7.336930 +يا -7.346350 +ور -7.348323 +ود -7.396992 +سا -7.400572 +با -7.404609 +وا -7.418316 +رو -7.424244 +_ر -7.468338 +بر -7.621806 +لي -7.672690 +_ج -7.686912 +ير -7.707178 +_ه -7.707984 +دي -7.708681 +مي -7.715292 +_خ -7.720155 +نا -7.733447 +اه -7.830730 +ها -7.837252 +لا -7.840599 +_ف -7.845533 +_پ -7.875259 +سي -7.948940 +ات -7.984607 +اب -8.047731 +ش_ -8.090260 +تر -8.093236 +يد -8.108121 +_ق -8.121194 +كا -8.126926 +_ع -8.172619 +مر -8.194312 +_گ -8.194741 +_و -8.197892 +وي -8.205490 +يل -8.210568 +بو -8.301760 +مو -8.313541 +ز_ -8.317093 +گر -8.319340 +از -8.338510 +ون -8.371948 +يت -8.392712 +بي -8.397458 +فر -8.397551 +نو -8.415555 +تي -8.419908 +ول -8.424693 +_ز -8.431380 +در -8.440799 +زي -8.442171 +تو -8.445977 +_ح -8.486413 +ب_ -8.513819 +وس -8.531383 +ره -8.532795 +هر -8.547165 +شت -8.554021 +يي -8.555164 +يك -8.573001 +س_ -8.574370 +ته -8.619618 +اش -8.624165 +خو -8.631728 +ع_ -8.638719 +رس -8.646240 +كر -8.662287 +مت -8.684699 +نه -8.745983 +او -8.771371 +دو -8.832333 +گا -8.834167 +شد -8.836274 +هم -8.856809 +شه -8.869126 +فت -8.880903 +مع -8.891228 +جا -8.909267 +دن -8.923508 +شا -8.923571 +رك -8.959493 +من -8.962595 +كي -8.965834 +نگ -8.968680 +ك_ -8.994896 +يم -8.995993 +يش -9.014297 +يس -9.017654 +زا -9.068160 +رت -9.079330 +كن -9.129237 +اف -9.140130 +سر -9.146646 +يز -9.159174 +عي -9.171021 +جم -9.175513 +_ي -9.186373 +گي -9.212776 +يو -9.213600 +خت -9.241172 +يه -9.243079 +قا -9.251049 +وم -9.261653 +شي -9.263007 +فا -9.265798 +خا -9.268366 +نت -9.272920 +اق -9.274732 +_چ -9.281219 +مه -9.295140 +اخ -9.302058 +شو -9.313211 +مل -9.327113 +تن -9.363039 +رش -9.365799 +رب -9.367426 +رم -9.375980 +مد -9.380035 +رف -9.388528 +قر -9.411279 +پي -9.412150 +اع -9.441475 +ف_ -9.460796 +_ط -9.460887 +وز -9.466617 +پا -9.474727 +رگ -9.514659 +لو -9.532740 +ق_ -9.533482 +في -9.558994 +شم -9.560406 +هن -9.566321 +كو -9.599725 +مس -9.612966 +شر -9.616161 +كت -9.616917 +رن -9.619392 +كش -9.634096 +_ص -9.641032 +لم -9.656497 +زم -9.668824 +له -9.669865 +ط_ -9.684897 +نس -9.687416 +نش -9.694797 +و_ -9.696779 +گ_ -9.704935 +بع -9.741084 +_ل -9.746882 +اك -9.747551 +مح -9.798178 +تم -9.802026 +عا -9.805387 +هي -9.807660 +حا -9.839860 +يگ -9.844758 +اح -9.852838 +سو -9.862756 +دگ -9.872743 +صو -9.902381 +قي -9.913629 +عل -9.929385 +جه -9.967827 +جو -9.978235 +نم -9.999184 +شك -10.004540 +نف -10.026206 +وج -10.026476 +بت -10.041197 +نج -10.043971 +وه -10.054374 +اص -10.062651 +دس -10.063702 +تح -10.072013 +بل -10.074648 +وش -10.078969 +زن -10.091143 +اج -10.098436 +پر -10.101059 +عد -10.114345 +كل -10.122273 +وب -10.133091 +خش -10.133799 +تب -10.141520 +وت -10.150441 +شن -10.157787 +بخ -10.161071 +عم -10.161453 +به -10.169772 +تف -10.188566 +هد -10.230572 +هس -10.231128 +گو -10.238808 +عن -10.245405 +چن -10.268572 +سط -10.284773 +حد -10.309856 +سل -10.314944 +هو -10.325579 +لت -10.327387 +مش -10.331033 +رز -10.349042 +يب -10.351727 +قع -10.354634 +جر -10.361247 +تش -10.361832 +ح_ -10.363833 +دم -10.369703 +كز -10.370807 +اط -10.388999 +طر -10.395697 +ج_ -10.407601 +بد -10.435575 +وع -10.437321 +بس -10.444299 +اث -10.455753 +جن -10.461122 +يخ -10.467141 +تق -10.468401 +عر -10.478726 +طو -10.496922 +سن -10.516137 +كس -10.523835 +نن -10.541204 +نظ -10.543331 +زه -10.543442 +سم -10.544576 +خ_ -10.553374 +تل -10.558896 +عه -10.572260 +كم -10.581428 +بن -10.581740 +سب -10.599365 +_غ -10.603942 +خي -10.608881 +نق -10.614708 +صل -10.639524 +مك -10.643785 +رخ -10.648238 +يق -10.653779 +مج -10.661740 +قد -10.663661 +تع -10.666640 +مق -10.677369 +لب -10.677430 +زد -10.680923 +قه -10.690872 +عت -10.708557 +حي -10.711723 +نر -10.727341 +خص -10.744465 +فو -10.764080 +صي -10.772967 +حم -10.774978 +رج -10.804724 +شگ -10.813999 +خر -10.814033 +زر -10.817004 +يج -10.826695 +تص -10.827907 +حت -10.829187 +صا -10.840217 +دل -10.868684 +قل -10.874655 +نب -10.899777 +مب -10.905096 +اگ -10.906517 +كه -10.936266 +تخ -10.937682 +يع -10.968977 +چه -10.978067 +بز -10.983158 +هل -10.998577 +ضا -11.019268 +غا -11.065289 +سه -11.078088 +تگ -11.079170 +نك -11.089669 +سك -11.092741 +گذ -11.094076 +يف -11.102443 +اپ -11.104481 +لس -11.107421 +سپ -11.112867 +زب -11.139952 +وق -11.142588 +رق -11.143845 +گل -11.148421 +صر -11.170797 +جد -11.173105 +ثر -11.179880 +لك -11.194610 +ظر -11.199133 +خل -11.204238 +حل -11.206951 +_ث -11.212435 +ص_ -11.213227 +مخ -11.220237 +لف -11.232487 +اغ -11.240578 +زش -11.243944 +مچ -11.256035 +غي -11.260073 +ثا -11.260755 +ضو -11.265582 +قو -11.330031 +قت -11.351557 +سف -11.355290 +فه -11.356601 +بق -11.363320 +طل -11.375770 +حس -11.379911 +نز -11.381937 +جي -11.385749 +وف -11.387386 +قب -11.409791 +جل -11.419755 +چا -11.426317 +حر -11.430862 +پو -11.435062 +_ژ -11.438503 +گز -11.439274 +لن -11.442106 +دش -11.461305 +صد -11.488301 +طق -11.491071 +طا -11.500272 +ثب -11.526756 +وط -11.527194 +ذا -11.528725 +دت -11.571781 +طب -11.577601 +قط -11.600602 +وص -11.604288 +پس -11.606482 +شخ -11.606944 +فع -11.623273 +عب -11.623332 +فق -11.627663 +بك -11.628660 +اض -11.634656 +مز -11.672675 +پن -11.685126 +رض -11.692097 +دك -11.705460 +هش -11.709115 +نط -11.725270 +وك -11.741607 +غر -11.748156 +تد -11.759737 +حو -11.761665 +لد -11.772837 +زو -11.773874 +گف -11.776927 +ظا -11.778943 +عض -11.783573 +هت -11.789004 +رح -11.798541 +چي -11.831379 +ث_ -11.836994 +خد -11.870938 +فل -11.912325 +صف -11.930722 +سخ -11.939136 +قن -11.944895 +وپ -11.945625 +حص -11.948917 +تج -11.949137 +ضي -11.954053 +قش -11.961792 +دب -11.968235 +مط -11.981134 +تغ -11.994225 +مص -12.001802 +حق -12.010719 +نخ -12.017462 +ژا -12.021384 +وض -12.023465 +طح -12.030113 +مپ -12.039129 +چو -12.052681 +دف -12.065487 +خس -12.072492 +طي -12.100611 +تك -12.101669 +هب -12.102321 +جز -12.112703 +وچ -12.113688 +بط -12.132447 +پ_ -12.140788 +حك -12.147412 +رل -12.173961 +لق -12.175588 +_ض -12.184270 +دد -12.202229 +شب -12.215824 +يچ -12.229726 +ژي -12.235968 +طه -12.249977 +رچ -12.250248 +هز -12.265946 +ذر -12.272531 +پد -12.298053 +لع -12.306192 +ذش -12.312775 +ژه -12.313435 +مث -12.331940 +قس -12.334139 +ضر -12.342485 +گش -12.368993 +مم -12.381294 +وگ -12.383371 +رع -12.399496 +ژو -12.403605 +فن -12.404609 +سع -12.409640 +يح -12.416207 +كد -12.421380 +دخ -12.427388 +فز -12.454479 +_ذ -12.473633 +ض_ -12.482511 +پل -12.496681 +غ_ -12.498181 +حض -12.501077 +گس -12.514022 +لح -12.534869 +صن -12.537069 +نع -12.545572 +لگ -12.553680 +عو -12.565535 +عث -12.603078 +هه -12.604577 +بش -12.613140 +يط -12.613488 +جب -12.616859 +گه -12.617324 +شع -12.618372 +زگ -12.621988 +وح -12.625379 +چ_ -12.650410 +مگ -12.653391 +رپ -12.655182 +جس -12.655421 +طع -12.672977 +لز -12.706389 +سد -12.708867 +په -12.709115 +ذي -12.713090 +گن -12.735656 +غل -12.746182 +لر -12.756785 +_ظ -12.761022 +رص -12.767337 +هف -12.785272 +چك -12.795245 +فس -12.802241 +نص -12.808208 +فض -12.816737 +حب -12.841406 +لط -12.842494 +يژ -12.842630 +وژ -12.848220 +دق -12.849313 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pnb b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pnb new file mode 100644 index 00000000000..9b6c933db3e --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pnb @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 5176813 +#UNIQUE_BIGRAMS 1042 +#UNSEEN_LOG_PROB -14.816416 +#EXPECTED_SCORE -7.912754 +ں_ -5.079490 +_ا -5.113635 +ي_ -5.127070 +ے_ -5.531615 +ا_ -5.552546 +ان -6.037963 +يا -6.050930 +_ن -6.067880 +اں -6.102677 +ر_ -6.111398 +ن_ -6.270532 +_ك -6.296890 +اي -6.394255 +_پ -6.439834 +ہ_ -6.495096 +_م -6.542056 +وں -6.562561 +_س -6.613498 +_ب -6.618528 +ني -6.695378 +_ج -6.768904 +_و -6.826096 +نا -6.855452 +ري -6.858424 +ار -6.874324 +_ت -6.889509 +ل_ -6.907831 +ال -7.032753 +_د -7.035566 +ند -7.068703 +لي -7.101318 +_ل -7.112034 +نو -7.242346 +يي -7.257447 +را -7.294985 +او -7.339518 +_ر -7.349073 +_ہ -7.353350 +دي -7.386960 +ين -7.387193 +ور -7.388220 +_گ -7.478177 +ون -7.508963 +دا -7.515577 +تا -7.529515 +وا -7.593634 +ت_ -7.602326 +كي -7.716734 +كھ -7.726440 +يہ -7.751804 +لا -7.753006 +وي -7.764898 +يں -7.813346 +ہو -7.846765 +_ش -7.856937 +جا -7.868418 +مي -7.905958 +ام -7.993162 +تو -8.007864 +اس -8.035603 +كا -8.048826 +_چ -8.054149 +دے -8.056892 +ات -8.083103 +رو -8.107645 +م_ -8.139747 +س_ -8.156110 +ھ_ -8.168244 +پن -8.181776 +يو -8.188914 +گي -8.220735 +ما -8.231835 +ھا -8.232422 +نگ -8.271845 +_ص -8.287521 +يل -8.319036 +با -8.320550 +ول -8.342364 +تھ -8.344989 +ست -8.351347 +ھي -8.362769 +ہن -8.378635 +يت -8.395146 +ہر -8.402106 +ير -8.421372 +تي -8.426061 +لو -8.433557 +وہ -8.435292 +پا -8.439447 +د_ -8.466005 +كر -8.479569 +سا -8.498393 +وب -8.502834 +يك -8.507390 +بر -8.530081 +اب -8.531421 +رك -8.549894 +_ف -8.573481 +لے -8.584351 +ك_ -8.586599 +تے -8.590783 +و_ -8.591643 +ع_ -8.624043 +نڈ -8.629330 +سي -8.635413 +جي -8.636965 +مر -8.675301 +بن -8.675301 +_ع -8.707560 +لع -8.723256 +اد -8.731999 +شہ -8.759151 +وك -8.771399 +يس -8.800180 +در -8.800925 +_ض -8.816923 +صو -8.823474 +ہا -8.838449 +ضل -8.848948 +دو -8.854097 +ٹي -8.859524 +گر -8.863153 +نے -8.886835 +اہ -8.892122 +اپ -8.909304 +_ٹ -8.914001 +من -8.929680 +اك -8.931992 +كو -8.944158 +رن -8.957258 +ڈ_ -8.989616 +ب_ -9.009730 +سن -9.024312 +دھ -9.025764 +پر -9.058377 +بي -9.075415 +پي -9.076619 +رت -9.091288 +وج -9.124812 +نہ -9.131205 +بو -9.135588 +ٹر -9.153727 +نت -9.155315 +_ق -9.166161 +ہي -9.216336 +_خ -9.218326 +تر -9.227485 +يٹ -9.273483 +يد -9.280572 +بہ -9.284911 +اج -9.294504 +كس -9.308749 +_ح -9.317969 +سل -9.320817 +شا -9.336407 +ج_ -9.348164 +رہ -9.357275 +نس -9.358372 +وس -9.391118 +يم -9.403332 +سك -9.418318 +_ي -9.423858 +بھ -9.438093 +نج -9.444680 +رے -9.468569 +سو -9.507999 +رد -9.509014 +جن -9.529875 +سم -9.572102 +رس -9.590246 +ھے -9.609057 +ٹ_ -9.609710 +نٹ -9.638963 +ود -9.657749 +رم -9.661581 +ڑا -9.672911 +گھ -9.674959 +ز_ -9.678381 +بے -9.711877 +ڈا -9.716324 +ڈي -9.726676 +مل -9.750724 +چو -9.751925 +سد -9.763020 +سر -9.767143 +چھ -9.772739 +پہ -9.774690 +كل -9.790152 +دت -9.793363 +زي -9.799310 +مو -9.801298 +ھر -9.816798 +نك -9.818810 +گ_ -9.832974 +اق -9.833228 +ہہ -9.849076 +رب -9.852937 +وڈ -9.868223 +سٹ -9.880780 +وم -9.883673 +يش -9.884199 +ہل -9.887891 +لك -9.888948 +ھن -9.901953 +عا -9.914539 +فر -9.915615 +اف -9.928050 +عل -9.928865 +ڑي -9.937037 +لہ -9.941963 +يے -9.944707 +جو -9.946906 +از -9.959337 +_ز -9.971037 +جہ -9.971876 +ھو -9.978046 +گو -9.988200 +ڑھ -9.992452 +پو -10.011886 +لم -10.046245 +وٹ -10.052150 +بل -10.062841 +گا -10.095703 +چا -10.116899 +رل -10.123102 +ہت -10.126525 +زا -10.132143 +كہ -10.161526 +اگ -10.170491 +ہڑ -10.205601 +مب -10.206918 +ٹب -10.249368 +في -10.276072 +_ڈ -10.283000 +چي -10.286476 +طا -10.294154 +سے -10.300466 +ھل -10.307866 +خا -10.318147 +فو -10.328860 +يز -10.333527 +وت -10.336767 +شي -10.366627 +قا -10.380305 +ہد -10.408436 +ڑ_ -10.412229 +گن -10.413368 +جر -10.419462 +يڈ -10.424815 +مس -10.450323 +_ط -10.456967 +لن -10.459319 +وچ -10.469952 +پ_ -10.486649 +پھ -10.491455 +ش_ -10.495473 +ڑے -10.500309 +ف_ -10.507593 +كم -10.536276 +دس -10.542082 +پت -10.549580 +تل -10.572738 +كن -10.591536 +ٹا -10.595842 +وڑ -10.612762 +جھ -10.615382 +دن -10.651236 +بع -10.695378 +گل -10.706977 +لد -10.713982 +مہ -10.734727 +صد -10.758668 +مد -10.764474 +صي -10.764959 +حا -10.766900 +ٹھ -10.774202 +عر -10.775667 +كے -10.781051 +لط -10.788919 +دہ -10.796334 +كت -10.811279 +كد -10.823347 +كٹ -10.825369 +چن -10.828406 +رج -10.828913 +وپ -10.845739 +اڑ -10.867963 +طن -10.872658 +ھد -10.880517 +ڈے -10.885780 +شر -10.891063 +يڑ -10.903287 +بد -10.906493 +وگ -10.941689 +وش -10.942237 +اچ -10.955458 +شن -10.955458 +شك -10.970477 +سپ -10.971037 +لڑ -10.974958 +قي -10.989050 +پڑ -10.990183 +دك -10.994725 +عد -11.003280 +يج -11.004425 +حم -11.010161 +چ_ -11.011886 +لگ -11.015920 +جد -11.024022 +جے -11.025764 +قب -11.029255 +وے -11.029837 +مش -11.039777 +اش -11.043890 +اع -11.070013 +جگ -11.079631 +نچ -11.083859 +مت -11.086281 +ق_ -11.126213 +تح -11.126836 +وو -11.141241 +حص -11.155792 +فا -11.160250 +مگ -11.167923 +مك -11.171133 +اص -11.171776 +يب -11.180158 +اخ -11.191193 +مح -11.199689 +لب -11.230161 +ٹو -11.236198 +صل -11.259922 +اح -11.261974 +گڑ -11.262658 +خل -11.282653 +اٹ -11.291008 +مق -11.291008 +ےا -11.296606 +گے -11.298008 +خ_ -11.303633 +جم -11.307866 +تص -11.312111 +عے -11.324922 +ظا -11.331371 +وز -11.331371 +تن -11.345805 +نن -11.368468 +تم -11.395991 +چڑ -11.403521 +غا -11.411849 +زم -11.418699 +كز -11.437900 +قے -11.437900 +صر -11.439447 +رط -11.457359 +ح_ -11.470743 +شت -11.493061 +پس -11.501924 +طو -11.503542 +خت -11.507593 +رچ -11.512470 +رپ -11.518998 +نم -11.525556 +عم -11.526377 +رٹ -11.535448 +يپ -11.537932 +رگ -11.540420 +يگ -11.555440 +خو -11.584244 +چل -11.606665 +فل -11.611889 +رف -11.617132 +وع -11.621516 +تظ -11.639185 +حك -11.644529 +تہ -11.644529 +يخ -11.656174 +جت -11.657074 +طر -11.660679 +يف -11.660679 +ہم -11.667008 +چك -11.669729 +يق -11.676099 +اڈ -11.691686 +رق -11.697228 +لس -11.707443 +فت -11.708375 +شم -11.713046 +ٹك -11.727148 +قہ -11.734727 +ڈھ -11.744256 +پل -11.745212 +سب -11.746169 +تع -11.782032 +_غ -11.807778 +دل -11.808777 +لج -11.843177 +صا -11.846252 +نر -11.847278 +عي -11.872135 +ڈو -11.876320 +قر -11.878417 +لپ -11.879466 +قت -11.880517 +دش -11.905958 +ڈر -11.914539 +رح -11.917770 +نل -11.918849 +پع -11.921009 +ٹن -11.926422 +كج -11.941689 +جز -11.950486 +صہ -11.968242 +قل -11.979452 +ٹل -11.989616 +زر -11.996432 +بت -12.024022 +لت -12.025183 +مپ -12.031003 +سط -12.032170 +وف -12.038018 +پچ -12.059269 +عو -12.059269 +جس -12.064034 +اے -12.072412 +ہے -12.078425 +مع -12.088100 +رڈ -12.101509 +ٹم -12.108876 +قع -12.128707 +ٹے -12.133708 +سف -12.153886 +نش -12.171776 +حر -12.191193 +گد -12.196416 +نع -12.197724 +خر -12.226817 +يچ -12.230830 +زن -12.237544 +مج -12.259239 +ھٹ -12.281265 +مط -12.309279 +ھڑ -12.310694 +رش -12.317791 +فغ -12.324922 +دم -12.333527 +نق -12.356727 +ھت -12.356727 +عہ -12.377337 +حب -12.380305 +تب -12.413368 +تك -12.414890 +پك -12.416412 +طي -12.419462 +خي -12.425582 +كش -12.428651 +قو -12.430189 +مز -12.434810 +زب -12.453446 +قس -12.469162 +اط -12.485051 +ضر -12.510842 +غي -12.517363 +غر -12.518998 +نظ -12.525556 +ڈن -12.537104 +بچ -12.543745 +بك -12.545410 +عظ -12.545410 +حس -12.550416 +ھڈ -12.580825 +زو -12.582534 +لر -12.585956 +كڑ -12.589387 +مص -12.594549 +قن -12.603193 +قد -12.617132 +وق -12.625913 +غل -12.627676 +حد -12.650788 +چر -12.654377 +ص_ -12.659777 +دد -12.667008 +مڈ -12.676099 +لٹ -12.696302 +ظم -12.711176 +پٹ -12.713046 +ہز -12.718670 +قص -12.718670 +ء_ -12.720550 +مغ -12.720550 +نز -12.722432 +پد -12.741390 +سج -12.747127 +كپ -12.749044 +لف -12.774202 +ہس -12.778112 +اء -12.785963 +مے -12.789905 +سس -12.793858 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pol b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pol new file mode 100644 index 00000000000..583f888bbee --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pol @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 363290598 +#UNIQUE_BIGRAMS 592 +#UNSEEN_LOG_PROB -18.276760 +#EXPECTED_SCORE -7.722307 +ie -5.156749 +e_ -5.440086 +a_ -5.593676 +_p -5.779438 +ni -5.874387 +_s -6.201152 +ow -6.288581 +y_ -6.417571 +st -6.461954 +ze -6.519562 +po -6.597110 +i_ -6.608520 +_w -6.646903 +al -6.669577 +ro -6.750190 +cz -6.806139 +_z -6.828420 +rz -6.850472 +ia -6.859359 +na -6.867764 +ch -6.882393 +o_ -6.904181 +za -6.909554 +ra -6.923182 +an -6.930617 +zy -6.945775 +pr -6.988613 +u_ -7.043758 +_k -7.054659 +ta -7.060212 +la -7.062322 +ac -7.097710 +wi -7.118654 +_n -7.141294 +wa -7.144013 +os -7.146604 +_m -7.166696 +ko -7.215281 +je -7.233870 +_t -7.238520 +_o -7.251797 +_d -7.286008 +or -7.304160 +od -7.314712 +dz -7.329149 +m_ -7.352706 +_r -7.354416 +mi -7.365001 +ej -7.390124 +sz -7.390271 +_j -7.420754 +h_ -7.423073 +l_ -7.437362 +ki -7.441602 +es -7.457262 +si -7.501740 +en -7.527818 +ar -7.537705 +er -7.541168 +ci -7.543350 +to -7.546385 +ol -7.565771 +on -7.595952 +yc -7.600692 +te -7.603566 +ka -7.610794 +_b -7.674304 +zi -7.679502 +go -7.708670 +ed -7.710184 +le -7.718529 +li -7.744183 +ja -7.748361 +wy -7.759826 +j_ -7.775110 +ny -7.787391 +eg -7.796745 +lo -7.809459 +ad -7.825261 +_c -7.835027 +ne -7.836878 +at -7.865097 +z_ -7.893294 +re -7.911035 +ic -7.928110 +wo -7.928179 +c_ -7.931169 +ty -7.940297 +ec -7.942135 +zn -7.957077 +az -7.989927 +ak -7.989953 +sk -8.009911 +sc -8.030094 +em -8.052169 +ez -8.078208 +do -8.116554 +_l -8.137194 +w_ -8.153355 +t_ -8.179867 +no -8.193477 +tr -8.198787 +as -8.211381 +da -8.251798 +ku -8.266508 +in -8.282889 +zo -8.298730 +ob -8.305630 +oz -8.323211 +ek -8.323655 +ok -8.340534 +aw -8.342869 +_g -8.375964 +el -8.402231 +aj -8.432622 +k_ -8.434131 +mo -8.437238 +ym -8.466676 +ma -8.485623 +_a -8.498674 +_u -8.503492 +is -8.508032 +cj -8.518083 +kt -8.545976 +by -8.561994 +pi -8.587698 +yl -8.593042 +am -8.594407 +dn -8.597600 +ry -8.600024 +lu -8.616426 +ce -8.617823 +we -8.622189 +il -8.624458 +ys -8.626387 +oc -8.659051 +sp -8.760526 +pa -8.848590 +n_ -8.865668 +ws -8.889232 +ot -8.893470 +tu -8.911735 +om -8.953640 +ln -8.959344 +dy -8.974268 +d_ -9.033491 +io -9.042744 +_i -9.057405 +et -9.104943 +cy -9.119527 +gr -9.141161 +de -9.153494 +bi -9.175537 +tw -9.181615 +ru -9.184211 +op -9.196922 +oj -9.199320 +wn -9.211626 +s_ -9.215464 +kr -9.215675 +nt -9.221404 +yn -9.246321 +so -9.285792 +ji -9.294614 +ly -9.314418 +ud -9.315539 +sa -9.328185 +ik -9.334855 +me -9.367332 +_f -9.368413 +ca -9.378095 +sl -9.413544 +ur -9.431643 +dl -9.439990 +uz -9.471554 +js -9.476810 +zw -9.493319 +be -9.573182 +ns -9.575110 +ba -9.586985 +nc -9.611276 +lk -9.637112 +og -9.661122 +ga -9.671816 +yw -9.672238 +us -9.674994 +im -9.679491 +br -9.682229 +du -9.686274 +tk -9.695210 +ho -9.697398 +sw -9.702921 +yk -9.712015 +uj -9.712055 +ew -9.745374 +ii -9.751677 +fi -9.779024 +zk -9.784201 +nk -9.795037 +ep -9.795386 +mu -9.802011 +un -9.806360 +gl -9.809924 +iu -9.812910 +yt -9.815366 +bo -9.818568 +rt -9.825856 +kl -9.839806 +gi -9.847706 +tn -9.890374 +pe -9.902341 +rw -9.920864 +co -9.926202 +se -9.934432 +ri -9.943205 +rs -9.963382 +_e -9.966212 +r_ -9.969162 +zd -9.990528 +ap -10.006403 +ag -10.038832 +dr -10.039664 +zc -10.055733 +bu -10.072436 +iz -10.076302 +uc -10.103835 +uk -10.107703 +zl -10.108737 +sy -10.123205 +ls -10.135673 +um -10.158267 +pl -10.160852 +nn -10.170879 +ut -10.172991 +ks -10.174445 +it -10.185940 +ub -10.283313 +rc -10.283799 +ul -10.310449 +yd -10.310540 +pu -10.319134 +zu -10.360850 +jn -10.365411 +sn -10.375183 +my -10.396623 +yj -10.436307 +nd -10.445747 +ju -10.447670 +rn -10.460013 +b_ -10.463608 +wc -10.472456 +ug -10.476666 +_h -10.479912 +iw -10.491562 +eb -10.524104 +rm -10.527757 +rd -10.530445 +ab -10.553691 +dw -10.569849 +wl -10.581620 +su -10.585770 +ha -10.594751 +bl -10.613231 +yz -10.635832 +zt -10.648920 +zb -10.685737 +kc -10.694397 +yb -10.759070 +au -10.772609 +wr -10.829377 +zm -10.851271 +gd -10.918418 +sr -10.926658 +up -10.956248 +yp -10.982153 +nu -10.997562 +jo -11.000732 +fo -11.007543 +zp -11.008532 +ck -11.020667 +hi -11.047901 +ge -11.062656 +mn -11.083919 +dc -11.086577 +dk -11.105116 +di -11.107112 +g_ -11.124038 +zg -11.171263 +cu -11.177192 +id -11.201488 +wz -11.241100 +yg -11.292448 +ds -11.323326 +of -11.326106 +rk -11.336564 +gu -11.359566 +zj -11.384129 +rg -11.387828 +sj -11.403803 +ig -11.441256 +pn -11.458118 +kz -11.464444 +eu -11.503596 +fe -11.518105 +he -11.538148 +mp -11.546831 +fa -11.547021 +sm -11.555698 +af -11.574626 +cn -11.593658 +ng -11.676986 +kw -11.679745 +jm -11.701456 +zr -11.733064 +ti -11.779914 +fr -11.781214 +ef -11.793101 +ir -11.813425 +ke -11.831576 +ea -11.867257 +hn -11.885952 +jd -11.901516 +rl -11.911548 +py -11.926032 +ml -11.926063 +yr -11.927764 +lt -11.934525 +lc -11.966239 +bn -11.975757 +p_ -11.985837 +zs -11.986852 +nf -12.002360 +gn -12.016128 +oi -12.030161 +gm -12.074314 +dp -12.083540 +hc -12.195161 +ps -12.203528 +fu -12.236052 +ua -12.261428 +jw -12.273803 +wk -12.273862 +wp -12.276697 +bs -12.292588 +pc -12.333322 +lm -12.348195 +jc -12.360798 +uw -12.399839 +wd -12.407111 +db -12.420981 +dm -12.446622 +ij -12.451018 +lb -12.493988 +tl -12.551997 +dd -12.552593 +ip -12.597714 +ct -12.603041 +rp -12.626605 +ib -12.635320 +hr -12.672429 +eo -12.705097 +ai -12.732916 +_x -12.765146 +hu -12.800972 +vi -12.848566 +th -12.910692 +wu -12.918387 +ld -12.955254 +ll -12.965191 +f_ -12.975517 +mk -13.023847 +mc -13.045574 +kn -13.045809 +wt -13.057374 +ms -13.145126 +dj -13.187361 +rb -13.217040 +_v -13.254426 +jl -13.263496 +jb -13.327167 +yf -13.388615 +bk -13.424317 +mb -13.488547 +oh -13.508215 +cl -13.528391 +hw -13.534174 +ei -13.544718 +nw -13.582070 +lp -13.587779 +lw -13.596853 +xi -13.605737 +ou -13.668520 +tz -13.679367 +x_ -13.726711 +fl -13.746333 +lg -13.766999 +gw -13.817704 +hy -13.904193 +hl -13.913906 +nz -13.953930 +xv -13.979040 +pt -14.000488 +bc -14.084574 +ix -14.096699 +bj -14.113635 +rr -14.122461 +dt -14.122745 +ss -14.149414 +ee -14.152448 +tc -14.158971 +cm -14.193375 +jk -14.249119 +ve -14.286405 +lz -14.306380 +jp -14.357481 +ue -14.398230 +oa -14.404503 +iv -14.405192 +pk -14.422267 +aa -14.463748 +oo -14.468152 +dg -14.496511 +tt -14.503227 +va -14.507198 +ae -14.551520 +jr -14.639090 +sf -14.680201 +oe -14.684586 +gz -14.748189 +nr -14.764123 +if -14.799584 +mt -14.810383 +lf -14.839311 +kk -14.860064 +uo -14.906996 +yo -14.915439 +sb -14.942818 +vo -14.972003 +ay -14.986755 +tp -14.993087 +sh -15.004398 +tf -15.138343 +ao -15.153316 +av -15.176070 +ts -15.185658 +fy -15.203828 +hm -15.211944 +tm -15.229382 +bw -15.233507 +pp -15.317121 +mw -15.371638 +ui -15.384660 +uf -15.437253 +_y -15.501384 +tv -15.504519 +ph -15.548376 +nl -15.677909 +ey -15.678534 +ht -15.710970 +gh -15.741973 +ex -15.773213 +cw -15.798113 +cb -15.820689 +kp -15.854407 +v_ -15.907851 +rf -15.909805 +ft -15.942943 +ov -15.947203 +mm -15.956263 +mf -15.966907 +oy -15.972003 +cr -16.010022 +wb -16.025038 +kb -16.059881 +pz -16.068588 +ah -16.110400 +zz -16.186250 +ev -16.207430 +lh -16.223445 +ff -16.235037 +ya -16.235344 +jt -16.274787 +sd -16.278255 +vp -16.334901 +qu -16.532667 +nb -16.545525 +eh -16.643352 +jg -16.650279 +cd -16.651914 +bp -16.687937 +fg -16.694239 +ye -16.877172 +nh -16.891102 +tb -16.891584 +xx -16.926279 +gt -16.967925 +mz -16.979168 +yi -17.000358 +wj -17.004007 +cf -17.036737 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/por b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/por new file mode 100644 index 00000000000..8f243299d79 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/por @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 441029966 +#UNIQUE_BIGRAMS 604 +#UNSEEN_LOG_PROB -19.801801 +#EXPECTED_SCORE -7.321810 +o_ -4.952120 +a_ -5.075079 +s_ -5.119466 +e_ -5.715729 +_p -5.849190 +es -5.947017 +_c -6.031175 +ra -6.082500 +co -6.221516 +en -6.330211 +ar -6.345872 +te -6.356692 +os -6.364427 +nt -6.405309 +er -6.428510 +ca -6.429024 +_a -6.430758 +re -6.454726 +as -6.491736 +_e -6.555788 +ta -6.564583 +or -6.565643 +_s -6.572631 +do -6.575376 +r_ -6.645271 +ma -6.690926 +_d -6.717715 +de -6.719239 +_m -6.733308 +an -6.756401 +m_ -6.824266 +to -6.844960 +ad -6.845861 +qu -6.902176 +da -6.955102 +st -7.001554 +ao -7.015245 +ri -7.032221 +is -7.034385 +_t -7.042778 +se -7.069693 +al -7.070032 +in -7.077815 +me -7.091070 +po -7.105651 +_f -7.120614 +ia -7.130599 +ue -7.157451 +pa -7.201908 +em -7.225920 +ro -7.235085 +na -7.261020 +on -7.262774 +_q -7.286892 +om -7.293813 +ic -7.406896 +_n -7.450293 +am -7.451715 +ci -7.504912 +ti -7.527475 +el -7.533636 +nd -7.549874 +tr -7.582065 +pr -7.586301 +sa -7.594762 +id -7.611145 +pe -7.637125 +ac -7.646342 +_r -7.695003 +di -7.782588 +ve -7.787876 +io -7.792038 +it -7.820021 +ss -7.851085 +no -7.862741 +l_ -7.866573 +li -7.875878 +la -7.878777 +mo -7.895666 +_i -7.927758 +ei -7.959621 +_v -7.975742 +so -7.988804 +ai -7.997508 +u_ -8.007503 +nc -8.008209 +le -8.017574 +ir -8.063107 +at -8.072209 +ec -8.073129 +um -8.113685 +ua -8.134785 +si -8.162068 +_l -8.200115 +ce -8.208604 +im -8.216755 +_u -8.234770 +ou -8.288869 +lo -8.298732 +vi -8.367241 +rt -8.385392 +fo -8.413241 +va -8.418879 +_o -8.432858 +_b -8.470105 +ui -8.535646 +ni -8.542165 +ns -8.573681 +ol -8.575391 +un -8.575414 +mi -8.582177 +il -8.602553 +ha -8.616901 +_g -8.621244 +fi -8.635589 +mp -8.661227 +su -8.682153 +i_ -8.701868 +tu -8.704942 +ho -8.732830 +oi -8.775287 +iv -8.798409 +ur -8.838153 +gu -8.854912 +et -8.868629 +eu -8.879750 +ga -8.897810 +ul -8.901656 +eg -8.903541 +od -8.911606 +ne -8.933100 +be -8.940889 +lh -8.948232 +nh -8.950237 +br -8.953545 +go -9.008885 +oc -9.040238 +ap -9.049169 +us -9.064652 +ut -9.064686 +fe -9.104086 +vo -9.148188 +ed -9.162567 +mu -9.165369 +ba -9.186533 +_j -9.196325 +av -9.206302 +fa -9.263095 +gr -9.279978 +_h -9.298950 +ge -9.299547 +rm -9.317863 +ab -9.325851 +ev -9.354767 +ag -9.360441 +ig -9.409165 +cu -9.417800 +sc -9.424023 +oe -9.429886 +rr -9.443981 +mb -9.474016 +ex -9.474204 +sp -9.483424 +ov -9.492832 +he -9.538052 +cr -9.542924 +ob -9.597406 +rc -9.688209 +du -9.692867 +gi -9.731775 +iz -9.771747 +lu -9.824230 +op -9.842974 +ch -9.853316 +lt -9.854881 +rn -9.863668 +ep -9.878101 +og -9.903015 +bo -9.924786 +rd -9.944425 +ot -10.000048 +nu -10.000993 +ea -10.001533 +pi -10.034387 +au -10.054322 +ie -10.057250 +ip -10.082139 +z_ -10.102433 +ud -10.106127 +jo -10.115064 +ng -10.117391 +ru -10.135465 +pl -10.151088 +za -10.156383 +bi -10.178018 +ze -10.189264 +cl -10.199470 +pu -10.288248 +uc -10.310603 +ez -10.315560 +ub -10.323426 +lm -10.359411 +ju -10.410890 +ja -10.417818 +az -10.430786 +if -10.435798 +rs -10.458182 +ef -10.482186 +n_ -10.530797 +fr -10.569589 +sm -10.614792 +oa -10.636015 +rg -10.664864 +bl -10.698232 +nf -10.763890 +up -10.765045 +nv -10.767107 +fu -10.788512 +ct -10.789120 +hi -10.870034 +xi -10.876809 +lg -10.909028 +eb -10.913293 +ib -10.924865 +ix -10.970798 +je -11.000929 +eq -11.007447 +ug -11.015401 +of -11.069504 +bu -11.110938 +iu -11.123735 +aq -11.162185 +lv -11.186429 +af -11.202389 +dr -11.234076 +t_ -11.268322 +xa -11.271222 +rv -11.273471 +eo -11.275246 +ej -11.341534 +y_ -11.483712 +rq -11.576340 +d_ -11.652411 +xp -11.653907 +zi -11.663097 +xe -11.694823 +oj -11.712904 +uz -11.851873 +hu -11.947949 +xo -11.952467 +th -12.041516 +xt -12.063162 +fl -12.081221 +vr -12.087941 +ld -12.101592 +ae -12.131266 +nq -12.149950 +rp -12.183352 +ll -12.308685 +uv -12.312091 +bs -12.313689 +_w -12.318677 +gn -12.376156 +aj -12.417183 +ox -12.436854 +gl -12.446974 +g_ -12.452794 +zo -12.455161 +oo -12.467503 +rb -12.484799 +b_ -12.542761 +sd -12.558520 +k_ -12.573018 +sf -12.585530 +xc -12.585671 +rl -12.589698 +sq -12.646810 +lb -12.759630 +ee -12.764835 +lc -12.882028 +sh -12.896023 +pt -12.923003 +ls -13.031060 +h_ -13.053717 +iq -13.075520 +lq -13.089195 +c_ -13.106921 +wi -13.113777 +cn -13.118133 +cc -13.134248 +_k -13.150961 +ck -13.172333 +wa -13.197464 +oz -13.253959 +ax -13.269670 +ps -13.312590 +ow -13.313956 +bj -13.314455 +nj -13.319128 +tt -13.322609 +rf -13.324957 +dm -13.328924 +_z -13.330742 +p_ -13.401789 +ke -13.413591 +pc -13.494979 +uo -13.510470 +nn -13.516782 +ay -13.641909 +sl -13.653328 +sb -13.669987 +ii -13.714366 +w_ -13.750021 +ry -13.758383 +x_ -13.760243 +bt -13.780906 +uj -13.791767 +nr -13.820064 +ux -13.837388 +_x -13.837532 +ok -13.860119 +ey -13.867434 +tl -13.887917 +dv -13.901519 +tw -13.968790 +rk -14.011803 +ki -14.081887 +oq -14.149772 +lp -14.150842 +we -14.195315 +uf -14.225012 +_y -14.318560 +yo -14.326872 +vu -14.347045 +ts -14.462602 +nz -14.469412 +f_ -14.497587 +ds -14.570369 +ly -14.574035 +oh -14.588791 +nk -14.616296 +gh -14.623627 +ew -14.676697 +ka -14.681846 +hn -14.692980 +lf -14.710326 +ty -14.757386 +ph -14.783902 +wo -14.793253 +tm -14.793531 +tc -14.795111 +zu -14.829367 +kp -14.844396 +gm -14.855796 +xu -14.933917 +ij -14.977787 +ah -15.025321 +ak -15.039132 +sg -15.083306 +ny -15.088084 +ht -15.130165 +ws -15.141471 +oy -15.156088 +aw -15.162672 +ff -15.166517 +dy -15.178234 +mm -15.232360 +dq -15.283629 +nl -15.295815 +db -15.306513 +ft -15.324789 +pp -15.358475 +ik -15.375478 +ms -15.469709 +ya -15.492903 +ks -15.496225 +hr -15.524317 +xv -15.533907 +uq -15.573078 +ji -15.593150 +by -15.609248 +wn -15.635650 +ys -15.636483 +sr -15.648363 +bm -15.677554 +sv -15.722832 +v_ -15.725843 +my -15.744219 +zm -15.747095 +tn -15.787598 +ye -15.799302 +pm -15.804722 +gs -15.866701 +sn -15.896322 +gt -15.898918 +cp -15.989870 +bb -16.051854 +vd -16.097916 +yl -16.163152 +mt -16.193230 +dw -16.209498 +bc -16.218949 +md -16.231730 +dn -16.318359 +cy -16.319429 +ko -16.323716 +bv -16.358749 +wr -16.415949 +ky -16.431766 +cd -16.465707 +dd -16.468967 +wh -16.484180 +lk -16.487783 +ku -16.488685 +zz -16.502893 +sy -16.652569 +yn -16.664753 +fc -16.683223 +dj -16.703677 +tv -16.714541 +kl -16.750157 +bd -16.770858 +_テ -16.773787 +gg -16.786673 +bn -16.810162 +dg -16.811667 +pn -16.818834 +ln -16.838251 +sw -16.847865 +ek -16.858709 +rz -16.894129 +aa -16.898918 +ww -16.944812 +tp -16.956413 +yd -17.145497 +bg -17.225451 +cs -17.242596 +hm -17.286895 +yt -17.306911 +nb -17.313822 +mc -17.328284 +sk -17.335299 +dt -17.351620 +uk -17.369234 +yw -17.417666 +nm -17.476106 +ym -17.502590 +yu -17.593473 +cb -17.594121 +tz -17.599308 +tb -17.642160 +hy -17.645510 +yr -17.669178 +pd -17.672591 +kn -17.680128 +j_ -17.689087 +nx -17.709975 +テ_ -17.715597 +rx -17.774520 +hl -17.775988 +kh -17.777457 +vy -17.835187 +fm -17.868461 +tf -17.910558 +td -17.911364 +xx -17.966432 +eh -17.973150 +dl -17.991788 +gb -18.005495 +kr -18.050077 +pf -18.071544 +uy -18.072445 +wl -18.099753 +cm -18.108971 +rj -18.142654 +ml -18.162672 +xy -18.198632 +mn -18.198632 +py -18.219448 +pb -18.225451 +cq -18.241581 +fs -18.254822 +gd -18.281673 +rw -18.369788 +yc -18.403418 +vl -18.437852 +km -18.483881 +rh -18.519085 +wt -18.570369 +wc -18.593473 +dc -18.596064 +vc -18.646180 +gw -18.650212 +lz -18.665093 +hz -18.677382 +kk -18.756300 +xf -18.781136 +yi -18.797438 +hc -18.807909 +bp -18.833658 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pus b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pus new file mode 100644 index 00000000000..f4a8140261b --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/pus @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 6591619 +#UNIQUE_BIGRAMS 1622 +#UNSEEN_LOG_PROB -14.218168 +#EXPECTED_SCORE -8.314772 +ه_ -4.801599 +ي_ -4.846256 +_ا -5.533980 +ان -5.763981 +_ك -6.157580 +_و -6.185472 +ن_ -6.190418 +_م -6.308252 +و_ -6.325386 +_د -6.536101 +ر_ -6.552689 +اي -6.602310 +ې_ -6.615384 +د_ -6.679850 +ت_ -6.686913 +ار -6.700871 +ل_ -6.712255 +_ب -6.757005 +ور -6.886734 +_پ -6.902358 +ست -6.909182 +وا -6.924467 +_خ -6.956946 +_س -6.963186 +ا_ -7.081664 +ون -7.098482 +_ن -7.117321 +وي -7.174095 +ول -7.187528 +ري -7.197420 +_ه -7.233361 +لي -7.242015 +را -7.244436 +اس -7.248058 +_ت -7.249622 +ال -7.253124 +_ل -7.325561 +ره -7.335284 +لا -7.370090 +نو -7.389483 +ند -7.395403 +ام -7.436744 +يا -7.459100 +دا -7.501581 +م_ -7.519822 +_ش -7.596411 +رو -7.601077 +كو -7.654669 +تا -7.684120 +ني -7.698596 +ين -7.720310 +لو -7.793346 +_ر -7.812555 +نه -7.833270 +ما -7.865830 +يو -7.903118 +با -7.930728 +ړي -7.943495 +كا -7.999020 +كړ -7.999693 +اد -8.006037 +ات -8.029063 +ته -8.032841 +او -8.077431 +تو -8.091809 +دو -8.132872 +ير -8.133671 +سي -8.177925 +له -8.184469 +دي -8.185615 +بر -8.213345 +مو -8.235800 +وم -8.238045 +نا -8.283490 +ده -8.313047 +اف -8.328654 +_ج -8.351134 +خو -8.370054 +مي -8.370199 +پا -8.423383 +وه -8.447554 +_ز -8.470739 +يد -8.480555 +يل -8.539518 +وس -8.540006 +يك -8.548750 +سو -8.571051 +شو -8.584095 +تي -8.587795 +_ي -8.606015 +ود -8.615599 +مر -8.620673 +غا -8.637919 +_ح -8.640188 +سر -8.665026 +كي -8.675011 +غه -8.675100 +اب -8.694646 +سا -8.727203 +پل -8.748602 +_ع -8.753978 +_څ -8.759279 +رت -8.780585 +بي -8.783282 +فغ -8.823668 +زي -8.843438 +هغ -8.844042 +_ځ -8.855756 +من -8.880196 +خپ -8.900239 +اك -8.911788 +پو -8.914426 +س_ -8.917492 +تر -8.930675 +_ډ -8.932813 +نس -8.937634 +رد -8.949163 +ها -8.952954 +مل -8.970085 +ب_ -8.977678 +_غ -8.995441 +در -9.026493 +كر -9.050199 +ك_ -9.103380 +مه -9.125824 +غو -9.128272 +وړ -9.129375 +ړه -9.144778 +وك -9.147755 +يت -9.149494 +_ق -9.183323 +اړ -9.211203 +_ف -9.212371 +از -9.216661 +رك -9.239896 +ۍ_ -9.263915 +يس -9.264588 +_چ -9.265531 +ښت -9.270118 +يي -9.291903 +_ټ -9.314440 +يم -9.317649 +اع -9.338751 +رې -9.340737 +كل -9.344574 +هم -9.371286 +اه -9.398797 +وو -9.405312 +مد -9.412155 +لت -9.436213 +عل -9.437579 +تل -9.447630 +پر -9.458060 +هي -9.458214 +رس -9.480712 +خا -9.483999 +كن -9.499441 +دل -9.500234 +_ګ -9.504837 +لم -9.508818 +بل -9.530182 +ټو -9.539273 +ز_ -9.544984 +اش -9.555157 +دې -9.559775 +خه -9.563413 +پي -9.569886 +بو -9.588638 +لې -9.601332 +مت -9.636264 +زم -9.637309 +شت -9.642898 +تن -9.689125 +لر -9.692199 +نګ -9.704747 +ېر -9.705295 +رن -9.705843 +خب -9.718880 +_ښ -9.730732 +t_ -9.738938 +كه -9.743058 +ځا -9.759469 +وز -9.776452 +ږي -9.782800 +لك -9.789370 +شر -9.796944 +سل -9.831624 +دغ -9.840024 +خت -9.880299 +اخ -9.880299 +قا -9.886501 +_گ -9.912843 +وږ -9.936025 +خل -9.942687 +جا -9.943764 +حا -9.946137 +_b -9.961985 +ږ_ -10.011279 +مخ -10.012861 +s_ -10.014897 +به -10.029234 +رم -10.030836 +شا -10.034275 +هر -10.050199 +زا -10.056012 +لس -10.065596 +نې -10.065831 +نت -10.065831 +ړا -10.079975 +_ژ -10.079975 +ش_ -10.094738 +كس -10.099772 +جو -10.104102 +عا -10.106030 +de -10.107478 +a_ -10.117655 +سې -10.121064 +نظ -10.130846 +فر -10.130846 +څخ -10.133794 +يه -10.149618 +وښ -10.158846 +اغ -10.168638 +_ص -10.171159 +وخ -10.188166 +وب -10.189699 +دن -10.203569 +ف_ -10.204602 +يز -10.207963 +يږ -10.216792 +نك -10.216792 +مش -10.218877 +مس -10.228823 +دم -10.236460 +كې -10.239102 +هن -10.246794 +اق -10.267148 +وت -10.268767 +غل -10.273907 +er -10.276619 +ګر -10.280153 +رز -10.297401 +فت -10.298228 +گر -10.300434 +حم -10.304580 +وې -10.311795 +اح -10.343862 +پښ -10.353852 +چا -10.377241 +_ط -10.377532 +ړ_ -10.384244 +ړو -10.403384 +ri -10.409325 +نن -10.409623 +نځ -10.415290 +كم -10.424585 +ړې -10.426994 +لل -10.436062 +كت -10.445187 +ښا -10.447630 +ق_ -10.450996 +es -10.471671 +ډي -10.481337 +رګ -10.482590 +il -10.498649 +رځ -10.501502 +خي -10.504361 +شد -10.504678 +ع_ -10.508499 +ch -10.512970 +لب -10.512970 +_d -10.524529 +خد -10.538134 +فا -10.546293 +لن -10.550226 +ګو -10.554169 +مح -10.564739 +لپ -10.567393 +ډې -10.573383 +bi -10.578729 +سم -10.581410 +بد -10.583087 +ګه -10.588469 +ځي -10.594548 +_c -10.595564 +جن -10.599973 +مړ -10.619467 +ld -10.631569 +څو -10.643073 +رب -10.645875 +ځو -10.652906 +بې -10.656788 +دد -10.660325 +شي -10.662097 +هد -10.685696 +te -10.693649 +ظر -10.703834 +عر -10.711888 +طا -10.712255 +سه -10.725535 +قو -10.728503 +ځ_ -10.729989 +مع -10.735949 +ګ_ -10.748320 +تم -10.796944 +ډو -10.798502 +قي -10.798502 +هو -10.800062 +رل -10.821290 +يش -10.832022 +اص -10.861853 +شم -10.869203 +تې -10.879887 +جه -10.881125 +عم -10.889404 +هې -10.895228 +صا -10.904428 +زه -10.917492 +نگ -10.929394 +كش -10.931957 +رف -10.950895 +نش -10.966139 +ht -10.970963 +زو -10.974922 +يب -10.982873 +حك -10.990869 +رخ -10.997565 +عي -10.998908 +في -11.001150 +nt -11.002496 +ېد -11.010150 +e_ -11.045796 +o_ -11.049039 +نډ -11.065362 +ېږ -11.075245 +un -11.080923 +مې -11.086147 +تړ -11.089005 +چي -11.093302 +_m -11.099052 +نړ -11.108686 +جر -11.108686 +ژو -11.120333 +جم -11.120333 +بن -11.131092 +an -11.146390 +os -11.148873 +تح -11.156846 +شن -11.179511 +ړل -11.186125 +وګ -11.186635 +sc -11.194308 +al -11.195334 +اج -11.197902 +قر -11.225413 +وج -11.227510 +تخ -11.228035 +مك -11.249722 +rs -11.251856 +نم -11.257203 +لګ -11.266339 +وش -11.269577 +صل -11.270658 +طر -11.272282 +na -11.272823 +co -11.273907 +du -11.277162 +سن -11.282059 +ح_ -11.283695 +خ_ -11.285879 +وپ -11.302367 +ګړ -11.314580 +ma -11.315137 +نج -11.315695 +وع -11.323526 +hr -11.325772 +r_ -11.325772 +مب -11.328021 +if -11.329710 +ژب -11.331401 +سپ -11.335920 +ګل -11.338185 +ft -11.344431 +en -11.354139 +نۍ -11.355859 +رش -11.364489 +قت -11.368534 +شه -11.376659 +ic -11.388932 +ېو -11.395403 +كښ -11.401903 +يې -11.411410 +ړۍ -11.413797 +غي -11.423383 +شك -11.424585 +يق -11.437275 +تګ -11.443357 +اچ -11.448242 +گا -11.448853 +جل -11.460525 +ښو -11.467326 +هل -11.467946 +_a -11.469807 +ظا -11.472914 +ro -11.480399 +بت -11.500551 +وټ -11.506269 +ځك -11.508181 +عد -11.513290 +اط -11.519702 +وژ -11.524207 +ښي -11.543024 +قد -11.544984 +عن -11.550882 +گي -11.551539 +اپ -11.552854 +صو -11.560766 +it -11.563413 +زن -11.566729 +زړ -11.569387 +مق -11.576722 +حد -11.583423 +_p -11.583423 +تق -11.591505 +حق -11.594887 +ښك -11.603034 +ط_ -11.611912 +be -11.614654 +تص -11.616028 +l_ -11.616715 +چو -11.622224 +si -11.625678 +sa -11.643073 +دس -11.647279 +تج -11.655022 +ca -11.658555 +عت -11.658555 +مج -11.674206 +اڼ -11.682096 +زې -11.692199 +mi -11.708954 +_g -11.722943 +تع -11.729617 +ra -11.732593 +ښې -11.743058 +پن -11.743809 +em -11.743809 +حي -11.750580 +گو -11.758900 +يخ -11.761937 +يگ -11.766505 +ګي -11.768031 +op -11.771087 +حت -11.771853 +پخ -11.772618 +ss -11.786468 +وق -11.791890 +تب -11.794220 +نب -11.795776 +صي -11.799672 +ډا -11.800452 +څن -11.802796 +نف -11.805144 +ig -11.808280 +رح -11.816151 +جي -11.829631 +څ_ -11.834418 +حب -11.839222 +وض -11.847264 +ږد -11.847264 +يع -11.850493 +ېن -11.853730 +wa -11.865115 +رج -11.875768 +لف -11.877414 +gr -11.884017 +رگ -11.884845 +چن -11.888159 +زد -11.893145 +_s -11.895645 +گ_ -11.903170 +سب -11.907368 +زر -11.919186 +هس -11.920882 +پې -11.923431 +ګټ -11.935382 +aw -11.941395 +ټه -11.941395 +py -11.948298 +yr -11.948298 +gh -11.948298 +ځه -11.953497 +ضا -11.955234 +دف -11.956973 +گف -11.958714 +غې -11.959586 +رغ -11.962203 +وځ -11.967453 +حس -11.970963 +ګې -11.972721 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/quz b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/quz new file mode 100644 index 00000000000..7a0e8d5c119 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/quz @@ -0,0 +1,483 @@ +#TOTAL_BIGRAMS 2523627 +#UNIQUE_BIGRAMS 479 +#UNSEEN_LOG_PROB -17.945139 +#EXPECTED_SCORE -6.922185 +a_ -4.355008 +an -5.230336 +qa -5.358722 +ta -5.469735 +i_ -5.475523 +aq -5.546850 +ma -5.586955 +pi -5.681606 +na -5.712119 +ll -5.763024 +ch -5.776561 +ay -5.874549 +_k -5.936676 +n_ -5.959582 +ka -5.968897 +la -5.994037 +ha -6.014106 +ap -6.047937 +un -6.091947 +pa -6.181512 +in -6.203251 +wa -6.396173 +is -6.400803 +ku -6.405302 +_p -6.478604 +ni -6.489350 +_s -6.497539 +am -6.576960 +as -6.673618 +_n -6.673793 +_m -6.678704 +_l -6.720595 +sq -6.786593 +q_ -6.839493 +y_ -6.847173 +at -6.895563 +ru -6.906494 +ar -6.914955 +_a -6.923604 +_h -6.989344 +ri -7.005340 +qt -7.009753 +_w -7.031801 +_c -7.039752 +hu -7.044197 +im -7.081417 +ti -7.084440 +u_ -7.090193 +mi -7.092688 +_r -7.115337 +yu -7.148856 +ya -7.159360 +il -7.270506 +uk -7.290772 +su -7.354552 +si -7.361586 +nt -7.414050 +uy -7.431609 +_i -7.526549 +_q -7.543300 +hi -7.545968 +ak -7.548855 +ac -7.581661 +ki -7.590229 +ra -7.657658 +s_ -7.777846 +al -7.785268 +st -7.805972 +li -7.813797 +tu -7.829445 +m_ -7.834917 +ip -7.845528 +aw -7.889179 +k_ -7.933772 +us -8.040053 +uq -8.050018 +ur -8.051079 +qi -8.090115 +sa -8.108142 +iq -8.108300 +ik -8.122090 +it -8.142946 +_u -8.159850 +uw -8.202661 +um -8.247477 +rq -8.267420 +ic -8.273067 +_t -8.294805 +nk -8.354178 +sp -8.358675 +up -8.370356 +pu -8.377373 +_d -8.381371 +_y -8.389208 +qu -8.406368 +wi -8.436948 +iy -8.448885 +di -8.520553 +ir -8.574016 +o_ -8.696737 +ut -8.711519 +nq -8.813797 +ns -8.830616 +yk -8.848688 +nc -8.883363 +tr -8.924993 +nu -8.931677 +_j -9.018250 +mu -9.121772 +pr -9.148451 +ym -9.150398 +sy -9.194600 +p_ -9.248867 +ul -9.263198 +qh -9.275191 +io -9.373766 +os -9.437345 +uc -9.496403 +yt -9.524758 +es -9.628631 +nm -9.683044 +lq -9.700537 +iw -9.701489 +on -9.769215 +e_ -9.783251 +_b -9.784763 +sh -9.810713 +sk -9.846054 +yn -9.848161 +lu -9.898015 +w_ -9.904028 +to -9.911716 +np -9.940076 +re -9.984558 +r_ -9.999110 +yp -10.006148 +er -10.024489 +wp -10.116051 +en -10.146829 +ws -10.196275 +sw -10.219943 +hk -10.250955 +je -10.254443 +ro -10.275545 +ia -10.319430 +mp -10.334114 +bu -10.352682 +l_ -10.363939 +wy -10.416880 +ad -10.545968 +el -10.546823 +ph -10.597296 +qm -10.601731 +or -10.610642 +kh -10.631349 +rk -10.638622 +kl -10.671810 +se -10.723069 +_g -10.788298 +no -10.793362 +yq -10.807636 +co -10.813797 +yl -10.869393 +hw -10.870462 +nd -10.924993 +kr -11.058833 +yr -11.067395 +do -11.067395 +rm -11.071080 +yw -11.073542 +ci -11.077243 +ue -11.100904 +vi -11.105935 +om -11.112249 +ca -11.114782 +oq -11.149424 +lo -11.188916 +gu -11.217219 +ea -11.229520 +mo -11.244699 +da -11.288357 +ja -11.291219 +ua -11.324553 +pe -11.345226 +ju -11.351188 +du -11.387484 +rt -11.390550 +_f -11.448485 +_e -11.462936 +le -11.467786 +cr -11.480798 +gr -11.485708 +de -11.493928 +id -11.503855 +py -11.505516 +ig -11.544260 +ne -11.545968 +ib -11.552822 +ce -11.570100 +va -11.570100 +h_ -11.580567 +iv -11.592875 +he -11.610642 +wq -11.621409 +ji -11.652357 +ov -11.693420 +te -11.714398 +nw -11.724035 +qn -11.727908 +ve -11.729849 +kw -11.807636 +_v -11.811740 +ed -11.826198 +kc -11.834525 +sm -11.847107 +ry -11.879050 +lp -11.881205 +qk -11.900745 +dr -11.902933 +dy -11.938392 +aj -11.963287 +qs -11.965571 +ep -11.983979 +ba -11.993272 +qp -12.002625 +qo -12.004972 +d_ -12.004972 +ol -12.014402 +lk -12.021515 +ho -12.038249 +jo -12.050321 +z_ -12.050321 +ec -12.055179 +t_ -12.067395 +rp -12.082192 +wr -12.094640 +rr -12.117320 +ny -12.137784 +me -12.142946 +nl -12.153325 +bl -12.190252 +yc -12.198289 +ge -12.206371 +go -12.233644 +so -12.241928 +rn -12.264252 +be -12.264252 +ie -12.284074 +mm -12.321623 +tt -12.339289 +ks -12.351188 +ab -12.357174 +ke -12.402881 +br -12.402881 +pt -12.409086 +lt -12.418444 +ag -12.424717 +yo -12.443700 +ly -12.505516 +ms -12.528975 +bo -12.528975 +ui -12.542553 +ga -12.545968 +ez -12.545968 +cu -12.584073 +j_ -12.619609 +ng -12.645015 +sc -12.701013 +fa -12.712478 +au -12.727908 +ah -12.747431 +ii -12.751367 +by -12.791334 +fr -12.803543 +rd -12.819984 +ot -12.824124 +bi -12.824124 +th -12.836615 +po -12.900745 +qe -12.918339 +rw -12.931677 +eh -12.958728 +rl -12.986296 +qq -12.990943 +eo -13.009679 +ql -13.014402 +_o -13.092142 +rc -13.142946 +fe -13.184918 +yi -13.190252 +nr -13.200978 +ub -13.228148 +rh -13.239161 +et -13.239161 +hq -13.278383 +kt -13.301283 +eq -13.312871 +ey -13.318700 +mb -13.384424 +eg -13.390550 +hh -13.453286 +wm -13.479165 +ae -13.479165 +oc -13.492280 +km -13.505516 +ev -13.525600 +ih -13.539147 +ud -13.539147 +ob -13.580567 +za -13.594642 +qr -13.601731 +av -13.608856 +kk -13.652357 +sl -13.674610 +ys -13.689638 +em -13.743505 +ai -13.775214 +wl -13.849215 +ps -13.874750 +rs -13.883363 +af -13.892028 +rg -13.900745 +of -13.900745 +gi -13.927217 +eb -13.954184 +op -13.990943 +lw -14.038249 +nz -14.038249 +zo -14.087158 +nn -14.107196 +fi -14.117320 +sn -14.158543 +vo -14.211785 +_z -14.233644 +sr -14.244699 +ky -14.255840 +if -14.255840 +xi -14.289787 +ew -14.301283 +ct -14.301283 +ww -14.312871 +kn -14.348204 +oa -14.409086 +uz -14.446888 +od -14.446888 +gn -14.472651 +wu -14.539147 +wk -14.594642 +oj -14.608856 +rb -14.637711 +lf -14.637711 +ck -14.652357 +_x -14.667154 +og -14.712478 +ft -14.712478 +ug -14.727908 +oi -14.727908 +ld -14.743505 +hr -14.759273 +oo -14.775214 +lm -14.824124 +pl -14.824124 +b_ -14.840803 +uj -14.874750 +fo -14.892028 +lb -14.927217 +lg -14.927217 +ij -14.945139 +lv -14.981665 +ex -15.000281 +fl -15.019140 +pc -15.038249 +iz -15.077243 +az -15.117320 +lh -15.179604 +oh -15.200978 +f_ -15.244699 +ee -15.244699 +qw -15.267067 +jt -15.267067 +ou -15.289787 +cn -15.336330 +ej -15.360177 +g_ -15.360177 +wn -15.409086 +wt -15.409086 +yh -15.434177 +dp -15.459712 +ao -15.459712 +we -15.485708 +c_ -15.485708 +ko -15.485708 +hn -15.539147 +rv -15.539147 +wo -15.539147 +nj -15.566628 +uh -15.623211 +kp -15.623211 +ei -15.712478 +fu -15.712478 +oz -15.743505 +ef -15.775214 +ok -15.775214 +ty -15.807636 +xo -15.807636 +zc -15.840803 +ow -15.840803 +aa -15.874750 +ml -15.874750 +iu -15.874750 +ts -15.909515 +oy -15.981665 +qy -16.057614 +qc -16.057614 +jr -16.137784 +ls -16.137784 +xa -16.137784 +eu -16.267067 +ye -16.312871 +tn -16.312871 +cl -16.312871 +tp -16.360177 +pm -16.409086 +yz -16.409086 +ax -16.409086 +oe -16.459712 +uv -16.459712 +tq -16.512180 +د_ -16.566628 +gl -16.623211 +mr -16.623211 +pb -16.682105 +ds -16.682105 +lc -16.682105 +gd -16.807636 +tb -16.874750 +lj -16.874750 +sj -16.874750 +gt -16.945139 +tw -16.945139 +mk -16.945139 +hp -17.019140 +v_ -17.019140 +ht -17.097142 +tl -17.179604 +ox -17.179604 +zi -17.179604 +ze -17.267067 +rf -17.360177 +zm -17.360177 +fg -17.360177 +ln -17.360177 +mq -17.459712 +dw -17.459712 +sb -17.459712 +xv -17.459712 +_ع -17.459712 +عب -17.459712 +بد -17.459712 +tc -17.566628 +rx -17.566628 +x_ -17.566628 +dl -17.682105 +pp -17.682105 +nh -17.682105 +pk -17.682105 +sv -17.682105 +vg -17.682105 +_م -17.682105 +مح -17.682105 +حم -17.682105 +مد -17.682105 +cq -17.807636 +xt -17.807636 +ss -17.945139 +kq -17.945139 +ix -17.945139 +dc -17.945139 +ff -17.945139 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/roh b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/roh new file mode 100644 index 00000000000..ef84ebd94e7 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/roh @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 13550574 +#UNIQUE_BIGRAMS 555 +#UNSEEN_LOG_PROB -19.467586 +#EXPECTED_SCORE -7.343146 +s_ -4.791833 +a_ -4.952103 +n_ -5.601232 +un -5.657922 +_p -5.819567 +er -5.845558 +_c -5.910843 +_s -5.916060 +as -6.007821 +r_ -6.055035 +ra -6.238805 +an -6.253172 +da -6.272228 +_d -6.283654 +in -6.336370 +ta -6.361634 +nt -6.396243 +ch -6.448816 +la -6.450575 +na -6.463483 +iu -6.517130 +al -6.576563 +en -6.579941 +sc -6.668150 +zi -6.752375 +ar -6.808802 +re -6.949383 +_i -6.953890 +cu -6.957948 +st -6.993879 +t_ -7.041402 +te -7.047572 +_t -7.057933 +ur -7.066384 +ia -7.112196 +pe -7.113625 +ad -7.133176 +li -7.161749 +i_ -7.169944 +es -7.184598 +ns -7.204811 +_l -7.231123 +is -7.239690 +_v -7.255757 +eg -7.274295 +l_ -7.280555 +_m -7.281399 +ma -7.353045 +_f -7.355013 +ls -7.363035 +ve -7.410935 +ai -7.429022 +on -7.452122 +tg -7.468320 +_a -7.480742 +co -7.482778 +sa -7.488770 +_r -7.512427 +az -7.518681 +si -7.521163 +ha -7.521945 +di -7.529420 +_e -7.557124 +ri -7.591928 +de -7.666495 +pr -7.690816 +it -7.715443 +ss -7.746064 +va -7.768825 +il -7.789193 +us -7.800755 +ti -7.824162 +ts -7.836763 +et -7.844475 +at -7.844965 +tu -7.903030 +el -7.909287 +gn -7.909569 +ni -7.914133 +_b -7.936572 +vi -7.950963 +_g -7.955052 +tr -7.961142 +qu -7.962867 +rt -7.979995 +nd -7.990084 +ro -8.015067 +rs -8.017051 +pa -8.039762 +gi -8.052595 +he -8.059367 +ca -8.067255 +se -8.069656 +ua -8.069684 +me -8.090370 +su -8.090457 +ir -8.153904 +av -8.183375 +_q -8.232419 +ic -8.238676 +or -8.266831 +um -8.275581 +e_ -8.298695 +g_ -8.316709 +pl -8.373661 +id -8.394648 +d_ -8.445296 +le -8.471246 +_n -8.480533 +ec -8.480685 +ga -8.512797 +du -8.529263 +ut -8.534978 +mi -8.541945 +em -8.558708 +fi -8.563977 +po -8.689739 +ue -8.695686 +be -8.699089 +nc -8.749382 +am -8.751124 +iz -8.773034 +mp -8.781208 +nz -8.797270 +iv -8.808300 +ie -8.835960 +ul -8.843570 +gl -8.861533 +za -8.873169 +pu -8.880877 +ed -8.882233 +ll -8.896470 +ol -8.912080 +o_ -8.918402 +_u -8.932639 +au -8.947280 +_o -8.983845 +fe -8.995861 +no -9.003819 +ct -9.017162 +ci -9.121224 +tt -9.138821 +gr -9.143209 +ds -9.166513 +lu -9.193749 +sp -9.203948 +im -9.224946 +rm -9.244832 +fu -9.256962 +ru -9.258526 +aj -9.259570 +ja -9.272746 +ba -9.277033 +hi -9.277231 +fa -9.287308 +om -9.308821 +mu -9.378472 +os -9.383867 +_h -9.386929 +ge -9.412893 +nn -9.424918 +pi -9.437412 +ne -9.460479 +to -9.466191 +vu -9.519736 +sv -9.535847 +ac -9.565469 +mo -9.567891 +ev -9.569184 +ce -9.580878 +gu -9.585533 +ov -9.591270 +cz -9.592832 +cr -9.656880 +ap -9.713766 +nu -9.735656 +ot -9.767131 +ag -9.769731 +ud -9.802347 +uc -9.822642 +ig -9.830473 +rd -9.834258 +oc -9.843228 +m_ -9.843325 +bl -9.843815 +c_ -9.857182 +ng -10.106301 +ex -10.114894 +lt -10.137981 +fr -10.147731 +zr -10.166575 +nf -10.171232 +rr -10.202755 +pp -10.236267 +mm -10.238451 +op -10.277033 +ez -10.287773 +bu -10.292840 +rn -10.309497 +cl -10.327716 +eu -10.328948 +ep -10.338428 +ab -10.350749 +mb -10.356042 +ui -10.365843 +so -10.371755 +dr -10.377126 +ff -10.410790 +if -10.423601 +rv -10.474650 +bi -10.496325 +p_ -10.506975 +od -10.510077 +zz -10.529931 +up -10.554379 +lo -10.598928 +hu -10.623576 +ea -10.630817 +h_ -10.640132 +rg -10.653961 +ub -10.698205 +cs -10.699797 +do -10.768523 +vo -10.770568 +ib -10.778961 +io -10.783083 +u_ -10.806726 +og -10.812267 +v_ -10.819561 +ef -10.897638 +ip -10.917269 +ei -11.009295 +cc -11.041023 +uz -11.065001 +rc -11.081749 +br -11.151965 +uv -11.160957 +hn -11.172460 +z_ -11.175412 +ms -11.204262 +je -11.269523 +gs -11.291505 +vr -11.297924 +af -11.321981 +fo -11.361494 +nv -11.386074 +ob -11.396369 +of -11.457333 +lm -11.494634 +ze -11.558387 +rz -11.577783 +mn -11.645068 +ug -11.687279 +vs -11.718512 +uf -11.755949 +ho -11.842837 +hs -11.883289 +rp -11.909262 +_z -11.991411 +xi -11.993580 +rq -11.996187 +uo -12.024296 +eq -12.056133 +vl -12.059310 +lv -12.063405 +ps -12.164374 +xe -12.196995 +pz -12.210556 +oj -12.221192 +_j -12.233957 +xa -12.261921 +fl -12.292573 +f_ -12.322799 +zu -12.358695 +_k -12.430343 +rl -12.435052 +lg -12.438594 +oz -12.465438 +x_ -12.474499 +sr -12.515677 +xp -12.521926 +bo -12.585942 +sl -12.708857 +eo -12.710997 +xt -12.731849 +go -12.763332 +zo -12.803107 +sg -12.813800 +hl -12.823800 +ke -12.895000 +pt -12.898247 +_w -12.924494 +ki -12.958835 +sf -12.994014 +ox -12.999235 +lp -13.053415 +zs -13.057040 +bs -13.078061 +eb -13.094729 +th -13.099394 +cn -13.127701 +ax -13.151723 +rf -13.240640 +oo -13.345337 +ld -13.348665 +ey -13.397230 +ix -13.402985 +dm -13.421555 +wi -13.500791 +sm -13.510698 +b_ -13.528201 +nq -13.563856 +k_ -13.628456 +gm -13.629804 +ou -13.633859 +xc -13.689036 +y_ -13.731849 +sb -13.774479 +we -13.790984 +ys -13.812267 +sk -13.846361 +rb -13.895811 +lc -13.922013 +jo -13.943658 +nl -13.970751 +oi -14.037215 +sd -14.143029 +rk -14.246836 +lf -14.352001 +cd -14.420388 +oa -14.427408 +vg -14.434463 +aq -14.446298 +ck -14.467849 +dv -14.511942 +ka -14.549744 +sq -14.609702 +eh -14.669483 +wa -14.694671 +ee -14.703166 +bb -14.708857 +sw -14.728955 +ik -14.752271 +oh -14.767038 +lr -14.800067 +dl -14.803107 +ko -14.809208 +gh -14.878069 +hr -14.884496 +ju -14.884496 +fs -14.900688 +ii -14.907216 +tl -14.913773 +vn -14.936963 +ow -14.967337 +bj -14.977605 +tn -15.044392 +oe -15.047994 +lb -15.066142 +nk -15.073465 +w_ -15.121995 +q_ -15.172214 +aa -15.180098 +pc -15.204011 +fq -15.308146 +ph -15.312472 +zd -15.334299 +nr -15.369923 +uj -15.415726 +tz -15.420388 +sh -15.511942 +ay -15.537033 +rx -15.562568 +ew -15.572910 +ku -15.593819 +zl -15.609702 +yo -15.620388 +nb -15.625761 +hm -15.658428 +ae -15.686226 +ao -15.686226 +ht -15.697497 +hw -15.726066 +dg -15.755213 +pd -15.778961 +ft -15.803107 +sn -15.840102 +iq -15.858961 +df -15.865302 +ux -15.865302 +xu -15.897435 +_y -15.910491 +bd -15.917064 +fp -15.936963 +ry -15.998364 +nm -16.012371 +bv -16.026515 +tc -16.055226 +tw -16.055226 +nj -16.106888 +tm -16.114422 +ye -16.129608 +ak -16.208035 +ek -16.224245 +cq -16.240640 +pb -16.257222 +wo -16.265586 +ah -16.406448 +ok -16.415726 +lz -16.502026 +rw -16.542104 +ty -16.625761 +tf -16.691851 +pf -16.703166 +lh -16.714571 +mv -16.761113 +xs -16.761113 +tp -16.772987 +bt -16.833870 +ws -16.833870 +ml -16.858961 +ks -16.858961 +tv -16.871672 +wn -16.910491 +gg -16.910491 +pn -16.923666 +dq -16.936963 +kh -17.033639 +hf -17.121995 +wh -17.137262 +dh -17.137262 +cp -17.152692 +zf -17.232419 +dn -17.232419 +ly -17.232419 +hg -17.282460 +ln -17.282460 +nh -17.299533 +wf -17.316811 +dw -17.316811 +np -17.369923 +dp -17.369923 +ih -17.443923 +oy -17.443923 +gt -17.482397 +rh -17.502026 +pg -17.542104 +hc -17.562568 +aw -17.625761 +ya -17.625761 +yn -17.625761 +py -17.669483 +kl -17.669483 +gb -17.691851 +sz -17.691851 +ww -17.714571 +cf -17.714571 +td -17.714571 +xn -17.761113 +yr -17.833870 +sy -17.833870 +tk -17.858961 +fc -17.858961 +mc -17.858961 +xo -17.884496 +uk -17.884496 +uy -17.910491 +gc -17.936963 +ny -17.963930 +gd -17.963930 +fd -17.963930 +dy -17.963930 +md -18.019425 +dt -18.047994 +fb -18.106888 +bn -18.106888 +_x -18.137262 +wl -18.168289 +fm -18.168289 +bc -18.168289 +cm -18.232419 +mg -18.265586 +cb -18.299533 +by -18.299533 +kr -18.334299 +wy -18.369923 +ej -18.443923 +j_ -18.443923 +uh -18.443923 +my -18.482397 +mh -18.521926 +kt -18.604388 +tb -18.647457 +fn -18.691851 +vf -18.691851 +gk -18.691851 +zk -18.784960 +ky -18.784960 +mf -18.833870 +kn -18.833870 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ron b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ron new file mode 100644 index 00000000000..f26f4827576 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ron @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 204442839 +#UNIQUE_BIGRAMS 578 +#UNSEEN_LOG_PROB -19.950927 +#EXPECTED_SCORE -7.304625 +e_ -4.978669 +a_ -5.034744 +i_ -5.482642 +re -5.676411 +at -5.855140 +ar -5.888469 +in -5.921313 +_c -5.924440 +_p -5.962420 +te -5.964046 +_a -6.085292 +ti -6.098717 +ta -6.118777 +ri -6.227537 +st -6.272121 +nt -6.275699 +t_ -6.312968 +_d -6.400585 +_s -6.411242 +ca -6.448621 +ul -6.479830 +or -6.620014 +ra -6.647987 +l_ -6.671420 +er -6.689388 +an -6.754514 +_m -6.813858 +le -6.817614 +en -6.829970 +tr -6.858504 +ea -6.885716 +es -6.900160 +_i -6.909395 +ma -6.926005 +ce -6.959336 +it -7.009987 +al -7.037098 +pr -7.079966 +_f -7.167799 +r_ -7.176726 +un -7.199000 +ni -7.237187 +di -7.276305 +ic -7.312614 +ie -7.317808 +ac -7.329394 +el -7.409177 +co -7.412525 +lu -7.416469 +_t -7.430730 +de -7.434322 +ru -7.448679 +_e -7.450617 +u_ -7.455138 +na -7.462099 +ur -7.462500 +il -7.482229 +ne -7.501842 +ia -7.503581 +pe -7.511657 +ii -7.515034 +as -7.526116 +li -7.529532 +n_ -7.552401 +tu -7.577819 +on -7.602630 +_r -7.631554 +ec -7.634804 +me -7.667520 +pa -7.689788 +ci -7.744453 +_l -7.750537 +la -7.784981 +cu -7.792255 +ro -7.834273 +lo -7.846060 +si -7.852103 +im -7.854454 +ui -7.854560 +to -7.922880 +oa -7.947229 +sa -7.959207 +_v -7.981326 +is -7.997074 +ut -8.014446 +se -8.033838 +mi -8.062013 +_n -8.071207 +ct -8.156584 +nd -8.158199 +su -8.217814 +da -8.237380 +po -8.242720 +ai -8.298228 +_o -8.325659 +_u -8.330932 +rt -8.353270 +et -8.359401 +om -8.368869 +pu -8.385625 +ei -8.389574 +am -8.414574 +_b -8.435178 +ve -8.448261 +nu -8.450980 +fo -8.469698 +os -8.470682 +ol -8.482665 +um -8.522906 +fi -8.534642 +em -8.572288 +ns -8.575074 +vi -8.576757 +sc -8.586416 +du -8.587572 +nc -8.594738 +va -8.606498 +iu -8.663102 +sp -8.666361 +s_ -8.687307 +za -8.715965 +mu -8.739111 +au -8.739331 +lt -8.748628 +fa -8.774127 +m_ -8.791414 +ap -8.804839 +oc -8.825737 +mp -8.830519 +ad -8.885586 +ot -8.888385 +ga -8.905234 +no -8.924697 +io -8.932196 +d_ -8.954879 +iv -8.961833 +us -8.966605 +fe -8.969716 +_g -8.972630 +rm -8.987923 +bi -8.994650 +do -9.009848 +ba -9.027560 +zi -9.044757 +ed -9.058210 +cr -9.132181 +uc -9.135653 +c_ -9.148920 +ir -9.151883 +ep -9.160625 +az -9.206426 +mo -9.207794 +op -9.210221 +so -9.272980 +ez -9.273604 +iz -9.280877 +pl -9.294102 +vo -9.298988 +ch -9.346125 +ua -9.359380 +bu -9.368135 +pi -9.373624 +eg -9.396166 +up -9.442279 +ev -9.478700 +ge -9.478856 +hi -9.493258 +ex -9.579735 +oi -9.629814 +gi -9.653426 +rs -9.656880 +pt -9.675254 +id -9.697440 +gr -9.757797 +av -9.766289 +od -9.767254 +ze -9.804896 +ju -9.873160 +ng -9.971340 +ab -9.992290 +cl -9.997648 +eu -10.029325 +gu -10.037927 +_j -10.058165 +ip -10.074082 +ob -10.077501 +mb -10.081747 +ag -10.088907 +z_ -10.122457 +dr -10.149293 +rc -10.159635 +aj -10.189954 +ig -10.271741 +bl -10.284112 +ub -10.287582 +af -10.320628 +rn -10.338836 +ef -10.350656 +ou -10.366610 +ib -10.393069 +eb -10.394056 +rg -10.397755 +br -10.407105 +ud -10.419645 +dz -10.431855 +rd -10.433104 +ov -10.445420 +of -10.506409 +if -10.523923 +_z -10.532343 +mn -10.615866 +be -10.633447 +nf -10.669611 +o_ -10.712245 +og -10.722499 +he -10.779792 +fu -10.818047 +p_ -10.829546 +bo -10.869716 +eo -10.957179 +zu -11.009201 +v_ -11.052324 +fr -11.111751 +fl -11.112579 +uz -11.127042 +_h -11.163143 +rv -11.204793 +xi -11.214838 +rb -11.233017 +sf -11.274526 +nv -11.326823 +ee -11.330725 +jo -11.402360 +ja -11.450387 +ug -11.453748 +vr -11.480599 +oz -11.551310 +zo -11.585296 +cc -11.604110 +xp -11.623171 +b_ -11.659531 +go -11.697932 +g_ -11.700819 +ld -11.776360 +ho -11.776360 +tf -11.867421 +ha -11.907062 +uv -11.958315 +sm -11.984355 +vu -12.059475 +xe -12.060258 +gh -12.088285 +xt -12.141173 +j_ -12.221226 +nz -12.262480 +xa -12.357564 +ps -12.385233 +ij -12.427446 +rl -12.542085 +bs -12.550950 +sl -12.563839 +lb -12.573914 +lc -12.580383 +eh -12.585578 +zv -12.589574 +ej -12.638141 +ln -12.663829 +lv -12.683841 +ae -12.689564 +dm -12.737962 +bt -12.740471 +hn -12.740809 +je -12.741969 +rp -12.802840 +ax -12.810981 +gl -12.813621 +sd -12.904354 +uf -12.908743 +rf -12.925774 +cd -12.968630 +_w -12.979759 +f_ -12.979816 +y_ -12.996270 +lm -13.028868 +nl -13.031406 +ih -13.043807 +uj -13.112204 +x_ -13.121796 +ox -13.129174 +nr -13.131579 +td -13.149677 +zb -13.220990 +ue -13.229369 +rz -13.240117 +oo -13.266924 +k_ -13.268804 +xc -13.276345 +ji -13.297646 +mt -13.315880 +dv -13.520654 +pd -13.541874 +rh -13.556338 +_k -13.694794 +tc -13.701077 +ss -13.715339 +sn -13.726008 +gn -13.777994 +th -13.795246 +we -13.817487 +h_ -13.847234 +ll -13.856625 +oe -13.993218 +md -14.016418 +tl -14.017822 +ah -14.022395 +tb -14.032884 +xu -14.061916 +jl -14.089945 +cm -14.119910 +ft -14.155911 +ke -14.180071 +vs -14.194949 +hr -14.229912 +ck -14.275785 +ki -14.297788 +ix -14.335222 +sh -14.359491 +tm -14.372155 +cv -14.426902 +cs -14.431885 +rk -14.460235 +nj -14.509090 +lg -14.581637 +_x -14.614715 +wa -14.636477 +oh -14.649476 +zd -14.674354 +w_ -14.691803 +ow -14.696105 +sr -14.735410 +dl -14.745841 +hu -14.806020 +ew -14.814129 +ls -14.815349 +pn -14.829045 +ux -14.831924 +lp -14.847234 +nn -14.850358 +wi -14.870298 +vl -14.880691 +ay -14.905599 +ao -14.908418 +tn -14.911894 +tv -14.965296 +gd -15.051095 +ok -15.071118 +pc -15.075010 +nk -15.088715 +vd -15.103297 +tt -15.223688 +ws -15.253700 +lz -15.275366 +yo -15.347379 +bd -15.382819 +_y -15.401329 +cn -15.501869 +gm -15.559658 +rr -15.565463 +tw -15.641338 +dn -15.663509 +ko -15.667543 +lf -15.670852 +ph -15.674908 +wo -15.734832 +ka -15.752254 +ms -15.770677 +ty -15.775419 +nm -15.805818 +ff -15.827403 +dc -15.849732 +ht -15.859768 +ly -15.895025 +fd -15.909720 +ry -15.921497 +uh -16.007209 +pp -16.022629 +cf -16.041068 +xv -16.078180 +ak -16.104788 +ek -16.168330 +mr -16.172494 +mv -16.198262 +jb -16.211053 +by -16.232083 +fm -16.270616 +mf -16.293672 +dd -16.337411 +sk -16.340921 +mm -16.350914 +bc -16.355048 +ya -16.368121 +ik -16.374102 +ds -16.377102 +ey -16.397669 +oj -16.418533 +zn -16.459281 +ks -16.544402 +ku -16.552518 +ml -16.575077 +dj -16.578526 +zz -16.631991 +bn -16.689750 +uu -16.707010 +bv -16.723715 +zg -16.751475 +ts -16.803798 +bb -16.807031 +ny -16.832335 +dg -16.869030 +sz -16.930284 +rj -16.968686 +dw -16.997020 +df -17.006280 +sy -17.030638 +sb -17.030638 +db -17.034422 +ys -17.088469 +bm -17.107276 +ww -17.109270 +zl -17.128353 +kl -17.216953 +qu -17.279570 +lk -17.281817 +gs -17.286322 +aa -17.320565 +cj -17.332162 +wn -17.361570 +gy -17.437197 +dy -17.438450 +np -17.444731 +oy -17.445990 +gb -17.461190 +kt -17.470131 +pv -17.485589 +vc -17.485589 +jm -17.497292 +xo -17.534320 +gv -17.669013 +my -17.670484 +jp -17.679344 +vn -17.679344 +sj -17.683795 +ye -17.694233 +pj -17.715339 +xy -17.758499 +gt -17.793341 +py -17.794945 +uo -17.837284 +wh -17.842251 +kr -17.848899 +mw -17.850566 +hl -17.869030 +_q -17.936466 +tp -17.938237 +gp -17.943564 +fc -17.985070 +jd -17.988737 +uk -17.997944 +lh -18.018408 +ky -18.039166 +fg -18.135447 +nb -18.160039 +aw -18.164179 +xx -18.185058 +nh -18.201981 +fp -18.208379 +sv -18.260609 +zm -18.294239 +yl -18.312502 +sw -18.371108 +lj -18.417298 +zy -18.424728 +pg -18.462464 +hc -18.573699 +yr -18.598694 +yw -18.618438 +yp -18.641338 +wr -18.697229 +bp -18.761632 +mc -18.761632 +sq -18.764772 +mg -18.793341 +pf -18.799767 +lr -18.838938 +kf -18.855578 +pm -18.875803 +sg -18.906683 +aq -18.906683 +qa -18.963266 +hy -18.970498 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/rus b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/rus new file mode 100644 index 00000000000..2a618fa0f21 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/rus @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 294248769 +#UNIQUE_BIGRAMS 1163 +#UNSEEN_LOG_PROB -15.798092 +#EXPECTED_SCORE -8.064453 +и_ -5.487624 +_п -5.900734 +ст -6.028722 +е_ -6.110398 +_с -6.134665 +а_ -6.229347 +о_ -6.319889 +ни -6.469055 +я_ -6.470914 +ен -6.491880 +ра -6.591558 +то -6.616658 +но -6.647171 +ко -6.698516 +ро -6.762310 +ов -6.776407 +го -6.789824 +_в -6.841609 +од -6.900420 +_о -6.904099 +ре -6.912452 +на -6.934997 +по -6.946337 +ос -6.976312 +пр -6.985020 +_к -7.019507 +ан -7.030301 +та -7.042803 +ер -7.045312 +ор -7.149940 +м_ -7.192585 +_н -7.236410 +_д -7.244947 +ол -7.262427 +ет -7.267485 +ле -7.279784 +ал -7.285779 +ли -7.286272 +во -7.295424 +_р -7.323159 +т_ -7.357960 +ь_ -7.372744 +те -7.399915 +_м -7.414407 +ом -7.452809 +_г -7.458481 +ль -7.460564 +ии -7.477265 +ел -7.494825 +ои -7.499801 +_б -7.501080 +от -7.525855 +_т -7.555002 +да -7.562748 +ва -7.565060 +ри -7.570131 +де -7.570701 +ка -7.575607 +ес -7.577491 +ти -7.589710 +ве -7.617762 +ны -7.622889 +ит -7.628040 +ме -7.657715 +х_ -7.681220 +ск -7.690342 +ог -7.701739 +ла -7.703653 +не -7.711281 +ис -7.715576 +ы_ -7.718773 +ин -7.758158 +ат -7.793513 +_и -7.805595 +он -7.844069 +ло -7.844958 +л_ -7.859583 +ем -7.906767 +об -7.925608 +ия -7.926025 +у_ -7.931165 +со -7.942966 +ед -7.958649 +ил -7.961497 +че -7.988596 +ас -7.995730 +ть -8.006347 +тр -8.028979 +_ч -8.042035 +ак -8.042738 +ие -8.071537 +тв -8.076799 +ав -8.090154 +се -8.156739 +за -8.182344 +в_ -8.183732 +ки -8.190247 +ар -8.207091 +_у -8.245976 +бо -8.273981 +_з -8.290745 +мо -8.307922 +ся -8.316179 +до -8.324845 +ми -8.355082 +им -8.368991 +ви -8.371944 +аз -8.409500 +же -8.413423 +ма -8.421135 +бы -8.440076 +пе -8.446698 +ди -8.455309 +нн -8.475017 +сл -8.526728 +ру -8.538157 +еи -8.563119 +_а -8.604765 +ам -8.618826 +ля -8.620184 +ик -8.627406 +к_ -8.630031 +вы -8.637748 +ци -8.650372 +ча -8.704843 +н_ -8.725061 +тс -8.725949 +_э -8.755333 +ег -8.768805 +ду -8.824738 +си -8.825085 +ю_ -8.825974 +ок -8.848272 +из -8.854368 +ек -8.860973 +ьн -8.868983 +_л -8.881726 +ыл -8.885999 +сс -8.923998 +дн -8.927864 +ых -8.942268 +_е -8.972829 +ое -9.021677 +сп -9.041832 +вл -9.046460 +па -9.058976 +чи -9.061331 +нт -9.090233 +ад -9.111289 +ев -9.137646 +ич -9.145705 +ры -9.152322 +их -9.155167 +гр -9.166933 +ае -9.169288 +хо -9.173283 +эт -9.177069 +чт -9.204281 +кр -9.214413 +ив -9.232284 +ту -9.239823 +_ф -9.246157 +са -9.264890 +ож -9.285324 +д_ -9.300007 +вн -9.306729 +кт -9.313389 +ые -9.339757 +оз -9.353120 +бр -9.367878 +ая -9.376817 +вс -9.379146 +ше -9.385976 +ыи -9.387512 +уд -9.387877 +уч -9.407927 +оп -9.409624 +ще -9.410024 +тн -9.411409 +р_ -9.422092 +ир -9.441996 +св -9.457762 +ио -9.517342 +лу -9.538487 +ез -9.557849 +це -9.578236 +ус -9.578847 +му -9.580606 +аб -9.599379 +ги -9.607851 +бе -9.613593 +вр -9.626638 +ач -9.630573 +нс -9.634672 +сь -9.635748 +ты -9.639579 +ят -9.646471 +пи -9.653767 +зн -9.674247 +зв -9.676704 +га -9.688583 +ку -9.706278 +жи -9.709325 +рт -9.720057 +дл -9.753595 +бл -9.773368 +ут -9.775253 +зо -9.780323 +оч -9.783379 +ид -9.802722 +аи -9.807648 +ур -9.819335 +ах -9.828698 +рн -9.839452 +уп -9.840611 +иц -9.843041 +сн -9.846215 +ря -9.874162 +_ж -9.899478 +ее -9.910147 +ня -9.912083 +ым -9.930653 +су -9.932429 +жд -9.940636 +ну -9.951003 +ши -9.952406 +жн -9.966470 +ию -9.981603 +уж -9.998601 +ют -10.014828 +зд -10.015366 +ац -10.023590 +иг -10.032856 +ую -10.034851 +ап -10.056700 +с_ -10.072330 +нд -10.096690 +пу -10.126875 +би -10.136639 +_х -10.139976 +фи -10.146064 +гл -10.170006 +рс -10.173246 +др -10.182247 +дс -10.190353 +аж -10.190823 +чн -10.191924 +ьс -10.195306 +пл -10.219882 +щи -10.221912 +зи -10.224878 +оя -10.235851 +ул -10.243718 +ук -10.255243 +иа -10.276504 +кл -10.281450 +еж -10.310644 +дв -10.318317 +лю -10.319895 +уб -10.323510 +_ц -10.332522 +ге -10.344597 +_я -10.348600 +еч -10.370774 +мп -10.392601 +яв -10.395821 +ба -10.402455 +бу -10.430782 +уг -10.438654 +рм -10.449521 +рв -10.452637 +ды -10.455258 +нц -10.458545 +мя -10.504614 +ву -10.510745 +лс -10.529183 +ыс -10.543981 +ср -10.573622 +мн -10.591349 +еб -10.602032 +нк -10.617308 +мы -10.621458 +аю -10.623104 +ыв -10.624140 +ке -10.633791 +жа -10.639919 +нь -10.644299 +ех -10.646278 +_ш -10.655043 +зы -10.655264 +аг -10.664561 +яе -10.672621 +ьк -10.681669 +рг -10.698330 +гд -10.709411 +тк -10.720975 +ьш -10.726356 +ун -10.733342 +ыт -10.745512 +еп -10.760166 +з_ -10.761706 +ян -10.781453 +фо -10.842127 +бщ -10.850124 +ош -10.866709 +оо -10.886119 +фе -10.895438 +см -10.922399 +вт -10.923160 +кс -10.924065 +тя -10.951552 +зе -10.955353 +кж -10.970601 +еш -10.974325 +зу -10.987076 +ял -11.024274 +гу -11.028006 +ша -11.040210 +ью -11.045547 +зм -11.069498 +ум -11.073402 +ье -11.079936 +иб -11.098092 +лн -11.105927 +рь -11.151719 +г_ -11.162119 +ущ -11.169023 +уз -11.187360 +ох -11.220636 +ещ -11.225418 +яз -11.227333 +лл -11.238235 +зр -11.255117 +рк -11.282897 +ца -11.283985 +ща -11.307937 +ец -11.311422 +аш -11.316490 +вя -11.324792 +иш -11.341425 +ха -11.345617 +вк -11.355925 +кв -11.364354 +рж -11.381558 +уе -11.394686 +ям -11.403146 +шл -11.412057 +лж -11.419263 +ющ -11.423687 +ьз -11.474710 +пы -11.479476 +яд -11.488337 +ео -11.492569 +сы -11.526880 +ьт -11.561079 +аф -11.571352 +пп -11.598204 +лы -11.605870 +эк -11.607185 +иж -11.607537 +мм -11.618965 +яб -11.641469 +сч -11.646412 +ьм -11.675385 +нг -11.675385 +ащ -11.680107 +оф -11.710988 +оц -11.718470 +фр -11.733183 +бн -11.748757 +ощ -11.749786 +рд -11.759222 +ын -11.787749 +ья -11.793081 +ях -11.836997 +ип -11.840499 +фа -11.840949 +кц -11.842784 +ыш -11.857155 +шк -11.869614 +ык -11.878392 +_ю -11.885275 +юд -11.891279 +юч -11.917535 +ух -11.930969 +лк -11.931007 +кн -11.955217 +хр -11.967063 +ою -11.978355 +дж -11.984263 +рш -11.985177 +рх -12.020327 +бс -12.044895 +еа -12.047279 +уш -12.051123 +бъ -12.059115 +дя -12.059805 +ау -12.062005 +мл -12.072779 +ч_ -12.079469 +дп -12.091979 +ув -12.102268 +ц_ -12.110505 +сш -12.112718 +ьб -12.115958 +вм -12.124295 +шо -12.149668 +рр -12.157621 +нч -12.171305 +ящ -12.177311 +шн -12.188893 +ып -12.196375 +ея -12.197526 +мь -12.199508 +дь -12.210946 +хи -12.253692 +дк -12.255489 +вь -12.260219 +хн -12.262217 +яю -12.275525 +вп -12.280736 +ъе -12.286065 +вш -12.298682 +ыр -12.300436 +цы -12.337132 +тм -12.351025 +нф -12.365285 +бя -12.404939 +вз -12.435955 +тч -12.462635 +тд -12.472688 +сх -12.479196 +зл -12.479532 +б_ -12.499721 +зя -12.509466 +дм -12.573741 +e_ -12.578682 +тл -12.602848 +сб -12.606176 +яч -12.628884 +ыб -12.684183 +п_ -12.684442 +уа -12.697051 +ищ -12.708360 +сд -12.732115 +юб -12.744107 +пн -12.746902 +юн -12.764603 +рл -12.769627 +яс -12.772849 +цу -12.775528 +рб -12.776800 +вг -12.804137 +зб -12.814448 +мс -12.816109 +нв -12.821706 +пя -12.830286 +жу -12.838954 +чь -12.849264 +яр -12.867835 +чк -12.875479 +фу -12.901352 +ii -12.904920 +еф -12.907404 +зг -12.940322 +яж -12.940669 +чл -12.941865 +ыч -12.946463 +ху -12.962223 +оэ -12.974404 +жб -12.996231 +дд -13.004024 +эл -13.008421 +чш -13.015427 +юж -13.032168 +юл -13.047985 +шт -13.088494 +фл -13.118353 +нщ -13.123464 +лг -13.123551 +юз -13.130174 +ею -13.159257 +мк -13.160379 +чу -13.164109 +рп -13.171554 +вд -13.178310 +юр -13.181131 +гн -13.188435 +зк -13.192148 +ыд -13.202556 +бх -13.255944 +цк -13.256232 +i_ -13.258440 +шь -13.279737 +s_ -13.298039 +иф -13.301698 +_x -13.302887 +_t -13.303878 +яц -13.307353 +цо -13.330702 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/san b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/san new file mode 100644 index 00000000000..faf7bc42af0 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/san @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 578367 +#UNIQUE_BIGRAMS 664 +#UNSEEN_LOG_PROB -14.994855 +#EXPECTED_SCORE -7.128409 +त_ -3.670935 +न_ -4.919359 +_अ -4.984437 +_त -5.066732 +_इ -5.135124 +इत -5.394167 +_भ -5.466110 +_क -5.654916 +य_ -5.711696 +प_ -5.785724 +र_ -5.892512 +वत -5.977033 +_ए -6.029023 +भव -6.147272 +_न -6.177104 +द_ -6.191434 +म_ -6.220157 +_व -6.289682 +अप -6.330853 +_स -6.382362 +क_ -6.451191 +ष_ -6.488557 +_ज -6.498671 +व_ -6.536841 +_म -6.585359 +_य -6.664880 +_प -6.665892 +_द -6.909505 +ल_ -6.909505 +तम -7.117871 +रत -7.139162 +ण_ -7.207566 +गर -7.264725 +जन -7.268182 +थ_ -7.296136 +_र -7.319453 +पर -7.359037 +कर -7.368487 +तद -7.406916 +रण -7.437722 +_ग -7.452064 +अत -7.453375 +नग -7.456877 +तन -7.510904 +तथ -7.533371 +एक -7.536609 +नम -7.636310 +श_ -7.693509 +स_ -7.704394 +कत -7.726940 +_ब -7.773665 +_श -7.790134 +नन -7.825910 +दश -7.896667 +एत -7.899047 +ह_ -7.937666 +यत -7.953655 +अन -8.066815 +यद -8.068823 +तष -8.072847 +_आ -8.147980 +भर -8.149395 +सम -8.162201 +तय -8.165062 +मन -8.182348 +ग_ -8.238497 +भग -8.239251 +इद -8.246050 +गत -8.285200 +एष -8.287537 +मह -8.296920 +तत -8.303998 +दन -8.331858 +रज -8.338302 +षय -8.421381 +कल -8.426522 +जत -8.442921 +दव -8.443789 +बह -8.460387 +_उ -8.462146 +अय -8.471855 +वर -8.533371 +वन -8.613172 +_ल -8.640784 +मत -8.646770 +वष -8.692477 +ज_ -8.748235 +एव -8.751457 +पत -8.792898 +धन -8.821954 +उप -8.849304 +अध -8.851607 +नक -8.916418 +सध -8.932172 +चन -8.945638 +वद -8.945638 +धर -8.961717 +यथ -8.962961 +मप -8.996967 +गव -9.014921 +यन -9.023983 +कव -9.051513 +पद -9.082281 +नव -9.091777 +हन -9.106827 +सर -9.112338 +_च -9.130398 +ध_ -9.137405 +नय -9.161486 +सत -9.187429 +धक -9.188884 +मय -9.194719 +वल -9.219785 +वय -9.242269 +पन -9.265109 +रच -9.266644 +दर -9.274347 +थव -9.274347 +यग -9.286757 +णन -9.296136 +रम -9.300848 +_ह -9.334271 +गण -9.347210 +तव -9.385069 +सन -9.439453 +लय -9.458631 +हस -9.465669 +अह -9.495967 +कथ -9.528757 +दत -9.549169 +रय -9.594731 +कन -9.602467 +नस -9.604407 +जय -9.614149 +थप -9.655796 +हव -9.659826 +मर -9.665892 +तप -9.671984 +मक -9.671984 +जल -9.686298 +शर -9.686298 +वव -9.736484 +हर -9.736484 +भष -9.742882 +नत -9.742882 +वह -9.749308 +वश -9.749308 +कम -9.773119 +वस -9.821954 +जव -9.833287 +गह -9.835564 +हल -9.856223 +हत -9.872499 +कद -9.877183 +पय -9.893698 +रष -9.915214 +वध -9.915214 +लक -9.951801 +चत -9.964206 +तह -9.979234 +लन -9.989341 +_घ -10.020092 +वच -10.030490 +लख -10.043594 +दय -10.051513 +भ_ -10.086343 +रग -10.089058 +ख_ -10.119258 +ऽप -10.122035 +सह -10.155784 +शष -10.161486 +सव -10.196182 +मल -10.210888 +_फ -10.216813 +फल -10.243780 +रर -10.258983 +अथ -10.274347 +दप -10.286757 +चर -10.289877 +इय -10.305575 +मस -10.324642 +बल -10.324642 +तर -10.334271 +हद -10.340726 +ट_ -10.347210 +शल -10.347210 +कष -10.373441 +भम -10.386738 +भत -10.400159 +नद -10.417112 +भय -10.417112 +जग -10.420527 +रव -10.427380 +_ध -10.448139 +रप -10.469200 +कच -10.479848 +यम -10.479848 +दह -10.490574 +शव -10.508631 +मव -10.512269 +आग -10.579383 +कऽ -10.583205 +पण -10.594731 +सद -10.618064 +शस -10.637800 +सय -10.641780 +लत -10.678101 +शत -10.682194 +वक -10.682194 +यज -10.686298 +तस -10.694542 +रल -10.711173 +सक -10.711173 +चय -10.753608 +नर -10.766586 +अव -10.775303 +हम -10.788479 +मद -10.792898 +धय -10.806235 +रक -10.847005 +वज -10.865501 +रस -10.922457 +खत -10.922457 +कस -10.932172 +शन -10.966700 +जध -10.971701 +षण -10.971701 +पल -10.981754 +वभ -10.981754 +णय -11.012343 +टक -11.022685 +_ख -11.033101 +घर -11.043594 +यष -11.054163 +शय -11.059477 +नश -11.064810 +अभ -11.075537 +शक -11.080930 +रध -11.097232 +दद -11.130398 +वम -11.141626 +अम -11.147272 +यण -11.175841 +नप -11.181624 +मम -11.187429 +यस -11.199111 +नष -11.199111 +आद -11.234735 +थय -11.240759 +सभ -11.240759 +कट -11.246808 +नल -11.271261 +सश -11.277440 +लल -11.289877 +तल -11.289877 +गन -11.296136 +मध -11.308736 +हय -11.321447 +दस -11.327845 +लभ -11.327845 +च_ -11.334271 +उभ -11.360266 +कश -11.373441 +तक -11.380074 +खन -11.400159 +भज -11.406916 +दष -11.413705 +आध -11.427380 +दम -11.441186 +वण -11.448139 +टर -11.455125 +ई_ -11.462146 +इम -11.476290 +लप -11.476290 +णत -11.526916 +घट -11.549169 +बद -11.556663 +चप -11.594731 +कप -11.602467 +यव -11.602467 +बस -11.610244 +यक -11.625926 +टन -11.633831 +शध -11.649773 +गज -11.657810 +सस -11.674020 +अस -11.698682 +नभ -11.715361 +मट -11.732235 +आय -11.732235 +पज -11.740746 +घ_ -11.749308 +चद -11.766586 +हभ -11.766586 +पठ -11.766586 +षत -11.784074 +लह -11.801776 +ढ_ -11.819698 +अश -11.837845 +शख -11.847005 +नट -11.856223 +रभ -11.874839 +सख -11.874839 +बध -11.903221 +कय -11.903221 +लघ -11.932172 +नह -11.941953 +नज -11.941953 +धत -11.981754 +सच -11.991879 +आह -12.002074 +लव -12.002074 +चक -12.033101 +_ऐ -12.043594 +ऐत -12.043594 +टल -12.064810 +लम -12.064810 +आर -12.097232 +मघ -12.119258 +यय -12.119258 +हक -12.130398 +चल -12.152941 +आव -12.152941 +di -12.187429 +मज -12.187429 +भस -12.210888 +मख -12.234735 +कक -12.234735 +आन -12.246808 +तट -12.258983 +रद -12.271261 +भई -12.271261 +मण -12.271261 +एन -12.296136 +_ऋ -12.296136 +ब_ -12.296136 +डश -12.347210 +जर -12.360266 +फर -12.373441 +आच -12.373441 +दख -12.386738 +गम -12.400159 +अज -12.427380 +धव -12.441186 +भद -12.455125 +उद -12.469200 +णष -12.483414 +सग -12.483414 +इट -12.497770 +खल -12.497770 +रन -12.497770 +ic -12.512269 +रह -12.526916 +षद -12.541713 +आक -12.541713 +पट -12.556663 +सल -12.556663 +st -12.571770 +जभ -12.571770 +धम -12.571770 +जक -12.571770 +हध -12.587037 +शम -12.587037 +सफ -12.602467 +थन -12.633831 +ठ_ -12.633831 +जम -12.633831 +तऽ -12.649773 +गय -12.649773 +हब -12.665892 +नच -12.682194 +दढ -12.682194 +खक -12.698682 +गप -12.698682 +लड -12.715361 +गष -12.715361 +अल -12.715361 +_ट -12.715361 +मच -12.715361 +फ_ -12.715361 +_d -12.732235 +is -12.732235 +tr -12.732235 +ri -12.732235 +ct -12.732235 +t_ -12.732235 +डग -12.732235 +ऋष -12.732235 +ड_ -12.766586 +वग -12.784074 +लद -12.801776 +_ओ -12.819698 +ओड -12.819698 +बज -12.819698 +लग -12.837845 +रथ -12.837845 +दब -12.893698 +पड -12.912807 +लई -12.932172 +गद -12.932172 +तड -12.932172 +हष -12.932172 +घष -13.012343 +भन -13.012343 +पप -13.033101 +घन -13.075537 +आस -13.075537 +मग -13.075537 +nd -13.075537 +णम -13.097232 +खर -13.097232 +बर -13.097232 +आश -13.119258 +शश -13.119258 +पस -13.141626 +e_ -13.164346 +षध -13.164346 +डत -13.164346 +ठन -13.164346 +डन -13.187429 +रब -13.187429 +हप -13.187429 +हट -13.210888 +भभ -13.210888 +षक -13.210888 +गभ -13.210888 +षभ -13.210888 +शण -13.258983 +खद -13.258983 +भप -13.283645 +षस -13.283645 +आप -13.283645 +दल -13.308736 +गध -13.334271 +ठक -13.334271 +_ई -13.360266 +कफ -13.360266 +यल -13.360266 +ठत -13.386738 +आभ -13.386738 +टत -13.413705 +पध -13.413705 +तज -13.469200 +वप -13.469200 +आज -13.526916 +पथ -13.526916 +_i -13.526916 +in -13.526916 +जप -13.526916 +सऽ -13.556663 +_औ -13.556663 +औष -13.556663 +रफ -13.587037 +हण -13.618064 +भट -13.649773 +टट -13.682194 +रख -13.682194 +रश -13.682194 +गड -13.682194 +तए -13.715361 +घल -13.715361 +_ष -13.715361 +षड -13.715361 +an -13.715361 +नध -13.749308 +रठ -13.749308 +घत -13.749308 +डय -13.749308 +उव -13.784074 +ळभ -13.784074 +पम -13.819698 +इल -13.819698 +णभ -13.819698 +a_ -13.819698 +मथ -13.819698 +नड -13.819698 +_t -13.856223 +th -13.856223 +he -13.856223 +सई -13.856223 +दध -13.856223 +ia -13.856223 +पष -13.893698 +पश -13.893698 +णब -13.932172 +बन -13.932172 +धभ -13.932172 +यप -13.932172 +तभ -13.932172 +बण -13.971701 +एश -14.012343 +ईक -14.012343 +पव -14.012343 +णप -14.054163 +मठ -14.054163 +धप -14.054163 +णक -14.054163 +बभ -14.097232 +शह -14.097232 +पक -14.097232 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/sin b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/sin new file mode 100644 index 00000000000..9031003cc5a --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/sin @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 2653218 +#UNIQUE_BIGRAMS 1113 +#UNSEEN_LOG_PROB -15.217613 +#EXPECTED_SCORE -7.647850 +ය_ -4.637586 +_ව -5.231237 +න_ -5.237131 +_ස -5.261808 +_ක -5.425324 +_ප -5.453329 +ම_ -5.670815 +ට_ -5.691685 +ත_ -5.754408 +ර_ -5.855779 +_අ -5.896271 +_ම -5.967773 +ව_ -6.000123 +කර -6.085095 +_ල -6.114142 +ද_ -6.263039 +ක_ -6.288145 +_න -6.412923 +හ_ -6.431576 +ස_ -6.532712 +තර -6.607728 +_ද -6.617999 +_ත -6.832322 +ල_ -6.871706 +_හ -6.929125 +වය -6.961085 +_ය -7.041464 +_ග -7.044404 +_බ -7.113878 +ණ_ -7.114481 +නය -7.308386 +අත -7.339400 +වන -7.398264 +රම -7.402950 +සම -7.415427 +වත -7.421753 +_ඇ -7.462124 +නව -7.465003 +_ර -7.495489 +ග_ -7.537502 +ගන -7.593112 +පර -7.607144 +_එ -7.679204 +ලස -7.713603 +රන -7.717718 +ලබ -7.761057 +ලක -7.761765 +සද -7.784243 +ඩ_ -7.832013 +මම -7.853608 +වට -7.865606 +මට -7.867764 +ඇත -7.868399 +රය -7.914489 +සය -7.922119 +ළ_ -7.931911 +_ආ -7.932974 +රණ -7.980249 +නම -7.990031 +හක -8.000158 +_ජ -8.016539 +මය -8.076336 +කය -8.100310 +මහ -8.151032 +යන -8.164386 +_ඉ -8.188137 +දන -8.190200 +බ_ -8.193539 +වස -8.245564 +_භ -8.245894 +ලය -8.247546 +පළ -8.265003 +හර -8.267012 +පව -8.274401 +යට -8.274737 +කල -8.279967 +සට -8.307094 +කට -8.311060 +වර -8.345489 +තය -8.357031 +දව -8.372446 +තව -8.409684 +සඳ -8.425674 +වම -8.447338 +පහ -8.454760 +_ඔ -8.468562 +ඳහ -8.474739 +යත -8.474739 +වඩ -8.491667 +වල -8.572576 +හත -8.592378 +නත -8.626140 +ජන -8.633463 +ඔහ -8.636490 +ලද -8.657415 +නස -8.666445 +මන -8.678870 +තන -8.685347 +අන -8.686915 +මත -8.693653 +සව -8.739631 +තබ -8.740561 +සර -8.783524 +දක -8.799668 +දය -8.826324 +වද -8.833996 +මණ -8.835486 +බහ -8.853231 +_උ -8.863578 +පස -8.875787 +කව -8.890937 +වශ -8.902080 +පල -8.910690 +රග -8.913571 +ණය -8.915145 +භව -8.919878 +නග -8.928595 +යද -8.952641 +ගය -8.971351 +ලන -9.000575 +බව -9.013163 +සහ -9.013725 +හද -9.033535 +නක -9.071355 +ඳ_ -9.093462 +වව -9.099713 +පම -9.117725 +අව -9.119236 +රජ -9.135964 +පය -9.145786 +රව -9.148561 +දහ -9.157228 +රක -9.167509 +පද -9.175033 +බඳ -9.175348 +කළ -9.179756 +ළබ -9.240293 +වධ -9.268520 +හට -9.272550 +මර -9.280644 +සල -9.294918 +රද -9.303138 +ධ_ -9.311751 +ඉත -9.329483 +එහ -9.335091 +මග -9.344605 +යව -9.347790 +මල -9.361674 +හම -9.369566 +මද -9.371005 +ගම -9.376416 +යක -9.385479 +තල -9.386571 +ටය -9.398264 +දර -9.400100 +රප -9.415984 +සන -9.443737 +හස -9.460111 +තක -9.465868 +ගත -9.468562 +දම -9.482108 +මව -9.483665 +බල -9.485612 +තළ -9.491080 +ගර -9.536795 +පත -9.555495 +බද -9.561235 +රත -9.563702 +යග -9.587349 +යම -9.589861 +ප_ -9.593217 +රස -9.598688 +වක -9.599531 +දශ -9.601220 +_ඒ -9.610541 +කම -9.627645 +එක -9.635841 +ශල -9.654563 +ගණ -9.656317 +රට -9.664678 +ලව -9.673976 +පන -9.680654 +කත -9.698615 +බන -9.700876 +යය -9.719092 +තම -9.725983 +ජ_ -9.732907 +වහ -9.750129 +රහ -9.777547 +අඩ -9.789527 +ලට -9.818202 +ළම -9.878856 +ජව -9.911475 +කන -9.923042 +ඒව -9.924627 +කඩ -9.970260 +ජය -9.975725 +ධන -9.996681 +හන -9.997237 +කස -9.999462 +හල -10.008955 +නප -10.031542 +සක -10.034390 +කණ -10.040104 +පට -10.053332 +බය -10.062606 +ලප -10.077217 +හග -10.117725 +ෂ_ -10.123172 +සත -10.126815 +උප -10.128640 +ජත -10.156917 +ඟ_ -10.178810 +දල -10.190835 +නර -10.194654 +_ශ -10.217778 +ගල -10.231441 +තහ -10.245894 +ශය -10.245894 +ශෂ -10.268520 +_ච -10.292870 +ආක -10.296285 +සග -10.317638 +ඉහ -10.319721 +ටව -10.323200 +හව -10.335091 +මජ -10.361316 +ණක -10.386571 +ධක -10.427920 +හය -10.429419 +දග -10.442979 +ශ_ -10.449048 +මස -10.463562 +බණ -10.485223 +නහ -10.497747 +යර -10.499321 +වග -10.538412 +හළ -10.539221 +ගව -10.544084 +මක -10.566997 +ශක -10.588605 +චන -10.619923 +දස -10.628505 +ආය -10.637139 +ළය -10.656317 +ජල -10.673088 +නද -10.685571 +ආද -10.694554 +මඟ -10.724602 +ඇම -10.731981 +අප -10.738470 +කක -10.758111 +ටත -10.774210 +ටන -10.826571 +ඳව -10.829537 +අද -10.829537 +අධ -10.836480 +ටම -10.858522 +එස -10.867637 +යල -10.903642 +මළ -10.933107 +භග -11.005039 +ධර -11.008395 +සප -11.010637 +හණ -11.017384 +වළ -11.035531 +ටහ -11.037816 +චර -11.049293 +අර -11.055066 +සළ -11.063187 +අය -11.078392 +_ධ -11.089013 +උද -11.096138 +අග -11.110493 +පණ -11.121354 +ලම -11.121354 +ඉද -11.132297 +බම -11.150723 +භෂ -11.160647 +කග -11.179440 +රධ -11.184494 +හඳ -11.237336 +ඹ_ -11.239964 +යජ -11.242597 +ගඩ -11.255832 +යහ -11.265170 +ලත -11.271877 +ශප -11.277266 +ගට -11.285386 +නල -11.305889 +ඩව -11.336497 +අම -11.337904 +එව -11.342132 +ආහ -11.344958 +තට -11.353470 +ඩන -11.363464 +ණන -11.376416 +බට -11.389485 +ටක -11.420449 +ඩය -11.426422 +වණ -11.430919 +නන -11.436937 +ගහ -11.461261 +පක -11.473579 +බර -11.495391 +අශ -11.498534 +එය -11.508005 +ළඳ -11.514353 +නට -11.535181 +ළව -11.535181 +ධප -11.538412 +e_ -11.540030 +යණ -11.543272 +ආග -11.566173 +සභ -11.569474 +භම -11.571127 +ෂය -11.577761 +යළ -11.586095 +වච -11.589442 +ආස -11.599531 +රර -11.602910 +වෂ -11.630228 +ලග -11.647568 +රල -11.659832 +ධය -11.659832 +ඩග -11.665120 +බධ -11.670427 +තත -11.688260 +ළඹ -11.691853 +තප -11.719092 +ළද -11.741259 +උත -11.748725 +චල -11.750597 +අක -11.792417 +කප -11.794347 +පථ -11.796280 +බග -11.809881 +සධ -11.819676 +ෂව -11.827559 +දළ -11.829537 +නශ -11.835486 +යධ -11.841460 +ඳන -11.841460 +කහ -11.863578 +රඟ -11.886041 +ථව -11.906770 +ෂණ -11.906770 +ආව -11.908859 +ණත -11.932044 +ණව -11.949143 +එර -11.977368 +ඩම -12.019640 +ටස -12.028699 +කථ -12.070185 +ඇය -12.091384 +දඩ -12.103298 +ගක -12.105692 +රච -12.108091 +අස -12.110493 +සබ -12.122566 +ජධ -12.129858 +ආල -12.132297 +භත -12.142095 +දප -12.149487 +ළත -12.154436 +දද -12.156917 +අල -12.159403 +ඒක -12.166884 +_t -12.179440 +කශ -12.181965 +රඹ -12.187027 +ජක -12.222968 +නඩ -12.230787 +සජ -12.241280 +ලඛ -12.265170 +th -12.270534 +ඳම -12.284029 +රළ -12.303138 +භ_ -12.305889 +ළහ -12.311406 +ඹර -12.314172 +ළක -12.319721 +නධ -12.322504 +හජ -12.333687 +ශර -12.333687 +රඳ -12.344958 +ජග -12.347790 +සස -12.350627 +ථ_ -12.353470 +පඩ -12.353470 +ණම -12.359172 +තග -12.379310 +භජ -12.402674 +ළග -12.402674 +_ළ -12.405621 +කඳ -12.411534 +හප -12.414499 +භර -12.417471 +_ඕ -12.426422 +he -12.435430 +ලහ -12.435430 +යස -12.441466 +ණස -12.450569 +ශන -12.456669 +ඕන -12.459729 +ටල -12.472033 +බඩ -12.493822 +ටර -12.493822 +_ඡ -12.509589 +සච -12.512763 +අභ -12.525531 +අහ -12.528740 +ඡය -12.548149 +අණ -12.561235 +පජ -12.584424 +ච_ -12.621635 +ඉර -12.628505 +ඛන -12.631953 +ඇර -12.635408 +හඩ -12.699067 +ධව -12.702687 +ඔව -12.709955 +වජ -12.717260 +ළට -12.724602 +ටළ -12.731981 +පෂ -12.750597 +යප -12.777069 +a_ -12.777069 +_s -12.788565 +d_ -12.796280 +පග -12.807930 +_ඓ -12.815750 +ඓත -12.815750 +ළල -12.815750 +_a -12.835486 +කබ -12.835486 +කද -12.835486 +භය -12.855496 +ඉට -12.883985 +ජප -12.883985 +ටප -12.896368 +රභ -12.921459 +ටද -12.929921 +_c -12.934170 +ෂත -12.955607 +ලභ -12.955607 +ඇග -12.968624 +රබ -12.968624 +තණ -12.972990 +_ඵ -12.977368 +උඩ -12.981760 +හභ -12.999462 +පබ -13.008395 +_ට -13.017384 +ටග -13.026429 +දට -13.030973 +_ඝ -13.040104 +ගබ -13.063187 +දච -13.072525 +එද -13.077217 +ගස -13.081924 +වබ -13.091384 +ඥ_ -13.091384 +an -13.096138 +එළ -13.115310 +ආර -13.134741 +ළඟ -13.139639 +na -13.139639 +රඛ -13.179440 +_r -13.189565 +er -13.194654 +ඝත -13.215190 +ඵල -13.230787 +ඛ_ -13.246555 +ෂධ -13.251849 +චත -13.251849 +_l -13.251849 +ශව -13.257163 +ණජ -13.262496 +_d -13.267849 +බස -13.300393 +එත -13.316944 +_ඖ -13.328085 +ඖෂ -13.328085 +ඉජ -13.328085 +බක -13.333687 +s_ -13.333687 +ගඹ -13.339312 +_ඊ -13.362032 +ඩක -13.379310 +ආධ -13.396797 +තද -13.408574 +මප -13.426422 +පඨ -13.426422 +t_ -13.444494 +සඟ -13.462795 +ධත -13.468947 +ri -13.468947 +වභ -13.468947 +දණ -13.475126 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/slk b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/slk new file mode 100644 index 00000000000..4384bc30066 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/slk @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 161761350 +#UNIQUE_BIGRAMS 593 +#UNSEEN_LOG_PROB -17.953874 +#EXPECTED_SCORE -7.744367 +e_ -5.344295 +_p -5.641287 +a_ -5.797391 +_s -6.075695 +st -6.143812 +i_ -6.240924 +pr -6.419029 +ne -6.419942 +u_ -6.444900 +y_ -6.461926 +o_ -6.466591 +_v -6.518705 +ov -6.519610 +po -6.662101 +to -6.664105 +ra -6.666428 +_n -6.676201 +na -6.711305 +re -6.723727 +ie -6.724411 +en -6.732493 +ni -6.821410 +ch -6.872483 +ko -6.903278 +_m -6.903511 +la -6.909462 +al -6.911028 +m_ -6.913109 +_t -6.943930 +_k -6.950803 +ro -6.960352 +ri -7.057083 +_z -7.061353 +ta -7.097447 +_d -7.104557 +t_ -7.164721 +or -7.184389 +om -7.194155 +at -7.221647 +va -7.224339 +ve -7.261494 +te -7.268482 +ed -7.272231 +an -7.299838 +ia -7.315603 +od -7.341748 +ho -7.386132 +ci -7.397227 +le -7.397444 +li -7.426878 +ti -7.438832 +_a -7.450242 +ak -7.458035 +h_ -7.465290 +_r -7.480089 +os -7.491478 +_o -7.492436 +ol -7.510909 +no -7.539996 +lo -7.540778 +sk -7.587075 +ad -7.598127 +av -7.620216 +as -7.656407 +el -7.657914 +ac -7.664720 +me -7.670271 +_b -7.679851 +vy -7.681062 +vo -7.697883 +do -7.699716 +l_ -7.699857 +in -7.732060 +_c -7.744473 +ic -7.780893 +ka -7.784593 +es -7.786722 +er -7.787408 +za -7.789060 +ej -7.815796 +si -7.821462 +ob -7.828023 +de -7.846681 +ny -7.848643 +tr -7.878285 +ma -7.887179 +it -7.948789 +j_ -7.961664 +da -7.966376 +et -7.999991 +mi -8.002041 +je -8.009404 +ok -8.011016 +zi -8.011354 +ar -8.013611 +oz -8.040114 +ce -8.059408 +kt -8.074364 +ky -8.090826 +ku -8.129620 +v_ -8.131663 +oc -8.151299 +dn -8.166678 +vi -8.187988 +bo -8.190700 +is -8.192356 +am -8.196327 +il -8.236355 +sl -8.243803 +mo -8.246545 +ke -8.269160 +_u -8.272669 +aj -8.303888 +yc -8.311487 +di -8.329460 +k_ -8.344743 +se -8.362140 +az -8.384108 +on -8.409351 +em -8.434867 +_h -8.491093 +eh -8.501890 +sp -8.505091 +ot -8.528799 +n_ -8.537973 +_j -8.573656 +ca -8.602527 +_l -8.615408 +d_ -8.647980 +_i -8.668018 +tu -8.701317 +so -8.712019 +nu -8.724331 +zn -8.733678 +ec -8.780722 +ud -8.804236 +ze -8.826907 +pa -8.830504 +oj -8.848431 +ek -8.855875 +eb -8.916977 +ik -8.920793 +sa -8.942147 +r_ -8.955802 +ru -8.973110 +tn -8.979854 +ou -9.036154 +us -9.054489 +ez -9.061782 +cn -9.073103 +lu -9.079216 +_f -9.123370 +nt -9.131530 +ut -9.132586 +by -9.134890 +s_ -9.150194 +uj -9.184665 +ys -9.188823 +im -9.198442 +pe -9.214211 +ty -9.220512 +ln -9.239924 +iv -9.245077 +ns -9.276879 +vs -9.276967 +vn -9.283161 +kr -9.286487 +ck -9.290592 +bu -9.290911 +co -9.315391 +ur -9.324620 +tv -9.327207 +be -9.328978 +du -9.330080 +ba -9.339808 +oh -9.346115 +ym -9.346434 +ry -9.350644 +uc -9.353547 +ju -9.359528 +hl -9.375789 +br -9.395529 +mu -9.408823 +sv -9.431990 +op -9.455160 +_e -9.489136 +ja -9.490939 +up -9.516741 +zd -9.527107 +dl -9.545171 +ap -9.562007 +pl -9.566361 +uz -9.586187 +su -9.602659 +z_ -9.626514 +iz -9.630425 +hr -9.642962 +c_ -9.652427 +dy -9.654833 +dr -9.691900 +nc -9.695208 +kl -9.696029 +pi -9.721742 +zo -9.735453 +tk -9.758690 +ha -9.764085 +ev -9.848378 +id -9.853261 +ah -9.861186 +dz -9.874938 +rn -9.927816 +ab -9.967135 +bi -9.988005 +sn -9.998221 +nk -10.022282 +yt -10.050937 +lk -10.090860 +ly -10.109588 +ep -10.115670 +uk -10.160594 +fi -10.194116 +hu -10.205223 +sm -10.209578 +ul -10.212199 +ji -10.217594 +bl -10.297950 +rt -10.312178 +zv -10.371139 +rs -10.384132 +vr -10.387759 +rv -10.390606 +vl -10.433870 +fo -10.472010 +mn -10.477582 +my -10.488573 +ii -10.494273 +rm -10.496513 +au -10.504459 +bn -10.506170 +ds -10.512814 +hy -10.543020 +dv -10.553249 +rc -10.556187 +un -10.573411 +ip -10.588534 +um -10.620752 +ir -10.632214 +vz -10.653289 +ls -10.700950 +ub -10.737453 +zu -10.756304 +iu -10.802450 +nd -10.810020 +js -10.820417 +vu -10.831783 +uv -10.858526 +cu -10.880417 +yr -10.902918 +zk -10.910417 +nn -10.967438 +zm -10.967706 +uh -10.968403 +jn -10.986457 +yv -10.986800 +eu -10.990624 +jo -11.019789 +he -11.020475 +io -11.021438 +yb -11.031932 +zs -11.043143 +_g -11.064205 +dp -11.064797 +lm -11.116947 +ps -11.124176 +yz -11.137455 +bc -11.141720 +tl -11.164058 +kc -11.172659 +pu -11.184504 +hn -11.190807 +fa -11.201458 +cl -11.207752 +dk -11.210306 +ga -11.225196 +yk -11.233075 +ib -11.240630 +jm -11.263163 +sy -11.274784 +zr -11.277106 +gi -11.287301 +rd -11.339640 +og -11.350731 +zh -11.376251 +zb -11.385742 +eg -11.393063 +cs -11.467280 +zl -11.480879 +ua -11.482001 +hc -11.488983 +kd -11.500022 +ct -11.546004 +kv -11.551963 +rk -11.557920 +jt -11.566632 +fe -11.567011 +vk -11.607931 +ij -11.612895 +ea -11.612895 +gr -11.615999 +ex -11.618072 +rz -11.624844 +yn -11.628680 +dc -11.643840 +rg -11.649788 +p_ -11.672782 +b_ -11.694491 +go -11.705470 +yh -11.709496 +nf -11.736266 +kn -11.738702 +zp -11.749133 +ge -11.778284 +ef -11.785444 +ml -11.808452 +ng -11.846406 +mb -11.889372 +ht -11.901092 +ih -11.904302 +dm -11.922397 +rh -11.933552 +ki -11.939180 +hi -11.943003 +pn -11.980029 +jv -12.004556 +sc -12.013341 +bs -12.013747 +db -12.054441 +yp -12.089577 +nz -12.091600 +lh -12.121809 +fu -12.137475 +vd -12.138158 +ss -12.139204 +ts -12.164570 +ag -12.194565 +zy -12.196489 +mt -12.206866 +lt -12.236041 +bj -12.290358 +of -12.343505 +gu -12.374000 +mp -12.387556 +km -12.408593 +py -12.464556 +rb -12.464858 +lb -12.511016 +ig -12.556227 +ai -12.590032 +pt -12.592839 +yu -12.618745 +_w -12.630460 +xi -12.632384 +jd -12.646839 +eo -12.665028 +yd -12.666767 +fr -12.668798 +vc -12.765963 +ao -12.799968 +hv -12.841979 +vt -12.886735 +sr -12.953718 +mr -12.974312 +jh -13.006096 +sh -13.016774 +f_ -13.016848 +g_ -13.044913 +bd -13.057555 +jl -13.082094 +af -13.085346 +lv -13.122246 +mc -13.125590 +tc -13.143152 +mk -13.186727 +tb -13.192226 +nv -13.208934 +vb -13.231573 +zt -13.255010 +we -13.274320 +rl -13.295594 +ms -13.345129 +pc -13.355374 +ax -13.428514 +yl -13.432354 +oi -13.443538 +tm -13.467785 +vp -13.498731 +th -13.503488 +xt -13.534794 +vh -13.547552 +kz -13.555905 +gy -13.561393 +gl -13.619586 +yj -13.621159 +ug -13.650677 +lz -13.685269 +dh -13.730254 +oo -13.771066 +if -13.790648 +lc -13.817437 +bk -13.821950 +ll -13.840932 +x_ -13.879795 +hk -13.888966 +dt -13.893031 +rp -13.911877 +dd -13.975820 +jk -13.978265 +hs -14.077461 +jc -14.102658 +ld -14.209778 +w_ -14.220124 +bv -14.222509 +oa -14.275469 +fl -14.345222 +hm -14.348939 +ue -14.364093 +tz -14.383214 +bm -14.418519 +xp -14.425371 +tf -14.462339 +cy -14.518585 +sf -14.572107 +gm -14.613761 +mz -14.620260 +ow -14.633120 +gn -14.635842 +sd -14.644725 +ay -14.695881 +ee -14.714943 +ph -14.741326 +ae -14.750884 +wa -14.774436 +xu -14.846701 +jb -14.848016 +wi -14.852494 +yo -14.859105 +uf -14.864948 +nm -14.866813 +sz -14.871884 +jp -14.888019 +ft -14.896699 +fy -14.898604 +lf -14.977258 +cb -14.987072 +td -15.008666 +cc -15.019586 +nl -15.062277 +ew -15.103757 +cv -15.139043 +pk -15.146787 +cr -15.147434 +mf -15.173235 +ks -15.244845 +_y -15.276707 +ei -15.302065 +oe -15.408981 +tt -15.410145 +nj -15.424586 +ox -15.505249 +pp -15.541371 +zj -15.570587 +uo -15.591132 +hd -15.609742 +nh -15.733044 +cm -15.738398 +xn -15.785476 +wo -15.829461 +xe -15.858840 +ey -15.861493 +mm -15.871617 +nb -15.892081 +gt -15.933343 +jr -15.966082 +hz -16.027309 +ux -16.029693 +rr -16.047705 +sb -16.064109 +zz -16.134865 +bt -16.171260 +jz -16.184483 +ff -16.226264 +bz -16.252483 +bh -16.266477 +qu -16.294877 +xa -16.304229 +zc -16.321655 +nr -16.389708 +ya -16.419105 +gh -16.464354 +rf -16.491215 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/slv b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/slv new file mode 100644 index 00000000000..cf47577836f --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/slv @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 174562727 +#UNIQUE_BIGRAMS 573 +#UNSEEN_LOG_PROB -19.148508 +#EXPECTED_SCORE -7.617036 +i_ -5.393478 +a_ -5.462306 +o_ -5.539059 +_p -5.621402 +e_ -5.806360 +_s -6.006681 +st -6.299873 +pr -6.312774 +ra -6.431318 +po -6.504776 +_n -6.527799 +re -6.528739 +na -6.540388 +ni -6.550144 +je -6.699081 +ko -6.700471 +_t -6.753529 +en -6.773289 +ov -6.795307 +no -6.805757 +_d -6.815751 +_v -6.857189 +an -6.865039 +te -6.868245 +_k -6.876367 +ve -6.900512 +li -6.905243 +_z -6.938613 +la -7.018388 +al -7.046352 +ne -7.050996 +ja -7.081542 +_o -7.089397 +ri -7.100379 +el -7.172375 +il -7.184692 +os -7.192117 +ed -7.197570 +m_ -7.200317 +od -7.204382 +ti -7.207859 +_m -7.239066 +em -7.239840 +nj -7.241502 +av -7.247521 +ka -7.253611 +lo -7.254309 +ta -7.268210 +se -7.304646 +aj -7.326133 +le -7.407197 +me -7.408584 +sk -7.418088 +h_ -7.420877 +za -7.437830 +or -7.508565 +va -7.517627 +lj -7.534224 +to -7.545020 +ro -7.555332 +et -7.557635 +er -7.568575 +is -7.595964 +_b -7.596250 +j_ -7.601726 +di -7.616506 +ar -7.632530 +ih -7.648957 +ce -7.651695 +de -7.666766 +mo -7.668187 +l_ -7.681517 +vi -7.718666 +ci -7.724756 +ol -7.769126 +da -7.769299 +vo -7.776153 +do -7.820367 +as -7.821578 +u_ -7.823540 +at -7.828230 +t_ -7.840517 +_i -7.841205 +_l -7.842523 +ij -7.890848 +r_ -7.921924 +ot -7.922860 +_r -7.932125 +tr -7.940910 +ma -7.945560 +es -7.946233 +ak -7.950481 +in -7.955915 +ob -7.991959 +v_ -8.000069 +im -8.028965 +ic -8.062425 +it -8.068635 +ga -8.084307 +om -8.084958 +bi -8.091050 +ev -8.099573 +ji -8.110194 +ik -8.135348 +am -8.137885 +go -8.167372 +eg -8.191845 +sa -8.232641 +ek -8.240919 +ad -8.257801 +jo -8.283966 +_c -8.303477 +tu -8.317707 +sl -8.322231 +bo -8.322686 +ke -8.337062 +_u -8.361930 +iz -8.374017 +ec -8.374259 +oc -8.380718 +n_ -8.427746 +dn -8.456258 +ki -8.491075 +ca -8.498086 +ud -8.507755 +ej -8.508757 +dr -8.525824 +mi -8.561821 +az -8.565445 +on -8.589754 +si -8.607977 +og -8.645956 +_g -8.648680 +_a -8.656993 +sp -8.659611 +zi -8.682138 +ju -8.691490 +k_ -8.706168 +ez -8.721269 +oj -8.723825 +ac -8.739550 +vn -8.748523 +c_ -8.758542 +so -8.832283 +d_ -8.839674 +vs -8.853359 +oz -8.873262 +ze -8.881128 +ru -8.901851 +zn -8.905080 +_j -8.920403 +ok -8.921313 +kr -8.986836 +s_ -8.987386 +tn -8.992857 +op -9.046273 +pa -9.073398 +be -9.096258 +iv -9.107817 +ah -9.165634 +_e -9.169679 +ku -9.199966 +sn -9.220590 +cn -9.228337 +tv -9.234945 +up -9.258026 +us -9.266685 +js -9.269437 +ns -9.269671 +ln -9.316228 +br -9.363271 +pi -9.369008 +sv -9.399812 +vl -9.413657 +gr -9.420697 +ir -9.434704 +eb -9.457562 +pe -9.476000 +gl -9.510480 +rj -9.555237 +bl -9.579825 +vr -9.583885 +nc -9.613748 +rn -9.704164 +ap -9.712537 +ds -9.776211 +zd -9.811170 +kl -9.829138 +pl -9.833150 +rs -9.833550 +ba -9.851014 +_f -9.863101 +uj -9.905425 +ep -9.907709 +ug -9.909966 +gi -9.940159 +_h -9.944461 +ag -9.952046 +ab -9.991767 +nt -10.006212 +ur -10.010647 +bn -10.012310 +ho -10.036670 +dv -10.065118 +sm -10.067679 +hk -10.091097 +rv -10.092053 +id -10.100803 +rt -10.116415 +sc -10.138575 +eh -10.182347 +zb -10.189442 +lu -10.215285 +kt -10.275053 +zv -10.278929 +rz -10.332899 +nu -10.343823 +un -10.355492 +nd -10.363080 +ub -10.375510 +z_ -10.377179 +dl -10.413363 +nk -10.420017 +ge -10.429150 +zo -10.441705 +uz -10.464517 +ig -10.474136 +cu -10.507578 +tk -10.551567 +tj -10.565427 +sr -10.578751 +ip -10.614713 +mu -10.633929 +zg -10.635598 +jn -10.649377 +co -10.686092 +mb -10.707519 +um -10.717239 +uc -10.726843 +cl -10.768362 +fi -10.768693 +cj -10.816853 +du -10.818434 +zm -10.842149 +g_ -10.853252 +zu -10.856834 +db -10.861439 +ut -10.908496 +rg -10.911469 +zl -10.944363 +vz -11.032674 +ks -11.055887 +dp -11.068575 +su -11.083760 +mn -11.099977 +dj -11.104228 +lc -11.156885 +rk -11.167111 +ul -11.187436 +hi -11.200058 +ml -11.206176 +zr -11.218235 +rd -11.246008 +fo -11.266690 +vp -11.276210 +b_ -11.383315 +pu -11.393395 +zp -11.400505 +km -11.496360 +zk -11.507121 +bu -11.512930 +gu -11.512978 +uk -11.531868 +mr -11.537655 +mp -11.550067 +zj -11.560089 +io -11.583207 +ha -11.591344 +kj -11.592671 +ib -11.631974 +jb -11.645811 +lk -11.667046 +bc -11.707491 +hr -11.715112 +p_ -11.716307 +kd -11.730588 +ck -11.741130 +bj -11.746203 +pn -11.761129 +lt -11.836350 +rm -11.842073 +vc -11.856773 +ia -11.868962 +ps -11.880165 +oh -11.880913 +dk -11.923201 +ms -11.923265 +fe -11.966832 +rb -11.976691 +rl -11.991759 +bs -12.024577 +kv -12.071009 +vk -12.096299 +rc -12.118545 +ng -12.159624 +kn -12.173568 +vt -12.173950 +ct -12.213792 +jv -12.216425 +ls -12.269911 +fr -12.306661 +hn -12.331450 +cr -12.363668 +hu -12.436014 +vu -12.437939 +ef -12.446586 +dg -12.448664 +fa -12.472233 +ht -12.475524 +kc -12.483074 +af -12.497913 +ua -12.502605 +he -12.509384 +tl -12.514017 +eo -12.556301 +nf -12.571614 +lg -12.591421 +pt -12.607732 +dz -12.659781 +of -12.673970 +uv -12.698426 +rh -12.726605 +uh -12.758664 +ou -12.766878 +lm -12.766993 +sb -12.778096 +ea -12.792858 +zs -12.810620 +dm -12.856834 +th -12.885377 +sj -12.911055 +oi -12.942654 +nn -13.008415 +mt -13.019009 +gm -13.037026 +bv -13.045246 +dd -13.046854 +jp -13.057735 +ie -13.101956 +jt -13.115754 +gn -13.140542 +bd -13.162273 +jm -13.205805 +bm -13.211831 +y_ -13.304778 +jk -13.306870 +jd -13.440969 +vj -13.461612 +ts -13.482838 +pk -13.542627 +zh -13.549348 +ld -13.561687 +au -13.590554 +eu -13.632971 +nz -13.700460 +fu -13.706634 +ch -13.718283 +_w -13.727222 +lz -13.736667 +jh -13.761359 +oo -13.784379 +dt -13.802686 +ei -13.833482 +mc -13.872119 +ae -13.940898 +hc -13.944672 +lb -13.960264 +vg -13.966865 +mk -13.999657 +jc -14.006713 +f_ -14.034181 +nv -14.049794 +vd -14.057947 +hv -14.130798 +ee -14.142708 +sd -14.172766 +rp -14.184106 +oa -14.197552 +ll -14.202218 +hl -14.238660 +gt -14.281797 +ai -14.309553 +if -14.322533 +sh -14.345575 +cb -14.365324 +td -14.490427 +pc -14.490427 +mv -14.509576 +dh -14.528788 +vb -14.618658 +ss -14.634547 +zt -14.688299 +np -14.725877 +bt -14.727222 +zz -14.732612 +w_ -14.799619 +dc -14.850228 +jl -14.883066 +ii -14.944281 +vm -14.973497 +ue -14.988464 +wi -15.019421 +ft -15.040434 +tm -15.057806 +wa -15.069410 +ay -15.072539 +yo -15.196155 +ew -15.206430 +lp -15.288066 +_y -15.291707 +lf -15.357496 +ry -15.372844 +ao -15.374950 +fl -15.391906 +tc -15.415191 +we -15.456214 +tt -15.511505 +x_ -15.571011 +pd -15.574636 +cv -15.591268 +oe -15.598631 +fj -15.642768 +nl -15.658926 +ow -15.672243 +cc -15.789987 +gh -15.926414 +md -16.041548 +ty -16.125914 +fs -16.150352 +fg -16.169108 +ex -16.179498 +iu -16.182568 +tp -16.240899 +pj -16.303691 +ey -16.346436 +mz -16.353339 +jr -16.407627 +vh -16.426429 +ly -16.470027 +ph -16.501119 +wo -16.501886 +gj -16.516533 +pp -16.528202 +lv -16.538392 +ye -16.562187 +rr -16.563787 +nr -16.572621 +jz -16.631816 +mm -16.637703 +tf -16.666643 +sf -16.707629 +tw -16.710285 +zc -16.737119 +ny -16.874351 +cm -16.884315 +bz -16.917691 +ax -16.985780 +ws -17.009573 +ys -17.043780 +kb -17.050495 +gd -17.068558 +ui -17.104210 +bk -17.117075 +aa -17.150352 +xp -17.151554 +ox -17.160002 +cd -17.166066 +oy -17.246028 +nh -17.313081 +cy -17.336143 +gs -17.490427 +ff -17.495000 +mj -17.521189 +_x -17.522745 +kp -17.543120 +dy -17.565389 +wh -17.619282 +ya -17.647851 +ix -17.671811 +yl -17.687427 +sy -17.766302 +pg -17.773691 +mw -17.784846 +fp -17.786713 +bb -17.843895 +by -17.915646 +ww -17.927959 +nb -17.952906 +rf -17.978291 +gz -17.986853 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/snd b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/snd new file mode 100644 index 00000000000..03f83d155b6 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/snd @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 2098036 +#UNIQUE_BIGRAMS 1147 +#UNSEEN_LOG_PROB -14.025684 +#EXPECTED_SCORE -8.016759 +ي_ -4.469866 +ن_ -4.707190 +_ا -5.048094 +و_ -5.693051 +ان -5.739431 +_م -5.942616 +_س -5.949994 +_ڪ -6.133474 +اه -6.166581 +ر_ -6.209496 +هي -6.255354 +ري -6.442905 +_و -6.463997 +يو -6.477965 +_پ -6.553331 +هن -6.636063 +ين -6.695473 +اي -6.707999 +ا_ -6.754981 +ار -6.766689 +ند -6.861057 +يي -6.903811 +_ج -6.961603 +ل_ -6.997090 +ء_ -7.027718 +يا -7.169400 +ت_ -7.175252 +ال -7.279188 +_ه -7.298111 +ون -7.303097 +م_ -7.340947 +ما -7.386819 +_ت -7.387164 +د_ -7.419878 +سا -7.437769 +ور -7.438366 +نه -7.474598 +لي -7.503006 +_ر -7.511513 +سن -7.546952 +_ن -7.556370 +را -7.617849 +ڪي -7.642644 +وا -7.659923 +ڪر -7.671092 +_ش -7.674881 +لا -7.696827 +_ب -7.754016 +جي -7.755947 +تا -7.766839 +_ك -7.786896 +ه_ -7.788720 +_ل -7.840579 +_ع -7.867948 +ڻ_ -7.887354 +ام -7.893227 +وي -7.902905 +يء -7.913808 +س_ -7.936876 +اد -7.960318 +دا -7.982234 +اب -7.989206 +ني -8.051511 +ب_ -8.096538 +دي -8.106927 +_ح -8.128510 +مي -8.134681 +رو -8.142627 +اس -8.170885 +كا -8.183225 +دو -8.193052 +اڻ -8.263149 +نڌ -8.282932 +_د -8.312576 +هو -8.347763 +ٿي -8.362172 +ير -8.386588 +يڪ -8.394897 +_ٿ -8.460238 +پا -8.461207 +يل -8.474843 +سي -8.485894 +_ق -8.509507 +ڪا -8.522597 +_ڏ -8.543740 +_ٻ -8.555852 +ست -8.562076 +شا -8.586187 +_گ -8.590422 +نا -8.605877 +با -8.610708 +_خ -8.617984 +عل -8.645257 +تي -8.683478 +يم -8.687725 +مو -8.700255 +پي -8.701686 +هه -8.727978 +_چ -8.728561 +ول -8.747943 +جن -8.751791 +پو -8.797872 +ره -8.803699 +هر -8.812328 +مل -8.814184 +نج -8.822254 +اء -8.827868 +ڏي -8.860098 +ڪو -8.878751 +رن -8.898304 +_ف -8.898961 +ها -8.904552 +كي -8.913145 +ات -8.961003 +در -8.970286 +_ص -8.973393 +بي -8.978240 +پن -9.029064 +يت -9.035906 +يد -9.040606 +ڪ_ -9.046775 +چي -9.070610 +جو -9.084355 +ڻي -9.101251 +ڌ_ -9.127548 +ڪن -9.149249 +اڪ -9.164558 +قي -9.192448 +دس -9.213522 +رت -9.214747 +ڌي -9.222942 +نگ -9.242385 +گه -9.243635 +لو -9.270562 +تر -9.279082 +يس -9.286792 +من -9.327741 +ف_ -9.357654 +او -9.362625 +تع -9.377184 +نو -9.385438 +_ط -9.397445 +_ڇ -9.404418 +سم -9.412362 +ٽي -9.427434 +حا -9.435506 +بر -9.479989 +مر -9.513270 +ٽ_ -9.526902 +جا -9.537595 +سل -9.539640 +رس -9.571201 +ڪت -9.573295 +ز_ -9.575917 +از -9.582755 +خا -9.594935 +گر -9.598129 +وء -9.631011 +پر -9.633193 +ڙي -9.641408 +مد -9.684892 +عر -9.688292 +ٻي -9.701973 +مح -9.705987 +جه -9.712896 +وم -9.714050 +شه -9.726812 +سو -9.742631 +قا -9.756244 +_ز -9.777210 +اع -9.785682 +لم -9.806467 +تو -9.819456 +حم -9.824435 +ڏه -9.826931 +عا -9.839476 +اچ -9.845157 +اص -9.856587 +بن -9.859778 +صو -9.864899 +مت -9.880370 +له -9.882965 +نڊ -9.888168 +_ڳ -9.892084 +وج -9.913808 +نس -9.918459 +يش -9.944648 +لن -9.950759 +اف -9.964434 +قت -9.969251 +ھي -10.002018 +دن -10.003429 +ٽر -10.005548 +ڙ_ -10.019754 +سر -10.026911 +_ڀ -10.035545 +فا -10.045685 +هڙ -10.051511 +صل -10.058093 +عي -10.058093 +في -10.067655 +رڻ -10.089216 +اح -10.092215 +ق_ -10.107306 +وڏ -10.109584 +يق -10.120259 +_ي -10.121791 +يٽ -10.125627 +هڻ -10.133329 +حي -10.141073 +اھ -10.148859 +_ڊ -10.148859 +ڊ_ -10.148859 +وق -10.155118 +رڪ -10.155118 +ڪس -10.160617 +نن -10.160617 +په -10.164558 +وس -10.164558 +ڪل -10.201326 +ڳو -10.216791 +زا -10.217610 +تن -10.223353 +اڳ -10.233251 +ش_ -10.237396 +عم -10.246556 +ع_ -10.252415 +بل -10.264206 +شر -10.277800 +ج_ -10.277800 +ڪم -10.290663 +طر -10.319370 +مش -10.323769 +رد -10.324651 +ڊي -10.337050 +شي -10.340612 +لڪ -10.341504 +وٽ -10.342397 +سٽ -10.343290 +اج -10.344183 +وڃ -10.362172 +لك -10.383141 +يڻ -10.398837 +اخ -10.408151 +گ_ -10.416585 +وٺ -10.421292 +چو -10.439320 +يب -10.440275 +ح_ -10.449861 +خت -10.451786 +زي -10.453714 +يع -10.482939 +فر -10.482939 +سگ -10.490833 +ود -10.496782 +مس -10.497776 +رف -10.500762 +ڳا -10.500762 +_ٽ -10.500762 +ڏن -10.502756 +يز -10.502756 +اق -10.508755 +تم -10.514779 +مب -10.525888 +نظ -10.531984 +عو -10.536062 +ڄا -10.538106 +_ڄ -10.542201 +يج -10.556628 +ٺ_ -10.557665 +گي -10.571201 +بو -10.580648 +وه -10.586980 +اپ -10.591217 +يه -10.601864 +رم -10.633193 +ھن -10.643056 +طو -10.644156 +لف -10.649669 +_ض -10.656312 +ٻو -10.664101 +جڏ -10.668571 +_ذ -10.670812 +وع -10.682065 +قو -10.683195 +رك -10.693407 +وڪ -10.702545 +صا -10.705987 +مه -10.707136 +مع -10.717520 +دڙ -10.722159 +وب -10.726812 +ڏا -10.731481 +تح -10.732651 +مق -10.733821 +ڃي -10.738513 +بد -10.739688 +وڙ -10.750310 +وف -10.772992 +شن -10.776606 +لع -10.782650 +ڻه -10.788720 +نم -10.798484 +وڌ -10.802163 +هل -10.831936 +قر -10.833190 +چا -10.834445 +يخ -10.845790 +لق -10.845790 +_ٺ -10.848323 +هت -10.850861 +رپ -10.855950 +دب -10.864899 +لد -10.871325 +بت -10.894699 +ڙه -10.907851 +_غ -10.911820 +قب -10.914472 +مخ -10.917129 +حڪ -10.921123 +طا -10.923792 +جم -10.937213 +يٺ -10.939912 +زن -10.939912 +ڙو -10.949399 +خ_ -10.954848 +كڻ -10.954848 +ڏو -10.961689 +حب -10.965809 +_ڌ -10.974085 +زم -10.981017 +عد -10.982408 +ٻا -10.997793 +رب -10.997793 +ٺي -11.002018 +ڌا -11.002018 +عب -11.011923 +لڳ -11.017614 +يف -11.031941 +فت -11.043506 +گا -11.055164 +ڪج -11.077281 +ڻو -11.080255 +نڪ -11.081745 +وز -11.086223 +ڇپ -11.093717 +رج -11.098233 +سڪ -11.113387 +ڇڏ -11.116437 +خو -11.124091 +ضل -11.137971 +يڊ -11.141073 +هج -11.147298 +نڍ -11.147298 +تل -11.153551 +رح -11.169301 +ڀر -11.185225 +نٽ -11.194864 +هم -11.201326 +خي -11.201326 +حس -11.202946 +ٽا -11.224175 +ڪٽ -11.225821 +نت -11.245721 +خل -11.247391 +عن -11.252415 +ڳي -11.282932 +گھ -11.284646 +شڪ -11.300168 +فو -11.310610 +قد -11.317613 +صد -11.326416 +ص_ -11.344183 +وش -11.349556 +ظ_ -11.360363 +ڌو -11.362172 +مج -11.362172 +اڙ -11.382223 +ذر -11.391429 +لل -11.393278 +ضر -11.408151 +چ_ -11.434554 +ڀا -11.459511 +ڇا -11.471177 +ثر -11.473131 +لس -11.477046 +حص -11.482939 +لط -11.516792 +قل -11.516792 +ٽن -11.520828 +صف -11.524875 +غي -11.526902 +صي -11.530966 +_ھ -11.535042 +ڪه -11.535042 +مٿ -11.541176 +ڻا -11.547337 +اض -11.553525 +وت -11.553525 +جد -11.557665 +پڙ -11.565980 +بع -11.568066 +سب -11.572248 +كن -11.572248 +ٺا -11.572248 +اٿ -11.591217 +حق -11.591217 +ڊا -11.599729 +ده -11.606145 +اش -11.612591 +اگ -11.623397 +رٽ -11.623397 +دگ -11.629921 +وچ -11.643056 +جت -11.645257 +مط -11.654094 +فق -11.654094 +خر -11.662986 +دم -11.667453 +ضي -11.674179 +حو -11.678680 +ظا -11.683195 +به -11.683195 +نھ -11.701400 +ڪڏ -11.712896 +رق -11.712896 +ٽو -11.729145 +حر -11.740865 +خد -11.743220 +هڪ -11.747943 +لت -11.766988 +غل -11.786289 +صر -11.786289 +ظر -11.791155 +فظ -11.835701 +چڪ -11.838217 +فح -11.853403 +گذ -11.855950 +تب -11.868751 +وڻ -11.876487 +جب -11.881667 +رڊ -11.884264 +گو -11.905211 +جل -11.910496 +ثا -11.918459 +شت -11.921123 +كر -11.921123 +تص -11.939912 +مپ -11.942616 +طي -11.942616 +هد -11.942616 +قس -11.948040 +ٿا -11.948040 +غا -11.950759 +ڏس -11.953484 +بق -11.975468 +لب -11.978240 +ڌر -11.981017 +ڪڙ -11.981017 +بخ -12.006255 +خب -12.006255 +نق -12.014766 +يپ -12.029064 +يگ -12.029064 +سف -12.031941 +شو -12.037712 +اڏ -12.043506 +جھ -12.052241 +ٽڪ -12.052241 +مڪ -12.055164 +ذا -12.055164 +عت -12.063970 +بب -12.069871 +شم -12.069871 +پ_ -12.075795 +ڍي -12.093717 +اك -12.096726 +ڃا -12.099741 +جر -12.114912 +اث -12.121025 +نص -12.121025 +پل -12.133329 +ظم -12.142627 +وغ -12.158258 +رز -12.161404 +ڳ_ -12.161404 +قع -12.164558 +نل -12.177241 +قن -12.180429 +اٽ -12.180429 +پٽ -12.190036 +ٻڌ -12.190036 +رل -12.206192 +سڄ -12.212705 +حد -12.212705 +زو -12.219248 +ڀي -12.219248 +حض -12.232424 +شع -12.235736 +تق -12.235736 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/som b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/som new file mode 100644 index 00000000000..cbec705cef9 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/som @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 10959479 +#UNIQUE_BIGRAMS 559 +#UNSEEN_LOG_PROB -18.712771 +#EXPECTED_SCORE -7.131949 +a_ -4.350195 +aa -4.722173 +ay -5.363384 +da -5.508278 +ad -5.582131 +an -5.660729 +n_ -5.785618 +ha -5.832129 +o_ -5.877169 +ka -5.955436 +_d -6.017204 +y_ -6.045078 +ma -6.059712 +_s -6.077381 +al -6.131764 +la -6.144028 +ar -6.210195 +oo -6.225706 +_a -6.293228 +ga -6.297478 +_w -6.321659 +ya -6.374132 +wa -6.383740 +_i -6.471197 +ee -6.493774 +_m -6.509531 +ii -6.565460 +na -6.779357 +ba -6.814338 +i_ -6.843294 +iy -6.864658 +ax -6.883839 +in -6.888370 +ta -6.897899 +_k -6.948818 +ah -6.956760 +sa -6.995104 +sh -7.056843 +d_ -7.057965 +u_ -7.068546 +ra -7.082270 +xa -7.084823 +ag -7.154548 +dh -7.175594 +_h -7.194694 +ab -7.260949 +_l -7.291784 +so -7.305441 +as -7.343397 +_b -7.347121 +id -7.359996 +_g -7.366411 +is -7.425380 +am -7.447819 +ey -7.490077 +di -7.528327 +hi -7.543154 +yo -7.608934 +uu -7.633034 +li -7.641578 +e_ -7.689931 +_c -7.718481 +gu -7.763910 +_x -7.806065 +do -7.824149 +ir -7.827883 +ca -7.832557 +_q -7.904949 +qa -7.926821 +si -7.945585 +he -7.966868 +_t -8.086289 +lk -8.110424 +r_ -8.131939 +nt -8.159452 +ac -8.212623 +rk -8.218454 +il -8.241854 +ri -8.250648 +ho -8.254100 +en -8.257158 +ku -8.308735 +eg -8.327600 +mi -8.374405 +_y -8.398901 +nk -8.425946 +dd -8.434163 +l_ -8.435074 +_j -8.466999 +le -8.476064 +om -8.556944 +ig -8.594462 +yn -8.603394 +de -8.618629 +_u -8.654834 +or -8.669767 +lo -8.705756 +ed -8.716736 +ug -8.722173 +ki -8.722396 +ur -8.751262 +mu -8.761109 +ci -8.789661 +ul -8.815939 +re -8.822062 +_n -8.866285 +go -8.892321 +ye -8.896455 +ti -8.936527 +xi -8.941891 +yi -8.942668 +b_ -8.948834 +we -8.951178 +ys -8.979072 +ji -9.022979 +bi -9.031702 +xu -9.033632 +s_ -9.035704 +on -9.053989 +aq -9.059457 +er -9.063114 +ol -9.133380 +ge -9.154155 +ud -9.170826 +el -9.172116 +dk -9.180723 +og -9.212467 +_f -9.214264 +aw -9.218650 +bo -9.226278 +od -9.251410 +af -9.256393 +un -9.259778 +ib -9.278541 +ob -9.279358 +ow -9.286904 +_r -9.315890 +no -9.316562 +qo -9.319250 +to -9.334807 +su -9.339746 +im -9.377510 +rt -9.378386 +du -9.379613 +ni -9.437765 +us -9.448854 +ko -9.465416 +ro -9.469610 +sk -9.473161 +fa -9.484998 +me -9.492754 +be -9.508679 +x_ -9.537738 +nu -9.576209 +at -9.601348 +nd -9.641999 +st -9.722674 +wl -9.728916 +ub -9.895452 +uq -9.910702 +um -9.923557 +ia -9.929193 +bu -9.984530 +ru -10.024692 +ux -10.028536 +ak -10.035704 +uw -10.073502 +qd -10.087040 +se -10.102298 +rs -10.155055 +ih -10.160167 +f_ -10.170446 +es -10.211687 +g_ -10.228171 +yu -10.238312 +wu -10.246604 +kh -10.258488 +ll -10.273236 +te -10.293910 +mo -10.300701 +ja -10.344017 +bt -10.347443 +hu -10.377773 +yd -10.378824 +ne -10.426398 +_e -10.433663 +oy -10.475408 +h_ -10.519749 +oc -10.564300 +fi -10.572696 +ke -10.595327 +ox -10.638952 +ex -10.642104 +_o -10.649697 +sl -10.660736 +cd -10.680692 +rr -10.686103 +bk -10.699394 +m_ -10.710160 +xo -10.729028 +cu -10.763853 +ix -10.763853 +oq -10.784137 +xd -10.793920 +co -10.803534 +gt -10.811082 +je -10.837576 +xe -10.862359 +gi -10.891570 +it -10.911970 +of -10.944288 +dl -10.947404 +xw -10.956792 +lm -11.004133 +jo -11.078760 +ls -11.087613 +ic -11.096810 +c_ -11.097098 +_p -11.115381 +mb -11.126521 +os -11.187231 +yb -11.249967 +fu -11.267059 +gm -11.276825 +pu -11.288631 +sb -11.288631 +tl -11.294240 +th -11.311200 +tu -11.315219 +bs -11.316898 +ce -11.331072 +rg -11.339916 +uk -11.375148 +wi -11.419531 +w_ -11.464579 +io -11.507242 +aj -11.522651 +fk -11.562707 +eh -11.585585 +if -11.588420 +wo -11.619975 +ld -11.626620 +rb -11.648006 +sm -11.665432 +et -11.677454 +ny -11.681340 +fr -11.694805 +bd -11.695678 +rc -11.709278 +bn -11.712809 +qe -11.719896 +xm -11.727464 +qi -11.752681 +nn -11.762252 +pi -11.786693 +ik -11.786693 +lb -11.787158 +mk -11.795089 +op -11.796961 +uf -11.798836 +lu -11.802593 +ek -11.805889 +eb -11.807775 +ns -11.845063 +rl -11.862604 +ot -11.908930 +mm -11.915017 +gs -11.932920 +q_ -12.003052 +cs -12.017715 +dn -12.056440 +uc -12.069394 +rd -12.071093 +db -12.083037 +gg -12.103167 +ry -12.103167 +rn -12.161674 +yk -12.222655 +uh -12.227066 +ly -12.334467 +ut -12.337870 +cy -12.358461 +xk -12.527695 +gn -12.544112 +qu -12.555161 +xb -12.557539 +rj -12.605957 +ft -12.629954 +hd -12.671430 +uj -12.677454 +ng -12.700927 +rm -12.705316 +xy -12.709719 +km -12.712367 +cm -12.713251 +mg -12.724789 +wg -12.738217 +ai -12.755409 +fo -12.763624 +fe -12.796961 +t_ -12.801653 +sn -12.812976 +dw -12.928295 +ec -12.960460 +ou -12.960460 +em -12.983730 +br -13.003052 +nb -13.019354 +rx -13.033632 +iq -13.043601 +ij -13.049169 +ju -13.059246 +ds -13.066004 +wr -13.067133 +lg -13.109551 +wd -13.152056 +fd -13.160468 +oj -13.239744 +fh -13.296888 +bb -13.338552 +ea -13.356389 +dg -13.471291 +yg -13.573499 +df -13.578321 +xs -13.605957 +nl -13.629120 +nf -13.630788 +hn -13.632459 +ap -13.644209 +hl -13.723898 +qt -13.723898 +xt -13.747240 +hk -13.747240 +ef -13.754499 +sc -13.756319 +ht -13.756319 +pl -13.785763 +bg -13.812029 +ql -13.812029 +ws -13.829170 +ch -13.875901 +p_ -13.911970 +qs -13.911970 +mn -13.942732 +hr -13.948964 +ej -13.986932 +hm -14.021541 +pr -14.034737 +ve -14.039162 +_v -14.059246 +fl -14.079614 +tr -14.107226 +pa -14.116549 +nc -14.118889 +gy -14.128288 +iw -14.152056 +bl -14.195851 +rw -14.213248 +fg -14.215751 +hs -14.225805 +dy -14.248685 +wh -14.258971 +yl -14.261555 +cl -14.282388 +lp -14.285014 +ss -14.287644 +gh -14.303527 +yr -14.324980 +au -14.344017 +by -14.355009 +cn -14.368868 +bc -14.371655 +cw -14.388496 +bj -14.402682 +qb -14.402682 +vi -14.425674 +k_ -14.440232 +ok -14.457898 +gd -14.509159 +xg -14.540186 +az -14.578321 +xn -14.620804 +sy -14.630788 +eq -14.634132 +rf -14.654357 +ln -14.664577 +ua -14.695678 +iv -14.702681 +sg -14.716791 +dr -14.770966 +qm -14.785763 +_z -14.793219 +nm -14.831087 +po -14.854294 +zi -14.866040 +ze -14.873923 +gp -14.951048 +rq -14.984796 +cb -14.993358 +sw -14.993358 +oh -15.010636 +mt -15.014988 +ct -15.019354 +mc -15.077337 +dm -15.095657 +wf -15.109551 +nj -15.109551 +ew -15.156857 +sr -15.161674 +bq -15.176223 +qn -15.200801 +fp -15.210750 +ep -15.246125 +ym -15.251250 +dj -15.292919 +qr -15.324980 +ei -15.330393 +jk -15.374449 +qy -15.374449 +mh -15.385676 +kr -15.385676 +va -15.391322 +j_ -15.414132 +fs -15.414132 +gl -15.419892 +ov -15.490858 +bw -15.496933 +ts -15.540186 +wm -15.546472 +sf -15.571895 +lx -15.591260 +ms -15.644209 +jn -15.650966 +pe -15.657755 +ev -15.657755 +ck -15.706196 +ie -15.706196 +gr -15.720340 +v_ -15.741820 +ae -15.785763 +tf -15.823433 +bh -15.823433 +up -15.893823 +vo -15.901860 +za -15.901860 +lf -15.918070 +nq -15.918070 +lq -15.993358 +ty -16.001972 +cr -16.054759 +wq -16.054759 +mr -16.100274 +sx -16.118889 +oa -16.137748 +lw -16.137748 +hf -16.195851 +xq -16.195851 +sp -16.215751 +rp -16.225805 +sj -16.266735 +mp -16.308860 +av -16.319587 +sd -16.341282 +cq -16.352253 +pt -16.396991 +lh -16.396991 +tt -16.431480 +lc -16.443161 +nh -16.503033 +eu -16.515311 +eo -16.552786 +kn -16.630788 +hc -16.685236 +uv -16.727464 +fn -16.741820 +lt -16.893823 +dp -16.926244 +ue -16.926244 +kt -16.942732 +nz -17.081895 +xf -17.137748 +md -17.195851 +fb -17.215751 +hy -17.256393 +fy -17.256393 +iu -17.277151 +hq -17.298213 +hb -17.319587 +jy -17.319587 +bm -17.341282 +fq -17.363308 +ui -17.408396 +yj -17.454939 +oi -17.478785 +dc -17.527695 +mx -17.552786 +xl -17.604316 +js -17.657755 +tk -17.657755 +nr -17.770966 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/spa b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/spa new file mode 100644 index 00000000000..16bc029f338 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/spa @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 487097908 +#UNIQUE_BIGRAMS 597 +#UNSEEN_LOG_PROB -19.906760 +#EXPECTED_SCORE -7.282803 +s_ -5.001716 +o_ -5.125873 +a_ -5.252707 +e_ -5.723579 +es -5.789721 +_p -5.915053 +_c -5.942583 +en -5.981225 +on -5.999765 +os -6.124253 +er -6.151353 +n_ -6.188169 +ra -6.257179 +ar -6.344929 +as -6.354869 +ue -6.363843 +an -6.384356 +re -6.412530 +co -6.459366 +nt -6.470663 +ci -6.477218 +_d -6.526239 +_e -6.539822 +de -6.551474 +io -6.582746 +ta -6.624344 +_a -6.629539 +te -6.639890 +r_ -6.695252 +or -6.709413 +do -6.717625 +_l -6.753769 +ad -6.803892 +_s -6.821336 +_m -6.861111 +al -6.902978 +st -6.920591 +na -6.940482 +ro -6.965074 +qu -7.015770 +to -7.033841 +ca -7.047721 +ri -7.092200 +ia -7.098973 +in -7.134160 +la -7.148965 +lo -7.168979 +l_ -7.198900 +ma -7.231283 +_t -7.236221 +da -7.280903 +po -7.305028 +_q -7.325337 +ie -7.328204 +ic -7.331047 +pa -7.355583 +ti -7.364272 +el -7.373259 +un -7.388144 +tr -7.423531 +ac -7.485562 +no -7.527296 +di -7.555213 +le -7.566250 +si -7.592125 +me -7.601299 +id -7.643806 +_r -7.681386 +se -7.687461 +_f -7.737478 +pr -7.763201 +ec -7.767355 +li -7.794911 +is -7.815772 +ne -7.816330 +nd -7.853587 +om -7.911541 +_h -7.936991 +pe -7.973706 +mi -7.978928 +so -7.987440 +_i -8.021935 +mo -8.023707 +nc -8.054104 +am -8.096470 +_u -8.099856 +ni -8.154906 +sa -8.181434 +ce -8.242295 +cu -8.250912 +em -8.255942 +_v -8.257450 +vi -8.354123 +it -8.400330 +rt -8.413384 +_n -8.414561 +ab -8.421196 +ol -8.482339 +ha -8.503658 +ba -8.513003 +_o -8.532094 +ve -8.552478 +_g -8.565417 +im -8.574146 +at -8.593329 +ed -8.616037 +ll -8.635817 +bi -8.656333 +mp -8.686057 +_b -8.691051 +eg -8.713215 +gu -8.746738 +br -8.784940 +il -8.807387 +oc -8.820669 +ur -8.820881 +tu -8.838147 +us -8.866604 +pu -8.867960 +ga -8.877952 +su -8.878511 +ua -8.882565 +ct -8.892931 +d_ -8.894942 +ir -8.949503 +go -8.953158 +fi -8.975532 +mb -8.994328 +fu -9.014261 +va -9.023787 +ns -9.046024 +mu -9.079818 +ui -9.101430 +gr -9.137700 +od -9.140331 +sp -9.149187 +iv -9.163367 +ig -9.171064 +ob -9.184891 +ch -9.196092 +et -9.210266 +ul -9.219259 +za -9.239934 +rm -9.276890 +rr -9.293123 +_j -9.305398 +ot -9.313111 +bl -9.356513 +ho -9.384158 +vo -9.412051 +uc -9.428753 +jo -9.452020 +ea -9.506649 +lu -9.507921 +ev -9.551282 +pi -9.577772 +sc -9.594626 +fe -9.640856 +ap -9.647378 +pl -9.654406 +ex -9.665707 +nu -9.674166 +ge -9.679285 +rs -9.686489 +cr -9.722726 +du -9.763174 +be -9.793266 +gi -9.818034 +rc -9.829460 +y_ -9.844355 +ud -9.853078 +ju -9.889199 +ib -9.915622 +ru -9.916175 +op -9.922179 +ip -9.936309 +rg -9.965340 +je -9.968939 +rd -9.974748 +lt -9.980589 +ay -10.000417 +fo -10.003530 +iz -10.004312 +ut -10.007439 +hi -10.009083 +ag -10.016900 +au -10.027986 +ep -10.037533 +aj -10.048096 +ej -10.095520 +um -10.110494 +ja -10.113076 +ub -10.127827 +cl -10.132245 +rn -10.143447 +ez -10.199080 +bo -10.201387 +z_ -10.231294 +eb -10.231412 +ng -10.297308 +he -10.313658 +zo -10.320508 +if -10.364496 +fa -10.369299 +fr -10.383678 +i_ -10.397122 +cc -10.431170 +dr -10.432901 +og -10.433970 +eo -10.440306 +av -10.543001 +bu -10.553544 +ov -10.563195 +nf -10.592219 +up -10.620546 +ef -10.639603 +xi -10.668645 +yo -10.692713 +sm -10.769752 +ai -10.857613 +uy -10.875825 +nv -10.876899 +nz -10.891864 +ug -10.894867 +az -10.969349 +lm -10.996352 +af -11.103113 +lg -11.131889 +oy -11.241000 +of -11.243447 +ye -11.288980 +xp -11.426581 +ij -11.428743 +sd -11.442023 +rv -11.445290 +eq -11.449989 +gl -11.511514 +t_ -11.523775 +iu -11.583394 +hu -11.595441 +uv -11.612061 +eu -11.671511 +pt -11.712829 +rl -11.770786 +rq -11.825515 +lv -11.829993 +ei -11.854626 +xt -11.859339 +uj -11.953145 +nq -12.010525 +ld -12.033572 +ey -12.074810 +ya -12.111431 +rp -12.115013 +rz -12.144251 +aq -12.151644 +ae -12.161135 +fl -12.179332 +ah -12.249520 +ee -12.258345 +lc -12.304239 +_w -12.363875 +_z -12.365062 +m_ -12.369679 +th -12.371953 +sl -12.403330 +k_ -12.444605 +gn -12.488898 +ou -12.677029 +_k -12.687580 +oj -12.727277 +rb -12.757189 +ox -12.828775 +yu -12.865792 +bs -12.889733 +g_ -12.933271 +pc -12.951244 +lb -12.952159 +b_ -12.957497 +oe -12.966335 +vu -13.024721 +sf -13.031525 +ki -13.039657 +nj -13.042378 +uo -13.052156 +uz -13.065551 +u_ -13.082741 +oo -13.107779 +h_ -13.120462 +oz -13.128795 +ls -13.129564 +cn -13.133711 +c_ -13.186355 +nm -13.240018 +sh -13.278230 +ck -13.283861 +uf -13.287825 +wa -13.307748 +oa -13.332343 +_y -13.332679 +bt -13.341906 +ii -13.371043 +nn -13.411553 +bj -13.420845 +xc -13.443829 +ke -13.507007 +ax -13.514890 +ss -13.518048 +dm -13.519126 +oh -13.558498 +iq -13.588210 +_x -13.610857 +rf -13.617914 +lq -13.674258 +rk -13.719166 +wi -13.728784 +ps -13.785161 +sq -13.794094 +x_ -13.802406 +lp -13.806644 +oq -13.837919 +p_ -13.842437 +zu -13.863604 +tb -13.915428 +ry -13.973178 +we -14.020433 +mn -14.055102 +ow -14.083666 +lr -14.128583 +oi -14.130122 +tt -14.131716 +lf -14.137576 +eh -14.148400 +tl -14.168057 +sg -14.197691 +ji -14.220465 +xa -14.226812 +w_ -14.299782 +zq -14.341598 +hn -14.393624 +ak -14.442048 +ix -14.470812 +ka -14.501810 +ao -14.526062 +nr -14.531663 +dv -14.554857 +zc -14.579083 +ts -14.618696 +ok -14.696144 +ly -14.699214 +ny -14.748419 +ph -14.766962 +xo -14.804354 +ew -14.835622 +f_ -14.846663 +nk -14.854626 +dq -14.919598 +xv -14.956320 +xu -14.959147 +tm -14.963683 +gh -15.024869 +wo -15.048362 +ms -15.051678 +ds -15.058636 +sb -15.083923 +dy -15.104436 +ty -15.130972 +zg -15.223352 +sr -15.239760 +nl -15.249188 +ff -15.257054 +tw -15.294296 +ze -15.334728 +uq -15.356929 +tc -15.358173 +hr -15.358297 +gm -15.374059 +sn -15.392668 +ik -15.422535 +mm -15.431538 +rj -15.476067 +ft -15.498104 +bc -15.593875 +ko -15.629616 +v_ -15.630968 +aw -15.687365 +ht -15.687834 +ks -15.749480 +zi -15.762592 +bb -15.780318 +gt -15.807408 +pp -15.854012 +tn -15.860870 +my -15.906895 +nh -15.911087 +lz -15.915107 +xe -15.966145 +ys -16.041654 +by -16.051075 +ws -16.091039 +xh -16.137469 +wn -16.171823 +sv -16.180156 +sk -16.218488 +dw -16.250689 +aa -16.277259 +yl -16.277730 +gs -16.296917 +ux -16.390758 +dn -16.424748 +ky -16.427878 +lk -16.437572 +yn -16.438624 +dd -16.457424 +ln -16.556427 +kl -16.567603 +ih -16.587298 +ku -16.587590 +bv -16.591972 +dg -16.601365 +cy -16.636540 +dj -16.698190 +bm -16.702605 +ek -16.704186 +fg -16.747523 +bd -16.753401 +bn -16.766879 +zz -16.788845 +wh -16.807408 +uu -16.823806 +vd -16.887375 +uk -16.896379 +j_ -17.012971 +cd -17.038261 +xx -17.136829 +cs -17.139392 +hm -17.148830 +ym -17.164408 +mc -17.178398 +mt -17.186327 +yd -17.188538 +tv -17.205896 +sy -17.223918 +dh -17.231191 +yw -17.235298 +wr -17.256474 +nb -17.292156 +ww -17.315155 +gb -17.354818 +yr -17.359791 +fm -17.414104 +sw -17.414104 +gd -17.518674 +tz -17.530962 +qa -17.562720 +zm -17.601659 +yc -17.634429 +py -17.668577 +hy -17.676001 +hl -17.784826 +gg -17.840046 +zt -17.851908 +uh -17.851908 +pn -17.879497 +tp -17.903260 +fp -17.903987 +zy -17.925209 +cq -17.940773 +ml -18.017286 +lh -18.029914 +dl -18.053893 +kh -18.081559 +iy -18.165279 +rx -18.169639 +rh -18.174888 +fy -18.248612 +vy -18.279378 +yp -18.306007 +pb -18.308890 +pg -18.344922 +kn -18.351842 +xy -18.361785 +tg -18.418767 +vr -18.435470 +zn -18.458757 +wl -18.481342 +pd -18.499887 +zh -18.502085 +kr -18.509802 +fs -18.538836 +_テ -18.565016 +cb -18.609338 +dt -18.632021 +rw -18.657513 +hz -18.671048 +pm -18.679727 +fb -18.687209 +cf -18.755038 +iw -18.776157 +hw -18.813877 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/sqi b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/sqi new file mode 100644 index 00000000000..01905e3a86d --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/sqi @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 25295142 +#UNIQUE_BIGRAMS 661 +#UNSEEN_LOG_PROB -18.096527 +#EXPECTED_SCORE -7.460362 +e_ -4.299839 +sh -5.638723 +er -5.693676 +te -5.730910 +_p -5.811430 +t_ -5.913454 +he -6.005492 +re -6.056319 +es -6.107467 +et -6.110652 +a_ -6.122846 +je -6.170070 +_n -6.213457 +n_ -6.271578 +_d -6.274075 +r_ -6.275841 +_k -6.343165 +en -6.361408 +_s -6.381220 +ar -6.401442 +it -6.427813 +i_ -6.449567 +dh -6.568099 +ri -6.587747 +pe -6.713606 +in -6.734145 +ti -6.753449 +_m -6.800615 +nd -6.845816 +ht -6.920788 +is -6.958333 +ne -7.006551 +an -7.031644 +ve -7.039433 +me -7.048109 +or -7.074088 +ra -7.105184 +_t -7.114117 +_v -7.168511 +at -7.185089 +_a -7.210552 +ke -7.261992 +mi -7.303023 +li -7.317924 +im -7.317970 +s_ -7.328978 +on -7.399794 +_e -7.465018 +si -7.481140 +ta -7.493451 +nj -7.493852 +de -7.513247 +_f -7.523138 +ur -7.539014 +ua -7.567369 +gj -7.568895 +se -7.655961 +_b -7.695741 +ll -7.730289 +as -7.734485 +pa -7.755136 +_g -7.774749 +ni -7.776999 +le -7.781020 +ko -7.813625 +al -7.854964 +_l -7.894724 +ng -7.897455 +tu -7.923971 +_r -7.933950 +di -7.935583 +ja -7.937875 +ku -7.946488 +ga -7.960498 +to -8.003379 +ik -8.024876 +j_ -8.044017 +ma -8.062210 +st -8.078275 +ro -8.120666 +ka -8.215079 +il -8.220308 +em -8.228547 +un -8.233294 +tr -8.293310 +pr -8.306106 +rr -8.335411 +la -8.412118 +aj -8.437974 +th -8.447838 +_i -8.451807 +po -8.456527 +vi -8.488987 +ev -8.497331 +hi -8.527551 +ek -8.530037 +be -8.539768 +m_ -8.577203 +nt -8.583841 +mb -8.595553 +ak -8.609141 +ha -8.612709 +do -8.649957 +ba -8.657100 +rt -8.667939 +ji -8.670284 +sa -8.688452 +hu -8.688946 +_j -8.698961 +k_ -8.725513 +qe -8.726672 +ej -8.745593 +o_ -8.751923 +na -8.762585 +ul -8.763477 +ed -8.771231 +ci -8.793151 +_c -8.802339 +el -8.802798 +hk -8.804327 +uk -8.805832 +kt -8.828548 +kr -8.843745 +_h -8.852155 +lo -8.870110 +fi -8.885025 +ij -8.887427 +ru -8.915381 +_q -8.925994 +om -8.963712 +oh -8.985171 +ia -8.986241 +ol -8.993665 +mu -8.997012 +os -9.013925 +ip -9.034893 +um -9.088687 +dr -9.112545 +lu -9.120015 +du -9.124688 +jt -9.152721 +ir -9.153338 +pu -9.168978 +_z -9.186218 +jo -9.224942 +so -9.263332 +ot -9.274344 +us -9.281744 +rs -9.298778 +oj -9.316124 +qi -9.333202 +ki -9.333975 +ad -9.338399 +am -9.344578 +rm -9.351081 +mo -9.358588 +tj -9.358999 +da -9.366396 +ep -9.374626 +nu -9.377279 +io -9.379063 +yr -9.432013 +fa -9.441499 +h_ -9.459496 +u_ -9.502824 +bi -9.535254 +pi -9.538219 +ez -9.545190 +eg -9.545659 +hm -9.550056 +l_ -9.566871 +_o -9.567174 +zi -9.608339 +no -9.621622 +ut -9.625761 +su -9.631269 +_u -9.645908 +iz -9.662821 +gr -9.663099 +ry -9.666385 +av -9.671911 +id -9.748779 +op -9.760753 +d_ -9.782589 +oi -9.786311 +ty -9.806087 +rg -9.818497 +br -9.830079 +iv -9.838566 +hq -9.843326 +fe -9.848574 +ce -9.866192 +fu -9.876181 +va -9.883165 +rd -9.906513 +bu -9.912712 +ho -9.914582 +ze -9.947599 +rk -9.948332 +ap -9.990238 +gu -9.991689 +eh -10.008099 +od -10.027731 +fo -10.028148 +az -10.034116 +rb -10.053986 +ye -10.054714 +ca -10.110684 +aq -10.134014 +ue -10.149673 +ge -10.172923 +hj -10.187149 +og -10.191811 +ju -10.211625 +pt -10.216707 +jn -10.229728 +zo -10.234118 +bo -10.237006 +yt -10.294223 +nc -10.295584 +ok -10.302338 +pj -10.303923 +af -10.334012 +ft -10.350002 +go -10.350225 +rf -10.358288 +hp -10.372810 +ks -10.378038 +rj -10.403614 +ud -10.423685 +uh -10.431776 +zh -10.439042 +ie -10.502824 +vo -10.530058 +ig -10.541573 +dj -10.546597 +za -10.547024 +ic -10.556699 +ns -10.569122 +ys -10.577813 +kj -10.642895 +up -10.675078 +qy -10.685466 +ec -10.731658 +uf -10.741682 +fs -10.747161 +lt -10.770384 +bl -10.804556 +fr -10.814177 +uj -10.827382 +ov -10.835801 +pl -10.864756 +vj -10.941081 +mp -10.986993 +ag -10.998732 +ah -11.005048 +iu -11.053926 +sp -11.058541 +mj -11.095503 +ug -11.098001 +sk -11.103513 +zu -11.104894 +ac -11.109801 +ih -11.118144 +mt -11.153955 +mr -11.194682 +of -11.230550 +kl -11.252785 +ib -11.265225 +rc -11.291433 +eq -11.304068 +ea -11.320602 +ab -11.331291 +qu -11.369562 +ls -11.373037 +oz -11.400527 +au -11.409807 +ub -11.420398 +eu -11.422745 +ob -11.498939 +sm -11.533013 +lj -11.542849 +hy -11.579732 +ef -11.595001 +dy -11.614362 +rn -11.622071 +p_ -11.637070 +rp -11.641801 +eo -11.663469 +ps -11.679655 +if -11.691679 +lm -11.694134 +g_ -11.708186 +oq -11.717760 +zg -11.731658 +c_ -11.749025 +hv -11.759863 +vr -11.761248 +xh -11.779378 +sl -11.842069 +sj -11.913317 +rv -11.926799 +co -11.931470 +uz -11.977647 +oc -11.989890 +ai -11.990122 +nk -12.002238 +fl -12.013748 +iq -12.019894 +y_ -12.055867 +ny -12.056353 +hd -12.082335 +vd -12.085306 +jy -12.093757 +z_ -12.095004 +vl -12.132157 +qa -12.140631 +q_ -12.186418 +gi -12.210814 +dm -12.212708 +uq -12.269584 +tm -12.271556 +gl -12.272967 +fj -12.284587 +_x -12.292290 +nx -12.371072 +bj -12.396370 +cd -12.443245 +zb -12.459536 +sn -12.475688 +sy -12.550014 +nf -12.557902 +rh -12.609720 +lb -12.625852 +eb -12.655719 +hs -12.663099 +hf -12.691113 +zj -12.728171 +ch -12.802823 +v_ -12.806905 +jm -12.822932 +hn -12.827900 +uc -12.829560 +js -12.834967 +vu -12.852998 +rl -12.865713 +ck -12.910679 +kz -12.938616 +ii -12.939064 +b_ -12.982717 +yl -13.049325 +rq -13.055139 +yj -13.076165 +ts -13.077150 +oe -13.085554 +zm -13.103513 +cm -13.108541 +by -13.124751 +cu -13.154605 +jv -13.186152 +sf -13.218404 +xi -13.253064 +ym -13.277207 +qj -13.339692 +_w -13.365945 +lk -13.384733 +ex -13.390846 +ds -13.408102 +kn -13.419929 +_y -13.426822 +yn -13.478615 +gt -13.551382 +ss -13.557558 +uv -13.568603 +zy -13.571377 +sv -13.600127 +my -13.612932 +f_ -13.657192 +yp -13.669773 +jr -13.728945 +ms -13.729720 +zv -13.735154 +lq -13.762634 +py -13.810178 +cl -13.815924 +xe -13.820868 +ou -13.830806 +oa -13.838305 +yk -13.845003 +oo -13.858494 +nv -13.868696 +yq -13.879830 +ei -13.936826 +tb -13.945798 +qo -13.966648 +ui -13.998966 +jz -14.023451 +tn -14.033936 +ae -14.052229 +lg -14.053198 +jd -14.061951 +db -14.066836 +jq -14.143208 +zr -14.154605 +td -14.175559 +gn -14.200040 +nz -14.210814 +gd -14.222760 +ld -14.246952 +hb -14.271556 +rz -14.304645 +wa -14.319727 +ax -14.343244 +lp -14.357540 +sq -14.400064 +sc -14.478615 +ln -14.508878 +x_ -14.528962 +ox -14.589542 +lf -14.607939 +wi -14.619377 +bs -14.619377 +hr -14.620813 +tl -14.657192 +nn -14.660142 +tt -14.664579 +mn -14.711243 +w_ -14.734376 +qt -14.768990 +zn -14.819218 +bn -14.835801 +ay -14.837470 +xj -14.854265 +xo -14.857647 +cr -14.857647 +sd -14.876395 +ew -14.905856 +cf -14.939512 +xv -15.013041 +tp -15.018710 +ee -15.035851 +ix -15.088531 +tv -15.141146 +lc -15.200040 +yo -15.237006 +ow -15.243629 +we -15.270429 +mm -15.330262 +ux -15.346804 +ao -15.353952 +xu -15.375611 +dl -15.380469 +sg -15.382904 +ey -15.390233 +fy -15.407482 +ly -15.407482 +ph -15.437539 +cj -15.473416 +gm -15.478615 +fq -15.496960 +tk -15.523579 +aa -15.534365 +ff -15.539789 +ct -15.539789 +lv -15.558934 +kh -15.561690 +wo -15.617942 +ky -15.623690 +ya -15.623690 +hc -15.638161 +nr -15.743734 +sb -15.797941 +pn -15.814280 +yh -15.861038 +pp -15.874681 +qv -15.881551 +gh -15.884998 +dn -15.902359 +hg -15.927021 +bd -15.930579 +е_ -15.988731 +uo -16.003642 +nl -16.014928 +yf -16.033936 +nh -16.157729 +lr -16.166092 +md -16.226035 +gp -16.230413 +jk -16.239210 +pd -16.265928 +tf -16.325570 +km -16.339692 +aw -16.344429 +jl -16.358737 +tw -16.378038 +dv -16.382904 +yc -16.412448 +oy -16.437539 +cc -16.442610 +mc -16.447699 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/srd b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/srd new file mode 100644 index 00000000000..c0668859e19 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/srd @@ -0,0 +1,224 @@ +#TOTAL_BIGRAMS 123984 +#UNIQUE_BIGRAMS 220 +#UNSEEN_LOG_PROB -13.597866 +#EXPECTED_SCORE -6.855426 +u_ -4.759923 +a_ -4.892234 +_c -4.932886 +s_ -5.180436 +_s -5.321277 +_p -5.552926 +t_ -5.584404 +i_ -5.640764 +st -5.762447 +e_ -5.826377 +un -5.924734 +es -6.000186 +ch -6.210711 +da -6.216756 +cu -6.292261 +sa -6.296827 +hi -6.302327 +tu -6.379666 +_e -6.403109 +us -6.420945 +pr -6.433965 +_d -6.486209 +nt -6.502997 +os -6.504053 +ar -6.527477 +as -6.574389 +_n -6.640184 +an -6.665947 +li -6.734919 +nu -6.748618 +na -6.799557 +_m -6.825717 +_l -6.853705 +in -6.867226 +o_ -6.874035 +ro -6.876767 +_t -6.882247 +at -6.924027 +on -6.942515 +er -6.954010 +n_ -6.959792 +ia -6.989057 +_f -7.003915 +ca -7.064926 +it -7.098020 +re -7.120513 +is -7.127004 +te -7.135160 +ta -7.138435 +du -7.145007 +no -7.158243 +rd -7.171602 +_u -7.183392 +so -7.222827 +en -7.228051 +di -7.272336 +_i -7.323605 +co -7.365206 +ra -7.376763 +ti -7.408042 +im -7.410019 +ma -7.413983 +_a -7.417957 +su -7.516782 +gu -7.529625 +ss -7.557851 +tz -7.606911 +po -7.655352 +si -7.657700 +nd -7.695793 +to -7.698207 +me -7.727502 +_b -7.729970 +ru -7.734919 +nn -7.734919 +ba -7.757403 +zi -7.827037 +et -7.834986 +ae -7.842979 +ne -7.861803 +de -7.922615 +ri -7.968510 +or -7.980215 +om -8.006905 +pu -8.024977 +mb -8.037151 +se -8.037151 +ot -8.049430 +fa -8.106013 +dd -8.118895 +mu -8.138435 +al -8.212435 +ic -8.229796 +ai -8.233294 +ad -8.268743 +_g -8.316168 +io -8.319882 +ni -8.400158 +ig -8.431954 +fi -8.472711 +pa -8.489342 +mp -8.506166 +ed -8.514653 +le -8.544755 +ua -8.549107 +id -8.549107 +ng -8.657700 +nc -8.690976 +do -8.700626 +ol -8.715223 +he -8.720122 +ur -8.759923 +sc -8.775136 +tr -8.780243 +bi -8.800853 +rt -8.806052 +em -8.919794 +lu -8.954010 +be -9.006905 +ie -9.012904 +eg -9.012904 +gi -9.018928 +ul -9.018928 +iu -9.061813 +pe -9.151610 +mi -9.283170 +ag -9.327337 +am -9.327337 +pi -9.327337 +la -9.327337 +gh -9.342366 +r_ -9.372900 +gn -9.388413 +ge -9.388413 +og -9.435979 +od -9.468583 +rr -9.485166 +ui -9.493530 +_r -9.493530 +vi -9.501942 +ga -9.625174 +lo -9.739885 +cr -9.886371 +mo -9.931110 +ll -9.954010 +ab -10.112440 +au -10.125379 +op -10.151610 +ub -10.205549 +fu -10.247369 +fe -10.247369 +ep -10.261583 +ir -10.275938 +ov -10.334832 +oc -10.365206 +eu -10.365206 +ci -10.396232 +oi -10.476851 +ve -10.493530 +el -10.527477 +lt -10.562242 +ut -10.562242 +il -10.579944 +ze -10.616014 +ud -10.653008 +bo -10.671867 +um -10.749869 +za -10.790511 +rn -10.811270 +uf -10.811270 +ls -10.853705 +uo -10.875400 +fo -10.942515 +uv -11.138435 +oa -11.164907 +_o -11.164907 +_v -11.275938 +br -11.305085 +rc -11.365206 +bu -11.396232 +rs -11.460363 +ns -11.493530 +gr -11.527477 +ap -11.527477 +tt -11.634392 +rm -11.634392 +aj -11.710341 +ce -11.790511 +ou -11.790511 +ug -11.875400 +go -11.875400 +bl -11.875400 +_z -12.061813 +c_ -12.061813 +eo -12.112440 +sm -12.164907 +ip -12.275938 +rv -12.275938 +ue -12.396232 +ib -12.396232 +ea -12.460363 +ei -12.527477 +ev -12.527477 +if -12.597866 +jo -12.597866 +ez -12.597866 +bb -12.671867 +oe -12.671867 +ju -12.832332 +ec -13.012904 +ii -13.012904 +rg -13.112440 +up -13.334832 +nf -13.460363 +gl -13.460363 +d_ -13.460363 +iv -13.597866 +ac -13.597866 +iz -13.597866 +sp -13.597866 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/srp b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/srp new file mode 100644 index 00000000000..bf24ea6ac40 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/srp @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 67668766 +#UNIQUE_BIGRAMS 877 +#UNSEEN_LOG_PROB -17.510704 +#EXPECTED_SCORE -7.789821 +а_ -4.874937 +е_ -5.405390 +и_ -5.666749 +_п -5.718910 +о_ -5.935234 +_с -6.167390 +ра -6.184895 +на -6.253749 +ст -6.288098 +ко -6.368028 +у_ -6.466532 +_к -6.486674 +ни -6.589138 +та -6.702026 +по -6.707121 +_о -6.712607 +но -6.795914 +_н -6.820937 +пр -6.830784 +ре -6.857156 +је -6.937487 +ов -6.944713 +_д -6.954860 +иј -6.964104 +ан -6.979470 +ла -7.003837 +ли -7.004075 +ва -7.054366 +ка -7.057237 +ма -7.086424 +ој -7.091154 +ти -7.111595 +ен -7.126588 +м_ -7.126934 +ри -7.142486 +ве -7.148224 +ос -7.201195 +_и -7.239582 +во -7.280810 +_м -7.290724 +од -7.299759 +_б -7.310670 +им -7.336467 +ин -7.350570 +не -7.352067 +_в -7.354672 +ом -7.376134 +ав -7.411471 +_т -7.423305 +ро -7.428136 +ог -7.431312 +ор -7.445509 +ал -7.474666 +ви -7.495552 +то -7.536697 +ед -7.564863 +ск -7.596241 +да -7.619829 +ме -7.621990 +ад -7.622011 +ја -7.627313 +ар -7.644593 +_р -7.653035 +ат -7.757489 +ак -7.775636 +те -7.790494 +ај -7.811361 +ил -7.852337 +ис -7.865920 +_у -7.868584 +би -7.869830 +ди -7.888568 +ло -7.930977 +_г -7.932054 +_з -7.942532 +ер -8.000645 +ик -8.008013 +ци -8.023535 +ет -8.033405 +ем -8.037365 +тр -8.038732 +он -8.040348 +де -8.055560 +н_ -8.066337 +ле -8.090105 +за -8.164929 +ио -8.183832 +ру -8.202928 +ње -8.228304 +ас -8.233249 +го -8.259044 +ел -8.264541 +гр -8.269546 +г_ -8.276563 +_ј -8.277562 +до -8.282964 +из -8.293949 +ки -8.308806 +мо -8.310883 +об -8.314408 +са -8.323976 +их -8.330096 +ао -8.336050 +ек -8.343559 +_а -8.363594 +ит -8.378644 +ју -8.379460 +ји -8.387334 +ок -8.412096 +аз -8.414531 +ол -8.415986 +х_ -8.426503 +св -8.457038 +се -8.523040 +ив -8.539129 +ј_ -8.541201 +т_ -8.602803 +дн -8.623036 +ам -8.624340 +ке -8.631636 +шт -8.638374 +д_ -8.641444 +ес -8.641640 +др -8.645477 +зи -8.662040 +сл -8.687182 +па -8.696085 +ку -8.712913 +вр -8.728103 +ањ -8.750929 +иц -8.755529 +кр -8.771130 +тв -8.781826 +ич -8.827170 +от -8.842365 +бр -8.856240 +ми -8.875707 +га -8.897543 +бо -8.901412 +че -8.912783 +к_ -8.919725 +си -8.955709 +ту -8.971397 +вн -8.983153 +ац -9.029149 +уг -9.044929 +ну -9.070034 +ез -9.072626 +ус -9.134810 +р_ -9.142357 +оп -9.170619 +_ч -9.180828 +пе -9.186779 +ач -9.193692 +ег -9.225896 +_л -9.237882 +ир -9.238268 +ба -9.246442 +ут -9.250059 +чи -9.257543 +иш -9.264082 +ев -9.269217 +зн -9.285795 +бе -9.289313 +ња -9.317098 +ду -9.317928 +_ф -9.321143 +нс -9.323982 +_ш -9.326184 +лу -9.336099 +сн -9.342396 +сп -9.360655 +ео -9.383256 +_њ -9.388562 +уп -9.398312 +ећ -9.423506 +ца -9.423697 +це -9.431964 +рв -9.453865 +нт -9.466163 +же -9.471450 +ље -9.527022 +ап -9.534640 +гу -9.563193 +чк -9.583430 +пу -9.592784 +ча -9.607078 +пи -9.611224 +ун -9.611391 +_ц -9.637304 +жи -9.653095 +жа -9.699263 +ља -9.709901 +_е -9.717887 +зв -9.738082 +тн -9.760819 +иг -9.761374 +ењ -9.766415 +ћи -9.774609 +гл -9.777038 +су -9.785067 +зе -9.790286 +уч -9.798522 +уд -9.810705 +ђе -9.817249 +оз -9.820002 +чн -9.820060 +њи -9.824345 +ср -9.829651 +ук -9.872395 +рс -9.874856 +ул -9.879628 +лн -9.880029 +_ж -9.901850 +ше -9.902891 +ги -9.926597 +ур -9.959313 +еђ -9.962245 +уј -9.990551 +љу -10.017611 +фи -10.027168 +ће -10.031468 +кт -10.044896 +со -10.070779 +ош -10.090494 +оч -10.094568 +рн -10.098466 +ге -10.098793 +ож -10.125195 +уб -10.142562 +му -10.142778 +_х -10.143044 +ву -10.147535 +бл -10.202244 +рт -10.207780 +см -10.210758 +пл -10.229680 +с_ -10.234089 +аш -10.251085 +еб -10.266391 +јс -10.270441 +ђу -10.270757 +рж -10.271257 +дв -10.277542 +ум -10.282791 +еш -10.284944 +аг -10.304250 +вљ -10.311628 +ељ -10.312929 +в_ -10.317384 +еп -10.318799 +јо -10.320461 +ши -10.351907 +уж -10.354473 +кл -10.356512 +шк -10.369653 +аљ -10.386506 +нд -10.387049 +ша -10.420143 +зо -10.425909 +дс -10.430551 +бу -10.478170 +ћа -10.481245 +јн -10.501346 +тк -10.519351 +нц -10.524210 +зб -10.524681 +ид -10.527825 +ип -10.534165 +хо -10.551851 +уш -10.558813 +ућ -10.590448 +уз -10.658668 +аф -10.675934 +пс -10.688404 +ођ -10.702120 +еч -10.703008 +рб -10.706673 +аж -10.706851 +зу -10.707315 +вл -10.718659 +ољ -10.728428 +ћ_ -10.732957 +рк -10.770711 +рш -10.773172 +кв -10.792516 +ађ -10.793461 +з_ -10.836203 +њу -10.842649 +тс -10.874635 +зл -10.890251 +ау -10.921337 +дњ -10.971611 +рг -11.005573 +ић -11.022592 +рм -11.047646 +аб -11.062662 +нк -11.089542 +цр -11.112583 +ђа -11.129009 +мн -11.137870 +мп -11.149592 +л_ -11.154152 +кс -11.172143 +зм -11.215592 +зд -11.251735 +љи -11.255483 +мб -11.268888 +аћ -11.273261 +лт -11.275426 +еј -11.285183 +чу -11.291529 +ец -11.298063 +фо -11.313174 +фе -11.318990 +пш -11.335368 +цу -11.341994 +еж -11.363573 +мљ -11.391538 +зг -11.414225 +ув -11.433496 +хр -11.433614 +рп -11.433968 +јв -11.447301 +фр -11.456918 +ш_ -11.473737 +рд -11.476833 +жн -11.493887 +оц -11.501161 +ха -11.521763 +шњ -11.529746 +рз -11.533786 +фа -11.554542 +ињ -11.566519 +_љ -11.574105 +фс -11.587296 +кц -11.598425 +бн -11.608842 +дл -11.616854 +шћ -11.633015 +ц_ -11.670398 +мр -11.681910 +оћ -11.703648 +ах -11.708206 +мс -11.732304 +ои -11.735935 +км -11.770004 +дб -11.772612 +пт -11.794786 +чл -11.801848 +рх -11.808715 +тп -11.835112 +дм -11.850540 +хи -11.878363 +иљ -11.926431 +нг -11.935505 +зр -11.987280 +гд -12.004434 +еф -12.020642 +оф -12.028013 +ех -12.042151 +п_ -12.061157 +шл -12.101063 +иж -12.130204 +бј -12.163266 +уа -12.195803 +иб -12.209470 +гн -12.214021 +хв -12.219705 +хе -12.231447 +рл -12.232780 +пн -12.240704 +књ -12.248774 +ух -12.249605 +ћу -12.263899 +рц -12.304304 +кљ -12.305491 +мл -12.320461 +лм -12.345541 +б_ -12.442368 +фу -12.461240 +ч_ -12.533976 +иф -12.558587 +шн -12.581273 +љ_ -12.594266 +уо -12.621952 +њо -12.637355 +бљ -12.644709 +сц -12.697545 +иа -12.709348 +тл -12.711062 +лб -12.754599 +нз -12.795696 +вс -12.800251 +еа -12.829592 +чј -12.844255 +јб -12.879808 +ћн -12.887058 +љн -12.901177 +рђ -12.904933 +уц -12.918734 +јк -12.932335 +дг -12.941363 +оу -12.955010 +жу -12.986328 +оњ -13.020288 +шу -13.028459 +вј -13.038289 +рч -13.077927 +жб -13.145287 +кн -13.146060 +_џ -13.157898 +вц -13.164734 +лс -13.184644 +еу -13.187824 +аи -13.236376 +пљ -13.254597 +љс -13.260233 +ii -13.288752 +љо -13.292812 +_ћ -13.309598 +хн -13.319371 +нф -13.321552 +хт -13.326800 +ае -13.342880 +вг -13.353552 +ху -13.417196 +ох -13.431964 +јм -13.455481 +чо -13.456918 +вш -13.503449 +јч -13.594925 +жј -13.714211 +ђи -13.727452 +ое -13.728609 +рњ -13.731216 +сх -13.791306 +шп -13.816307 +лк -13.818153 +ф_ -13.833633 +оа -13.843941 +тх -13.849281 +уђ -13.883348 +жњ -13.888189 +јп -13.928507 +вк -13.939184 +i_ -13.976500 +сј -13.978564 +џе -14.016220 +фл -14.060338 +мј -14.067641 +чв -14.068739 +_ђ -14.077559 +мц -14.086061 +сф -14.126672 +нч -14.153617 +тњ -14.194603 +вд -14.211087 +бз -14.240910 +лг -14.248359 +дј -14.295596 +тм -14.358694 +нр -14.359142 +нп -14.372646 +шљ -14.453088 +ж_ -14.469438 +џо -14.473313 +јз -14.516631 +иђ -14.518131 +тз -14.528171 +ђо -14.531197 +шв -14.573715 +лд -14.577880 +уе -14.620743 +пј -14.653885 +јт -14.655535 +нб -14.658840 +_x -14.694574 +_i -14.699104 +фт -14.706495 +ћо -14.750479 +жр -14.780166 +кћ -14.783168 +пц -14.804362 +зј -14.814156 +рј -14.855904 +хл -14.864146 +уњ -14.876919 +нв -14.902809 +јл -14.908699 +цо -14.913297 +шо -14.947918 +јц -14.956704 +џа -14.975813 +жд -15.047646 +уи -15.051261 +уф -15.075349 +e_ -15.096854 +гв -15.097602 +кш -15.103594 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ssw b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ssw new file mode 100644 index 00000000000..b879ebc4b8e --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ssw @@ -0,0 +1,197 @@ +#TOTAL_BIGRAMS 60711 +#UNIQUE_BIGRAMS 193 +#UNSEEN_LOG_PROB -12.567742 +#EXPECTED_SCORE -6.562466 +e_ -4.772677 +i_ -4.782453 +_k -4.800220 +si -4.823581 +a_ -4.906677 +_n -5.020848 +ts -5.041047 +ku -5.134783 +ng -5.549820 +kh -5.590462 +ut -5.692454 +_l -5.779840 +an -5.824928 +o_ -5.928221 +nt -5.984283 +u_ -5.996369 +fu -6.146519 +tf -6.226112 +ka -6.273121 +un -6.331250 +la -6.350512 +ma -6.391818 +le -6.529921 +ha -6.569998 +el -6.579058 +ba -6.592754 +he -6.611221 +lo -6.646496 +ge -6.663258 +_u -6.729799 +um -6.760387 +ak -6.778535 +at -6.807521 +nk -6.823581 +ph -6.870080 +ek -6.903828 +be -6.918127 +mu -6.921004 +ny -6.953032 +ye -6.955980 +en -6.961892 +on -6.976781 +ik -7.044180 +om -7.056780 +ab -7.079099 +lu -7.088770 +em -7.095254 +no -7.098507 +in -7.128119 +ga -7.134783 +_e -7.154961 +li -7.182311 +wa -7.206676 +_b -7.267618 +se -7.308470 +ul -7.327428 +na -7.331250 +ti -7.484529 +ni -7.523348 +ho -7.554280 +ne -7.567742 +es -7.581331 +_s -7.604268 +al -7.670502 +_a -7.675351 +_f -7.734852 +et -7.745012 +ap -7.839822 +ve -7.872862 +ya -7.912390 +_w -7.935474 +nd -7.935474 +kw -7.947156 +ko -8.108311 +ke -8.182311 +_y -8.189231 +fo -8.274960 +iv -8.312241 +ri -8.358289 +af -8.373970 +_t -8.397817 +_i -8.430239 +il -8.446727 +fr -8.455042 +wo -8.463406 +dl -8.463406 +im -8.514631 +ok -8.532118 +we -8.567742 +hu -8.585890 +ub -8.651266 +ob -8.680217 +te -8.699846 +je -8.729799 +ze -8.739923 +bo -8.900986 +eb -8.935474 +uv -8.947156 +me -8.970807 +nj -8.970807 +ca -9.007027 +ep -9.148203 +go -9.175425 +_m -9.203170 +va -9.217245 +gu -9.231459 +so -9.274960 +us -9.289757 +hi -9.350512 +it -9.397817 +mt -9.430239 +ay -9.446727 +mi -9.446727 +is -9.463406 +am -9.480279 +dv -9.497353 +as -9.497353 +ic -9.532118 +mn -9.549820 +yo -9.549820 +fa -9.567742 +hl -9.604268 +mp -9.622884 +zi -9.660852 +ta -9.660852 +dz -9.660852 +uy -9.699846 +od -9.719745 +vw -9.719745 +mb -9.739923 +uf -9.760387 +gi -9.781146 +uz -9.823581 +to -9.889670 +sa -9.889670 +fw -9.958933 +su -10.056780 +ib -10.082315 +iz -10.108311 +uk -10.134783 +bu -10.161750 +av -10.217245 +ow -10.245814 +yi -10.245814 +ah -10.274960 +up -10.430239 +ol -10.430239 +bi -10.497353 +_j -10.497353 +bh -10.567742 +sw -10.567742 +ja -10.680217 +nc -10.719745 +os -10.802207 +mh -11.031689 +tu -11.082315 +ns -11.366108 +st -11.430239 +ci -11.430239 +ms -11.497353 +ou -11.497353 +th -11.497353 +h_ -11.497353 +vu -11.567742 +ki -11.567742 +uj -11.567742 +ez -11.567742 +if -11.719745 +eo -11.719745 +ev -11.802207 +_o -11.802207 +de -11.802207 +_p -11.889670 +mk -11.982780 +vi -11.982780 +ud -11.982780 +iy -12.082315 +eh -12.082315 +ov -12.082315 +id -12.189231 +lw -12.189231 +aw -12.189231 +jo -12.189231 +oh -12.189231 +hr -12.304708 +vo -12.304708 +ac -12.430239 +ug -12.430239 +gc -12.430239 +tj -12.567742 +ig -12.567742 +ip -12.567742 +s_ -12.567742 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/swa b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/swa new file mode 100644 index 00000000000..25900d71c1b --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/swa @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 7643138 +#UNIQUE_BIGRAMS 539 +#UNSEEN_LOG_PROB -19.093144 +#EXPECTED_SCORE -7.153838 +a_ -3.945046 +i_ -4.795225 +_k -4.832343 +wa -5.134454 +_m -5.360131 +ka -5.368825 +o_ -5.829700 +ku -5.932861 +an -6.001306 +li -6.136485 +ma -6.138465 +in -6.339372 +ha -6.345222 +at -6.389272 +u_ -6.409909 +ik -6.426585 +e_ -6.499787 +ak -6.543841 +na -6.554677 +_w -6.626359 +_h -6.665429 +ki -6.683010 +am -6.721573 +ni -6.725883 +hi -6.831795 +ti -6.837115 +al -6.863534 +ba -6.929418 +ng -6.966660 +ta -7.009065 +ia -7.032596 +il -7.059335 +_u -7.072334 +_a -7.107510 +kw -7.197874 +is -7.215327 +en -7.242681 +_i -7.289220 +ri -7.297451 +ch -7.309198 +_n -7.315378 +sh -7.344808 +ar -7.353765 +mb -7.435477 +nd -7.463088 +ji -7.473249 +_s -7.491034 +zi -7.504372 +di -7.517766 +si -7.557537 +sa -7.564452 +da -7.564630 +mi -7.579465 +la -7.594015 +hu -7.614473 +un -7.632077 +za -7.682486 +ya -7.684892 +as -7.721792 +ad -7.788292 +to -7.793266 +um -7.813718 +ny -7.821596 +ra -7.910356 +mw -7.939067 +im -7.941710 +uw -7.946032 +ab -7.962840 +ai -7.967746 +ut -7.981038 +we -7.987060 +mu -8.007947 +_b -8.013204 +tu -8.110794 +ke -8.134839 +yo -8.154766 +_t -8.194746 +le -8.205403 +pa -8.206184 +it -8.220976 +on -8.243625 +aa -8.249472 +_v -8.252750 +fa -8.281241 +_p -8.314085 +ye -8.317815 +_c -8.326817 +bi -8.339173 +_l -8.354661 +aj -8.373818 +us -8.374945 +ko -8.387154 +fu -8.411627 +id -8.424605 +mo -8.432083 +ne -8.456276 +aw -8.490966 +ul -8.494025 +gi -8.509832 +_y -8.512380 +_j -8.528321 +vi -8.531670 +ua -8.548392 +ja -8.559672 +az -8.560313 +he -8.616768 +ez -8.656966 +_z -8.665218 +bu -8.667289 +te -8.683572 +go -8.701691 +me -8.701848 +ay -8.718131 +pi -8.726822 +iw -8.756556 +gu -8.757046 +ga -8.843366 +uk -8.867937 +af -8.868907 +er -8.874300 +ok -8.906547 +se -8.918279 +ao -8.940365 +nz -8.942871 +_d -8.964773 +ib -8.970253 +em -9.009600 +uu -9.022598 +ot -9.032646 +el -9.045754 +io -9.081508 +ah -9.099930 +ap -9.106678 +ek -9.147308 +zo -9.154389 +re -9.176391 +iz -9.176609 +up -9.181095 +ir -9.196406 +ii -9.211321 +no -9.213785 +es -9.234329 +if -9.279480 +ug -9.307193 +vy -9.339479 +uf -9.343643 +ge -9.343643 +oj -9.356825 +do -9.374006 +wi -9.377391 +dh -9.377642 +st -9.391647 +ur -9.397491 +bo -9.469798 +de -9.499411 +uz -9.501462 +nc -9.506533 +ru -9.517697 +ub -9.538883 +gh -9.542397 +ho -9.565810 +lu -9.570395 +po -9.590337 +ju -9.607462 +iy -9.661315 +ip -9.671746 +eo -9.672824 +ea -9.677145 +su -9.681478 +du -9.691276 +ol -9.705705 +uh -9.717098 +pe -9.751829 +_r -9.782421 +om -9.810960 +_f -9.839729 +yi -9.845622 +eh -9.877937 +or -9.892395 +je -9.908269 +ij -9.953593 +mk -9.954904 +uo -9.956966 +n_ -9.962417 +ud -10.003096 +nj -10.004647 +uj -10.023187 +mt -10.029683 +be -10.038192 +ro -10.046752 +lo -10.068275 +au -10.092594 +fi -10.097343 +ag -10.113562 +iv -10.128698 +bw -10.145276 +ui -10.177046 +th -10.205515 +of -10.206853 +ic -10.226619 +mj -10.260949 +ns -10.269777 +ig -10.285240 +et -10.295641 +kr -10.354969 +ew -10.358682 +mp -10.366635 +_g -10.385196 +zu -10.393297 +_e -10.401698 +s_ -10.452370 +vu -10.474759 +oz -10.515623 +ac -10.570539 +uc -10.607757 +ms -10.664222 +og -10.689249 +so -10.710283 +mf -10.723627 +r_ -10.791927 +oa -10.828873 +os -10.833688 +nu -10.837137 +nt -10.848577 +mn -10.884166 +ae -10.895268 +pu -10.896347 +zw -10.911174 +mz -10.963359 +yu -10.985385 +iu -10.987683 +ze -11.004647 +fr -11.012034 +sk -11.178358 +ie -11.178795 +d_ -11.215579 +hw -11.309228 +op -11.311624 +bl -11.344624 +_o -11.346097 +l_ -11.425903 +ef -11.449992 +ue -11.459529 +ep -11.491781 +ml -11.504887 +wo -11.507082 +py -11.522548 +tw -11.558533 +uv -11.577444 +mm -11.604814 +t_ -11.606579 +m_ -11.663610 +jo -11.670361 +eg -11.695183 +ee -11.722988 +mr -11.722988 +y_ -11.728102 +od -11.731307 +av -11.737739 +lm -11.740966 +jw -11.813845 +co -11.814525 +oh -11.848229 +ih -11.868554 +ev -11.882027 +ed -11.935736 +ey -11.956591 +ow -11.979275 +oi -11.989984 +kt -11.991521 +tr -12.021028 +mc -12.048750 +rd -12.051952 +sw -12.089301 +eu -12.129332 +ov -12.133566 +h_ -12.138664 +sp -12.143780 +rn -12.149772 +gw -12.154067 +va -12.193308 +ob -12.230923 +pr -12.263035 +ll -12.302537 +oo -12.304446 +fe -12.311145 +fo -12.314025 +ve -12.318839 +lb -12.357939 +nn -12.365888 +oy -12.379904 +ej -12.437373 +ei -12.475565 +rt -12.532578 +ca -12.542679 +gr -12.575715 +eb -12.622560 +ou -12.655062 +k_ -12.656280 +br -12.663610 +sl -12.702084 +mh -12.805038 +yw -12.811808 +sm -12.871380 +ks -12.872795 +mv -12.911537 +g_ -12.952844 +dw -12.972432 +ec -12.975469 +fs -12.998455 +md -13.059990 +dr -13.076200 +uy -13.094244 +p_ -13.107510 +kl -13.112517 +sr -13.136113 +mg -13.146345 +oe -13.200398 +ce -13.207522 +oc -13.230923 +ss -13.234557 +hr -13.254709 +gl -13.369879 +pw -13.456343 +fy -13.466990 +w_ -13.479871 +c_ -13.517005 +rk -13.532578 +rs -13.552851 +ck -13.575715 +nk -13.641732 +ry -13.663610 +lt -13.695809 +lk -13.713449 +rl -13.731307 +rg -13.746793 +vo -13.749390 +ci -13.765071 +pt -13.832311 +ld -13.934996 +lc -13.973950 +ls -13.983091 +ty -14.010865 +lf -14.020244 +kn -14.068072 +tl -14.094244 +pl -14.114190 +kh -14.131024 +sc -14.254709 +cr -14.284533 +tt -14.322702 +hn -14.373881 +ht -14.406302 +fl -14.418650 +ds -14.486355 +rc -14.486355 +ex -14.584963 +rw -14.690808 +my -14.751991 +ts -14.757209 +f_ -14.810451 +rr -14.826815 +rm -14.832311 +lg -14.871380 +ph -14.877049 +xi -14.888454 +ct -14.917366 +z_ -14.923219 +ly -14.970916 +rb -15.007753 +nm -15.051952 +sy -15.137813 +bd -15.179233 +_q -15.193308 +x_ -15.200398 +rp -15.243682 +lh -15.251024 +cl -15.318839 +wn -15.342172 +tn -15.381918 +vv -15.431105 +b_ -15.456343 +bn -15.499411 +v_ -15.543806 +sb -15.608346 +cu -15.608346 +hm -15.627329 +wh -15.675909 +ps -15.685825 +dn -15.821340 +yl -15.843366 +nb -15.854506 +_x -15.877049 +ws -16.007753 +cc -16.007753 +sn -16.020244 +qa -16.084374 +qu -16.110846 +ft -16.151488 +nr -16.151488 +tk -16.165294 +rv -16.165294 +pp -16.179233 +nw -16.207522 +dd -16.207522 +xa -16.236377 +wl -16.251024 +ys -16.251024 +rh -16.295878 +nl -16.295878 +by -16.357939 +dy -16.373881 +dm -16.390000 +gb -16.406302 +ox -16.406302 +gs -16.406302 +nh -16.422790 +kf -16.456343 +fg -16.473416 +gg -16.490694 +lz -16.508182 +ix -16.525884 +ff -16.543806 +zz -16.617806 +cs -16.675909 +bc -16.715987 +dv -16.757209 +bh -16.757209 +ax -16.778271 +hd -16.778271 +lv -16.778271 +gt -16.799644 +hy -16.821340 +sd -16.821340 +ky -16.821340 +gn -16.843366 +bs -16.865734 +nv -16.888454 +cp -16.934996 +gm -16.958843 +kk -17.032844 +lw -17.032844 +yr -17.084374 +hl -17.084374 +ln -17.137813 +lp -17.137813 +nf -17.165294 +cb -17.165294 +tb -17.193308 +yk -17.193308 +aq -17.251024 +q_ -17.251024 +yn -17.251024 +rf -17.280771 +wu -17.311145 +pm -17.311145 +xe -17.311145 +cy -17.311145 +xv -17.342172 +bb -17.342172 +fd -17.373881 +yd -17.373881 +dp -17.373881 +pn -17.406302 +dg -17.406302 +zm -17.439469 +vd -17.473416 +hs -17.473416 +tv -17.508182 +cn -17.508182 +bm -17.508182 +wy -17.508182 +hf -17.617806 +zt -17.695809 +zr -17.736451 +ux -17.736451 +pf -17.778271 +j_ -17.778271 +xf -17.821340 +sv -17.821340 +fw -17.821340 +yy -17.865734 +cm -17.911537 +gy -17.911537 +cd -18.007753 +kc -18.007753 +ww -18.058379 +dk -18.058379 +wr -18.058379 +km -18.058379 +vt -18.110846 +pd -18.110846 +fm -18.342172 +wt -18.406302 +vr -18.406302 +yc -18.406302 +pb -18.543806 +hc -18.543806 +cv -18.543806 +ym -18.543806 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/swe b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/swe new file mode 100644 index 00000000000..fa15e9ab55c --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/swe @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 330716115 +#UNIQUE_BIGRAMS 648 +#UNSEEN_LOG_PROB -18.177237 +#EXPECTED_SCORE -7.452167 +t_ -5.291640 +r_ -5.344251 +n_ -5.488823 +ar -5.646386 +er -5.672591 +_s -5.704805 +an -5.719643 +a_ -5.747193 +en -5.853459 +de -5.887833 +et -6.260693 +ra -6.305625 +or -6.352026 +e_ -6.371923 +_f -6.401494 +_m -6.428075 +te -6.433913 +in -6.486662 +_o -6.505300 +at -6.568788 +_a -6.584647 +_d -6.622101 +s_ -6.637963 +st -6.726879 +_h -6.783978 +_t -6.794120 +va -6.803925 +tt -6.806970 +om -6.812580 +na -6.834504 +re -6.835793 +la -6.842571 +ta -6.885666 +_b -6.912927 +ll -6.925474 +me -6.951605 +d_ -7.007852 +ka -7.049553 +fo -7.053514 +oc -7.068277 +nd -7.128414 +m_ -7.144739 +ch -7.155754 +ti -7.166908 +_v -7.180938 +_k -7.180961 +al -7.210283 +sk -7.219898 +h_ -7.226857 +ng -7.237043 +so -7.245200 +ad -7.312164 +ha -7.327116 +il -7.366692 +sa -7.448022 +ge -7.478967 +g_ -7.491992 +li -7.525328 +ma -7.581706 +es -7.586467 +l_ -7.589600 +ed -7.591113 +ig -7.608344 +le -7.641349 +el -7.647583 +_e -7.679049 +ga -7.684380 +ns -7.685128 +ve -7.709918 +be -7.715249 +_i -7.724359 +ri -7.724788 +_p -7.743945 +nt -7.744604 +_l -7.754838 +on -7.759954 +un -7.761510 +_n -7.772830 +da -7.839225 +ag -7.871535 +_r -7.893499 +_g -7.940800 +as -8.002724 +am -8.049902 +_u -8.113353 +is -8.154870 +vi -8.219732 +rt -8.224324 +se -8.229883 +kt -8.233166 +rs -8.274962 +nn -8.300817 +ne -8.344558 +pe -8.390365 +ni -8.393969 +ko -8.398104 +ck -8.406209 +tr -8.430280 +ol -8.433308 +si -8.441596 +ro -8.511129 +to -8.534585 +av -8.548268 +ke -8.551993 +kr -8.564268 +ru -8.602987 +it -8.649229 +id -8.679118 +he -8.694712 +dr -8.699401 +ak -8.727567 +ut -8.736764 +k_ -8.756393 +ho -8.777906 +no -8.805561 +ev -8.810445 +fr -8.839079 +ba -8.882132 +ja -8.904087 +rn -8.910217 +ov -8.971870 +lo -8.976805 +ot -8.981377 +ik -9.001291 +pp -9.001987 +ts -9.049630 +fa -9.088496 +ss -9.100821 +mi -9.102559 +ef -9.122346 +mm -9.149712 +rd -9.155153 +vs -9.173391 +fi -9.209312 +mo -9.237512 +og -9.275892 +iv -9.278867 +di -9.333941 +ft -9.351838 +_j -9.363607 +em -9.367774 +pa -9.367930 +io -9.372225 +rk -9.384891 +bl -9.394409 +os -9.421815 +gr -9.433040 +up -9.451378 +pr -9.454564 +ls -9.499114 +bo -9.575096 +ur -9.581168 +ek -9.608307 +od -9.617650 +ki -9.619011 +gt -9.630431 +lt -9.637463 +sl -9.639417 +gs -9.657243 +ld -9.660339 +ku -9.671057 +gi -9.681031 +do -9.697656 +go -9.702926 +tv -9.712951 +lk -9.732847 +us -9.733285 +sp -9.738124 +nv -9.753020 +sv -9.755771 +ie -9.826845 +kl -9.835336 +br -9.856658 +ic -9.866178 +jo -9.877586 +rr -9.900706 +_c -9.911441 +v_ -9.922867 +gg -9.927291 +kv -9.930982 +op -9.958488 +nk -9.978945 +ul -10.029213 +rl -10.060036 +rg -10.145128 +ok -10.149214 +pl -10.163992 +eg -10.175438 +ds -10.176932 +lu -10.192774 +ac -10.197602 +tu -10.199227 +ap -10.211159 +i_ -10.214903 +ks -10.241241 +po -10.258696 +mn -10.258787 +ju -10.263160 +ia -10.267799 +p_ -10.268983 +nl -10.293602 +ce -10.310216 +bi -10.356907 +hu -10.382667 +rm -10.383792 +o_ -10.404089 +lj -10.408766 +ty -10.439792 +fl -10.453447 +rb -10.455608 +ea -10.512765 +yc -10.555357 +ud -10.587538 +tk -10.589492 +ob -10.602815 +kn -10.611812 +je -10.615373 +ir -10.712647 +pu -10.744003 +dd -10.747614 +gl -10.769827 +af -10.779839 +tn -10.797689 +sy -10.807374 +rh -10.808985 +lv -10.825617 +ec -10.829748 +mt -10.847474 +lm -10.853515 +hi -10.871750 +ex -10.882046 +fe -10.910891 +yg -10.914962 +tl -10.981434 +um -10.987019 +mp -10.989802 +dl -10.997175 +yr -10.997505 +uk -11.017016 +mu -11.021788 +ny -11.025503 +ly -11.046423 +ci -11.047392 +y_ -11.059221 +if -11.062473 +my -11.064555 +by -11.079252 +yd -11.096877 +sj -11.111087 +yt -11.115998 +sm -11.135316 +eb -11.154310 +gh -11.176755 +of -11.188395 +gn -11.219712 +nu -11.227975 +ys -11.229043 +ry -11.244031 +rj -11.269446 +lb -11.277177 +bu -11.280570 +ms -11.282481 +dn -11.291189 +tb -11.292677 +rv -11.299885 +su -11.312411 +th -11.341582 +nf -11.362220 +pi -11.371078 +gu -11.391055 +sn -11.391172 +oj -11.402263 +ab -11.406816 +vn -11.409282 +ff -11.438102 +ax -11.460031 +mb -11.472981 +_w -11.476233 +pt -11.512606 +eh -11.527711 +sb -11.553153 +im -11.572872 +dv -11.580413 +rf -11.622239 +ep -11.624152 +mr -11.627971 +dj -11.637029 +ln -11.653574 +c_ -11.675866 +bb -11.676851 +xt -11.723958 +gd -11.725582 +ca -11.731400 +uv -11.742732 +vu -11.769667 +au -11.781274 +ml -11.803587 +ai -11.842740 +ou -11.878526 +yn -11.897623 +ya -11.912294 +co -11.912782 +du -11.934901 +mf -11.949233 +fu -11.960820 +oa -11.977735 +fj -12.010423 +gj -12.024149 +ub -12.036777 +ky -12.075434 +sc -12.099449 +u_ -12.100618 +vr -12.104723 +vo -12.124923 +mh -12.131445 +ps -12.140044 +oh -12.151589 +ue -12.159310 +_y -12.161307 +kh -12.209046 +sh -12.209624 +f_ -12.229430 +ee -12.233605 +xe -12.247622 +tf -12.252638 +eu -12.267961 +nh -12.287870 +tj -12.290228 +ip -12.292458 +mg -12.328913 +fy -12.387194 +nc -12.387708 +x_ -12.402299 +tg -12.416561 +ae -12.422177 +eo -12.455038 +ua -12.482386 +vl -12.502825 +wa -12.516997 +lf -12.523044 +ug -12.550704 +hj -12.599306 +ib -12.602313 +vt -12.644901 +ei -12.644984 +j_ -12.653222 +wi -12.667200 +lp -12.695105 +ph -12.729858 +jl -12.751685 +tc -12.762981 +nr -12.769515 +yl -12.816856 +sd -12.819818 +ym -12.840978 +oe -12.846783 +db -12.891627 +nb -12.934252 +pf -12.940308 +pg -12.953604 +bj -12.973922 +ah -12.993390 +uc -12.999879 +xa -13.049647 +oi -13.056877 +df -13.074230 +vd -13.083856 +jd -13.096447 +pn -13.112236 +we -13.128395 +lg -13.144027 +aj -13.238466 +b_ -13.253255 +lh -13.273501 +ay -13.281601 +oo -13.314109 +tm -13.317624 +ey -13.372546 +yp -13.412751 +yk -13.442891 +dy -13.465461 +rp -13.487838 +ui -13.497795 +ej -13.504573 +sr -13.506095 +nj -13.554714 +iu -13.559920 +pd -13.565568 +cr -13.583342 +hr -13.607259 +ow -13.607586 +dg -13.611020 +mk -13.616763 +vg -13.617969 +sf -13.667909 +w_ -13.701977 +rc -13.726129 +cu -13.751955 +yf -13.790748 +md -13.819976 +lr -13.851096 +z_ -13.854129 +xi -13.863136 +ew -13.866585 +td -13.868150 +hn -13.889375 +nm -13.915021 +hy -13.941406 +xp -13.982687 +ih -13.991826 +pm -14.066276 +cl -14.073778 +bs -14.096294 +dk -14.127653 +dm -14.131876 +gf -14.156758 +yo -14.164508 +dt -14.212313 +hl -14.229053 +gm -14.235770 +ct -14.256282 +_z -14.291015 +uf -14.327141 +gy -14.431810 +cc -14.507720 +aa -14.521914 +mv -14.530251 +wo -14.542066 +ux -14.545505 +np -14.556079 +gb -14.600578 +ii -14.635127 +ix -14.638350 +qu -14.661451 +bt -14.705178 +js -14.725715 +sg -14.742597 +vj -14.776844 +ye -14.802917 +cy -14.852515 +dw -14.872004 +gp -14.878034 +ht -14.912597 +ox -14.946631 +kd -14.952843 +za -14.967164 +vb -14.974729 +km -15.025912 +tp -15.043630 +jn -15.050424 +kg -15.064854 +aw -15.068148 +oy -15.086699 +yh -15.150636 +ze -15.160827 +fs -15.166431 +dp -15.173991 +az -15.208096 +kj -15.221033 +dh -15.224704 +ez -15.235097 +hm -15.243872 +ao -15.247941 +jk -15.262271 +gv -15.286475 +mj -15.303133 +vk -15.336858 +zi -15.339388 +iz -15.341378 +kb -15.343553 +vv -15.405443 +_q -15.437025 +yb -15.522530 +pk -15.606443 +hs -15.640131 +jt -15.649518 +ws -15.649742 +cs -15.662582 +tz -15.674168 +sw -15.675309 +hw -15.686078 +nz -15.715821 +wn -15.733300 +kk -15.818210 +vf -15.826045 +xu -15.892954 +wh -15.942642 +_x -15.944291 +pv -15.947596 +yi -15.965070 +qv -15.975432 +xo -15.975713 +pb -16.017930 +xn -16.019378 +yv -16.025767 +kf -16.128903 +bn -16.130780 +uh -16.130780 +zo -16.137054 +cq -16.139256 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tam b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tam new file mode 100644 index 00000000000..649ccd4d072 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tam @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 9550637 +#UNIQUE_BIGRAMS 1092 +#UNSEEN_LOG_PROB -15.454689 +#EXPECTED_SCORE -7.639017 +த_ -4.466991 +_ப -5.125726 +ர_ -5.129449 +க_ -5.356991 +_வ -5.397263 +_அ -5.489336 +_க -5.550052 +_ம -5.578355 +ய_ -5.604722 +_த -5.743267 +ல_ -5.918706 +ன_ -5.932119 +வ_ -6.011333 +_ஒ -6.051957 +ஒர -6.095203 +_ச -6.105963 +_இ -6.148407 +வர -6.339460 +ட_ -6.366276 +_ந -6.569956 +ற_ -6.829871 +பத -6.877760 +தவ -6.929925 +அத -6.960716 +மக -6.992351 +இத -7.114342 +தர -7.131735 +பர -7.211206 +மத -7.280064 +கட -7.333101 +ச_ -7.356650 +_உ -7.365367 +டய -7.390239 +கற -7.508290 +வத -7.528229 +ம_ -7.530741 +தன -7.571392 +மன -7.601147 +னத -7.617073 +ள_ -7.635698 +ரய -7.638283 +_எ -7.659108 +_ஆ -7.674518 +தய -7.681726 +ரவ -7.722365 +அவ -7.746556 +தக -7.750031 +ண_ -7.751853 +வல -7.786520 +கர -7.801472 +றத -7.804913 +ரத -7.814504 +ரக -7.866189 +யத -7.871591 +யக -7.888459 +y_ -7.909370 +வக -7.989488 +கத -8.002445 +சல -8.019199 +ழ_ -8.032110 +ரச -8.148375 +மற -8.160642 +தல -8.187122 +கல -8.208723 +சர -8.229791 +ரம -8.233469 +வள -8.241265 +s_ -8.248321 +t_ -8.264163 +வன -8.279852 +வட -8.307199 +பற -8.336490 +கள -8.343097 +அர -8.354028 +யன -8.354424 +தம -8.375490 +மட -8.388491 +பல -8.440546 +நல -8.450235 +ay -8.498478 +ரண -8.503294 +பட -8.525443 +வய -8.529177 +லக -8.545057 +_t -8.564027 +றய -8.593249 +கய -8.639668 +சய -8.678752 +றவ -8.682595 +மழ -8.693810 +றக -8.696252 +சத -8.717460 +னவ -8.734731 +மல -8.735182 +e_ -8.757628 +th -8.764772 +லய -8.783687 +da -8.795586 +கவ -8.800697 +st -8.825839 +லவ -8.861228 +_f -8.882315 +_ர -8.902486 +மய -8.924116 +_s -8.935683 +வற -8.940499 +வச -8.959700 +டவ -8.964522 +ar -8.968224 +சம -8.973378 +ளவ -8.994487 +வழ -9.002445 +டக -9.009123 +லம -9.019591 +கம -9.026034 +சற -9.048254 +_ஏ -9.065713 +நட -9.070415 +இர -9.104351 +ளய -9.104850 +ரட -9.117548 +என -9.138168 +பக -9.155464 +es -9.185230 +ஆக -9.262075 +_a -9.271940 +_ஜ -9.286110 +சவ -9.296901 +யர -9.306243 +is -9.324238 +_i -9.387378 +இவ -9.396003 +யல -9.405396 +ழத -9.407138 +or -9.409397 +ry -9.426446 +ழக -9.438134 +தட -9.468205 +in -9.485967 +பச -9.506476 +to -9.520720 +னம -9.522718 +மண -9.534545 +சன -9.534993 +நத -9.539370 +ri -9.553262 +நற -9.556671 +ணம -9.561228 +_p -9.589811 +_m -9.590626 +ho -9.594708 +rs -9.594942 +ரப -9.600678 +ழய -9.605259 +ப_ -9.609147 +மர -9.622182 +தச -9.625042 +l_ -9.627310 +உர -9.630300 +கழ -9.654444 +r_ -9.672082 +டம -9.695312 +பன -9.708396 +_c -9.718796 +எத -9.720579 +டத -9.722620 +தத -9.723897 +ரள -9.749153 +nd -9.767995 +உல -9.796191 +d_ -9.803326 +er -9.808058 +எழ -9.811719 +பழ -9.820843 +டன -9.869470 +அண -9.882814 +eb -9.883527 +ணவ -9.903643 +an -9.912496 +op -9.914536 +fe -9.918039 +உட -9.920379 +கன -9.930220 +at -9.943099 +நர -9.960904 +அற -9.984889 +ne -9.989949 +கண -10.002600 +னய -10.009746 +am -10.011772 +_ய -10.026979 +பய -10.036149 +he -10.054826 +na -10.060784 +யம -10.062560 +லன -10.064338 +ic -10.081912 +ur -10.084042 +re -10.106348 +ரன -10.106515 +on -10.107181 +ua -10.121581 +சக -10.124782 +உத -10.125288 +றம -10.135277 +co -10.137317 +இட -10.146533 +நக -10.163411 +தற -10.165491 +un -10.190869 +அள -10.206847 +ch -10.217959 +யட -10.223366 +னட -10.236245 +sd -10.239163 +ma -10.263652 +ஜன -10.278773 +அட -10.322012 +vi -10.335221 +ஏத -10.337565 +al -10.338151 +ஜ_ -10.342656 +மந -10.360220 +தப -10.361014 +id -10.386063 +te -10.388694 +பண -10.393156 +se -10.398651 +_r -10.399875 +அழ -10.408883 +ru -10.409088 +ளத -10.413614 +mo -10.427277 +ழம -10.427901 +br -10.432069 +ணய -10.443804 +_d -10.445909 +வம -10.448229 +le -10.460734 +os -10.514298 +me -10.528285 +நய -10.528285 +ai -10.531188 +_ட -10.534545 +ரல -10.552808 +i_ -10.569009 +_ஓ -10.569698 +ig -10.582381 +உற -10.603848 +gh -10.608321 +ra -10.611390 +si -10.613045 +நம -10.616361 +ea -10.622063 +யவ -10.630659 +_n -10.632337 +ti -10.641960 +ரஜ -10.642684 +ழவ -10.649947 +om -10.651161 +ht -10.652863 +hu -10.654079 +de -10.664829 +லத -10.677391 +ul -10.684085 +_g -10.699325 +n_ -10.699577 +தண -10.701588 +rt -10.701588 +py -10.712192 +ll -10.717015 +au -10.717269 +yr -10.722875 +us -10.727478 +su -10.740858 +_w -10.745777 +ளம -10.762737 +நள -10.778836 +m_ -10.787354 +_h -10.794848 +be -10.795653 +உண -10.798610 +தள -10.820570 +ஆற -10.824400 +ha -10.828240 +_j -10.834570 +பவ -10.846481 +நன -10.858210 +hi -10.860455 +கச -10.869188 +னக -10.872866 +_b -10.891971 +மச -10.898588 +லற -10.903209 +லச -10.914536 +nn -10.923896 +டச -10.935091 +pa -10.937756 +அம -10.949657 +ஆச -10.953845 +இன -10.977103 +ut -10.977712 +ed -10.983818 +ss -10.986880 +nc -10.993332 +யற -10.997341 +இற -11.003530 +ணக -11.021630 +em -11.024145 +ளக -11.025404 +_ல -11.033613 +ov -11.064661 +di -11.065308 +gr -11.085518 +டப -11.097053 +la -11.118051 +பம -11.118051 +_v -11.119395 +il -11.119395 +ad -11.126807 +ot -11.128497 +நவ -11.129512 +en -11.133579 +ie -11.135957 +fu -11.145164 +it -11.151335 +pi -11.165838 +ya -11.175239 +அன -11.179437 +டல -11.198836 +h_ -11.211676 +mb -11.227888 +ஆத -11.243552 +கக -11.247954 +அல -11.266068 +இச -11.277272 +bu -11.282531 +nt -11.312953 +tu -11.316416 +லர -11.321432 +cl -11.322979 +ph -11.324916 +ta -11.331908 +டர -11.341675 +_ஊ -11.356650 +we -11.370582 +சட -11.370582 +தழ -11.395596 +இண -11.403349 +மள -11.414851 +ஏழ -11.436459 +_ஈ -11.439392 +ஆட -11.447807 +சப -11.448229 +கப -11.449073 +லட -11.454998 +மம -11.470346 +fr -11.478944 +oc -11.479375 +எள -11.497604 +eo -11.504610 +ஜல -11.514298 +வண -11.517838 +ew -11.531188 +sa -11.537011 +cs -11.560544 +_o -11.578911 +rd -11.604083 +இல -11.608793 +அக -11.640271 +ce -11.652863 +வவ -11.656759 +லப -11.667529 +io -11.679371 +ct -11.694812 +ஆன -11.702343 +உய -11.702343 +இய -11.705366 +ju -11.749413 +_l -11.755146 +li -11.758805 +றப -11.758805 +ஜய -11.776714 +யச -11.776714 +_ஹ -11.802382 +ஓட -11.803461 +mi -11.813213 +ணந -11.841206 +ws -11.851217 +ec -11.859613 +po -11.875985 +as -11.879395 +pr -11.894844 +ஆல -11.904077 +லல -11.918624 +னச -11.944587 +ஏர -11.954145 +யள -11.955344 +ve -11.975886 +பஜ -12.015363 +nu -12.015989 +ro -12.034881 +ஆவ -12.059171 +ir -12.062398 +ac -12.084534 +b_ -12.086503 +ஓர -12.092428 +ni -12.098377 +ja -12.099703 +றன -12.103022 +ap -12.138679 +ep -12.172448 +ளர -12.179437 +_e -12.190692 +மவ -12.192812 +இள -12.197062 +ou -12.215622 +a_ -12.227164 +டட -12.240259 +னல -12.249792 +ia -12.255689 +ரர -12.261611 +ts -12.263095 +ca -12.276523 +no -12.281779 +ஒள -12.282531 +sh -12.293106 +_ஐ -12.308348 +ஷ_ -12.309115 +ழல -12.349538 +ஈட -12.360617 +ஜத -12.369382 +யண -12.374186 +யப -12.374988 +lo -12.380616 +ளன -12.427277 +rc -12.439812 +அப -12.442332 +um -12.453303 +னந -12.495422 +ug -12.497168 +ஏன -12.498915 +ஏற -12.501541 +p_ -12.504171 +bs -12.524497 +sp -12.528954 +ளட -12.533425 +ணன -12.539707 +ol -12.555079 +gu -12.601264 +ee -12.602203 +யய -12.605024 +ob -12.609737 +னர -12.614465 +தந -12.626833 +fi -12.651890 +ழப -12.659689 +அச -12.664584 +ue -12.679371 +wo -12.692310 +pl -12.696315 +ty -12.697318 +bl -12.706375 +ly -12.718541 +ga -12.720579 +பள -12.724663 +பப -12.726710 +ஹர -12.734924 +tt -12.747335 +et -12.748374 +pu -12.758805 +fo -12.777775 +do -12.780961 +சச -12.780961 +ஆள -12.789491 +எர -12.798072 +_ஞ -12.809955 +சழ -12.813213 +நச -12.842870 +pt -12.878826 +ub -12.882244 +cu -12.885669 +ex -12.891397 +el -12.915703 +so -12.935683 +லந -12.936867 +ng -12.938052 +oo -12.942802 +tr -12.946374 +எம -12.949956 +ஒல -12.949956 +வஜ -12.952348 +dn -12.953546 +எட -12.954745 +c_ -12.957145 +நப -12.960753 +லள -12.977712 +ஊர -12.987493 +_y -13.010992 +ge -13.038689 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tam-rom b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tam-rom new file mode 100644 index 00000000000..de78396718a --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tam-rom @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 116065302 +#UNIQUE_BIGRAMS 699 +#UNSEEN_LOG_PROB -18.800479 +#EXPECTED_SCORE -7.275557 +th -5.134683 +a_ -5.239490 +an -5.407753 +i_ -5.596519 +u_ -5.604140 +al -5.832010 +na -5.837541 +ha -5.923824 +_a -5.936020 +am -5.937978 +aa -5.959998 +ai -6.015695 +_p -6.045319 +m_ -6.110735 +ka -6.157881 +la -6.270639 +l_ -6.297219 +va -6.311361 +ar -6.325680 +_k -6.348542 +_s -6.450253 +_t -6.452635 +ru -6.462379 +en -6.465570 +at -6.499917 +n_ -6.519044 +ra -6.534450 +ma -6.559938 +hu -6.581891 +_m -6.590387 +_v -6.622760 +pa -6.672105 +_n -6.692083 +_i -6.896731 +ad -6.916107 +_e -6.943090 +e_ -6.977006 +ir -6.982856 +il -6.988533 +um -7.001497 +ga -7.026527 +nd -7.086171 +av -7.100364 +in -7.115493 +ya -7.125716 +ng -7.189153 +da -7.189879 +kk -7.189959 +nn -7.197667 +ku -7.278223 +un -7.287681 +hi -7.333947 +di -7.351976 +iy -7.359737 +ll -7.395593 +ee -7.456018 +vi -7.474796 +tt -7.476163 +r_ -7.480579 +ri -7.530949 +nt -7.550991 +ak -7.581703 +ta -7.629899 +er -7.652965 +on -7.666301 +dh -7.708628 +du -7.778163 +po -7.790086 +it -7.846759 +ni -7.893278 +uk -7.895621 +or -7.900127 +s_ -7.949442 +_o -8.005497 +ur -8.014488 +sa -8.057134 +mi -8.076727 +pp -8.122806 +ut -8.153716 +ne -8.160649 +ch -8.174035 +ve -8.205399 +he -8.224248 +id -8.259468 +li -8.274231 +_u -8.307908 +el -8.310289 +ko -8.321690 +oo -8.334224 +ay -8.340802 +mu -8.362648 +ik -8.373605 +so -8.426928 +pu -8.431358 +ol -8.442463 +re -8.465944 +_r -8.471439 +ap -8.519692 +nu -8.548928 +ud -8.559037 +y_ -8.570805 +t_ -8.577713 +od -8.594134 +as -8.602101 +lu -8.626054 +_c -8.644847 +si -8.662609 +pe -8.685834 +ul -8.700590 +se -8.719580 +ag -8.744975 +tu -8.757444 +ti -8.761743 +ot -8.779375 +_d -8.847297 +et -8.862850 +d_ -8.863730 +_y -8.873817 +me -8.885386 +su -8.947385 +ki -8.995108 +es -8.997038 +ho -9.004730 +ye -9.004756 +pi -9.077673 +st -9.117688 +de -9.142538 +mo -9.164924 +ip -9.175537 +le -9.175982 +ja -9.186561 +sh -9.191233 +uv -9.206030 +iv -9.258264 +ba -9.267366 +ic -9.270464 +ro -9.271063 +o_ -9.321244 +_f -9.345994 +h_ -9.347531 +_b -9.349395 +mb -9.353699 +om -9.384672 +dd -9.416137 +mm -9.451886 +_l -9.452478 +nr -9.463914 +te -9.465812 +ed -9.479394 +is -9.495597 +vu -9.569095 +lo -9.569700 +ov -9.574355 +ke -9.578815 +yi -9.593606 +up -9.604441 +zh -9.629466 +g_ -9.667614 +rt -9.720996 +dr -9.721604 +ht -9.812724 +tr -9.839100 +ie -9.842177 +_g -9.883142 +yu -9.893343 +ig -9.911856 +_h -9.916797 +us -9.960342 +mp -9.975404 +ea -9.990599 +ep -9.995743 +gu -10.014854 +aj -10.033779 +nj -10.048500 +k_ -10.069423 +to -10.095215 +os -10.096276 +ey -10.097991 +ac -10.124193 +im -10.131035 +_w -10.147379 +co -10.157991 +ok -10.172091 +ge -10.282226 +rk -10.319766 +ly -10.323616 +ia -10.351342 +gi -10.361576 +em -10.381899 +nk -10.455460 +bu -10.494091 +_j -10.508431 +ow -10.523209 +no -10.524893 +ji -10.531170 +az -10.570664 +ry -10.599302 +p_ -10.602043 +ev -10.606455 +ou -10.689576 +pr -10.723389 +ec -10.773249 +io -10.824036 +ce -10.833214 +ct -10.845787 +eo -10.847343 +oa -10.877098 +wa -10.879953 +ss -10.895165 +ug -10.962920 +rr -10.978836 +pl -10.988047 +do -10.997266 +be -11.021144 +bi -11.117936 +fr -11.126692 +nc -11.146900 +ca -11.147238 +fi -11.162828 +op -11.184593 +ei -11.188823 +oi -11.194638 +rs -11.203932 +ex -11.220209 +nl -11.226689 +rn -11.322883 +ci -11.325050 +yo -11.342245 +au -11.347159 +og -11.380705 +ty -11.413965 +bo -11.417632 +w_ -11.423698 +fo -11.432328 +gs -11.432878 +ah -11.449955 +ae -11.454692 +ls -11.501712 +ab -11.502072 +pd -11.506550 +ij -11.534890 +ek -11.543844 +x_ -11.551509 +lv -11.555170 +c_ -11.580756 +iz -11.606881 +wn -11.610686 +nm -11.617465 +oc -11.618794 +uc -11.653771 +nb -11.668181 +go -11.676701 +we -11.725450 +ew -11.753072 +yr -11.756250 +ns -11.777301 +ts -11.829907 +ju -11.866013 +cs -11.884786 +sr -11.896681 +yy -11.906000 +ph -11.934812 +np -11.966099 +fe -11.971779 +bl -12.022281 +wi -12.037981 +rp -12.044582 +ub -12.047631 +uy -12.051425 +gh -12.065421 +oy -12.077942 +vo -12.078264 +ck -12.079125 +wo -12.080847 +eg -12.101783 +rc -12.124248 +tc -12.158161 +rv -12.200242 +hy -12.218964 +fu -12.221574 +br -12.235053 +rd -12.250233 +rg -12.261419 +lk -12.308940 +ks -12.324413 +gr -12.331827 +ws -12.361347 +vv -12.366326 +ps -12.367508 +lm -12.403556 +sc -12.405038 +sw -12.406522 +of -12.433084 +fa -12.442187 +jo -12.457765 +ny -12.477479 +dy -12.480317 +b_ -12.496890 +oz -12.503804 +rm -12.527092 +ff -12.557941 +f_ -12.622943 +sl -12.641567 +eb -12.642521 +hr -12.668504 +cr -12.697770 +lp -12.722255 +bh -12.723263 +hm -12.746821 +yt -12.790185 +hc -12.798131 +cl -12.812187 +ds -12.815409 +hk -12.824217 +lr -12.849864 +ms -12.850415 +gl -12.883847 +ez -12.901618 +ua -12.903522 +hn -12.907336 +pt -12.920575 +sp -12.949190 +rl -12.953917 +ue -12.969983 +lt -12.982604 +my -12.984819 +kr -13.000217 +lg -13.047842 +if -13.064356 +ob -13.069690 +ld -13.106051 +uz -13.108464 +cu -13.110881 +yw -13.120591 +je -13.133043 +ys -13.188823 +ib -13.218372 +v_ -13.250718 +rb -13.278856 +wh -13.287778 +df -13.341213 +vl -13.406117 +lh -13.413965 +j_ -13.425953 +tl -13.446898 +ln -13.504670 +tk -13.508431 +oh -13.570286 +sm -13.584874 +mn -13.606106 +qu -13.655935 +kt -13.665917 +dk -13.731017 +lb -13.807368 +_q -13.831084 +aw -13.900097 +gn -13.910395 +eh -13.926562 +ze -13.951552 +lw -13.982202 +xt -13.997165 +oj -14.005727 +bs -14.007773 +mk -14.058194 +tp -14.077405 +iu -14.082571 +ef -14.098618 +ui -14.230984 +yk -14.233855 +cc -14.248781 +sk -14.267290 +ft -14.288027 +dm -14.299009 +sy -14.326837 +ih -14.344312 +fl -14.374620 +z_ -14.379910 +tm -14.379910 +hs -14.399118 +yp -14.403421 +gg -14.433910 +ao -14.444956 +py -14.446066 +mr -14.478613 +by -14.479181 +kh -14.479181 +tw -14.495741 +wr -14.506694 +vy -14.547783 +yn -14.553152 +af -14.563347 +hw -14.613565 +yv -14.621689 +xa -14.643157 +hd -14.675318 +hh -14.681837 +bb -14.683799 +dv -14.688386 +sn -14.703562 +dl -14.707547 +ii -14.778435 +gm -14.788953 +sd -14.807368 +nf -14.813797 +uj -14.815230 +mg -14.831084 +lc -14.857409 +_z -14.858885 +bt -14.867034 +za -14.881969 +tn -14.901618 +hv -14.918456 +mt -14.928499 +tv -14.966198 +ym -14.985424 +ix -14.994322 +ej -15.027979 +kn -15.048895 +xp -15.050581 +uh -15.064143 +ax -15.105613 +hl -15.180259 +oe -15.182107 +yl -15.231941 +_x -15.241540 +sb -15.262884 +zi -15.301517 +rf -15.316656 +vd -15.329906 +lf -15.331955 +uo -15.339150 +nv -15.371455 +bg -15.399118 +yd -15.447176 +ox -15.450511 +lz -15.461687 +uu -15.465056 +gt -15.553152 +gy -15.557941 +rj -15.604247 +ww -15.625455 +ml -15.664948 +td -15.681837 +hg -15.687074 +wd -15.717559 +hp -15.769382 +wt -15.811651 +vt -15.903141 +dc -15.918456 +cy -15.937052 +wf -15.962225 +_т -15.992700 +yc -16.018872 +dg -16.033805 +та -16.077834 +ам -16.077834 +ми -16.077834 +ил -16.077834 +лн -16.077834 +на -16.077834 +ад -16.077834 +д_ -16.077834 +xi -16.137517 +sg -16.157366 +kd -16.164653 +ky -16.164653 +_க -16.207279 +pb -16.260931 +rh -16.274662 +dt -16.280586 +bd -16.318686 +kb -16.389482 +eq -16.395899 +ம_ -16.408819 +cv -16.430612 +கம -16.482022 +vc -16.482022 +km -16.493445 +iq -16.521235 +jp -16.544809 +zz -16.547188 +q_ -16.549570 +eu -16.556742 +xc -16.585790 +dp -16.648254 +zu -16.743238 +xy -16.751443 +gk -16.762456 +xo -16.770771 +gb -16.815947 +bj -16.836165 +yb -16.859624 +jm -16.871498 +iw -16.877472 +rw -16.877472 +kv -16.895544 +jn -16.895544 +md -16.916917 +zt -16.948011 +sv -16.957471 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tat b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tat new file mode 100644 index 00000000000..1b0f56ada55 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tat @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 187531067 +#UNIQUE_BIGRAMS 1267 +#UNSEEN_LOG_PROB -16.005335 +#EXPECTED_SCORE -8.074070 +н_ -5.352329 +а_ -5.762587 +ар -5.836178 +_б -5.853675 +_к -6.012579 +ан -6.128409 +е_ -6.182376 +ен -6.202626 +ла -6.254907 +ы_ -6.257127 +_т -6.277402 +ын -6.369819 +лә -6.423505 +р_ -6.504391 +ә_ -6.523712 +та -6.560692 +әр -6.713197 +га -6.734730 +ал -6.763497 +_а -6.771445 +ел -6.800595 +ка -6.959481 +_и -6.960071 +ер -6.975618 +да -7.014871 +нд -7.034108 +ән -7.063836 +_м -7.068198 +ре -7.094308 +на -7.136238 +ры -7.188203 +ны -7.202957 +ра -7.205923 +ле -7.228958 +ат -7.258984 +_с -7.291206 +к_ -7.356193 +бе -7.416582 +те -7.424018 +лы -7.445910 +дә -7.447853 +м_ -7.576628 +п_ -7.593183 +ит -7.621304 +ин -7.668645 +ге -7.671561 +ба -7.683194 +ул -7.709890 +ил -7.741522 +ма -7.778348 +әм -7.790610 +и_ -7.793667 +ак -7.804766 +ты -7.811120 +тә -7.826556 +_х -7.842788 +мә -7.862028 +не -7.862934 +гә -7.893020 +гы -7.895625 +_д -7.911263 +бу -7.933471 +ст -7.957058 +ия -7.991631 +ки -8.006819 +се -8.007310 +де -8.023285 +_я -8.039510 +нә -8.073435 +рг -8.074030 +т_ -8.075592 +ор -8.087351 +че -8.089028 +_у -8.091479 +лг -8.104274 +_р -8.127248 +һә -8.148676 +әт -8.155035 +ур -8.182432 +аш -8.196371 +ыр -8.225839 +ди -8.242662 +ми -8.244201 +нн -8.288882 +ик -8.294026 +_һ -8.295437 +ир -8.304043 +әк -8.316794 +сы -8.318116 +ек -8.323890 +ис -8.332215 +рә -8.348502 +әл -8.360829 +ас -8.378017 +са -8.389304 +ти -8.391070 +рт -8.415569 +он -8.425692 +ыл -8.433317 +ча -8.433517 +ке -8.459959 +аи -8.469924 +_ш -8.476050 +кы -8.495596 +ни -8.496404 +_п -8.496918 +_ж -8.512656 +рл -8.516398 +еш -8.519709 +кә -8.532899 +аг -8.574936 +ык -8.580680 +би -8.589232 +_ч -8.639848 +_г -8.645259 +ды -8.660743 +рн -8.670750 +ли -8.674913 +ез -8.682979 +ту -8.689110 +чы -8.691118 +лд -8.711381 +кл -8.712862 +шл -8.723349 +_ә -8.735322 +л_ -8.737868 +_э -8.747587 +ко -8.775741 +лл -8.784481 +ам -8.789864 +нч -8.790804 +кт -8.813365 +за -8.813441 +то -8.829741 +кү -8.838839 +_ү -8.840693 +сә -8.840919 +ид -8.863986 +рд -8.864423 +тл -8.866774 +ад -8.876778 +ро -8.878892 +әг -8.884153 +бә -8.903434 +ме -8.906214 +у_ -8.959770 +ыш -8.960674 +ри -8.997303 +ем -9.025306 +пр -9.038672 +_ө -9.046664 +рс -9.053152 +_н -9.056830 +я_ -9.059890 +әс -9.064780 +_ф -9.089143 +аз -9.091462 +ь_ -9.110642 +ва -9.114355 +им -9.119070 +нг -9.136045 +ха -9.136305 +шу -9.167285 +ес -9.177896 +ку -9.178992 +ыи -9.181876 +ру -9.188679 +ше -9.189945 +_о -9.194080 +ет -9.200312 +ү_ -9.210490 +шы -9.225613 +тө -9.230499 +ег -9.248733 +_е -9.255413 +ыг -9.259043 +зе -9.262474 +үз -9.270709 +нт -9.276146 +әи -9.284651 +тт -9.287836 +си -9.305013 +хә -9.317176 +жи -9.318792 +шк -9.361193 +ль -9.370273 +_в -9.375021 +чә -9.392690 +тк -9.422180 +ун -9.431330 +з_ -9.435809 +ав -9.444514 +ып -9.467128 +ш_ -9.473256 +үл -9.483099 +тү -9.484344 +ип -9.490158 +ци -9.496351 +тр -9.524931 +ио -9.533104 +эш -9.534586 +әү -9.542579 +иг -9.583109 +өн -9.586861 +шт -9.603270 +ап -9.627905 +ач -9.661235 +еп -9.669381 +кө -9.682463 +ом -9.689624 +ов -9.737550 +яс -9.745116 +шә -9.762905 +ян -9.789632 +өч -9.812382 +сп -9.823123 +үг -9.835567 +аб -9.840517 +в_ -9.855063 +со -9.862005 +уе -9.871587 +_з -9.876417 +өр -9.910802 +ук -9.941428 +бү -9.963279 +әп -9.969289 +ол -9.975642 +үр -9.986364 +сс -10.001953 +ау -10.014320 +ят -10.014376 +әб -10.017410 +уч -10.044225 +мо -10.061607 +рм -10.064512 +рү -10.087481 +уз -10.091135 +ша -10.107159 +нл -10.107990 +жә -10.122933 +ве -10.124212 +ед -10.126902 +зм -10.147181 +мө -10.162415 +ыз -10.174847 +лм -10.182640 +тн -10.184778 +ие -10.202228 +ть -10.204640 +с_ -10.220818 +әш -10.242816 +пу -10.243152 +бл -10.243422 +әд -10.245130 +су -10.256857 +ус -10.258854 +үт -10.266604 +_ю -10.272026 +вы -10.276551 +па -10.277659 +ос -10.286365 +яш -10.288577 +уг -10.292691 +зл -10.313168 +уб -10.316499 +пл -10.325740 +зи -10.335767 +хе -10.338036 +ыт -10.373632 +әз -10.385037 +ое -10.397123 +лу -10.401695 +ск -10.404675 +рк -10.416192 +но -10.425271 +яр -10.429286 +по -10.430698 +фи -10.434919 +зг -10.439410 +иб -10.440307 +ае -10.475429 +мы -10.476469 +зы -10.479223 +бы -10.500820 +үч -10.508577 +әч -10.526939 +як -10.531873 +гъ -10.552915 +из -10.592338 +кч -10.610252 +гр -10.619651 +жа -10.631123 +мм -10.650245 +лк -10.657484 +фа -10.657819 +өл -10.689701 +ог -10.693072 +ок -10.713621 +фе -10.736696 +әһ -10.758800 +ив -10.768725 +зә -10.787639 +ум -10.820693 +кс -10.831937 +зу -10.836629 +өс -10.839699 +сл -10.855063 +му -10.879609 +ьл -10.885258 +ым -10.900368 +ял -10.900427 +ев -10.902134 +ут -10.904565 +нс -10.905908 +жы -10.906720 +_л -10.906749 +от -10.907384 +уы -10.908671 +чи -10.925510 +мн -10.933973 +ыс -10.940550 +өз -10.944062 +әх -10.947446 +ые -10.954802 +иж -10.983565 +үп -11.002569 +иа -11.004654 +нк -11.015013 +фә -11.020531 +үм -11.031408 +әв -11.035261 +иц -11.041085 +кк -11.043178 +къ -11.060736 +ч_ -11.070216 +ич -11.072650 +ши -11.075004 +лү -11.117291 +мл -11.126343 +пе -11.152128 +уд -11.176546 +лн -11.200027 +зд -11.220496 +зн -11.224853 +фо -11.225885 +ло -11.227341 +аф -11.227858 +сү -11.250471 +оч -11.256481 +бо -11.256989 +шь -11.272834 +ья -11.278136 +ыч -11.279206 +ну -11.281790 +иф -11.286029 +шм -11.291437 +бр -11.306498 +це -11.337597 +әж -11.339450 +оц -11.349914 +өм -11.369016 +го -11.369158 +чр -11.376094 +нь -11.378201 +мп -11.378816 +хи -11.401778 +эл -11.426298 +их -11.450401 +вә -11.457566 +өи -11.467161 +еч -11.470323 +об -11.474082 +өт -11.478575 +үс -11.480465 +ги -11.507893 +уш -11.514720 +ду -11.521037 +юл -11.522711 +әя -11.525861 +чл -11.558252 +рб -11.574044 +рш -11.580132 +ъл -11.583670 +ац -11.596143 +яз -11.598455 +рь -11.603905 +ех -11.606181 +ви -11.630293 +кр -11.632929 +еб -11.650010 +сө -11.658018 +ая -11.679760 +аж -11.687326 +дө -11.701297 +үе -11.706841 +ах -11.715663 +оз -11.725529 +д_ -11.731272 +од -11.758947 +мк -11.773120 +иө -11.783117 +хо -11.788142 +бө -11.790593 +нц -11.811815 +уи -11.824538 +яг -11.828898 +яд -11.839910 +ъд -11.859530 +кш -11.866523 +оп -11.876699 +нф -11.880377 +үб -11.891003 +үн -11.900854 +лт -11.902266 +мг -11.907517 +ху -11.910654 +ыя -11.920193 +ът -11.921625 +юг -11.940157 +кн -11.945700 +пк -11.955138 +рч -11.956239 +во -11.967779 +еи -11.979412 +үд -11.990920 +нү -12.016637 +сь -12.021491 +лс -12.029123 +өб -12.067803 +ьә -12.068497 +гу -12.085916 +до -12.089533 +нб -12.121844 +хө -12.139195 +рр -12.140758 +ф_ -12.151918 +тс -12.163129 +чк -12.163659 +шү -12.163976 +др -12.165778 +өк -12.168679 +уж -12.171161 +см -12.171977 +хс -12.172261 +хн -12.173753 +шч -12.187108 +пи -12.192752 +эч -12.201675 +ях -12.239641 +иш -12.242620 +еф -12.273672 +кц -12.278939 +яб -12.281503 +пт -12.307980 +юн -12.316705 +хш -12.324063 +о_ -12.340368 +әф -12.383371 +тб -12.401737 +вт -12.422956 +фт -12.443935 +же -12.452275 +еа -12.462790 +ии -12.507333 +мб -12.510472 +пш -12.511595 +тм -12.557835 +ьд -12.580462 +һа -12.582018 +өг -12.585654 +һи -12.608726 +фы -12.647293 +уа -12.648429 +өп -12.659386 +нв -12.669024 +рө -12.685906 +хы -12.690069 +тд -12.709312 +ьт -12.724956 +ье -12.731063 +хм -12.746470 +мд -12.782386 +ию -12.825850 +жу -12.834646 +жд -12.836840 +эм -12.850411 +вр -12.853482 +ж_ -12.856047 +уп -12.875976 +әэ -12.881945 +мт -12.901118 +өш -12.903238 +эр -12.942729 +тч -12.950381 +чу -12.950564 +өх -12.964209 +дп -12.967779 +г_ -13.010117 +эк -13.031923 +мь -13.037604 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tel b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tel new file mode 100644 index 00000000000..ceafd9a5504 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tel @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 11241963 +#UNIQUE_BIGRAMS 1404 +#UNSEEN_LOG_PROB -15.615442 +#EXPECTED_SCORE -7.788511 +న_ -4.675568 +ల_ -4.883633 +_అ -5.429004 +_వ -5.478691 +_క -5.483581 +ద_ -5.492098 +_మ -5.547679 +క_ -5.571081 +ర_ -5.673497 +డ_ -5.838199 +_ప -5.948732 +_స -6.139248 +_చ -6.150800 +త_ -6.187676 +గ_ -6.257287 +_న -6.266802 +_త -6.465050 +య_ -6.516028 +ట_ -6.583162 +_ర -6.695042 +మర -6.765854 +వర -6.811193 +_ద -7.045011 +నక -7.107100 +_ఆ -7.111955 +_ఉ -7.122056 +లన -7.129997 +కల -7.150308 +_ఇ -7.264767 +రగ -7.375565 +అన -7.422391 +_ల -7.435149 +తల -7.437305 +దన -7.440979 +_జ -7.469240 +కడ -7.470264 +పర -7.499528 +చ_ -7.520722 +రక -7.536527 +రయ -7.536790 +_గ -7.549668 +వల -7.607308 +మద -7.647501 +వ_ -7.658582 +తర -7.665860 +మన -7.695481 +లగ -7.709165 +స_ -7.725152 +కన -7.782711 +కర -7.794999 +మ_ -7.838838 +_భ -7.842279 +లక -7.899840 +నల -7.929411 +చస -7.933703 +_బ -7.975533 +ప_ -8.086198 +తన -8.135869 +వడ -8.160076 +_ఎ -8.169503 +వట -8.174871 +రల -8.176392 +అద -8.202201 +సన -8.223178 +సమ -8.230291 +రణ -8.233377 +రత -8.330253 +వత -8.332071 +చద -8.335052 +యన -8.350928 +చన -8.356470 +లద -8.359628 +రచ -8.361484 +దల -8.376418 +దర -8.378082 +రవ -8.407063 +పల -8.427684 +అత -8.435038 +యల -8.456516 +గల -8.458366 +గర -8.511561 +ణ_ -8.569179 +లల -8.572057 +సర -8.572839 +చయ -8.573768 +పద -8.579451 +నడ -8.585354 +శల -8.615086 +రజ -8.615338 +దశ -8.636683 +గన -8.654051 +పడ -8.660995 +సక -8.669174 +కట -8.671422 +చర -8.675981 +ఇద -8.695374 +తద -8.702146 +జర -8.740439 +దక -8.784973 +డల -8.806933 +మడ -8.813616 +రప -8.819518 +_శ -8.838426 +రడ -8.845434 +చల -8.860983 +గత -8.861103 +వన -8.863970 +నమ -8.869541 +ఉట -8.872907 +అవ -8.899809 +డన -8.908416 +నన -8.925724 +పన -8.942295 +మల -8.944127 +కవ -8.956569 +కద -8.972210 +నవ -8.977311 +నట -9.005923 +ఉద -9.007970 +దవ -9.044570 +వధ -9.062984 +చడ -9.063259 +అయ -9.083515 +ఉడ -9.089585 +పట -9.097085 +టల -9.099265 +వద -9.100603 +లస -9.102577 +జ_ -9.124471 +ధన -9.124471 +పయ -9.160076 +టన -9.166329 +నగ -9.174241 +లప -9.175798 +_ఏ -9.187798 +వస -9.212100 +నద -9.226019 +జన -9.230406 +నచ -9.244348 +రన -9.280284 +బధ -9.311173 +టద -9.317301 +డద -9.327571 +తయ -9.343489 +యత -9.347580 +ధర -9.360851 +_ఒ -9.380646 +సగ -9.388968 +ఆయ -9.396991 +శ_ -9.408632 +రమ -9.438061 +అట -9.452914 +భర -9.455164 +సద -9.486581 +మత -9.496929 +ధ_ -9.515124 +నత -9.527383 +టక -9.527952 +నర -9.535265 +గద -9.535646 +_హ -9.536885 +తక -9.539270 +కత -9.541946 +లవ -9.552026 +సల -9.576999 +కస -9.582498 +సధ -9.585453 +నయ -9.604707 +ధక -9.612823 +ఒక -9.615438 +యగ -9.620480 +ఇత -9.622603 +భగ -9.624526 +పత -9.637858 +మచ -9.643492 +డత -9.675981 +పచ -9.678293 +జల -9.692133 +అధ -9.700864 +ఆమ -9.739945 +వవ -9.742911 +డవ -9.744231 +తగ -9.763510 +గడ -9.765519 +ధచ -9.782598 +తమ -9.783502 +లయ -9.783615 +యడ -9.813789 +కక -9.834496 +నస -9.835199 +సబ -9.853366 +డగ -9.853603 +లత -9.854791 +అల -9.856931 +బడ -9.866004 +శర -9.885901 +జవ -9.891740 +హర -9.896747 +వక -9.901281 +కప -9.918441 +గణ -9.920927 +నప -9.925412 +లచ -9.930663 +సహ -9.944885 +లట -9.945012 +రస -9.947417 +మట -9.949954 +మయ -9.951605 +వష -9.954403 +ణల -9.963215 +టర -10.004274 +తస -10.013530 +గచ -10.015654 +షయ -10.021244 +దట -10.046537 +భవ -10.047215 +తడ -10.053884 +టవ -10.077401 +దగ -10.100463 +చశ -10.103989 +కమ -10.114052 +జక -10.166772 +అమ -10.190569 +_య -10.194023 +జత -10.199596 +మహ -10.201106 +హ_ -10.221952 +మగ -10.226403 +దచ -10.229944 +యక -10.248713 +ఉప -10.265360 +దప -10.274550 +ష_ -10.275027 +మక -10.275663 +ళ_ -10.277095 +సత -10.277414 +జయ -10.284439 +ణగ -10.310277 +చత -10.334762 +వశ -10.351934 +బగ -10.355292 +టయ -10.366600 +రద -10.368804 +డక -10.373904 +ఎత -10.381758 +కగ -10.416415 +బల -10.426977 +వజ -10.438150 +ఇట -10.442251 +చవ -10.455345 +శత -10.500178 +దత -10.504832 +ధగ -10.510063 +వచ -10.511935 +కయ -10.528710 +షల -10.535360 +ఎద -10.546257 +హద -10.550101 +ఇల -10.568496 +యద -10.572790 +మజ -10.586340 +యర -10.601613 +ఆధ -10.609411 +ఆల -10.616848 +చప -10.621289 +రశ -10.626960 +డప -10.627975 +ఆద -10.631431 +ధల -10.636734 +సవ -10.643287 +పక -10.681345 +వగ -10.686623 +భమ -10.687258 +భ_ -10.693620 +కశ -10.698303 +చక -10.704500 +బయ -10.706000 +దడ -10.717622 +యవ -10.717839 +_ధ -10.718271 +భష -10.737642 +వహ -10.740713 +బ_ -10.758610 +_ఫ -10.770442 +యమ -10.772236 +నభ -10.783728 +_ఈ -10.804923 +దద -10.840955 +డయ -10.853484 +గట -10.856812 +ఎవ -10.859433 +తవ -10.859910 +ఎల -10.862058 +నజ -10.873328 +లమ -10.885658 +వయ -10.890522 +ఇక -10.919559 +హన -10.920553 +బద -10.923043 +అడ -10.927285 +కథ -10.930538 +పప -10.933797 +యట -10.939583 +సప -10.955040 +వప -10.968348 +లభ -10.971437 +సఘ -10.974016 +బర -10.979706 +ఆర -10.989066 +సయ -11.033297 +సచ -11.033566 +ఫల -11.047351 +ళల -11.059899 +రట -11.067040 +దహ -11.069244 +రర -11.073662 +_ఐ -11.099899 +ఏద -11.120894 +కళ -11.124042 +ఏమ -11.125188 +డట -11.131220 +గమ -11.143360 +గవ -11.145394 +మవ -11.178324 +గప -11.179812 +పగ -11.184583 +దయ -11.190869 +హళ -11.198992 +మప -11.200199 +డడ -11.209590 +పఠ -11.210502 +_ట -11.234420 +ఖ_ -11.235348 +గగ -11.241238 +అశ -11.243415 +చచ -11.245594 +వభ -11.268522 +బత -11.268839 +పవ -11.342573 +శన -11.371182 +ఇవ -11.377314 +శఖ -11.377997 +సభ -11.386904 +తట -11.397944 +టబ -11.413262 +చబ -11.415714 +శధ -11.434416 +డమ -11.456606 +_ఖ -11.479143 +ఏడ -11.483179 +దళ -11.485017 +డచ -11.486857 +రధ -11.505019 +శవ -11.512123 +నష -11.520016 +హయ -11.520769 +అస -11.538220 +చట -11.549331 +టచ -11.550485 +టగ -11.550485 +ణక -11.563633 +గక -11.581219 +హల -11.583581 +టడ -11.590292 +బహ -11.593064 +నబ -11.597830 +హక -11.603010 +గళ -11.605807 +పజ -11.606207 +బట -11.613426 +మమ -11.618260 +గయ -11.622704 +మస -11.661671 +ణమ -11.670847 +భచ -11.678398 +లర -11.699156 +శడ -11.703858 +ఉచ -11.705143 +_ఊ -11.712876 +ఆక -11.715463 +దమ -11.737204 +మధ -11.738080 +తప -11.744671 +ఠశ -11.747316 +ఘట -11.762841 +యబ -11.767307 +పణ -11.820620 +రహ -11.827133 +థల -11.830868 +గహ -11.837428 +వణ -11.841190 +ళన -11.852535 +ఆహ -11.867802 +కచ -11.868761 +ఖల -11.868761 +వళ -11.881294 +వమ -11.889548 +కణ -11.891009 +జగ -11.908663 +జమ -11.927535 +శష -11.943116 +బబ -11.945138 +రబ -11.946657 +హచ -11.953258 +_ఓ -11.967063 +ఖర -11.975307 +ణన -11.982041 +హమ -11.994030 +ళక -11.994554 +భజ -12.010880 +యచ -12.022579 +ఐద -12.029000 +ఆన -12.032222 +హజ -12.045180 +మణ -12.052248 +భల -12.052793 +ధమ -12.056068 +ఆట -12.057709 +అచ -12.085884 +రష -12.087559 +అర -12.089235 +యప -12.089235 +డర -12.090914 +_ఘ -12.092034 +దబ -12.093155 +అభ -12.108374 +ఆడ -12.113483 +మళ -12.129494 +ణచ -12.155604 +పస -12.167952 +సడ -12.172092 +లహ -12.172685 +భయ -12.176838 +టత -12.187573 +ఎప -12.190569 +గధ -12.192370 +_డ -12.195978 +సట -12.224560 +శగ -12.251840 +ఆవ -12.259999 +తత -12.271374 +వబ -12.296977 +థన -12.326994 +చగ -12.352941 +లబ -12.367108 +హత -12.367786 +టప -12.386217 +టమ -12.386904 +అగ -12.412562 +ఘ_ -12.423095 +ధవ -12.436549 +ఇడ -12.437972 +నధ -12.444395 +ఇర -12.445827 +శయ -12.449411 +ధత -12.454444 +కష -12.458050 +యస -12.460218 +హస -12.465289 +శమ -12.466014 +ఊర -12.506511 +రఫ -12.529848 +థ_ -12.573768 +సస -12.585551 +ఏక -12.587920 +మఖ -12.596637 +ఫర -12.602212 +శక -12.613426 +యధ -12.652553 +శస -12.653379 +ఈయ -12.667503 +యజ -12.674198 +థక -12.704714 +జబ -12.705571 +భత -12.735890 +జప -12.736766 +తజ -12.756167 +పష -12.760613 +రభ -12.773134 +లజ -12.778534 +ళశ -12.788487 +టట -12.790304 +యణ -12.794857 +పథ -12.806761 +రళ -12.812289 +లడ -12.838368 +ఎడ -12.842132 +గజ -12.848743 +అక -12.859194 +షక -12.869722 +జడ -12.899809 +భక -12.940591 +షధ -12.954785 +ఎట -12.956824 +ఆచ -12.958866 +దస -12.963984 +ఓడ -12.973242 +ళత -12.982560 +మబ -12.983599 +గస -13.004538 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tel-rom b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tel-rom new file mode 100644 index 00000000000..dc430a7405f --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tel-rom @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 112458238 +#UNIQUE_BIGRAMS 695 +#UNSEEN_LOG_PROB -18.258562 +#EXPECTED_SCORE -7.463226 +a_ -4.783945 +an -5.207563 +i_ -5.447812 +am -5.644678 +ra -5.752331 +ha -5.779784 +na -5.816005 +u_ -5.901152 +ar -6.045054 +_a -6.080245 +_s -6.196658 +m_ -6.225354 +at -6.232811 +ma -6.337140 +_p -6.371652 +al -6.437322 +ta -6.457061 +va -6.477150 +ad -6.528563 +ka -6.562764 +ya -6.569452 +e_ -6.639374 +_t -6.644752 +ch -6.690456 +_k -6.691318 +th -6.744820 +nd -6.764033 +da -6.791635 +_m -6.806854 +la -6.832939 +_v -6.880045 +sa -6.906383 +_n -6.910124 +in -6.946263 +_c -6.950846 +pa -7.049929 +ga -7.100940 +ti -7.144912 +as -7.182702 +_b -7.191260 +ri -7.201541 +ni -7.306177 +_r -7.309421 +hi -7.321128 +ak -7.423084 +en -7.427106 +es -7.427759 +di -7.450929 +sh -7.479620 +te -7.500803 +un -7.508529 +_d -7.521491 +nt -7.547175 +dh -7.605847 +av -7.679051 +t_ -7.704385 +r_ -7.711187 +st -7.717389 +s_ -7.723769 +it -7.726000 +ay -7.758405 +ng -7.762261 +he -7.788069 +_i -7.802723 +lu -7.814198 +du -7.818976 +de -7.825549 +h_ -7.837389 +vi -7.879280 +el -7.886026 +ll -7.898638 +ku -7.915011 +ag -7.925427 +re -7.932424 +nu -7.973666 +o_ -7.976320 +ur -7.981667 +ah -7.998884 +y_ -8.020827 +aa -8.029321 +ap -8.030958 +ne -8.030985 +d_ -8.060723 +_e -8.088941 +bh -8.095923 +pr -8.100507 +ru -8.121290 +_g -8.130701 +ee -8.202825 +n_ -8.211241 +ba -8.221176 +er -8.221237 +li -8.232591 +ve -8.238491 +pu -8.242433 +is -8.245794 +ja -8.277543 +gu -8.295770 +on -8.323300 +le -8.329650 +ed -8.336775 +nn -8.338127 +ud -8.374186 +_l -8.474355 +si -8.488633 +tt -8.521463 +l_ -8.529964 +et -8.630909 +tr -8.642151 +ai -8.646309 +me -8.647070 +ab -8.650932 +hu -8.678367 +il -8.700623 +ko -8.754167 +_y -8.774405 +dd -8.775844 +ir -8.778297 +ho -8.788597 +or -8.790868 +su -8.818147 +uk -8.823822 +_u -8.825346 +ev -8.827151 +nc -8.851240 +mi -8.859153 +_h -8.862374 +ik -8.890335 +lo -8.917385 +id -8.943030 +oo -8.947077 +tu -8.952863 +_j -8.986565 +od -8.987450 +pe -9.001176 +ut -9.003215 +ki -9.009589 +se -9.013177 +ac -9.046123 +to -9.055867 +sr -9.057411 +ot -9.111720 +_o -9.134856 +ro -9.142217 +rt -9.156246 +pi -9.172706 +mo -9.179414 +ty -9.211089 +ul -9.211241 +mu -9.215841 +ug -9.227329 +em -9.234081 +aj -9.237050 +ok -9.243630 +mm -9.299945 +_f -9.319090 +ic -9.330872 +iv -9.331699 +po -9.360772 +wa -9.410488 +dr -9.413845 +us -9.434059 +nk -9.484354 +pp -9.494756 +im -9.554507 +ip -9.581027 +kk -9.639438 +sy -9.653854 +ca -9.663726 +ks -9.675616 +dy -9.681293 +hr -9.698266 +ep -9.726266 +g_ -9.761464 +ol -9.791527 +hy -9.796287 +um -9.800033 +os -9.814400 +ia -9.814701 +gi -9.860167 +kr -9.876739 +vu -9.880676 +co -9.881644 +ea -9.898418 +ek -9.904725 +bo -9.922542 +ig -9.942954 +ry -9.944673 +up -9.946925 +ie -9.986929 +ct -9.993950 +ss -10.006880 +so -10.015273 +rs -10.017558 +go -10.017957 +bu -10.033443 +kh -10.046652 +ge -10.061024 +ce -10.063548 +ec -10.098452 +om -10.130104 +ke -10.140840 +_w -10.175136 +k_ -10.208324 +ju -10.210967 +ci -10.239716 +ye -10.248678 +ey -10.276413 +tv -10.284166 +io -10.284582 +be -10.297763 +rv -10.302681 +au -10.309339 +ov -10.313741 +ou -10.358009 +rm -10.390806 +op -10.410750 +mp -10.412568 +uv -10.413617 +hn -10.424860 +sw -10.462450 +hm -10.467055 +ly -10.470108 +do -10.473752 +ao -10.498370 +no -10.518101 +ub -10.582069 +ny -10.609786 +mb -10.621744 +fi -10.626360 +br -10.636943 +yi -10.641526 +kt -10.676288 +gr -10.677422 +iy -10.690973 +ow -10.745519 +yo -10.746973 +ij -10.781828 +p_ -10.823578 +yy -10.828188 +pl -10.830382 +rn -10.856356 +gh -10.923190 +og -10.938818 +ts -10.943510 +sm -11.032180 +ex -11.049586 +yu -11.146006 +fr -11.159793 +oj -11.160791 +oh -11.168920 +vy -11.169925 +bl -11.170339 +rd -11.176205 +ns -11.264844 +ht -11.292058 +tl -11.297279 +vv -11.302130 +nj -11.317436 +we -11.331517 +rc -11.353638 +ls -11.373174 +tn -11.396916 +c_ -11.443246 +oc -11.446537 +uc -11.456453 +ih -11.459701 +eg -11.463174 +bb -11.467381 +oa -11.491927 +x_ -11.531330 +ob -11.534371 +gs -11.561024 +sc -11.561877 +je -11.569420 +ph -11.575594 +jn -11.611511 +w_ -11.625387 +cc -11.627820 +rr -11.635800 +of -11.642756 +sv -11.675952 +ff -11.694200 +yd -11.705210 +gg -11.730532 +fo -11.733412 +fe -11.750637 +my -11.755774 +vo -11.774439 +pt -11.777318 +ji -11.812692 +cl -11.818518 +bi -11.818889 +aw -11.875992 +ms -11.877728 +rl -11.900599 +ck -11.900991 +fa -11.927031 +ew -11.952430 +sl -11.972190 +oy -11.981187 +wi -12.041559 +f_ -12.092529 +v_ -12.095221 +rg -12.125740 +sp -12.132291 +sk -12.134943 +wn -12.139335 +ds -12.142231 +nl -12.164084 +ys -12.166441 +ky -12.175433 +jo -12.230223 +ib -12.238631 +dv -12.260998 +cr -12.322486 +wo -12.355855 +uj -12.366655 +pd -12.409843 +rk -12.423873 +tm -12.480518 +cu -12.485954 +b_ -12.516446 +rj -12.518552 +eb -12.528978 +j_ -12.576299 +uh -12.578023 +df -12.578180 +iz -12.595385 +yr -12.603346 +oi -12.611190 +eh -12.633678 +eo -12.634168 +lp -12.673352 +nm -12.705724 +cy -12.707610 +ua -12.713975 +if -12.753824 +sn -12.774528 +jy -12.774708 +ue -12.776507 +cs -12.790073 +fu -12.816111 +vr -12.854740 +mh -12.885085 +tp -12.886250 +za -12.893651 +lt -12.902464 +py -12.934242 +lm -12.939272 +rh -12.965095 +tc -12.968381 +hw -13.003558 +ld -13.004612 +rb -13.032717 +lk -13.035515 +ws -13.055908 +dm -13.088389 +mr -13.132867 +rp -13.137253 +ml -13.269081 +gl -13.291029 +lg -13.295665 +uy -13.318811 +hj -13.396086 +gn -13.430797 +mv -13.431648 +jj -13.435622 +bd -13.482132 +mg -13.494220 +ej -13.495997 +ps -13.514493 +hk -13.579907 +dl -13.599519 +zi -13.615853 +td -13.640215 +hh -13.641526 +az -13.683105 +bs -13.694966 +mn -13.740594 +ln -13.751168 +qu -13.756484 +yw -13.760396 +wh -13.763604 +dp -13.784812 +dw -13.794623 +bn -13.821858 +ii -13.857213 +xt -13.898932 +hv -13.931033 +np -13.948369 +tw -13.954058 +_q -13.998720 +hs -13.999980 +_z -14.007989 +cp -14.017319 +mk -14.020727 +fl -14.023288 +dg -14.036592 +xp -14.118649 +nr -14.136559 +ui -14.154227 +gy -14.206140 +by -14.208567 +lb -14.230593 +nv -14.291029 +tk -14.313317 +ft -14.323801 +ax -14.339673 +rw -14.355721 +lh -14.356259 +yp -14.356797 +ef -14.364353 +hb -14.398300 +ze -14.438183 +af -14.478028 +vt -14.498074 +wr -14.508202 +md -14.508202 +ei -14.515995 +mc -14.518402 +lw -14.581793 +mt -14.593163 +z_ -14.611672 +yt -14.644809 +kl -14.672682 +jh -14.696327 +hc -14.712080 +hp -14.746929 +lf -14.768966 +nf -14.776867 +lv -14.791345 +kn -14.810386 +wd -14.837924 +ox -14.846213 +dt -14.889946 +xy -14.890725 +lr -14.923040 +xa -14.958544 +rf -14.966737 +sb -14.992433 +sd -15.001663 +db -15.028852 +jp -15.040046 +vn -15.040911 +sg -15.065334 +lz -15.070622 +bt -15.075043 +pn -15.079478 +ae -15.088389 +gt -15.096457 +dn -15.126429 +hl -15.168330 +bj -15.218315 +uu -15.239995 +kc -15.244968 +kv -15.248959 +xm -15.282312 +uz -15.286407 +lc -15.381774 +dc -15.466365 +yn -15.471019 +ix -15.492149 +ym -15.571137 +oe -15.581164 +wt -15.605263 +uw -15.633678 +hd -15.636290 +zo -15.685470 +iu -15.685470 +vs -15.708640 +km -15.708640 +uo -15.730794 +xi -15.787712 +nh -15.812599 +gm -15.828935 +gd -15.836421 +mw -15.888389 +pc -15.889946 +ww -15.915091 +zp -15.937459 +wl -15.939070 +jg -15.963454 +mj -15.986591 +gb -16.001663 +cb -16.037455 +iq -16.037455 +yl -16.042641 +vd -16.056564 +yb -16.070622 +pg -16.079478 +zv -16.084818 +gk -16.110003 +_x -16.133789 +xc -16.229114 +qa -16.248959 +తల -16.285382 +xx -16.335423 +eq -16.389463 +jr -16.400518 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tgk b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tgk new file mode 100644 index 00000000000..8700f68094c --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tgk @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 16339333 +#UNIQUE_BIGRAMS 1284 +#UNSEEN_LOG_PROB -15.925798 +#EXPECTED_SCORE -7.709395 +и_ -4.250385 +ар -5.343306 +да -5.781718 +_м -5.821675 +_д -5.966454 +д_ -5.996529 +р_ -6.055313 +он -6.201366 +хо -6.425477 +ан -6.500822 +_х -6.530242 +ни -6.544455 +_б -6.552778 +ма -6.638897 +ст -6.672097 +_а -6.709873 +ха -6.710548 +т_ -6.715133 +_к -6.734606 +н_ -6.748295 +та -6.765849 +ах -6.779801 +ас -6.806247 +ри -6.823984 +ра -6.825185 +ор -6.876944 +ои -6.882559 +а_ -6.907379 +ро -6.927907 +на -6.929434 +ти -6.935614 +ад -6.947782 +уд -7.046970 +_н -7.050091 +ам -7.115205 +ме -7.131702 +_с -7.132185 +ин -7.138578 +нд -7.140720 +ди -7.143388 +_т -7.161464 +ат -7.167798 +ли -7.192977 +ка -7.237345 +о_ -7.241334 +ба -7.251672 +аи -7.311032 +бо -7.358002 +си -7.375328 +до -7.382530 +ру -7.409627 +ки -7.433376 +_ш -7.469570 +то -7.512455 +ав -7.518093 +ол -7.556787 +ва -7.561600 +ох -7.563219 +ми -7.571542 +ис -7.577500 +ал -7.579965 +_р -7.597711 +ша -7.601547 +хи -7.602285 +_и -7.605944 +ур -7.632628 +мо -7.676353 +_ф -7.745176 +ия -7.760277 +ир -7.772774 +му -7.779102 +со -7.821555 +ом -7.839806 +са -7.899968 +ла -7.937420 +м_ -7.960658 +рд -7.991313 +иш -8.011768 +ун -8.021968 +ик -8.024495 +_о -8.049635 +ко -8.063504 +ум -8.066602 +од -8.087225 +но -8.123085 +ии -8.136763 +е_ -8.139076 +го -8.177543 +_г -8.196586 +ак -8.208394 +ку -8.225655 +ус -8.288951 +ху -8.295455 +шу -8.298621 +ил -8.321799 +_в -8.341798 +зи -8.341826 +ло -8.381116 +ид -8.392969 +_п -8.395881 +де -8.398232 +гу -8.422081 +во -8.459729 +би -8.481718 +ош -8.507354 +за -8.558601 +ши -8.576691 +аа -8.589423 +га -8.596241 +ер -8.607803 +ед -8.612599 +аб -8.644186 +_ҽ -8.695865 +уз -8.765052 +фа -8.766089 +лл -8.767858 +_з -8.809244 +бу -8.819220 +аз -8.866119 +от -8.882278 +зо -8.885030 +аф -8.906945 +им -8.914637 +л_ -8.927778 +я_ -8.935798 +оз -8.963784 +аъ -8.976582 +аш -8.981706 +ги -8.995160 +ех -9.000170 +ке -9.008422 +ду -9.032217 +фу -9.044427 +об -9.053312 +ҽи -9.058670 +ие -9.078248 +_я -9.079298 +ул -9.086480 +ух -9.109755 +иф -9.130291 +еб -9.134255 +яи -9.137285 +фт -9.140769 +их -9.144962 +ос -9.151525 +из -9.158572 +шт -9.181152 +ок -9.189480 +еш -9.203987 +_ч -9.265748 +ет -9.267706 +хр -9.296565 +з_ -9.301850 +к_ -9.320584 +су -9.321431 +_у -9.340595 +оя -9.342485 +аг -9.343346 +оҽ -9.344149 +як -9.364026 +ви -9.384004 +ҽа -9.384830 +бе -9.470932 +фи -9.486682 +иб -9.489790 +па -9.501646 +ек -9.515538 +дг -9.522210 +х_ -9.525720 +не -9.584567 +фе -9.587689 +ш_ -9.602577 +ен -9.603607 +уш -9.635346 +ов -9.655570 +ят -9.656995 +у_ -9.686303 +тр -9.703501 +ъ_ -9.705711 +ит -9.733328 +фо -9.736488 +тс -9.736638 +хт -9.745630 +ту -9.752392 +иг -9.764936 +б_ -9.790669 +рх -9.798353 +с_ -9.804025 +нх -9.818622 +ук -9.822214 +ҽо -9.866037 +рк -9.895588 +шо -9.899968 +шв -9.937918 +ҽу -10.012566 +нт -10.048395 +тх -10.074816 +ну -10.081975 +уф -10.143463 +ем -10.154088 +ча -10.162159 +те -10.164286 +ре -10.178029 +ле -10.179564 +мр -10.198115 +_э -10.247815 +еъ -10.251039 +по -10.261298 +ег -10.288647 +нг -10.289199 +рг -10.315400 +в_ -10.328056 +чу -10.330782 +яд -10.353013 +ес -10.363444 +кт -10.430221 +ут -10.509991 +чи -10.511537 +ел -10.515926 +уб -10.553383 +лм -10.601959 +рт -10.606495 +иҽ -10.623249 +оф -10.626735 +тт -10.633171 +ог -10.636540 +ar -10.637665 +ио -10.671827 +вр -10.682380 +вл -10.707260 +се -10.735584 +ян -10.742526 +пе -10.750110 +рз -10.784738 +нб -10.821176 +i_ -10.829989 +др -10.838856 +мм -10.852831 +лу -10.875046 +хе -10.881361 +хс -10.886199 +_л -10.893067 +мх -10.906394 +нр -10.911487 +ъи -10.947651 +уи -10.959910 +d_ -10.968554 +рб -10.970501 +сл -10.972629 +зу -10.995340 +шк -11.000577 +оъ -11.032587 +зе -11.047461 +кр -11.079394 +лх -11.107951 +сс -11.125204 +дд -11.132519 +нс -11.133709 +ев -11.139275 +ф_ -11.195110 +пу -11.202166 +r_ -11.214912 +аҽ -11.221222 +чо -11.231163 +ъд -11.235628 +ву -11.236053 +рр -11.256861 +зб -11.280607 +ае -11.286109 +уя -11.290526 +ha -11.294734 +_d -11.305868 +da -11.312814 +рм -11.312814 +лт -11.318666 +хм -11.338879 +см -11.352667 +рс -11.365423 +an -11.383237 +рв -11.384889 +оч -11.400797 +ъл -11.411820 +вв -11.418572 +ск -11.426327 +еи -11.457027 +бр -11.466990 +n_ -11.497045 +сф -11.506005 +sh -11.510635 +ъа -11.515796 +_е -11.523574 +еф -11.528260 +рн -11.545048 +_m -11.550335 +ув -11.552720 +ър -11.557769 +зд -11.571408 +зм -11.572483 +иа -11.582738 +ез -11.585449 +кс -11.587893 +г_ -11.596617 +хб -11.597437 +_b -11.633171 +_a -11.644999 +on -11.655214 +уҽ -11.669237 +ъо -11.682525 +шг -11.701220 +ср -11.718969 +лк -11.729425 +_k -11.769553 +зх -11.778210 +ое -11.787544 +_h -11.796311 +o_ -11.801659 +нҽ -11.829025 +мл -11.836432 +уа -11.837078 +am -11.847453 +кл -11.890719 +ro -11.897777 +ше -11.922927 +t_ -11.933940 +ad -11.939825 +пр -11.954468 +ив -11.955870 +эр -11.959734 +хх -11.968200 +мк -11.973516 +хш -12.016402 +пи -12.018598 +st -12.050828 +уг -12.054580 +оа -12.068544 +дх -12.069682 +хн -12.093409 +ън -12.106977 +эъ -12.113223 +or -12.113614 +шр -12.119888 +уъ -12.120281 +_s -12.130539 +нк -12.134106 +бх -12.135695 +еа -12.148065 +уо -12.148465 +na -12.150873 +съ -12.171090 +тм -12.175576 +m_ -12.187470 +ma -12.191182 +мч -12.208629 +бд -12.210302 +ъм -12.213234 +фг -12.216592 +кк -12.271411 +ъз -12.307210 +сб -12.312140 +ҽ_ -12.336594 +фз -12.356829 +бл -12.359611 +e_ -12.393890 +оп -12.394840 +хл -12.395316 +ео -12.397696 +ве -12.397696 +hu -12.415916 +мт -12.416399 +шх -12.429490 +яг -12.430464 +as -12.433880 +хд -12.435347 +me -12.436814 +to -12.463994 +a_ -12.473001 +мн -12.481056 +шн -12.490171 +nd -12.505491 +ba -12.549276 +ra -12.607045 +хк -12.643303 +шм -12.648963 +_n -12.670675 +is -12.677022 +рш -12.685721 +un -12.685721 +ът -12.689799 +ҽр -12.707408 +_t -12.721054 +do -12.730025 +уп -12.743888 +bo -12.760947 +мъ -12.761560 +mo -12.766474 +нв -12.769553 +ud -12.772021 +ku -12.790043 +кш -12.799454 +ta -12.805131 +at -12.817825 +вз -12.824214 +мз -12.827419 +ге -12.836432 +уч -12.841608 +нн -12.854629 +нъ -12.869749 +ъс -12.902501 +ho -12.910637 +ik -12.916769 +нз -12.924299 +va -12.934631 +av -12.942255 +um -12.947128 +мб -12.953417 +фр -12.954818 +ию -12.959031 +рф -12.968907 +ип -12.981706 +ya -12.983135 +нч -12.993179 +rd -13.018598 +мс -13.022267 +зн -13.023001 +рҽ -13.024472 +ях -13.027418 +рл -13.037776 +al -13.038518 +кд -13.051203 +ka -13.051953 +кх -13.085329 +_c -13.094567 +св -13.101535 +in -13.104642 +тл -13.106977 +рч -13.106977 +вс -13.107757 +нф -13.115572 +ап -13.132916 +тб -13.132916 +фх -13.140869 +ао -13.146462 +сх -13.159329 +ni -13.160137 +ее -13.166618 +бт -13.174759 +бк -13.177211 +км -13.177211 +уе -13.192834 +лб -13.206124 +зв -13.206958 +тъ -13.210302 +яб -13.214492 +эх -13.215331 +du -13.216171 +_o -13.217853 +зъ -13.227136 +еп -13.232225 +ki -13.233925 +di -13.251039 +ri -13.251900 +ai -13.262273 +п_ -13.280607 +юн -13.285007 +om -13.285889 +os -13.289420 +_i -13.292961 +er -13.295622 +es -13.308105 +тг -13.308105 +ch -13.320697 +us -13.321601 +od -13.323410 +яъ -13.363793 +бб -13.363793 +ч_ -13.374068 +зг -13.385361 +ak -13.386307 +сн -13.386307 +ач -13.393890 +mi -13.394840 +ir -13.405340 +ъу -13.408216 +ич -13.408216 +ji -13.413024 +ne -13.414951 +хз -13.416881 +_ю -13.422687 +ga -13.445161 +_f -13.447132 +тя -13.470995 +k_ -13.473001 +ъв -13.483076 +эҽ -13.490171 +oj -13.491187 +шд -13.495259 +гр -13.507546 +kh -13.518902 +hi -13.546104 +вг -13.556704 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tgl b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tgl new file mode 100644 index 00000000000..28eb9d0d477 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tgl @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 35498290 +#UNIQUE_BIGRAMS 645 +#UNSEEN_LOG_PROB -18.500227 +#EXPECTED_SCORE -7.183513 +an -4.410922 +ng -4.614987 +g_ -4.716441 +a_ -5.308421 +in -5.683973 +_a -5.724315 +n_ -5.765500 +_m -5.833289 +na -5.842063 +ma -5.864065 +la -5.936772 +_p -6.081791 +ka -6.119545 +_n -6.160853 +pa -6.236026 +o_ -6.242396 +al -6.244219 +ga -6.260469 +_k -6.320906 +ag -6.332054 +on -6.438176 +_s -6.614110 +ta -6.664960 +sa -6.760446 +at -6.775305 +_i -6.836910 +ya -6.859829 +ay -6.865898 +ar -6.906642 +it -6.935453 +il -6.961312 +s_ -6.971722 +am -6.972854 +as -6.975813 +ba -6.986674 +_b -7.066735 +ak -7.125426 +ra -7.160289 +ni -7.193811 +y_ -7.194787 +si -7.245621 +t_ -7.256505 +_t -7.257978 +li -7.298651 +_d -7.370191 +to -7.381342 +ha -7.389687 +ti -7.404098 +is -7.418856 +i_ -7.424786 +e_ -7.452972 +un -7.473504 +hi -7.509168 +da -7.531454 +_h -7.603061 +er -7.631485 +ab -7.631815 +ap -7.653524 +ah -7.658303 +mg -7.662027 +_l -7.706377 +l_ -7.733391 +wa -7.780214 +ri -7.786428 +di -7.817069 +nd -7.822165 +bi -7.839486 +yo -7.884068 +aw -7.898832 +iy -7.923307 +ul -7.993607 +en -8.017883 +no -8.025212 +re -8.095204 +_r -8.112624 +_g -8.140222 +mi -8.145966 +ro -8.145989 +lo -8.182279 +d_ -8.185837 +nt -8.186109 +pi -8.189130 +es -8.235389 +um -8.268831 +_c -8.323036 +st -8.323296 +gi -8.400584 +or -8.409028 +r_ -8.435630 +te -8.435841 +ki -8.469962 +tu -8.472645 +ad -8.493337 +ku -8.532304 +aa -8.557286 +ig -8.609968 +ik -8.652788 +po -8.681501 +ko -8.706750 +_u -8.805886 +os -8.808052 +su -8.847683 +le -8.866017 +se -8.869149 +oo -8.891151 +k_ -8.907237 +pe -8.929615 +us -8.984799 +im -8.997164 +el -9.023487 +mu -9.028487 +ny -9.029442 +gu -9.037450 +bu -9.047522 +ib -9.086495 +_e -9.105846 +de -9.108715 +me -9.119051 +ai -9.126573 +sy -9.126732 +do -9.132058 +ne -9.174403 +ip -9.184417 +gk -9.217084 +lu -9.251796 +w_ -9.255815 +so -9.296075 +go -9.303400 +ns -9.344633 +ol -9.374831 +_w -9.375884 +co -9.391794 +tr -9.394472 +pu -9.395375 +id -9.421083 +p_ -9.424151 +rt -9.436601 +ho -9.441736 +ot -9.482670 +ur -9.489402 +io -9.514925 +_f -9.517454 +bo -9.546244 +ia -9.556245 +pr -9.587829 +m_ -9.614055 +mp -9.621302 +ir -9.624667 +ic -9.637687 +ut -9.705596 +_o -9.723248 +et -9.761362 +ca -9.800222 +ce -9.800367 +mo -9.814643 +nu -9.842580 +ch -9.845344 +he -9.854157 +om -9.856679 +ea -9.859017 +ao -9.885528 +up -9.891229 +_j -9.909561 +od -9.929041 +gg -9.945457 +ac -9.954905 +be -9.961170 +em -9.962427 +ll -9.981118 +mb -10.050234 +ed -10.063307 +oy -10.073080 +rs -10.080806 +uk -10.087556 +ve -10.097228 +wi -10.121516 +th -10.144516 +gs -10.166067 +hu -10.178023 +ek -10.190697 +ie -10.215513 +ob -10.220111 +uh -10.225404 +au -10.227206 +_y -10.235707 +ye -10.251722 +uw -10.253755 +ou -10.289422 +ub -10.314096 +op -10.354815 +ci -10.363034 +vi -10.402316 +ge -10.403527 +ug -10.419190 +kt -10.422365 +ok -10.430755 +gp -10.455195 +ss -10.462861 +ec -10.491010 +ih -10.495110 +br -10.496108 +h_ -10.505589 +we -10.515192 +gr -10.517156 +gl -10.523483 +sp -10.544756 +ry -10.544817 +ly -10.545242 +yu -10.550962 +gt -10.584517 +ru -10.622263 +du -10.629327 +iw -10.675639 +ty -10.680433 +sh -10.692691 +bl -10.732795 +ct -10.736049 +_v -10.736118 +gh -10.737991 +ke -10.748370 +ua -10.770775 +rd -10.776253 +nc -10.779250 +fi -10.790651 +pl -10.807888 +gy -10.869662 +gb -10.870118 +ts -10.871641 +ee -10.891653 +ep -10.899939 +ue -10.917204 +og -10.922321 +qu -10.934359 +rn -10.940097 +ks -10.952768 +ui -11.041727 +eg -11.048426 +gd -11.051528 +ii -11.129415 +c_ -11.133609 +rm -11.151341 +eb -11.205689 +iv -11.216673 +ow -11.240173 +nn -11.253407 +ud -11.277317 +jo -11.325107 +rr -11.368075 +b_ -11.382650 +fo -11.385257 +dr -11.386779 +fe -11.450524 +oc -11.519004 +z_ -11.535920 +ey -11.541724 +fa -11.547308 +tt -11.553036 +rk -11.581151 +ck -11.583644 +u_ -11.583769 +ex -11.592276 +gm -11.597682 +ld -11.603994 +ov -11.626304 +va -11.654850 +kl -11.676238 +ju -11.682770 +ph -11.694441 +uo -11.699568 +dy -11.701326 +gn -11.731274 +sc -11.740701 +fr -11.762562 +ev -11.779464 +ff -11.782754 +eo -11.803523 +cr -11.811681 +tl -11.814899 +ja -11.818270 +eh -11.822533 +lt -11.857395 +rc -11.862380 +sm -11.864349 +rl -11.917597 +pp -11.940257 +mm -11.950032 +ez -11.973702 +kr -11.985190 +wo -11.997102 +cu -12.004598 +ae -12.042670 +sk -12.047995 +ds -12.067226 +by -12.071767 +uy -12.146081 +za -12.149955 +je -12.152173 +cl -12.158848 +ls -12.161452 +oa -12.174543 +av -12.221323 +iz -12.246183 +of -12.262265 +oh -12.262464 +ys -12.295998 +nl -12.302142 +rg -12.348655 +oi -12.374750 +f_ -12.377343 +yn -12.411475 +af -12.456451 +_q -12.458737 +ef -12.478547 +uu -12.478779 +kh -12.491127 +ew -12.497928 +uc -12.581649 +tw -12.597682 +kb -12.645836 +lb -12.684374 +aq -12.685177 +zo -12.720948 +ht -12.730583 +yr -12.771770 +if -12.882187 +ky -12.885259 +rb -12.916025 +sl -12.940736 +ei -12.981241 +pt -12.997767 +x_ -13.011461 +rv -13.039245 +bs -13.059919 +lm -13.123781 +eu -13.143137 +ms -13.154950 +my -13.200515 +az -13.207033 +tn -13.210881 +cq -13.246775 +nk -13.253903 +hr -13.275905 +lf -13.288456 +fl -13.300296 +pw -13.320526 +tm -13.338516 +iu -13.352900 +ps -13.355880 +nm -13.365284 +rp -13.383844 +xt -13.396936 +vo -13.413691 +lp -13.418578 +oe -13.445075 +uz -13.445528 +md -13.448251 +dd -13.490659 +_z -13.504290 +cs -13.538214 +nf -13.542087 +sw -13.558174 +tb -13.573947 +np -13.580404 +cy -13.598438 +hn -13.615170 +ws -13.665504 +nh -13.669206 +ox -13.672916 +wn -13.684106 +nz -13.686783 +tc -13.726997 +lk -13.746414 +yl -13.752571 +dm -13.754817 +fu -13.758755 +ze -13.795266 +yi -13.797000 +kd -13.863894 +ft -13.865713 +py -13.877286 +nv -13.880348 +yd -13.893276 +yw -13.894513 +oj -13.911321 +dl -13.911321 +ym -13.930230 +cc -13.946820 +wb -13.953897 +dg -13.960360 +bn -14.001761 +lc -14.016503 +uf -14.024608 +xi -14.046447 +ji -14.056799 +lv -14.063742 +cb -14.070718 +nb -14.095404 +bb -14.115462 +nr -14.160149 +xa -14.204729 +kw -14.207033 +wh -14.210881 +xe -14.260270 +pn -14.268267 +pb -14.302348 +gw -14.336412 +wt -14.376478 +rh -14.378208 +nj -14.384279 +ax -14.406171 +sd -14.469299 +rf -14.474841 +hy -14.522825 +fw -14.526657 +rw -14.528577 +pc -14.534352 +xp -14.567519 +kn -14.573452 +dw -14.609571 +dn -14.631066 +yb -14.663394 +v_ -14.665504 +aj -14.681434 +dv -14.729203 +yf -14.794688 +zi -14.797000 +hd -14.830948 +pd -14.833319 +bc -14.841648 +sb -14.859659 +xc -14.865713 +bd -14.870575 +ix -14.912574 +sq -14.916339 +yc -14.958418 +oz -14.961008 +lh -14.980584 +lg -15.034122 +bg -15.095404 +kp -15.095404 +pm -15.146081 +ij -15.174356 +yt -15.190982 +xy -15.192503 +yp -15.192503 +ej -15.313062 +bh -15.363570 +zu -15.367001 +mr -15.373887 +zz -15.375614 +cm -15.387759 +sr -15.448251 +gc -15.507599 +sn -15.519004 +j_ -15.573452 +mn -15.583394 +hm -15.607540 +lr -15.619767 +sg -15.636231 +pg -15.640377 +wr -15.648704 +fp -15.669735 +km -15.688929 +cp -15.706207 +lj -15.728099 +hl -15.732518 +eq -15.754817 +wd -15.857245 +dp -15.908819 +dh -15.983214 +dc -16.007105 +db -16.012468 +iq -16.084067 +mt -16.101107 +nw -16.162383 +bp -16.220159 +hw -16.232623 +wl -16.273891 +vp -16.316375 +mv -16.356732 +rq -16.405289 +dt -16.459194 +lw -16.459194 +tf -16.473916 +mf -16.477620 +zs -16.496284 +fs -16.534352 +_x -16.545971 +df -16.557684 +vy -16.613641 +zm -16.754817 +yg -16.777465 +fc -16.795844 +dj -16.809783 +vd -16.852428 +tz -16.876675 +sf -16.896371 +rz -16.911321 +uv -16.951963 +fy -17.020550 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tha b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tha new file mode 100644 index 00000000000..cd30ded9edc --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tha @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 1440887 +#UNIQUE_BIGRAMS 1499 +#UNSEEN_LOG_PROB -13.533256 +#EXPECTED_SCORE -8.446339 +น_ -5.040904 +าย -5.212565 +ม_ -5.244093 +_เ -5.271290 +คม -5.696299 +าค -5.705414 +ยน -5.743314 +เม -6.203276 +ษา -6.296685 +มษ -6.345923 +_ม -6.412425 +_ก -6.420121 +ร_ -6.439022 +นา -6.453077 +_ร -6.480799 +รา -6.574546 +_พ -6.736038 +ย_ -6.736999 +_ส -6.814444 +กา -6.861055 +ง_ -6.967550 +าร -6.977357 +า_ -7.150756 +ด_ -7.210598 +ก_ -7.219375 +กร -7.243751 +_น -7.261616 +ท_ -7.286098 +พฤ -7.302917 +_ค -7.316738 +มน -7.348206 +รม -7.423899 +_จ -7.493463 +ภา -7.544514 +_ป -7.563329 +ยก -7.563708 +_อ -7.615194 +_แ -7.665735 +_บ -7.681682 +ต_ -7.818281 +ะ_ -7.843816 +บ_ -7.857451 +ฤษ -7.867939 +อง -7.878503 +ระ -7.881569 +ษภ -7.906818 +_ว -7.908260 +ลา -7.924223 +_ต -7.955943 +_โ -7.984059 +ช_ -8.064867 +_ธ -8.127609 +หา -8.145359 +วา -8.167355 +_ท -8.169371 +นว -8.200549 +คร -8.203202 +อ_ -8.257934 +มก -8.263769 +ว_ -8.277684 +าน -8.305290 +ยา -8.306875 +ตร -8.330531 +at -8.359178 +ล_ -8.362470 +กน -8.438935 +ปร -8.440673 +_ห -8.462052 +จา -8.468067 +d_ -8.479101 +าก -8.479815 +_c -8.505785 +วน -8.508699 +รร -8.517111 +_m -8.521520 +รอ -8.552763 +te -8.556151 +ธน -8.566742 +เท -8.566742 +ed -8.573974 +_ช -8.588161 +ap -8.611860 +าช -8.631977 +ay -8.635955 +ma -8.642342 +บร -8.661675 +re -8.671031 +รก -8.703221 +y_ -8.722970 +ea -8.723816 +cr -8.725934 +_a -8.761993 +ละ -8.790084 +จก -8.800314 +แล -8.847040 +ตล -8.867003 +ชา -8.867003 +นย -8.871217 +อน -8.896283 +ศจ -8.911631 +บา -8.920337 +ฤศ -8.930072 +pr -8.930560 +ทย -8.931538 +l_ -8.933005 +พ_ -8.981273 +อก -9.001145 +_ข -9.007830 +_p -9.010409 +_ย -9.049665 +พร -9.053916 +ณ_ -9.057113 +สง -9.059782 +ri -9.084030 +เว -9.086749 +ชน -9.087838 +il -9.098227 +ปช -9.102624 +ดย -9.139983 +โด -9.145643 +าท -9.146210 +มว -9.150187 +เด -9.154745 +มา -9.163905 +t_ -9.172546 +าม -9.196431 +เอ -9.204678 +r_ -9.212379 +ไท -9.244207 +อบ -9.253955 +จ_ -9.284224 +งห -9.300547 +กษ -9.334404 +วล -9.338936 +ฯ_ -9.344784 +ทพ -9.349348 +ng -9.387063 +สม -9.399858 +สา -9.423727 +ทร -9.426480 +กฎ -9.438240 +ส_ -9.443808 +อา -9.462759 +ลง -9.483394 +กท -9.485546 +กก -9.493463 +ฎา -9.494907 +ถน -9.497076 +รฐ -9.501424 +_ล -9.514546 +ธร -9.520416 +ทม -9.525573 +กด -9.530748 +ซ_ -9.530748 +ชย -9.535198 +ลก -9.552387 +_ไ -9.553891 +สน -9.554644 +มถ -9.556151 +รษ -9.564466 +วฒ -9.569783 +จด -9.589703 +มห -9.594340 +รว -9.595888 +ป_ -9.615390 +หร -9.624055 +นธ -9.632772 +สเ -9.662486 +s_ -9.673891 +มอ -9.690341 +มช -9.691997 +_n -9.694484 +_ผ -9.694484 +pa -9.703638 +เส -9.714533 +ธ_ -9.728905 +หล -9.732308 +รบ -9.759821 +รง -9.778166 +_ศ -9.780806 +_ด -9.805681 +นต -9.809270 +ฒน -9.833730 +อด -9.846579 +ตา -9.846579 +าด -9.855827 +นป -9.865135 +นน -9.866069 +าล -9.876384 +ค_ -9.878267 +ษท -9.884879 +er -9.885826 +ดอ -9.887721 +ทอ -9.896283 +ag -9.903937 +เก -9.906818 +ge -9.931049 +คา -9.933005 +บอ -9.940856 +าง -9.989902 +e_ -9.992959 +ยง -10.003199 +วช -10.007315 +ขอ -10.015582 +ph -10.018695 +อซ -10.020774 +ชว -10.038566 +ะเ -10.038566 +an -10.050196 +คณ -10.054449 +กส -10.066208 +ne -10.073742 +กบ -10.081315 +อส -10.092204 +กต -10.104276 +ธา -10.107587 +ทก -10.109798 +กล -10.117563 +เร -10.122019 +กม -10.124252 +วง -10.133220 +าว -10.142244 +เล -10.142244 +_l -10.143376 +าเ -10.155887 +นก -10.180076 +ปท -10.183565 +la -10.202317 +ศ_ -10.202317 +ลอ -10.216543 +เซ -10.227305 +li -10.230910 +งา -10.232114 +นอ -10.242993 +ดล -10.246637 +รย -10.251511 +_ถ -10.267467 +มต -10.296134 +ษณ -10.301179 +เข -10.301179 +ew -10.315143 +ฐม -10.340883 +สท -10.353927 +ตน -10.376377 +นค -10.379041 +ทา -10.387063 +ผย -10.389747 +_ภ -10.391091 +เผ -10.393783 +มร -10.404600 +ชป -10.405958 +วต -10.418236 +พล -10.429239 +หม -10.432002 +et -10.437546 +าต -10.440326 +นเ -10.443111 +เช -10.444505 +คน -10.445901 +อล -10.450097 +en -10.454305 +วด -10.458526 +เป -10.461346 +พช -10.465587 +พน -10.484111 +ดง -10.492741 +กฯ -10.501424 +ชด -10.502876 +เห -10.510159 +วร -10.511619 +ศร -10.516011 +h_ -10.518947 +ws -10.521888 +hu -10.527788 +บต -10.532230 +รณ -10.541154 +uk -10.545636 +ke -10.545636 +es -10.575883 +sp -10.582009 +pe -10.592792 +_ใ -10.598991 +เต -10.603657 +ชช -10.614605 +is -10.617748 +ธป -10.622475 +ดา -10.625636 +หว -10.641542 +นท -10.644745 +w_ -10.652782 +เพ -10.654395 +นด -10.662486 +sh -10.672256 +_e -10.678806 +คว -10.680449 +รเ -10.687036 +it -10.690341 +ใน -10.706982 +gu -10.727207 +gl -10.730605 +ทท -10.732308 +คล -10.742564 +สว -10.742564 +าธ -10.756353 +ดต -10.759821 +ปด -10.759821 +พท -10.761558 +สอ -10.765039 +tt -10.768528 +โอ -10.775531 +ua -10.779046 +กอ -10.779046 +สต -10.782569 +on -10.787870 +งค -10.787870 +มพ -10.796748 +คอ -10.798530 +ยว -10.800314 +จน -10.802101 +_t -10.805681 +หน -10.805681 +ออ -10.812867 +สร -10.812867 +ขา -10.814670 +ปต -10.820090 +แด -10.820090 +ไอ -10.830992 +_s -10.832817 +ทช -10.838306 +เบ -10.838306 +ะบ -10.845657 +n_ -10.847501 +ss -10.851195 +โท -10.854899 +lo -10.867939 +าะ -10.871686 +a_ -10.875443 +ทศ -10.875443 +ยอ -10.879210 +ใจ -10.884879 +ลต -10.886773 +ฐ_ -10.888670 +จร -10.890570 +_j -10.898193 +ะท -10.900105 +บน -10.902020 +ลย -10.913561 +_d -10.921307 +_ฉ -10.929095 +ศก -10.929095 +อย -10.938890 +p_ -10.976726 +โล -10.982792 +งก -10.986851 +งเ -10.999094 +งน -11.007315 +วจ -11.009377 +อท -11.013511 +ดเ -11.013511 +จต -11.019734 +าพ -11.021814 +el -11.025984 +าห -11.025984 +แส -11.036461 +สย -11.042784 +ดน -11.049135 +th -11.053384 +สด -11.055514 +สพ -11.057646 +ow -11.064063 +อเ -11.064063 +าบ -11.070508 +ศา -11.074821 +_y -11.094391 +บม -11.094391 +ตท -11.098776 +ตว -11.100974 +มธ -11.107587 +รค -11.112012 +สบ -11.120904 +ll -11.123135 +si -11.123135 +วะ -11.127609 +ye -11.129851 +ชร -11.129851 +รป -11.132096 +รส -11.134345 +ญ_ -11.136598 +กย -11.145643 +c_ -11.152464 +าจ -11.152464 +ar -11.163905 +ผบ -11.163905 +ชล -11.168507 +ภ_ -11.168507 +ยม -11.173124 +สภ -11.175437 +ti -11.182401 +จง -11.184730 +_ฟ -11.198783 +เค -11.210598 +ลด -11.212973 +ทน -11.215352 +ถง -11.217734 +_ฮ -11.229707 +รน -11.229707 +แก -11.232114 +ลบ -11.241780 +าส -11.241780 +ปล -11.246637 +_ซ -11.286098 +hi -11.288601 +นส -11.291108 +โร -11.293619 +มจ -11.318974 +วท -11.324099 +ig -11.329243 +ฟ_ -11.329243 +ลม -11.339585 +จบ -11.360494 +อฟ -11.371063 +เฉ -11.373717 +รต -11.376377 +st -11.381710 +กจ -11.384384 +บก -11.387063 +ha -11.389747 +ข_ -11.400534 +gh -11.403243 +ดบ -11.403243 +ะช -11.408677 +ยร -11.411402 +_r -11.419607 +ju -11.438935 +ถา -11.444505 +ขต -11.447299 +ยส -11.452901 +สธ -11.455711 +นร -11.455711 +ro -11.469841 +มท -11.472684 +วย -11.472684 +ตอ -11.475532 +ษต -11.484111 +ชม -11.498524 +รพ -11.498524 +เน -11.501424 +คด -11.501424 +วก -11.504329 +na -11.513082 +นะ -11.513082 +ยเ -11.513082 +โค -11.516011 +นบ -11.524835 +io -11.539663 +ดร -11.539663 +ะธ -11.539663 +ศอ -11.542646 +ou -11.542646 +รธ -11.548633 +ตพ -11.554644 +นพ -11.554644 +ทห -11.560680 +กเ -11.575883 +ะว -11.578943 +ทว -11.591247 +งส -11.591247 +พธ -11.594340 +สข -11.594340 +ภม -11.594340 +ณะ -11.603657 +ทธ -11.606777 +รห -11.609903 +อต -11.609903 +าโ -11.619322 +สก -11.644745 +บด -11.644745 +แร -11.651171 +มค -11.670623 +แม -11.673891 +ดช -11.673891 +po -11.680449 +ษ_ -11.680449 +ตต -11.683739 +ลเ -11.690341 +ฉล -11.713692 +_ง -11.720434 +จย -11.720434 +ผก -11.727207 +ie -11.737427 +ปา -11.740849 +ยบ -11.740849 +wi -11.744280 +าญ -11.754622 +up -11.765039 +รศ -11.768528 +ฟน -11.768528 +ณพ -11.775531 +oc -11.775531 +ec -11.779046 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tsn b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tsn new file mode 100644 index 00000000000..4e02d621b64 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tsn @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 10733133 +#UNIQUE_BIGRAMS 561 +#UNSEEN_LOG_PROB -19.164772 +#EXPECTED_SCORE -6.974867 +a_ -4.256770 +e_ -4.962198 +o_ -5.061185 +ts -5.121185 +_t -5.435721 +le -5.518556 +ng -5.547458 +_m -5.717075 +se -5.727421 +la -5.834412 +_b -5.853575 +g_ -5.867030 +an -5.924429 +el -6.010197 +lo -6.050887 +en -6.086396 +na -6.160850 +et -6.237621 +mo -6.265714 +ha -6.295474 +_l -6.308284 +sa -6.384092 +on -6.506835 +ho -6.540748 +tl -6.545837 +ba -6.586208 +bo -6.606694 +ol -6.776635 +at -6.782646 +_n -6.831302 +_k -6.858246 +_s -6.861306 +ma -6.864357 +di -6.878932 +so -6.906693 +th -6.961725 +ka -6.996264 +_h -7.012261 +i_ -7.071648 +wa -7.118097 +he -7.180116 +al -7.183277 +go -7.199801 +me -7.225762 +re -7.251522 +li -7.302172 +ne -7.312156 +is -7.430222 +_d -7.458928 +or -7.505747 +eb -7.510715 +ot -7.548641 +it -7.584543 +hl -7.620832 +_p -7.643283 +be -7.660965 +ak -7.681652 +te -7.702695 +_e -7.721977 +ga -7.726069 +ta -7.740628 +pa -7.757806 +nt -7.762323 +ko -7.764221 +si -7.773632 +_f -7.835870 +ra -7.850563 +il -7.864059 +ok -7.877177 +ah -7.942833 +ar -7.973282 +ke -7.994275 +we -8.010371 +sh -8.026613 +_g -8.028998 +ro -8.106714 +oa -8.115298 +oh -8.172708 +eh -8.174144 +gw -8.176903 +ab -8.206297 +ag -8.225078 +kg -8.226768 +in -8.233873 +ap -8.245085 +ph -8.261985 +am -8.269556 +ek -8.281426 +fe -8.283477 +_r -8.338760 +_j -8.369193 +to -8.429596 +pe -8.431034 +_i -8.459567 +ae -8.547358 +no -8.560188 +er -8.625257 +om -8.631587 +es -8.652638 +em -8.670545 +ao -8.684469 +lh -8.689177 +op -8.702331 +fa -8.707604 +ny -8.708447 +ik -8.715549 +_a -8.739306 +hi -8.741203 +ut -8.749915 +os -8.776665 +as -8.829130 +sw -8.853669 +eo -8.900819 +mm -8.901847 +oe -8.921526 +kh -8.925311 +ti -8.936200 +po -9.023601 +ir -9.089733 +im -9.093106 +ef -9.096560 +ll -9.097665 +ya -9.098991 +hu -9.129156 +nn -9.175892 +oo -9.188620 +ri -9.265290 +ob -9.284357 +bi -9.287545 +ed -9.315535 +ip -9.463689 +um -9.475219 +od -9.484818 +of -9.530311 +ge -9.541086 +ye -9.636607 +og -9.641752 +ep -9.641967 +ai -9.658275 +eg -9.659578 +u_ -9.694458 +us -9.701715 +lw -9.732715 +ki -9.760310 +ja -9.813624 +ee -9.836915 +ch -9.837653 +je -9.874652 +_y -9.877430 +ad -9.885799 +fo -9.903842 +jo -9.966743 +fu -10.004215 +_c -10.014049 +s_ -10.066558 +kw -10.073059 +aa -10.083667 +_o -10.098770 +mp -10.120601 +hw -10.138519 +_w -10.140793 +oi -10.144897 +fi -10.155282 +bu -10.176436 +yo -10.197274 +af -10.234844 +ea -10.240037 +pi -10.245085 +nk -10.279922 +ni -10.288470 +ul -10.294366 +mi -10.320597 +ru -10.342245 +ib -10.378109 +su -10.402827 +n_ -10.458290 +st -10.460371 +ih -10.540185 +jw -10.548616 +au -10.570524 +ia -10.602978 +du -10.612206 +t_ -10.705189 +pu -10.773426 +tj -10.805543 +y_ -10.868984 +r_ -10.901783 +d_ -10.912624 +tu -10.922243 +_u -10.986243 +_q -10.999391 +ku -11.039286 +un -11.071322 +up -11.073059 +ei -11.084981 +lu -11.089660 +jh -11.106751 +da -11.113883 +l_ -11.137307 +ay -11.216017 +ur -11.300625 +ca -11.329044 +ic -11.358388 +ua -11.359448 +ou -11.424461 +nd -11.434843 +ec -11.437077 +va -11.438569 +io -11.460371 +rw -11.475602 +ov -11.504600 +uo -11.552244 +tt -11.565627 +de -11.572979 +ns -11.585730 +id -11.614522 +if -11.657732 +ve -11.660340 +gi -11.663388 +qe -11.688456 +oc -11.709459 +ac -11.719396 +pr -11.723936 +ie -11.759378 +mu -11.817864 +uk -11.830047 +bj -11.842827 +nc -11.853731 +iw -11.853731 +wi -11.907452 +tw -11.937188 +co -11.938770 +qa -11.941411 +ce -12.021295 +m_ -12.024091 +ow -12.030262 +gk -12.035896 +h_ -12.053500 +tr -12.125547 +fr -12.165126 +ui -12.204551 +k_ -12.230154 +wo -12.252936 +ud -12.276750 +ue -12.287461 +ew -12.345040 +iv -12.379004 +oj -12.389784 +vi -12.407201 +kr -12.416723 +ii -12.449429 +ij -12.467586 +eu -12.509294 +ry -12.533794 +ub -12.558717 +p_ -12.568482 +ss -12.589867 +oy -12.607375 +aj -12.610734 +ci -12.619166 +ck -12.625947 +rr -12.683143 +rt -12.769666 +_v -12.788563 +cr -12.805783 +bs -12.811569 +gs -12.824186 +rs -12.834949 +pl -12.846783 +nw -12.899214 +c_ -12.906419 +ji -12.910553 +bl -12.910553 +do -12.955756 +pt -13.003524 +gu -13.006840 +qo -13.062096 +uf -13.075957 +ig -13.075957 +ju -13.175659 +ct -13.265456 +br -13.278752 +b_ -13.308444 +iu -13.344341 +eq -13.345739 +ty -13.386901 +ev -13.386901 +gl -13.395566 +gh -13.420403 +ej -13.420403 +av -13.430755 +ps -13.451686 +aw -13.456211 +w_ -13.492931 +lt -13.499142 +ey -13.503819 +qh -13.510078 +gr -13.536986 +mb -13.538584 +ly -13.561152 +oq -13.619166 +cu -13.625947 +sp -13.653395 +yp -13.658600 +ft -13.660340 +aq -13.676088 +rd -13.711712 +nl -13.751942 +wn -13.828091 +ls -13.912624 +tc -13.935608 +ex -13.948300 +sc -13.965399 +x_ -14.009054 +ld -14.038155 +nu -14.054072 +kn -14.077118 +rn -14.086441 +dr -14.119554 +pp -14.141249 +rk -14.185643 +py -14.198221 +rc -14.205821 +zi -14.218577 +cy -14.226285 +nj -14.244432 +rl -14.252280 +uc -14.281426 +wh -14.311174 +v_ -14.322145 +uw -14.335977 +sl -14.364046 +ms -14.381153 +az -14.389784 +_z -14.389784 +ws -14.392672 +ze -14.415989 +iy -14.445675 +ds -14.482124 +nq -14.497587 +yw -14.503819 +yu -14.513218 +rv -14.570933 +ks -14.577491 +iz -14.594017 +sk -14.607375 +rg -14.614101 +sm -14.627647 +ys -14.641322 +cl -14.644761 +ug -14.711712 +kt -14.722573 +qu -14.751942 +rm -14.755655 +f_ -14.774367 +fs -14.804821 +kc -14.812536 +iq -14.925115 +cs -14.958963 +zo -14.967551 +sy -14.993624 +ff -14.993624 +ht -15.006840 +dp -15.047229 +lm -15.107640 +dy -15.165743 +cc -15.226285 +fl -15.231447 +td -15.241826 +yt -15.241826 +hn -15.305719 +nf -15.305719 +dd -15.322145 +nz -15.338760 +za -15.395566 +nb -15.497587 +gy -15.510078 +yi -15.567665 +rb -15.567665 +hr -15.574208 +uh -15.587384 +hy -15.594017 +yr -15.600680 +vo -15.627647 +bb -15.662081 +yb -15.669067 +qi -15.669067 +ox -15.770605 +kl -15.778139 +tp -15.793326 +lc -15.832006 +uu -15.847773 +ix -15.971864 +tm -15.989246 +cn -16.042685 +rp -16.051787 +uy -16.088781 +gb -16.098180 +by -16.098180 +bc -16.117163 +zh -16.146115 +dm -16.146115 +xy -16.195697 +bm -16.205821 +tn -16.216017 +my -16.226285 +dc -16.247043 +ky -16.257536 +sx -16.289479 +nv -16.311174 +dh -16.333200 +bw -16.344341 +xt -16.355568 +tf -16.389784 +dn -16.424831 +kj -16.460750 +lk -16.460750 +pm -16.485203 +hs -16.485203 +zu -16.510078 +ux -16.522678 +z_ -16.522678 +xp -16.522678 +gn -16.535389 +sb -16.548213 +oz -16.561152 +sr -16.587384 +xa -16.614101 +yl -16.627647 +md -16.641322 +dg -16.655128 +ax -16.669067 +ez -16.669067 +vd -16.697356 +_x -16.711712 +pd -16.816409 +hb -16.896136 +sq -16.946177 +cp -16.946177 +gt -16.963251 +hg -16.980528 +mw -16.998016 +gg -16.998016 +lf -16.998016 +bd -17.015718 +xu -17.033640 +xc -17.033640 +rf -17.051787 +wu -17.107640 +dv -17.107640 +lp -17.146115 +ym -17.146115 +pf -17.185643 +sv -17.185643 +gp -17.205821 +dl -17.205821 +xi -17.289479 +lg -17.355568 +sd -17.378288 +lv -17.424831 +kb -17.472925 +js -17.472925 +cz -17.522678 +kp -17.522678 +gm -17.522678 +dw -17.548213 +xf -17.548213 +gd -17.548213 +mc -17.574208 +zv -17.655128 +df -17.655128 +lb -17.655128 +tb -17.655128 +tz -17.683143 +kv -17.683143 +sf -17.711712 +q_ -17.740858 +hv -17.800979 +vt -17.800979 +ml -17.832006 +mf -17.832006 +yg -17.863715 +xb -17.896136 +yn -17.896136 +pc -17.896136 +vx -17.998016 +cb -17.998016 +tk -18.033640 +ql -18.033640 +bt -18.070166 +nm -18.107640 +np -18.107640 +cd -18.146115 +hm -18.226285 +gv -18.268105 +vm -18.268105 +mg -18.268105 +gz -18.268105 +sn -18.311174 +xm -18.311174 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tuk b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tuk new file mode 100644 index 00000000000..a52591f16e6 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tuk @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 18991905 +#UNIQUE_BIGRAMS 620 +#UNSEEN_LOG_PROB -19.595924 +#EXPECTED_SCORE -7.527487 +n_ -5.095921 +ar -5.447092 +yn -5.694782 +la -5.900169 +_b -5.910193 +y_ -5.914191 +le -6.026686 +an -6.033934 +a_ -6.052610 +in -6.074334 +er -6.079451 +i_ -6.143395 +da -6.312695 +_g -6.461526 +ya -6.482165 +r_ -6.485830 +en -6.519436 +ly -6.561026 +ry -6.614896 +ny -6.650694 +as -6.670870 +_y -6.694326 +il -6.704938 +ma -6.711132 +ri -6.762867 +_d -6.846154 +al -6.878743 +e_ -6.905386 +_h -6.911262 +me -6.941727 +nd -6.949077 +_t -6.955068 +_a -6.985568 +_s -6.988854 +ni -7.017799 +de -7.026775 +li -7.072758 +ra -7.089545 +un -7.094467 +ur -7.100835 +at -7.118745 +k_ -7.150436 +ta -7.176671 +ba -7.197470 +_m -7.200562 +sy -7.256982 +sa -7.265676 +yl -7.338460 +na -7.358229 +di -7.387642 +_o -7.414411 +et -7.458316 +ga -7.467201 +ol -7.479878 +ag -7.498659 +ir -7.506028 +bi -7.522205 +dy -7.523826 +is -7.530453 +ay -7.571450 +_e -7.601393 +ha -7.650366 +el -7.667685 +am -7.669601 +gi -7.749654 +yk -7.757457 +gy -7.804368 +ak -7.805659 +_u -7.868464 +be -7.897150 +es -7.899706 +bo -7.952055 +ka -7.962173 +ye -7.968267 +yy -7.969999 +or -7.970437 +ek -7.978251 +tu -7.978327 +rl -7.989481 +_k -8.020094 +p_ -8.061360 +ed -8.062152 +t_ -8.072093 +ge -8.075655 +rk -8.103465 +ti -8.107021 +tl -8.136666 +re -8.136816 +si -8.141163 +sl -8.151774 +ne -8.174617 +ad -8.187072 +ow -8.190485 +gu -8.201691 +du -8.213638 +do -8.242105 +ul -8.254602 +ik -8.259017 +kl -8.267864 +go -8.276341 +on -8.284608 +ci -8.285675 +yr -8.293042 +se -8.294234 +st -8.294949 +mi -8.301549 +us -8.332460 +_i -8.346090 +um -8.360749 +rd -8.381650 +ki -8.404506 +em -8.413414 +ey -8.437467 +m_ -8.462624 +yu -8.464367 +ll -8.483381 +rm -8.506428 +ap -8.530889 +nu -8.539286 +my -8.562993 +km -8.612470 +yg -8.638390 +zi -8.641602 +s_ -8.663736 +_c -8.673287 +iy -8.675583 +az -8.676267 +he -8.681591 +ah -8.696073 +ty -8.708508 +ys -8.709271 +eg -8.711849 +_n -8.745068 +ca -8.776902 +_p -8.794131 +ab -8.843526 +ky -8.864263 +yz -8.872250 +rt -8.954729 +mu -8.955558 +nl -8.989790 +lu -9.003917 +su -9.006415 +cy -9.015585 +ld -9.040970 +te -9.048874 +nc -9.101481 +l_ -9.124107 +ke -9.126950 +ez -9.134316 +z_ -9.140777 +lm -9.155387 +je -9.167348 +ji -9.194062 +im -9.202004 +kd -9.209720 +oy -9.270254 +ip -9.279666 +oz -9.334715 +uk -9.336531 +so -9.341401 +zy -9.343768 +ru -9.371074 +td -9.372231 +wa -9.393173 +sd -9.396651 +ec -9.410335 +nj -9.419097 +za -9.426762 +iz -9.455860 +_w -9.465173 +yo -9.466677 +to -9.512824 +yp -9.520558 +id -9.524077 +pa -9.526765 +ok -9.535758 +ze -9.543050 +sg -9.558261 +wl -9.566819 +it -9.569414 +os -9.579085 +ym -9.589347 +ml -9.589581 +ho -9.611579 +ig -9.616281 +tn -9.629097 +yd -9.673752 +ut -9.683339 +pl -9.686840 +uc -9.777935 +op -9.806560 +hu -9.821192 +ko -9.847824 +ce -9.878457 +uz -9.884404 +gt -9.914365 +we -9.916493 +tm -9.934220 +lk -9.948936 +ku -9.950965 +gl -9.983817 +up -10.006376 +ug -10.021771 +_r -10.061806 +nt -10.083814 +rn -10.155473 +_j -10.166693 +yt -10.179145 +hy -10.207517 +jy -10.239302 +ij -10.245467 +ja -10.247497 +sm -10.279241 +mo -10.290423 +ac -10.305533 +_z -10.354619 +pr -10.360399 +pe -10.391591 +lg -10.392407 +ej -10.411007 +_l -10.465173 +tr -10.473141 +zm -10.477358 +zu -10.496874 +pd -10.513990 +og -10.542710 +py -10.573749 +eb -10.585023 +yi -10.595681 +w_ -10.610331 +yj -10.646282 +wr -10.656300 +ss -10.663181 +aw -10.669477 +pu -10.676547 +bu -10.689535 +hl -10.711530 +ns -10.767370 +ls -10.769756 +ks -10.850767 +ng -10.884979 +nm -10.891602 +zl -10.906835 +uw -10.912095 +po -10.927251 +rg -10.943316 +oh -10.943914 +ln -10.970799 +rs -10.988285 +sp -11.031358 +g_ -11.055245 +gr -11.072155 +uy -11.081014 +om -11.094405 +zg -11.118523 +rb -11.135170 +hm -11.138592 +by -11.171504 +ep -11.224140 +ud -11.275376 +nk -11.311216 +ew -11.327523 +rc -11.331628 +gd -11.348366 +ju -11.377173 +uh -11.388940 +zd -11.407185 +wu -11.421700 +yh -11.424203 +cl -11.461205 +no -11.523127 +mm -11.530748 +ib -11.538636 +ob -11.577807 +ot -11.592041 +dd -11.622854 +tc -11.626933 +lo -11.629337 +_f -11.631987 +ro -11.642877 +hi -11.645065 +lt -11.660474 +lj -11.723040 +mc -11.760238 +yb -11.770817 +pj -11.782277 +oc -11.789788 +rh -11.838753 +aj -11.864015 +wy -11.873389 +ub -11.879101 +c_ -11.911509 +pi -11.938984 +ic -11.951265 +cu -11.953975 +mp -12.037413 +eh -12.049920 +mk -12.076249 +wi -12.086454 +mg -12.089100 +kt -12.097398 +hs -12.103068 +rz -12.117510 +cb -12.128012 +hn -12.162073 +od -12.270489 +wd -12.314695 +co -12.325182 +io -12.382030 +br -12.385278 +oj -12.420242 +jl -12.425247 +sc -12.497204 +lb -12.541351 +nr -12.546795 +nb -12.555000 +iw -12.570626 +gc -12.582226 +pg -12.587827 +bs -12.616162 +mr -12.630059 +kr -12.630059 +yc -12.631505 +jo -12.699101 +ia -12.706698 +fi -12.711276 +md -12.723554 +tg -12.726125 +nn -12.779603 +gz -12.837362 +ws -12.887135 +zb -12.903339 +lc -12.929175 +ps -12.976145 +bl -12.982894 +pk -13.013346 +wg -13.052177 +hb -13.061238 +rj -13.076906 +gn -13.111447 +wt -13.116835 +hr -13.184528 +nz -13.223231 +sj -13.262255 +d_ -13.279524 +fe -13.337317 +sk -13.339677 +wp -13.351539 +fo -13.374750 +rw -13.383653 +uj -13.431527 +rp -13.434047 +cm -13.535025 +gs -13.573402 +zo -13.588294 +pc -13.658263 +ww -13.661212 +ii -13.704161 +h_ -13.716379 +jr -13.727670 +sh -13.753665 +wo -13.907418 +eo -13.928583 +rr -13.935707 +sn -13.965777 +fr -14.025329 +zn -14.030405 +af -14.057348 +o_ -14.131757 +ts -14.187359 +np -14.200171 +kg -14.208775 +nh -14.218879 +cd -14.270489 +kc -14.341253 +_x -14.361898 +ms -14.363498 +kk -14.365100 +tb -14.379600 +bm -14.468075 +nw -14.595799 +wc -14.653360 +mb -14.661212 +dm -14.687028 +vi -14.715357 +cn -14.733866 +fa -14.790864 +sw -14.806016 +yw -14.895793 +ou -14.905086 +ea -14.966993 +dr -14.991529 +pn -14.991529 +lh -14.994006 +sb -15.008956 +ph -15.026596 +ih -15.067746 +xi -15.072973 +tj -15.281036 +gh -15.320900 +au -15.374750 +ef -15.400804 +jb -15.407392 +sr -15.417330 +wn -15.457782 +u_ -15.461205 +of -15.495887 +dh -15.502924 +ai -15.506456 +iu -15.535025 +fu -15.571551 +mz -15.597681 +b_ -15.605234 +mt -15.655319 +_v -15.707206 +ua -15.719450 +xv -15.723554 +kj -15.723554 +if -15.821329 +wh -15.893479 +pt -15.950063 +tk -15.954880 +hg -15.959713 +ct -15.974310 +kw -15.974310 +f_ -16.054760 +dw -16.065139 +ht -16.086124 +pm -16.118185 +ck -16.145458 +cs -16.162073 +dl -16.190197 +еr -16.207338 +rf -16.213097 +cg -16.230514 +zh -16.308517 +th -16.384465 +kb -16.430688 +kn -16.437414 +ix -16.471522 +x_ -16.471522 +oe -16.492381 +eu -16.556829 +nf -16.639722 +lp -16.647500 +mn -16.727670 +ds -16.735938 +hh -16.744253 +zk -16.786564 +zc -16.821329 +ae -16.839031 +tt -16.875101 +wk -16.884261 +sf -17.029134 +hc -17.039330 +zt -17.070357 +wz -17.167654 +lw -17.167654 +е_ -17.236367 +hd -17.296238 +bn -17.345991 +ie -17.397522 +gb -17.423994 +mе -17.450961 +yе -17.464636 +еn -17.549525 +tw -17.549525 +nе -17.564171 +lе -17.578968 +tf -17.655319 +dz -17.687028 +pp -17.687028 +zr -17.735938 +pb -17.735938 +iv -17.752617 +v_ -17.752617 +bе -17.875101 +kh -17.930954 +bh -17.950063 +а_ -17.969428 +еk -17.969428 +ch -18.029134 +ан -18.049598 +fl -18.091418 +ao -18.134487 +bd -18.178881 +ft -18.178881 +gf -18.201601 +_一 -18.201601 +一_ -18.201601 +ме -18.201601 +на -18.224685 +ei -18.224685 +еl -18.271991 +xx -18.296238 +mj -18.320900 +dе -18.320900 +gе -18.371526 +hе -18.423994 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tur b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tur new file mode 100644 index 00000000000..d3a1bbea0c7 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/tur @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 239445031 +#UNIQUE_BIGRAMS 583 +#UNSEEN_LOG_PROB -18.767863 +#EXPECTED_SCORE -7.435746 +i_ -5.146975 +in -5.563184 +n_ -5.623362 +r_ -5.858458 +ar -5.889715 +a_ -5.961394 +la -6.051912 +_b -6.061384 +an -6.077834 +er -6.080827 +e_ -6.082493 +il -6.198572 +ir -6.211912 +si -6.294118 +le -6.300356 +ri -6.308697 +li -6.325164 +_k -6.478304 +_s -6.482335 +k_ -6.521574 +_a -6.609842 +_o -6.626938 +de -6.646768 +_i -6.667855 +di -6.670455 +is -6.683850 +_y -6.686540 +en -6.735043 +da -6.782247 +ni -6.791998 +nd -6.848885 +ya -6.852865 +_d -6.859359 +ra -6.875093 +bi -6.884333 +ma -6.910243 +ti -6.925997 +_g -6.951425 +as -6.974903 +un -7.003919 +al -7.037061 +ol -7.046594 +ak -7.128654 +ka -7.134703 +_t -7.151151 +gi -7.247173 +ta -7.306028 +ik -7.316227 +sa -7.320074 +ki -7.340011 +on -7.373017 +ek -7.419386 +el -7.445045 +ur -7.447526 +u_ -7.482813 +or -7.494873 +me -7.502605 +te -7.505529 +re -7.521548 +ay -7.552424 +ci -7.563179 +et -7.564923 +mi -7.567809 +_e -7.568559 +ne -7.603044 +_h -7.615999 +es -7.655451 +st -7.714244 +im -7.730793 +ul -7.755314 +_m -7.791482 +ge -7.795919 +_c -7.856370 +iy -7.868504 +ad -7.874458 +m_ -7.878233 +ha -7.906987 +se -7.910513 +am -7.913360 +ye -7.924418 +ba -7.927450 +na -7.939525 +l_ -7.940928 +lu -7.943683 +du -8.002481 +ru -8.050612 +nu -8.075112 +ce -8.097233 +ed -8.109362 +t_ -8.117120 +at -8.139814 +yi -8.155847 +gu -8.182215 +em -8.213618 +yo -8.249467 +nl -8.254196 +ca -8.262895 +_u -8.273729 +kl -8.310930 +lm -8.324062 +us -8.325938 +ku -8.334982 +bu -8.354331 +ld -8.360027 +tu -8.362869 +iz -8.365275 +ke -8.373732 +ic -8.408038 +s_ -8.431664 +ig -8.452759 +rd -8.489751 +um -8.504415 +so -8.545929 +rl -8.594742 +su -8.610467 +z_ -8.611780 +kt -8.646431 +uz -8.665265 +mu -8.680672 +ko -8.722298 +ap -8.742132 +_v -8.745030 +be -8.783491 +it -8.791059 +ey -8.794773 +_p -8.801218 +yu -8.817794 +uk -8.840108 +az -8.845631 +ah -8.847802 +ac -8.870104 +ll -8.872390 +nc -8.877331 +ze -8.901639 +_f -8.925461 +ve -8.928898 +go -8.958339 +rk -8.962108 +ec -8.981980 +zi -9.018423 +_n -9.058185 +sl -9.086307 +rt -9.087175 +ok -9.098978 +uy -9.105986 +va -9.138885 +ag -9.140989 +yl -9.209122 +ev -9.214473 +hi -9.236714 +ml -9.280411 +do -9.303182 +za -9.307417 +ab -9.312791 +eg -9.314888 +og -9.324602 +oy -9.356598 +rm -9.358626 +cu -9.367852 +id -9.422744 +he -9.429886 +pa -9.435207 +sm -9.447380 +ug -9.480788 +p_ -9.529654 +fi -9.531499 +co -9.541517 +fa -9.568850 +to -9.598470 +rs -9.599399 +_r -9.609300 +gr -9.618146 +pi -9.623778 +bo -9.628210 +sk -9.640661 +lk -9.726365 +tl -9.741882 +oz -9.749196 +ut -9.771266 +ro -9.782721 +af -9.796972 +lg -9.853167 +av -9.883425 +ns -9.883459 +uc -9.895976 +_z -9.897131 +nm -9.914301 +nt -9.972888 +tt -10.012872 +os -10.042621 +ga -10.083981 +tm -10.134564 +ib -10.136747 +pl -10.172778 +ez -10.194295 +ih -10.199160 +ip -10.209667 +_l -10.223864 +gl -10.269958 +zl -10.291113 +y_ -10.302931 +lt -10.341694 +c_ -10.373944 +vi -10.399433 +rc -10.444673 +op -10.450833 +ks -10.469258 +nr -10.475962 +pr -10.532195 +tr -10.652832 +om -10.673958 +po -10.719933 +if -10.723647 +up -10.747418 +fe -10.834679 +yn -10.845359 +hu -10.850734 +eb -10.851402 +md -10.859271 +ny -10.869177 +ng -10.892410 +cl -10.893888 +ot -10.896826 +zu -10.910400 +sy -10.937132 +ud -10.967008 +no -11.008819 +ai -11.015177 +pe -11.018711 +yr -11.021689 +lo -11.032212 +nk -11.047535 +rg -11.072063 +yd -11.079580 +kr -11.096631 +oc -11.100674 +pt -11.123439 +ef -11.126184 +lc -11.168119 +tk -11.175472 +f_ -11.181882 +yg -11.214355 +zd -11.219849 +zm -11.220683 +eh -11.225262 +od -11.228020 +vr -11.242808 +vl -11.244883 +ep -11.296066 +aa -11.302824 +dd -11.347735 +rn -11.358484 +mo -11.379696 +h_ -11.418668 +kk -11.424751 +iv -11.436793 +d_ -11.459757 +ms -11.487117 +o_ -11.487688 +zo -11.545875 +fo -11.548580 +ji -11.582269 +vu -11.690899 +oj -11.691676 +rb -11.705897 +ub -11.743229 +ly -11.778778 +fu -11.805013 +sp -11.890110 +uv -11.904544 +km -11.957643 +ft -12.049616 +pm -12.060358 +kc -12.087634 +uh -12.097265 +ht -12.160817 +mp -12.170117 +ct -12.181938 +g_ -12.246668 +cm -12.304164 +ls -12.359829 +je -12.367259 +hr -12.368979 +ia -12.376905 +kn -12.416081 +uf -12.446833 +lb -12.510552 +hm -12.556379 +hl -12.562599 +v_ -12.566212 +tb -12.572070 +br -12.606338 +fr -12.616405 +ob -12.624562 +_j -12.640055 +dr -12.650515 +ps -12.677022 +nb -12.708737 +mc -12.709383 +dl -12.729211 +ua -12.763112 +zg -12.772146 +ii -12.790384 +tc -12.808336 +th -12.840766 +mk -12.869335 +ss -12.881059 +mh -12.881559 +fl -12.904936 +zy -12.915930 +bl -12.938125 +cr -12.939639 +ho -12.962156 +aj -12.974032 +sc -12.988649 +nn -12.999020 +gm -13.023243 +rh -13.047829 +sb -13.080127 +sv -13.086402 +ov -13.104384 +zc -13.105339 +_w -13.115035 +ys -13.117175 +mm -13.118836 +pu -13.134409 +of -13.144521 +ch -13.151631 +hs -13.191319 +ea -13.191770 +eo -13.227096 +vc -13.229929 +yb -13.230276 +bd -13.260881 +rv -13.273056 +hc -13.297173 +ym -13.302094 +ry -13.304957 +cb -13.308498 +nz -13.313273 +dy -13.386616 +mb -13.401273 +ts -13.404667 +yk -13.446160 +yv -13.466272 +ja -13.467295 +rf -13.526211 +gd -13.538060 +sr -13.555146 +rz -13.558559 +rj -13.588972 +sf -13.666447 +vv -13.775775 +sn -13.777128 +ph -13.796629 +nf -13.796800 +rr -13.799805 +hk -13.875570 +yf -13.898389 +fy -13.900138 +tv -13.909472 +hn -13.911235 +b_ -13.934535 +rp -13.936423 +kd -13.958890 +vd -13.961964 +j_ -13.995522 +mz -14.007280 +sh -14.016937 +pk -14.045993 +ln -14.072737 +oo -14.165127 +vg -14.176908 +my -14.228830 +io -14.230219 +ck -14.271804 +x_ -14.297294 +ty -14.339139 +sg -14.342891 +mr -14.362048 +tf -14.370829 +hp -14.376841 +ej -14.378508 +we -14.407283 +ae -14.426524 +yt -14.448044 +mn -14.456960 +ie -14.502523 +lz -14.524081 +ue -14.536054 +jo -14.543085 +wi -14.559722 +ou -14.570677 +w_ -14.604498 +oh -14.606902 +au -14.647303 +vs -14.663161 +hd -14.664568 +ei -14.674460 +vk -14.690939 +ee -14.723495 +lp -14.748817 +ij -14.799977 +ew -14.817615 +vo -14.829319 +vy -14.851591 +bb -14.854444 +hb -14.864294 +zz -14.906971 +wa -14.921669 +bz -14.963406 +eu -14.969386 +ow -14.984932 +ax -15.052326 +ky -15.057658 +nv -15.079189 +yc -15.084203 +oa -15.190812 +lh -15.212839 +td -15.281970 +ff -15.283411 +yh -15.302763 +tn -15.318188 +kp -15.421491 +cd -15.448179 +kb -15.475919 +ao -15.499171 +ju -15.516294 +uo -15.531053 +pc -15.558704 +dn -15.680618 +yy -15.696208 +wo -15.699410 +zk -15.704870 +jl -15.712938 +hv -15.721052 +bm -15.727248 +vz -15.728229 +kf -15.750643 +lf -15.763322 +ex -15.800320 +mt -15.824591 +ui -15.842535 +py -15.845015 +fs -15.904752 +fk -15.939165 +db -15.966297 +gz -16.001834 +mg -16.019736 +oe -16.040297 +lv -16.045993 +zn -16.073984 +oi -16.076063 +bn -16.079397 +ds -16.143812 +ws -16.177801 +_x -16.275742 +hy -16.305201 +iu -16.336769 +kv -16.354834 +cs -16.415686 +kh -16.420434 +gh -16.503642 +dv -16.512064 +zb -16.529627 +sd -16.547984 +vm -16.561324 +zs -16.575964 +cz -16.575964 +tw -16.652725 +vh -16.684103 +hh -16.774423 +tg -16.789360 +zt -16.794830 +gc -16.840059 +fg -16.847145 +gn -16.859271 +gs -16.926727 +ix -16.951712 +tp -16.966297 +jd -16.978694 +dt -16.984932 +zr -17.007777 +nj -17.063630 +dp -17.073568 +xi -17.216734 +bs -17.247342 +wh -17.248279 +uu -17.266213 +wn -17.290155 +ox -17.292087 +mf -17.297901 +zh -17.330300 +nh -17.349290 +by -17.362428 +dw -17.397367 +sw -17.401534 +pp -17.405713 +gt -17.486391 +bj -17.526780 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/uig b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/uig new file mode 100644 index 00000000000..87d5b453cf5 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/uig @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 12171371 +#UNIQUE_BIGRAMS 1016 +#UNSEEN_LOG_PROB -16.378878 +#EXPECTED_SCORE -7.709933 +_ي -4.681864 +ي_ -5.135792 +لي -5.656921 +يي -5.765035 +_ب -5.794850 +ن_ -5.803695 +ني -5.822347 +يل -5.879070 +ين -6.189938 +يا -6.229538 +ا_ -6.321180 +_ق -6.406780 +ان -6.509244 +ار -6.525990 +ە_ -6.543221 +يد -6.556570 +_ت -6.605708 +تي -6.631742 +ر_ -6.648697 +ري -6.656759 +دي -6.666575 +ير -6.681925 +قي -6.703345 +_م -6.771145 +پ_ -6.824421 +_ك -6.848629 +ەن -6.861307 +لا -6.871028 +ڭ_ -6.929253 +ەر -6.957540 +كي -6.995226 +سي -7.024634 +يە -7.041867 +يڭ -7.044368 +غا -7.056372 +بي -7.145762 +قا -7.197017 +لە -7.229307 +يك -7.278698 +ق_ -7.287412 +ەت -7.315609 +اي -7.333602 +يش -7.338140 +يق -7.352113 +ول -7.374263 +مە -7.429669 +ال -7.433516 +دا -7.435259 +ۇر -7.483847 +مي -7.536372 +دە -7.575267 +يم -7.582815 +يۇ -7.583065 +بو -7.648649 +_د -7.665757 +ۇن -7.668913 +_س -7.681122 +يس -7.691523 +ۇ_ -7.726216 +_ھ -7.733059 +ش_ -7.749315 +گە -7.782179 +شي -7.790973 +يت -7.793153 +تە -7.796575 +تا -7.807022 +ت_ -7.811888 +يو -7.838094 +يپ -7.876881 +يغ -7.880256 +دۇ -7.911137 +ما -7.977701 +چي -8.015235 +با -8.019842 +مۇ -8.023107 +ات -8.023631 +ك_ -8.067601 +را -8.076117 +كە -8.082689 +م_ -8.096087 +اش -8.116600 +اق -8.159913 +يز -8.187984 +يې -8.222334 +ەل -8.239499 +سا -8.252237 +ھە -8.304905 +ۇل -8.315703 +ل_ -8.348477 +زي -8.351339 +_ن -8.360737 +تۇ -8.365186 +ند -8.384030 +_خ -8.401359 +ەم -8.404087 +لۇ -8.457962 +لغ -8.473551 +_ش -8.499828 +_پ -8.510595 +اس -8.524931 +_چ -8.533822 +ز_ -8.537605 +ېر -8.617799 +رن -8.625315 +رل -8.625550 +ام -8.632354 +رە -8.632589 +يۆ -8.634001 +ەك -8.677987 +رۇ -8.708356 +ۋا -8.736442 +ەپ -8.754963 +قۇ -8.793153 +_ج -8.799530 +ەس -8.811675 +نا -8.813541 +اپ -8.832598 +لل -8.835411 +ۇپ -8.838609 +غۇ -8.866221 +ۇش -8.880787 +اد -8.886498 +يگ -8.919349 +ۇق -8.926656 +كې -8.985761 +لد -8.986662 +يۈ -8.987144 +نل -8.994621 +ۈر -9.002868 +چە -9.046890 +تت -9.129057 +ېل -9.129920 +ھا -9.145342 +ۈن -9.159100 +كۆ -9.170734 +پي -9.191930 +ۆز -9.199436 +از -9.205721 +بە -9.213160 +ەي -9.230713 +شق -9.242727 +پا -9.249781 +كۈ -9.259919 +تو -9.260937 +قو -9.262173 +رد -9.268373 +يچ -9.271520 +تۈ -9.273426 +ۇي -9.305617 +شل -9.340079 +ور -9.346932 +سە -9.368316 +اھ -9.377511 +خي -9.380431 +شۇ -9.380589 +گي -9.389863 +نە -9.404328 +جا -9.427811 +وي -9.428137 +تل -9.441921 +بۇ -9.450769 +ەھ -9.467035 +ېت -9.479081 +_ر -9.483062 +ست -9.498927 +ۋە -9.538134 +ون -9.545821 +ھي -9.563650 +كا -9.569132 +تې -9.624380 +قە -9.652341 +ەق -9.673964 +بې -9.685239 +ەش -9.686997 +لم -9.705384 +ۈچ -9.712626 +شە -9.716410 +ۇم -9.770459 +وق -9.791734 +لگ -9.794047 +س_ -9.803126 +جي -9.808643 +_ۋ -9.818991 +رغ -9.829845 +زا -9.836224 +پە -9.838935 +خە -9.850050 +اۋ -9.850707 +ېي -9.880787 +اڭ -9.881346 +رۈ -9.882129 +رت -9.901270 +چۈ -9.928502 +دې -9.936262 +سۇ -9.940100 +ۇس -9.953200 +ۆل -9.994077 +ەز -10.000377 +ەۋ -10.002199 +نۇ -10.027337 +ەد -10.050405 +رق -10.057966 +اك -10.059483 +ۈش -10.060748 +رگ -10.067347 +ۇچ -10.074997 +وم -10.075253 +مل -10.078325 +قل -10.092232 +لس -10.098196 +ۈم -10.134110 +تك -10.135042 +شت -10.137176 +رك -10.138111 +خا -10.138779 +مې -10.156257 +ۈپ -10.172854 +شا -10.189367 +قق -10.191029 +_گ -10.196304 +ۈز -10.205512 +يۋ -10.244235 +قت -10.260573 +غي -10.260864 +جە -10.269031 +ەڭ -10.281960 +_ز -10.289654 +اخ -10.298136 +سل -10.314495 +اب -10.333182 +مم -10.340233 +رم -10.351493 +نچ -10.358946 +تق -10.365030 +يج -10.371767 +ۆر -10.376959 +تن -10.378379 +كك -10.415941 +رس -10.416750 +پت -10.424875 +نغ -10.453177 +ۇت -10.455338 +خش -10.499956 +ڭي -10.500128 +لق -10.507529 +ۇز -10.516008 +وت -10.519658 +ۋي -10.533822 +پۇ -10.556670 +ەب -10.557027 +اغ -10.558993 +چا -10.585339 +وغ -10.647484 +دد -10.669710 +كت -10.696801 +سو -10.698572 +ېگ -10.704494 +ۇڭ -10.743385 +لۈ -10.753172 +_ل -10.757269 +نو -10.758500 +شن -10.763231 +زل -10.766324 +ۆت -10.772531 +لت -10.789635 +نت -10.807368 +ېم -10.813967 +نج -10.814394 +شك -10.815035 +ۈك -10.820169 +چ_ -10.828551 +ېن -10.843937 +گۈ -10.850269 +وڭ -10.852897 +ۈل -10.857288 +ېس -10.866332 +رو -10.880787 +ېڭ -10.900137 +كل -10.913793 +ۆپ -10.920210 +زم -10.932435 +وپ -10.936843 +وخ -10.960740 +زن -10.972839 +ۇغ -10.973792 +سۈ -10.980722 +يب -10.986963 +جۇ -10.995166 +چو -10.995892 +ېك -11.042133 +ېز -11.043383 +زگ -11.043883 +قې -11.054937 +زۇ -11.062015 +دۆ -11.095082 +سې -11.099756 +وش -11.107059 +نن -11.130518 +سۆ -11.194358 +مد -11.215906 +خت -11.228081 +نگ -11.235206 +ەخ -11.239499 +ۇد -11.244379 +نك -11.251008 +زۈ -11.264942 +كو -11.284323 +يخ -11.293517 +ۇھ -11.308771 +تۆ -11.318728 +نس -11.325404 +ھۆ -11.332723 +پل -11.333946 +ڭد -11.341924 +ۆك -11.368943 +ڭل -11.371139 +_غ -11.402883 +رب -11.408350 +ھې -11.419345 +ڭگ -11.461844 +دو -11.480689 +زە -11.491229 +من -11.498069 +غر -11.507011 +ۈت -11.549369 +بل -11.552926 +مك -11.561857 +ۈس -11.601455 +ۇۋ -11.606251 +ېخ -11.614032 +ب_ -11.641413 +اچ -11.655492 +لو -11.662391 +خۇ -11.674738 +سم -11.675901 +ۋر -11.695031 +د_ -11.700938 +گو -11.716012 +ېچ -11.718806 +اج -11.727220 +غە -11.728426 +ڭغ -11.731648 +ڭا -11.745825 +ېق -11.750719 +پۈ -11.791314 +غل -11.795521 +لې -11.812048 +سس -11.844373 +ۆي -11.847864 +ەگ -11.867217 +ېھ -11.878331 +دۈ -11.889080 +زد -11.891330 +و_ -11.908087 +ۆۋ -11.922279 +ھو -11.923199 +وب -11.945466 +غ_ -11.956259 +قن -11.971410 +مو -11.972362 +ېش -11.980003 +كۇ -12.002685 +ېد -12.018827 +مچ -12.141990 +ھۇ -12.161406 +غد -12.161949 +نې -12.171213 +رچ -12.177239 +شۈ -12.183290 +وس -12.192692 +چې -12.194914 +ۋې -12.198252 +زغ -12.199366 +لن -12.202156 +نق -12.224673 +قس -12.225808 +تر -12.230927 +گې -12.240646 +پو -12.245242 +ەچ -12.276069 +سپ -12.336703 +ېغ -12.344696 +ۇك -12.346546 +رز -12.383436 +زو -12.384704 +مۈ -12.388512 +گر -12.392967 +ەج -12.418047 +تچ -12.420644 +لك -12.421945 +يھ -12.422595 +_ف -12.426505 +ۈە -12.448863 +ھۋ -12.450189 +ۇب -12.474268 +ۋۇ -12.503565 +رش -12.527160 +چق -12.531364 +مگ -12.532066 +ود -12.558993 +ھ_ -12.560424 +ۈگ -12.580612 +سك -12.604035 +گۇ -12.623351 +رپ -12.643687 +ۆم -12.668166 +كس -12.676677 +خس -12.679007 +ھم -12.683679 +خو -12.686802 +ۇخ -12.686802 +وۋ -12.701727 +وز -12.708852 +گا -12.716809 +چل -12.721605 +ھك -12.736897 +چۇ -12.736897 +پن -12.737707 +چك -12.749086 +خې -12.757269 +پر -12.758090 +چچ -12.763025 +نۈ -12.782936 +قد -12.783772 +رۋ -12.783772 +رخ -12.784608 +وك -12.784608 +مپ -12.785444 +پپ -12.797208 +ېپ -12.798896 +نم -12.802279 +ۆس -12.803126 +پك -12.804821 +بۆ -12.811622 +ۆگ -12.830492 +ۆڭ -12.833085 +قچ -12.847864 +شر -12.853994 +ھق -12.862796 +لچ -12.866332 +تس -12.936146 +جر -12.940799 +در -12.986242 +ھر -13.003659 +خن -13.005607 +ڭن -13.006582 +نۆ -13.016370 +نز -13.020303 +كر -13.033163 +پې -13.039137 +ەغ -13.069383 +بۈ -13.074486 +ۆچ -13.080634 +شم -13.088872 +مر -13.102360 +ۇج -13.125477 +سق -13.126537 +نب -13.133976 +نپ -13.143598 +وگ -13.165212 +لب -13.178337 +كش -13.198252 +رې -13.198252 +مغ -13.207192 +تم -13.218446 +چر -13.235492 +بد -13.238926 +وھ -13.240072 +مب -13.273719 +في -13.285506 +پچ -13.297390 +ھت -13.304567 +كچ -13.319031 +وچ -13.354594 +اگ -13.369570 +ۆھ -13.379641 +ھل -13.379641 +پم -13.433701 +جب -13.442911 +بب -13.495329 +خب -13.496699 +كن -13.499441 +خل -13.506321 +جت -13.510465 +فو -13.553995 +ڭق -13.564009 +زچ -13.584247 +ېۋ -13.594474 +جۈ -13.601823 +ھب -13.603298 +رج -13.612176 +ۇا -13.642171 +ھس -13.643687 +شې -13.651292 +ڭخ -13.680563 +زر -13.715214 +ڭۈ -13.718406 +قم -13.737707 +ڭە -13.744198 +ج_ -13.750719 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ukr b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ukr new file mode 100644 index 00000000000..381b8b7354c --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/ukr @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 432382186 +#UNIQUE_BIGRAMS 1103 +#UNSEEN_LOG_PROB -16.121059 +#EXPECTED_SCORE -8.131147 +и_ -5.533303 +_п -5.836409 +і_ -5.885806 +_в -6.197838 +о_ -6.317545 +я_ -6.426811 +_с -6.447959 +ро -6.456870 +ст -6.457728 +а_ -6.618663 +у_ -6.628928 +на -6.683547 +но -6.884740 +ов -6.896006 +ко -6.908605 +ра -6.915053 +ни -6.976287 +ер -6.983816 +_р -6.986889 +_з -6.999681 +ві -7.020700 +пр -7.022196 +по -7.075514 +_д -7.075674 +ні -7.086759 +ти -7.123746 +ен -7.135841 +в_ -7.141923 +го -7.146330 +ан -7.163951 +_м -7.171214 +_н -7.175851 +ви -7.211756 +ва -7.261058 +_т -7.277643 +та -7.294493 +ор -7.296185 +ог -7.317780 +_к -7.319966 +ом -7.363846 +ід -7.397054 +ал -7.403068 +ня -7.429836 +ри -7.430124 +_б -7.457239 +во -7.457862 +ре -7.463622 +ли -7.467694 +то -7.477009 +ів -7.482960 +нн -7.489972 +од -7.493982 +ос -7.517269 +ь_ -7.544900 +ть -7.552063 +х_ -7.563989 +ін -7.578009 +_о -7.614838 +ав -7.626810 +е_ -7.676590 +ль -7.678433 +м_ -7.681004 +ин -7.684206 +ат -7.712203 +за -7.765541 +ці -7.770325 +ол -7.784275 +ла -7.821633 +ки -7.846009 +рі -7.880105 +ис -7.907017 +ло -7.913058 +он -7.946961 +ма -7.978768 +ит -7.982151 +лі -8.003598 +ас -8.007740 +ка -8.010410 +іс -8.021569 +до -8.028519 +их -8.030843 +мі -8.058803 +об -8.060456 +ьк -8.092200 +мо -8.121254 +бу -8.153270 +тр -8.166276 +ок -8.188045 +ку -8.222702 +ся -8.229070 +ди -8.238448 +ар -8.249509 +те -8.252696 +им -8.260029 +ті -8.261900 +іл -8.270854 +пе -8.273118 +ии -8.274683 +ик -8.281298 +ю_ -8.293350 +ак -8.293372 +кі -8.295148 +ив -8.298946 +ад -8.310940 +да -8.312224 +ле -8.322713 +сь -8.353583 +_у -8.398152 +ча -8.403087 +_г -8.414716 +_ч -8.423640 +ту -8.436033 +пі -8.438319 +ді -8.449257 +іи -8.460055 +ля -8.465404 +_а -8.487313 +ми -8.494953 +дн -8.500357 +д_ -8.502057 +вн -8.508523 +кр -8.522140 +де -8.524865 +му -8.540072 +не -8.569269 +оі -8.575725 +ув -8.617216 +от -8.640254 +іі -8.658623 +ме -8.677373 +ам -8.692202 +як -8.706888 +_я -8.713256 +_і -8.715009 +ве -8.756260 +іт -8.767388 +н_ -8.771010 +_л -8.800650 +ру -8.808970 +зн -8.826172 +к_ -8.841577 +є_ -8.868184 +ул -8.890941 +се -8.897495 +га -8.917612 +сп -8.921785 +же -8.922078 +оз -8.942870 +ну -8.954678 +сі -8.966980 +ьн -8.976509 +ел -8.991946 +ож -9.007872 +бо -9.013271 +ем -9.030344 +ук -9.047273 +ає -9.074131 +чн -9.074414 +чи -9.095983 +вс -9.109822 +ьс -9.111807 +ду -9.114871 +тв -9.117376 +ед -9.121105 +сл -9.122223 +че -9.122358 +аз -9.146384 +_ц -9.155001 +ою -9.185679 +ил -9.193312 +бі -9.195540 +со -9.216888 +ут -9.221118 +св -9.244256 +ек -9.268405 +нт -9.270079 +сн -9.270197 +си -9.284202 +па -9.308851 +ах -9.312457 +ац -9.314170 +хо -9.326499 +т_ -9.331486 +ет -9.335587 +су -9.375797 +_ф -9.377033 +ьо -9.380435 +гр -9.408046 +уд -9.415808 +аи -9.420166 +ют -9.428280 +ес -9.435859 +ез -9.442975 +оп -9.444495 +кл -9.455449 +кт -9.469616 +ше -9.471149 +оч -9.480293 +ба -9.507495 +аг -9.523446 +аі -9.539188 +ич -9.551679 +бл -9.567861 +р_ -9.585364 +нь -9.588232 +нс -9.591050 +ія -9.593466 +ио -9.595728 +_х -9.607622 +це -9.625981 +із -9.632449 +лю -9.632905 +вл -9.643546 +ач -9.654699 +с_ -9.656488 +ік -9.662206 +ур -9.691774 +са -9.708480 +зв -9.709150 +ск -9.717654 +пи -9.734226 +ши -9.736043 +нк -9.788818 +аб -9.839684 +іч -9.844970 +ир -9.855049 +рн -9.870192 +ус -9.872908 +ім -9.874524 +рт -9.881472 +ид -9.898695 +жи -9.916978 +_и -9.946285 +дл -9.958493 +_ж -9.961294 +бе -9.965284 +ап -9.981560 +аю -9.985015 +ву -9.996281 +уп -10.008709 +жн -10.022150 +би -10.042612 +иц -10.047275 +тн -10.049423 +єт -10.063211 +еи -10.067544 +оц -10.075767 +ж_ -10.078604 +жа -10.085392 +_ш -10.093028 +ць -10.093677 +гі -10.120385 +аж -10.124204 +ев -10.124949 +тя -10.170366 +зи -10.173388 +лу -10.180022 +ір -10.223102 +из -10.223294 +зо -10.230692 +иш -10.234365 +зу -10.236960 +що -10.250813 +ум -10.257933 +тк -10.259638 +пл -10.278938 +чо -10.296887 +ят -10.300085 +нц -10.305234 +др -10.310314 +рм -10.311903 +іш -10.343913 +_е -10.347265 +дж -10.364989 +фі -10.370178 +рс -10.374632 +ря -10.399920 +нд -10.400218 +бр -10.400367 +рш -10.429046 +іо -10.429120 +тт -10.448478 +зм -10.485398 +з_ -10.491231 +ує -10.507517 +ям -10.513464 +іх -10.529832 +ох -10.530671 +шо -10.544354 +еж -10.549535 +ун -10.552293 +уч -10.552794 +иб -10.563388 +пу -10.565773 +ьш -10.582222 +хі -10.609894 +ип -10.622410 +рг -10.628113 +фо -10.628488 +яд -10.642964 +дк -10.667853 +нш -10.669075 +дв -10.674453 +гл -10.674916 +іа -10.679156 +зі -10.725655 +юд -10.731390 +ян -10.750168 +уж -10.782645 +вж -10.801374 +еб -10.817891 +б_ -10.831039 +яв -10.846770 +_щ -10.856821 +іб -10.864246 +ец -10.872586 +зе -10.879421 +дп -10.885910 +гу -10.896639 +ха -10.911864 +аш -10.920550 +вт -10.929723 +зб -10.937051 +ег -10.944462 +чі -10.947154 +мп -10.950359 +зр -10.950584 +вч -10.953280 +жу -10.954856 +іж -10.958350 +л_ -10.960370 +дс -10.970738 +іє -10.972702 +ію -10.977235 +уг -10.982025 +ке -11.014443 +іг -11.016972 +вд -11.056462 +ги -11.057621 +дя -11.063479 +зд -11.077716 +ош -11.091229 +еч -11.112141 +ищ -11.113567 +ші -11.130627 +сц -11.136594 +кс -11.155932 +еп -11.183580 +ще -11.194995 +рк -11.201345 +шк -11.218710 +ша -11.227068 +зп -11.227884 +ця -11.237697 +рж -11.242337 +ую -11.247787 +вр -11.259314 +ге -11.270580 +фе -11.270976 +вк -11.293705 +уз -11.318058 +юч -11.326293 +ци -11.329793 +юв -11.340716 +_є -11.356745 +кв -11.362839 +иг -11.367478 +мл -11.422712 +ч_ -11.426013 +єв -11.430123 +уб -11.470247 +оє -11.472483 +ху -11.475083 +ьт -11.475606 +чу -11.476595 +іц -11.507356 +яг -11.543303 +шл -11.550160 +шн -11.555784 +кц -11.563277 +хн -11.568193 +иж -11.568284 +бн -11.571215 +єм -11.581864 +цт -11.598137 +рд -11.606508 +шт -11.618870 +ео -11.620455 +ню -11.622295 +ял -11.623505 +жл -11.643071 +г_ -11.645356 +вп -11.654825 +пн -11.656192 +жі -11.703692 +ях -11.729009 +шу -11.732150 +зг -11.745790 +дб -11.762931 +ух -11.806235 +цю -11.825216 +ш_ -11.828998 +ех -11.829034 +мк -11.832328 +фа -11.846487 +рх -11.848024 +фр -11.852631 +мн -11.859112 +зь -11.886807 +дм -11.889425 +оф -11.893748 +ои -11.903225 +аф -11.912983 +кн -11.915624 +ау -11.921449 +яч -11.922588 +єю -11.926336 +еа -11.941296 +хи -11.964631 +иє -11.991117 +жо -11.993782 +ьм -12.001341 +яє -12.016315 +рв -12.018016 +вв -12.029897 +ює -12.054708 +ия -12.098168 +кщ -12.099940 +тю -12.168526 +оя -12.182448 +нф -12.186594 +ое -12.196129 +щи -12.210622 +єі -12.211049 +ща -12.260468 +чк -12.261729 +дт -12.271974 +дь -12.272650 +см -12.290375 +_ю -12.296673 +уш -12.299075 +еш -12.314731 +хт -12.324812 +сх -12.329647 +хв -12.333982 +вз -12.337121 +гн -12.376232 +лк -12.429589 +мс -12.432261 +вя -12.433811 +уа -12.474685 +ая -12.481023 +рп -12.483275 +нг -12.489076 +еф -12.515676 +п_ -12.516242 +рю -12.518394 +фу -12.537667 +иі -12.583932 +ощ -12.595099 +ср -12.598716 +ея -12.598819 +вц -12.609310 +дд -12.610207 +єд -12.652031 +зк -12.679369 +хр -12.729836 +бс -12.748497 +ащ -12.753857 +рл -12.754064 +еі -12.761714 +нч -12.791636 +тл -12.816910 +ею -12.835714 +щі -12.846143 +зл -12.887919 +рч -12.893748 +шв -12.894002 +яр -12.913537 +жк -12.925532 +жд -12.926674 +дз -12.963458 +чл -12.968450 +чч -12.971582 +яю -12.991416 +ьб -12.996125 +юю -13.006054 +бх -13.007922 +жб -13.009737 +дч -13.032341 +вш -13.034467 +тс -13.043876 +i_ -13.067541 +яц -13.067942 +єн -13.075526 +ii -13.103180 +іщ -13.117253 +зя -13.130029 +іп -13.133563 +зт -13.163956 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/urd b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/urd new file mode 100644 index 00000000000..bac753daf1f --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/urd @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 45545586 +#UNIQUE_BIGRAMS 1674 +#UNSEEN_LOG_PROB -14.734752 +#EXPECTED_SCORE -8.195356 +_ا -5.016330 +ي_ -5.176435 +_م -5.342287 +ں_ -5.581978 +ر_ -5.737268 +ے_ -5.858194 +ا_ -6.012027 +يں -6.124042 +_ك -6.202412 +ان -6.226513 +مي -6.252316 +_ب -6.354298 +اي -6.407675 +ہ_ -6.436689 +ور -6.577584 +ن_ -6.597442 +ت_ -6.620286 +يا -6.797019 +ار -6.829405 +_ت -6.888420 +ري -6.906893 +او -6.967051 +_ہ -6.993672 +_س -6.999704 +د_ -7.151762 +_پ -7.159095 +ل_ -7.174575 +ال -7.175065 +م_ -7.267838 +_ج -7.268136 +ام -7.271627 +لا -7.280941 +لي -7.284117 +_د -7.297335 +را -7.320773 +وا -7.360199 +_ر -7.420734 +ات -7.435315 +_ن -7.453727 +يك -7.519165 +وں -7.527305 +_و -7.530218 +ہي -7.585428 +ني -7.613248 +ين -7.631680 +ك_ -7.654360 +دي -7.670280 +تا -7.679348 +_ل -7.692674 +ما -7.719669 +اد -7.733442 +_ع -7.796712 +كي -7.802195 +ہو -7.804148 +با -7.817069 +يے -7.862376 +يي -7.879430 +اس -7.903392 +_گ -7.912980 +كر -8.018242 +اب -8.043793 +نا -8.047526 +ست -8.084330 +سي -8.090378 +نے -8.092400 +_ش -8.148253 +يل -8.160617 +جا -8.161687 +رو -8.166402 +ہا -8.169938 +_ح -8.193919 +ھي -8.225351 +دا -8.239804 +ير -8.243668 +تھ -8.247956 +تي -8.306221 +_ف -8.318020 +اق -8.323966 +_خ -8.329509 +كا -8.341101 +سا -8.426243 +بھ -8.439125 +وي -8.443011 +ون -8.446090 +مل -8.468703 +س_ -8.471804 +نہ -8.475170 +كہ -8.481044 +گي -8.483377 +ع_ -8.494907 +_ق -8.518630 +رہ -8.518804 +ند -8.523319 +اك -8.557210 +اہ -8.585489 +ھا -8.587632 +يو -8.588071 +ب_ -8.598126 +_چ -8.611246 +پا -8.667154 +اف -8.671629 +مر -8.699486 +مو -8.740382 +يس -8.748111 +عل -8.751684 +رت -8.755101 +ف_ -8.792858 +فر -8.860211 +كو -8.890317 +زي -8.893989 +در -8.899530 +لو -8.900892 +بي -8.909518 +دو -8.928963 +گر -8.938535 +نگ -8.988599 +نو -9.004975 +ز_ -9.012497 +_ص -9.016756 +ول -9.020157 +مت -9.061379 +يم -9.080269 +كس -9.097674 +ٹر -9.100314 +وم -9.102193 +يٹ -9.106081 +رن -9.116750 +من -9.133394 +بر -9.136243 +يد -9.144376 +رك -9.166157 +_ز -9.183623 +ھ_ -9.185023 +دہ -9.185742 +سل -9.215770 +تے -9.217617 +يش -9.236275 +كھ -9.251061 +ٹ_ -9.293046 +رد -9.310318 +نس -9.310418 +_ط -9.311042 +و_ -9.327554 +لے -9.336803 +اپ -9.339632 +قا -9.344752 +قع -9.346483 +ٹي -9.354195 +از -9.381168 +عد -9.390130 +پن -9.397375 +اع -9.414847 +يت -9.423782 +ق_ -9.433343 +بل -9.444665 +بہ -9.445659 +وس -9.453677 +سر -9.481576 +_ي -9.485816 +شا -9.498889 +پي -9.577856 +لك -9.588571 +وج -9.589400 +مش -9.590059 +حا -9.597476 +سك -9.613118 +ود -9.615327 +زا -9.615601 +لم -9.627478 +بع -9.634007 +جم -9.638594 +تر -9.648602 +عا -9.662885 +بن -9.665532 +تم -9.667025 +شت -9.671822 +ہر -9.681856 +گ_ -9.696736 +لہ -9.700158 +ح_ -9.722917 +يہ -9.724042 +تع -9.738419 +رب -9.756032 +قي -9.762455 +رے -9.766588 +يز -9.771757 +خا -9.780366 +نت -9.783350 +ش_ -9.790037 +شي -9.815984 +_ڈ -9.827997 +نٹ -9.829120 +طا -9.839298 +تح -9.851508 +اں -9.861787 +مج -9.891324 +وت -9.896658 +عي -9.900074 +وب -9.904409 +جن -9.910463 +جو -9.911103 +وز -9.917338 +مس -9.939499 +وع -9.955828 +سو -9.958031 +شر -9.960144 +كن -9.968561 +في -9.973999 +ہن -9.975656 +ھے -9.994338 +مع -9.997962 +بو -10.000620 +سم -10.001724 +فا -10.003251 +صو -10.013855 +چھ -10.018940 +ہم -10.037563 +كت -10.047853 +تو -10.047920 +رف -10.054979 +گا -10.055990 +كم -10.057407 +ہل -10.059197 +خو -10.066516 +حك -10.084769 +پو -10.095022 +رم -10.097345 +عم -10.099498 +قب -10.100262 +مد -10.111327 +پر -10.122971 +اح -10.147049 +حم -10.157032 +جي -10.163375 +رس -10.169418 +صي -10.174461 +شہ -10.188364 +چا -10.207825 +طر -10.214245 +جہ -10.216618 +لن -10.226071 +ہت -10.229794 +نك -10.257211 +پہ -10.274763 +رق -10.275037 +اج -10.278967 +تہ -10.284053 +مہ -10.286346 +وگ -10.288087 +شن -10.291497 +صل -10.291736 +_ٹ -10.300657 +ج_ -10.301936 +سٹ -10.305981 +خت -10.308188 +مح -10.316081 +اص -10.320368 +سن -10.320975 +قو -10.334204 +تف -10.337725 +صد -10.358035 +وق -10.374508 +سے -10.387391 +ڈي -10.389429 +قت -10.393556 +نظ -10.403561 +اخ -10.416058 +ھو -10.419741 +ھر -10.424522 +قر -10.428882 +چي -10.429668 +فو -10.435008 +يب -10.435183 +ہے -10.447471 +ڈ_ -10.453322 +فت -10.462276 +گو -10.516738 +اش -10.525769 +مق -10.530774 +رٹ -10.544665 +كل -10.546227 +اگ -10.551827 +قہ -10.554349 +نڈ -10.557353 +خل -10.567748 +فص -10.571359 +وٹ -10.614309 +عر -10.631592 +_غ -10.655253 +وہ -10.661345 +دھ -10.664272 +رح -10.667823 +رج -10.681440 +ه_ -10.702241 +يف -10.709224 +يع -10.729894 +بڑ -10.745416 +لت -10.749392 +يق -10.756333 +بق -10.758252 +مط -10.766781 +حد -10.782094 +بت -10.784215 +كے -10.786955 +شم -10.806791 +سط -10.810711 +مب -10.811224 +ٹا -10.818527 +_ذ -10.824489 +اط -10.865150 +ڈا -10.879162 +طي -10.899288 +دن -10.905350 +لب -10.906505 +ٹھ -10.918165 +ظا -10.953910 +صا -10.958378 +يڈ -10.966595 +وش -10.969069 +عت -10.971802 +تن -10.979328 +عو -10.982017 +تق -10.995534 +چ_ -10.996311 +طو -11.004942 +نچ -11.009180 +يج -11.018283 +_ض -11.019729 +قد -11.035932 +تل -11.047820 +نج -11.055485 +صر -11.055754 +اٹ -11.059197 +يگ -11.072097 +وك -11.072643 +nt -11.079207 +بد -11.082363 +حر -11.084975 +غا -11.087041 +رل -11.090974 +لف -11.130479 +كش -11.143820 +نم -11.144465 +مك -11.148199 +لق -11.158661 +تب -11.172778 +شك -11.179962 +اض -11.183273 +لس -11.221186 +دد -11.230669 +زم -11.238684 +ھن -11.239833 +ہد -11.241289 +_c -11.241749 +پھ -11.247591 +وف -11.248438 +جھ -11.261443 +ڑي -11.272136 +لگ -11.282435 +كٹ -11.312008 +طح -11.317980 +چو -11.319922 +مخ -11.320651 +جر -11.331875 +حي -11.346648 +غي -11.348711 +گھ -11.351689 +فل -11.366081 +دم -11.369178 +عہ -11.388495 +in -11.391215 +جل -11.391300 +حق -11.414284 +ضا -11.437114 +ضر -11.440984 +ڑا -11.456478 +كچ -11.459597 +بك -11.480534 +حت -11.484886 +خي -11.500311 +تخ -11.501137 +جب -11.504630 +دگ -11.511734 +سپ -11.513215 +حس -11.519060 +ٹو -11.519990 +عظ -11.524182 +لل -11.530634 +ڑھ -11.535891 +نب -11.538715 +ہز -11.549404 +عن -11.553968 +لد -11.555398 +مز -11.560842 +مم -11.566307 +ظم -11.570925 +دل -11.572661 +n_ -11.583313 +ظر -11.585453 +لع -11.602983 +شو -11.603969 +عب -11.607424 +خ_ -11.608412 +زن -11.609501 +t_ -11.621028 +رپ -11.629333 +ض_ -11.634560 +خر -11.639504 +پس -11.641931 +ط_ -11.644971 +an -11.645580 +ھت -11.658117 +اء -11.664169 +نر -11.664478 +ثر -11.667669 +بح -11.673348 +نش -11.674590 +حص -11.683314 +يخ -11.697341 +un -11.720136 +يص -11.722275 +گل -11.722810 +شد -11.725811 +y_ -11.732155 +وڑ -11.737121 +زر -11.741127 +زب -11.759679 +لح -11.774251 +نق -11.774695 +پ_ -11.794249 +قل -11.804750 +لز -11.812135 +ص_ -11.813046 +طل -11.813160 +رز -11.822996 +وپ -11.830706 +ذر -11.836255 +جس -11.839850 +خص -11.843337 +تص -11.847300 +سف -11.848935 +اچ -11.850338 +دس -11.860196 +ظ_ -11.865269 +e_ -11.866687 +حو -11.881670 +نص -11.881670 +اث -11.897051 +ڑے -11.906748 +ضل -11.925724 +co -11.931651 +پڑ -11.946452 +اڑ -11.947828 +چك -11.949581 +جد -11.952466 +ou -11.957496 +on -11.960270 +ٹن -11.961786 +زو -11.962165 +چن -11.969895 +ty -11.974859 +نف -11.980480 +a_ -11.983427 +غر -11.985224 +گن -11.989082 +فہ -11.991014 +دت -11.997217 +اغ -11.997605 +_s -11.998512 +مذ -12.007353 +خط -12.023219 +اظ -12.034204 +ذا -12.038062 +ث_ -12.039662 +ھل -12.059671 +قص -12.070938 +وچ -12.072574 +رش -12.077084 +صف -12.088489 +حب -12.096928 +ء_ -12.102211 +تج -12.128634 +رض -12.129343 +خب -12.134461 +چل -12.142316 +ڑ_ -12.145039 +گز -12.148630 +حل -12.154683 +رگ -12.160037 +رچ -12.160327 +مص -12.164538 +رر -12.176365 +رڈ -12.184010 +لط -12.193474 +_ث -12.198378 +ثا -12.198378 +بچ -12.200464 +سب -12.200911 +لر -12.205392 +وے -12.206889 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/urd-rom b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/urd-rom new file mode 100644 index 00000000000..a66e14dfcc1 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/urd-rom @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 193758092 +#UNIQUE_BIGRAMS 1122 +#UNSEEN_LOG_PROB -17.186581 +#EXPECTED_SCORE -7.618783 +ha -5.260562 +i_ -5.506125 +a_ -5.615760 +_a -5.651258 +ar -5.670718 +n_ -5.678466 +_k -5.898777 +_m -5.952254 +r_ -5.963843 +an -6.050033 +_h -6.079732 +_s -6.087763 +y_ -6.151980 +ma -6.188928 +ai -6.282870 +al -6.423282 +ra -6.488304 +ay -6.531714 +at -6.580943 +in -6.591180 +am -6.605975 +ah -6.615474 +_b -6.624960 +e_ -6.640217 +t_ -6.644335 +ta -6.650170 +la -6.657782 +na -6.687262 +ka -6.739885 +sa -6.741729 +d_ -6.795135 +h_ -6.881129 +_p -6.919913 +_t -6.961766 +ba -6.991576 +ad -7.050489 +sh -7.111872 +_n -7.146804 +m_ -7.188154 +l_ -7.225016 +hi -7.231843 +is -7.302164 +kh -7.303314 +_d -7.322550 +ur -7.344658 +as -7.357962 +aa -7.369894 +wa -7.381903 +s_ -7.402798 +ab -7.436563 +_r -7.487327 +_j -7.496153 +ee -7.507771 +er -7.523122 +_i -7.630271 +me -7.645511 +da -7.648534 +_l -7.649330 +li -7.685567 +ri -7.705854 +ja -7.712011 +az -7.714961 +ak -7.802386 +re -7.814356 +ya -7.852120 +ho -7.867110 +ch -7.901257 +pa -7.926399 +za -7.948587 +on -7.987621 +_w -8.001739 +mu -8.020320 +oo -8.093437 +ki -8.101072 +he -8.109145 +st -8.114133 +il -8.142202 +hu -8.143002 +au -8.147167 +_f -8.154515 +_g -8.171730 +di -8.174993 +ti -8.197487 +or -8.218150 +_c -8.221467 +ni -8.232461 +fa -8.237562 +si -8.254484 +eh -8.255945 +b_ -8.281707 +en -8.284212 +th -8.335781 +_u -8.355609 +us -8.382253 +ir -8.402312 +ll -8.461722 +ye -8.461974 +te -8.510954 +k_ -8.518585 +de -8.522736 +ne -8.538281 +ia -8.582187 +mi -8.593423 +du -8.601793 +ap -8.617710 +ga -8.625605 +nd -8.640189 +_z -8.668231 +es -8.679391 +af -8.693958 +un -8.700200 +ul -8.703810 +ag -8.736408 +um -8.739797 +ed -8.744433 +z_ -8.784972 +ik -8.801721 +u_ -8.803257 +o_ -8.813297 +rd -8.836058 +ro -8.840960 +qa -8.845316 +id -8.882666 +aw -8.891698 +iy -8.899990 +le -8.916690 +_q -8.922647 +ei -8.924933 +aq -8.935510 +se -8.937480 +bh -8.989917 +ot -9.012619 +oh -9.040374 +f_ -9.053054 +bi -9.102401 +aj -9.142952 +mo -9.145159 +ht -9.166137 +_y -9.179813 +lo -9.197645 +it -9.207903 +gh -9.211262 +im -9.213400 +ke -9.215948 +to -9.220996 +zi -9.288164 +mm -9.289971 +ng -9.293873 +pe -9.312680 +ss -9.337803 +rn -9.348811 +ea -9.397423 +em -9.411014 +g_ -9.475247 +be -9.487787 +su -9.489520 +hm -9.495936 +if -9.540287 +ji -9.542573 +ey -9.577156 +so -9.580767 +rt -9.595000 +sl -9.603443 +ko -9.619390 +ol -9.647671 +uh -9.659834 +q_ -9.660341 +el -9.664262 +p_ -9.684106 +ac -9.718670 +do -9.731691 +ny -9.741218 +ou -9.747694 +bo -9.756156 +ic -9.756194 +_e -9.760205 +fi -9.766171 +nt -9.769676 +ry -9.795647 +io -9.925548 +vi -9.930248 +rh -9.939153 +_o -9.940484 +os -9.941274 +ib -9.959277 +pn -9.973026 +ve -10.001960 +iq -10.012227 +et -10.033264 +om -10.041166 +bu -10.042375 +qu -10.061742 +tu -10.065854 +ut -10.104547 +no -10.133261 +ns -10.143869 +iz -10.143987 +ku -10.163410 +oi -10.204306 +kr -10.207788 +od -10.213046 +ub -10.220809 +ud -10.222498 +ge -10.223673 +po -10.236192 +hr -10.239825 +og -10.273749 +ty -10.286303 +gi -10.290586 +pr -10.327634 +ly -10.339413 +tr -10.364146 +j_ -10.383292 +ek -10.387734 +gu -10.390170 +uj -10.395155 +co -10.412140 +ua -10.438246 +br -10.439362 +ow -10.440665 +rs -10.455059 +uk -10.457549 +_v -10.465296 +rm -10.504758 +fo -10.507335 +pu -10.511872 +nu -10.533583 +ph -10.549966 +hn -10.560655 +ae -10.591503 +ew -10.605542 +jh -10.607840 +hy -10.614455 +ie -10.659004 +ca -10.663658 +zr -10.680227 +rk -10.715299 +ru -10.721396 +ct -10.732501 +uc -10.752298 +sy -10.757212 +fr -10.760722 +hl -10.762040 +ec -10.772343 +ce -10.793491 +ju -10.799530 +sk -10.804821 +pl -10.808929 +ob -10.850586 +tt -10.851852 +c_ -10.871051 +nn -10.879695 +go -10.881211 +ze -10.881998 +sm -10.899784 +dr -10.901407 +pi -10.917504 +lt -10.919175 +we -10.946657 +mb -10.948775 +ky -10.999793 +eb -11.008725 +w_ -11.031549 +dh -11.041559 +wi -11.042249 +kt -11.043443 +oa -11.050564 +wo -11.060612 +fe -11.063573 +oc -11.063764 +lu -11.100209 +nk -11.121086 +zo -11.122812 +ok -11.133746 +sr -11.136660 +nc -11.157428 +bb -11.159505 +nh -11.164213 +ez -11.187572 +mr -11.212198 +ig -11.213647 +qi -11.214461 +rf -11.221627 +gr -11.245436 +lm -11.280309 +ws -11.309870 +hs -11.317261 +eo -11.321676 +ov -11.340590 +ip -11.342445 +uf -11.345736 +ts -11.372532 +ex -11.377951 +bl -11.384625 +op -11.398869 +mz -11.403259 +yo -11.409119 +ci -11.427460 +pp -11.449613 +nj -11.457674 +ft -11.483026 +je -11.507790 +uz -11.522347 +x_ -11.542195 +jo -11.547936 +lk -11.556746 +pt -11.561330 +dd -11.571174 +ih -11.601070 +rr -11.617540 +ef -11.620398 +rc -11.621617 +av -11.627872 +ao -11.629947 +rb -11.632450 +bd -11.661196 +ks -11.685662 +ps -11.690773 +eg -11.693384 +tn -11.705220 +ue -11.717303 +لي -11.722025 +up -11.735773 +nl -11.747093 +zu -11.747656 +iv -11.751553 +oz -11.765743 +ld -11.835868 +qt -11.843400 +of -11.861794 +tk -11.876668 +ep -11.890284 +qb -11.948304 +ls -11.961369 +uq -11.982305 +qe -12.002815 +cl -12.004405 +iw -12.043161 +ck -12.059150 +ال -12.080920 +eq -12.091214 +bt -12.095183 +rg -12.101845 +nw -12.102107 +pk -12.125138 +rz -12.134147 +qs -12.136224 +hw -12.141059 +sp -12.151793 +ij -12.161243 +ms -12.175639 +rw -12.177155 +fu -12.178190 +wn -12.199535 +ug -12.212976 +ff -12.217153 +lz -12.265825 +bn -12.275908 +lp -12.277238 +mj -12.278717 +uw -12.284723 +ml -12.328400 +oe -12.346821 +fs -12.353041 +sc -12.359287 +my -12.378903 +mp -12.400479 +hb -12.426066 +cr -12.454286 +gs -12.460483 +ds -12.469323 +yu -12.506360 +لل -12.512960 +hk -12.554460 +nz -12.570676 +ui -12.573759 +م_ -12.575940 +sw -12.591664 +bs -12.631552 +له -12.632687 +yn -12.639607 +mh -12.646179 +oj -12.660280 +oq -12.672478 +عل -12.697780 +hz -12.716401 +hd -12.734758 +zh -12.742697 +mn -12.744127 +tm -12.756954 +zm -12.779079 +ev -12.787477 +gl -12.791061 +sn -12.794972 +zz -12.795078 +py -12.825453 +سل -12.830326 +ln -12.838265 +rv -12.839683 +ww -12.840885 +gy -12.843619 +hf -12.868571 +yi -12.874710 +_ص -12.892377 +لم -12.899983 +صل -12.911181 +yy -12.915432 +rp -12.938743 +dm -12.942021 +va -12.943780 +وس -12.944249 +jy -12.955324 +_ا -12.968871 +tc -12.980017 +يه -12.984958 +rl -12.989796 +gn -13.015954 +jr -13.023126 +pd -13.032328 +sb -13.057371 +ys -13.061821 +wh -13.097139 +oy -13.101714 +fz -13.111302 +lb -13.118833 +tl -13.119230 +lf -13.140453 +md -13.149085 +يا -13.166779 +jj -13.168286 +rj -13.228614 +dy -13.233769 +gt -13.239662 +هع -13.259386 +هو -13.259386 +ي_ -13.271557 +km -13.275982 +cu -13.276277 +qo -13.308094 +nf -13.331697 +lh -13.339702 +yr -13.346356 +kb -13.348374 +pm -13.350083 +dn -13.408310 +hq -13.410578 +ه_ -13.422955 +df -13.423282 +lv -13.423282 +dl -13.429183 +v_ -13.449363 +cs -13.464602 +vo -13.477113 +np -13.533208 +sj -13.538868 +zy -13.548828 +yk -13.550613 +kk -13.550613 +mt -13.567689 +kl -13.589001 +cy -13.608398 +uy -13.612309 +gg -13.655084 +tw -13.664915 +wr -13.711898 +nv -13.765432 +dg -13.787582 +iu -13.788004 +_ع -13.830326 +qr -13.831845 +nq -13.841868 +rq -13.884698 +zw -13.891924 +lg -13.892151 +uo -13.941904 +cc -13.941904 +kn -13.970304 +sg -13.986408 +ej -14.016694 +fd -14.092189 +ii -14.096618 +mk -14.117111 +jl -14.149220 +jm -14.160357 +bz -14.160902 +fl -14.174330 +yd -14.180124 +ہ_ -14.217366 +zv -14.257927 +nm -14.298760 +ax -14.299060 +by -14.312026 +qm -14.319010 +ym -14.350394 +ux -14.401687 +xp -14.428034 +jd -14.429676 +qv -14.441556 +qd -14.452532 +wt -14.455205 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/uzb b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/uzb new file mode 100644 index 00000000000..49f4b448e8d --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/uzb @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 14522411 +#UNIQUE_BIGRAMS 1305 +#UNSEEN_LOG_PROB -14.136219 +#EXPECTED_SCORE -8.517072 +i_ -5.553373 +la -5.963753 +ar -6.050969 +a_ -6.062117 +an -6.285641 +и_ -6.397198 +sh -6.464699 +da -6.506827 +n_ -6.526547 +li -6.737968 +il -6.766465 +а_ -6.795735 +_b -6.816053 +ri -6.893129 +ga -6.897339 +in -6.955012 +ni -7.011584 +is -7.036147 +r_ -7.058021 +н_ -7.108715 +_t -7.168382 +at -7.178492 +_к -7.201980 +ла -7.204000 +ан -7.222931 +ti -7.265936 +ар -7.267267 +as -7.287486 +di -7.290422 +_m -7.291511 +ol -7.365578 +_б -7.387484 +да -7.390365 +ma -7.391148 +al -7.441702 +si -7.456771 +iy -7.479231 +ha -7.483901 +_k -7.489389 +_a -7.497570 +га -7.499384 +_s -7.503434 +ta -7.533138 +_o -7.541442 +_y -7.564312 +_q -7.586156 +hi -7.586404 +ch -7.631453 +ng -7.651747 +ad -7.657030 +or -7.673911 +ни -7.715400 +ин -7.736898 +ки -7.774252 +ил -7.792350 +ли -7.811013 +ra -7.819830 +id -7.899828 +os -7.907464 +bi -7.932364 +_h -7.951442 +bo -7.955234 +ir -7.962922 +_i -7.982160 +on -7.984448 +ya -7.984976 +ди -8.019850 +_х -8.042222 +am -8.071293 +qi -8.080084 +р_ -8.092097 +na -8.098481 +_у -8.121509 +ki -8.122228 +t_ -8.132953 +ри -8.179860 +ка -8.189195 +ik -8.196112 +qa -8.205084 +y_ -8.221477 +ма -8.227628 +_м -8.274262 +g_ -8.280426 +gi -8.293022 +yo -8.306294 +sa -8.334589 +un -8.356433 +b_ -8.356498 +mi -8.361325 +ka -8.394170 +_u -8.405208 +_e -8.428601 +k_ -8.445887 +h_ -8.457225 +ay -8.477867 +ид -8.481449 +ти -8.523455 +_d -8.527298 +_т -8.573896 +ир -8.587512 +ик -8.595176 +ho -8.597598 +нг -8.597675 +ха -8.601722 +ат -8.639216 +ам -8.668020 +to -8.669192 +ur -8.684356 +к_ -8.687138 +он -8.688859 +_а -8.701376 +б_ -8.702203 +_о -8.705102 +ig -8.712668 +_и -8.719184 +би -8.721195 +ib -8.723125 +_n -8.726362 +ул -8.728256 +ah -8.729732 +та -8.755733 +im -8.770711 +iq -8.774447 +ии -8.777670 +m_ -8.788304 +ку -8.794686 +l_ -8.807537 +mo -8.812576 +си -8.834540 +ун -8.838809 +ра -8.847111 +ro -8.869890 +иш -8.881416 +om -8.887708 +zi -8.891996 +lg -8.909373 +yi -8.910233 +бу -8.925175 +st -8.930012 +so -8.934914 +ол -8.935839 +er -8.942373 +hu -8.960272 +mu -8.961411 +av -8.970502 +ab -8.981557 +tu -8.988302 +ал -8.989312 +lo -8.990221 +иб -9.002855 +qo -9.013290 +_д -9.041646 +ак -9.058444 +ll -9.068330 +et -9.113728 +ot -9.114003 +аи -9.114994 +ak -9.120790 +ор -9.128609 +ис -9.133957 +м_ -9.140950 +fa -9.141399 +ги -9.168868 +ko -9.170527 +oy -9.172647 +то -9.173851 +ba -9.176550 +oz -9.177931 +_с -9.191400 +hl -9.208166 +us -9.222219 +са -9.239049 +ур -9.247115 +_э -9.247719 +т_ -9.269809 +ас -9.273063 +_x -9.277249 +_j -9.290251 +rl -9.302055 +им -9.313640 +ke -9.314715 +зи -9.347216 +ми -9.354676 +г_ -9.356628 +nd -9.362175 +бо -9.372739 +_r -9.380333 +на -9.400534 +ul -9.416059 +iz -9.428601 +из -9.432303 +ро -9.442704 +en -9.445056 +ад -9.449286 +ag -9.450606 +aq -9.457085 +rg -9.489996 +tl -9.492641 +_f -9.493071 +od -9.502047 +q_ -9.507026 +бе -9.509413 +мо -9.522432 +уз -9.531814 +ст -9.541479 +ши -9.560857 +rd -9.583315 +де -9.583848 +ek -9.588277 +ке -9.591415 +oq -9.602416 +az -9.621696 +ер -9.625222 +ja -9.639334 +vo -9.651108 +нд -9.662012 +um -9.662334 +be -9.675840 +лг -9.681457 +ld -9.683907 +su -9.690787 +иг -9.693005 +ек -9.746359 +it -9.772448 +bu -9.772621 +ок -9.780114 +ба -9.795304 +o_ -9.803182 +_p -9.804691 +va -9.817811 +te -9.820593 +ко -9.829243 +rt -9.829695 +_н -9.832319 +аб -9.838400 +de -9.843502 +ит -9.872914 +_v -9.880198 +_c -9.895445 +ml -9.897149 +el -9.908371 +rn -9.917180 +yu -9.919969 +чи -9.929140 +no -9.929625 +ш_ -9.936617 +af -9.946778 +za -9.989261 +му -9.995028 +ен -10.025352 +од -10.035534 +рг -10.044214 +ku -10.054108 +ах -10.056433 +nl -10.058021 +ту -10.074315 +аг -10.078176 +ав -10.082694 +ия -10.084850 +d_ -10.088523 +og -10.089280 +лд -10.099053 +ло -10.107906 +do -10.108344 +ук -10.109112 +s_ -10.116923 +uc -10.124223 +_ж -10.144769 +л_ -10.154813 +_g -10.159237 +xo -10.173277 +_р -10.174769 +хо -10.177758 +ум -10.196054 +ud -10.209636 +_п -10.213169 +uv -10.214349 +nt -10.214467 +yd -10.227986 +me -10.232640 +ok -10.234194 +z_ -10.253467 +_ш -10.277310 +оз -10.297922 +uz -10.310230 +рд -10.314399 +з_ -10.330426 +лл -10.340438 +op -10.345082 +ос -10.345082 +vi -10.346375 +ob -10.370370 +_я -10.375904 +pa -10.376432 +зб -10.395708 +аш -10.401206 +rk -10.410640 +hk -10.414160 +ои -10.426549 +qu -10.441253 +hq -10.442220 +es -10.442911 +ic -10.453459 +tr -10.479320 +ош -10.486428 +ет -10.490996 +ша -10.507532 +tg -10.508545 +kl -10.517254 +ye -10.517691 +ov -10.528068 +ча -10.531593 +аз -10.531593 +ru -10.562208 +_z -10.568078 +су -10.578066 +re -10.579129 +yl -10.584306 +со -10.586137 +ом -10.601336 +ва -10.603189 +xa -10.608608 +uq -10.616072 +ax -10.631749 +тг -10.652546 +зо -10.661368 +oi -10.674297 +жа -10.674459 +td -10.675596 +nc -10.688162 +em -10.697040 +_в -10.700012 +sl -10.707301 +рн -10.707468 +уч -10.715630 +lm -10.723671 +xi -10.731926 +ел -10.733279 +jo -10.736495 +от -10.739210 +_е -10.739889 +ns -10.748409 +ху -10.751317 +_ф -10.752859 +hg -10.769236 +рт -10.769236 +ут -10.770451 +во -10.777060 +тл -10.785627 +oh -10.787557 +шл -10.791249 +_ч -10.798132 +ов -10.804158 +bl -10.834131 +кл -10.835401 +рл -10.846883 +ме -10.879263 +о_ -10.883385 +тд -10.900564 +но -10.904176 +if -10.909135 +нл -10.912960 +v_ -10.914685 +rm -10.928366 +ij -10.929140 +rq -10.935352 +уш -10.936325 +zo -10.938468 +fo -10.941591 +шк -10.945504 +fi -10.946876 +я_ -10.947660 +хи -10.950213 +ql -10.955727 +vl -10.966024 +_г -10.976995 +ut -10.978197 +еи -10.983015 +лм -10.984020 +ub -10.996753 +ju -11.002244 +нт -11.010418 +kt -11.010418 +uh -11.012469 +уи -11.016579 +po -11.034388 +tt -11.046313 +rs -11.048205 +до -11.049258 +u_ -11.051575 +па -11.051786 +фа -11.052419 +_l -11.061520 +le -11.082263 +aj -11.084203 +oj -11.084850 +eg -11.099162 +ey -11.103527 +ус -11.132228 +ss -11.134683 +ji -11.136471 +sm -11.138485 +уд -11.139157 +с_ -11.151533 +ne -11.153568 +ух -11.161966 +чу -11.164244 +iv -11.166297 +рк -11.169726 +в_ -11.172016 +кд -11.172933 +шу -11.174769 +go -11.176838 +lq -11.177528 +д_ -11.180983 +zl -11.229058 +hr -11.240310 +gl -11.253589 +mk -11.274108 +gu -11.282250 +мл -11.282745 +du -11.294924 +sp -11.295173 +об -11.311240 +uy -11.314525 +ае -11.330042 +ht -11.340567 +uk -11.357149 +иу -11.365775 +ят -11.368924 +zb -11.380532 +vr -11.383713 +кк -11.385573 +yt -11.386902 +ре -11.406185 +kk -11.414025 +ян -11.434501 +zm -11.444156 +vc -11.446927 +аъ -11.457504 +qt -11.463383 +за -11.467035 +ug -11.476345 +ks -11.477478 +nu -11.495722 +ру -11.497444 +ez -11.514781 +дл -11.519439 +ич -11.521190 +мм -11.545334 +лк -11.556661 +tn -11.559357 +pu -11.572004 +qd -11.573214 +нч -11.575941 +hd -11.579889 +ac -11.583848 +lu -11.616540 +уг -11.617788 +те -11.618413 +dr -11.622792 +эд -11.626557 +dd -11.633168 +oa -11.639176 +md -11.647757 +qs -11.655427 +mm -11.660564 +pr -11.662817 +жи -11.663783 +сс -11.675434 +he -11.684888 +аф -11.690130 +pi -11.711960 +жу -11.715630 +xs -11.718640 +жо -11.727372 +ег -11.736834 +шг -11.748750 +ed -11.753888 +тт -11.760766 +гу -11.792130 +еб -11.794951 +jl -11.799902 +zg -11.807003 +eb -11.809141 +ev -11.844871 +rb -11.856613 +зл -11.857350 +ув -11.869193 +рс -11.880760 +ви -11.882635 +вл -11.903034 +сл -11.932243 +ог -11.932243 +еч -11.933020 +sr -11.934963 +_л -11.939638 +gr -11.942764 +ea -11.946680 +пр -11.949820 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/vie b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/vie new file mode 100644 index 00000000000..fcfedd419fc --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/vie @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 173158326 +#UNIQUE_BIGRAMS 935 +#UNSEEN_LOG_PROB -20.081349 +#EXPECTED_SCORE -6.629060 +_t -4.628468 +ng -4.639193 +g_ -4.927176 +n_ -4.993447 +i_ -5.168762 +_c -5.169921 +an -5.268749 +_n -5.297115 +_d -5.448687 +on -5.451819 +nh -5.603960 +c_ -5.788828 +ha -5.839692 +ie -5.998761 +th -6.027396 +uo -6.065916 +h_ -6.091531 +ho -6.115439 +t_ -6.120365 +ch -6.163363 +u_ -6.178565 +en -6.181358 +oi -6.203402 +a_ -6.214599 +hu -6.232226 +hi -6.293980 +_h -6.388236 +tr -6.483969 +o_ -6.521842 +m_ -6.596831 +ua -6.610966 +_l -6.690870 +_m -6.737820 +_k -6.754433 +_v -6.803001 +ai -6.842139 +_b -6.853484 +y_ -6.942446 +un -7.000918 +oc -7.003771 +in -7.051980 +da -7.083622 +kh -7.175677 +am -7.177476 +at -7.191424 +_g -7.196079 +_p -7.197232 +cu -7.210912 +ac -7.212930 +ca -7.217156 +ay -7.236421 +ph -7.278208 +na -7.348109 +du -7.441232 +_s -7.446542 +oa -7.499136 +ao -7.599489 +p_ -7.625811 +ba -7.706379 +gi -7.713694 +do -7.714840 +ta -7.730495 +ia -7.760970 +ro -7.781573 +au -7.782178 +eu -7.811974 +ti -7.819494 +qu -7.831216 +_q -7.832532 +he -7.857056 +uc -7.929858 +mo -7.930529 +e_ -7.976468 +uy -7.982191 +la -8.002841 +ot -8.106135 +di -8.110670 +ra -8.119601 +et -8.124475 +va -8.235768 +gu -8.289105 +co -8.292500 +ye -8.321338 +ru -8.337922 +ma -8.365394 +sa -8.374075 +to -8.430281 +ga -8.563244 +ap -8.577025 +tu -8.680910 +vi -8.710142 +lo -8.735707 +_r -8.745647 +ri -8.781674 +vo -8.821828 +lu -8.845885 +em -8.851913 +de -8.949779 +_x -8.985002 +gh -9.103473 +bi -9.104509 +go -9.152130 +_a -9.199506 +ic -9.305818 +ki -9.317142 +no -9.379892 +li -9.411316 +re -9.430120 +ep -9.524773 +mi -9.595202 +_o -9.632542 +ne -9.666528 +nu -9.690891 +so -9.801686 +mu -9.828426 +eo -9.829297 +vu -9.834613 +ec -9.894477 +le -9.935293 +io -10.008917 +op -10.076400 +bo -10.091783 +om -10.129373 +er -10.132372 +xu -10.140681 +te -10.228115 +s_ -10.239804 +si -10.312615 +ke -10.370028 +su -10.403300 +iu -10.415833 +im -10.417827 +ar -10.436190 +be -10.478262 +bu -10.485530 +_u -10.586131 +ni -10.647513 +d_ -10.686649 +uu -10.731288 +al -10.775205 +r_ -10.854514 +_y -10.893478 +ui -11.002732 +me -11.035655 +xa -11.046381 +ut -11.089811 +l_ -11.090211 +or -11.191188 +el -11.282169 +st -11.313782 +ue -11.330313 +es -11.335105 +nd -11.377479 +it -11.394806 +us -11.485901 +_i -11.530455 +is -11.560841 +xe -11.587977 +se -11.631723 +_w -11.663072 +as -11.691311 +il -11.737932 +_f -11.779153 +ea -11.806796 +up -11.806945 +ll -11.861178 +ol -11.862294 +nt -11.863318 +id -11.884551 +_j -11.981924 +_e -12.032929 +ae -12.077318 +k_ -12.090520 +um -12.141669 +ve -12.143439 +ur -12.199393 +ge -12.221704 +ce -12.232529 +ou -12.273851 +ad -12.281795 +xi -12.282293 +ip -12.327783 +sh -12.334782 +pa -12.338272 +os -12.352537 +ir -12.458749 +rt -12.562680 +ed -12.589748 +tt -12.642736 +po -12.661884 +pe -12.697856 +xo -12.698742 +ka -12.756203 +wa -12.781791 +ag -12.867173 +sc -12.877292 +ss -12.880556 +rd -12.900420 +pi -12.934844 +v_ -12.935758 +br -12.971046 +oo -12.971046 +ck -12.982800 +ns -13.003861 +ry -13.025650 +nc -13.027527 +ak -13.076202 +oe -13.085007 +lb -13.091974 +ul -13.100510 +rl -13.117218 +rn -13.128888 +pt -13.200647 +nn -13.209933 +rs -13.248089 +wi -13.248413 +ei -13.258910 +dr -13.266526 +ld -13.276494 +yo -13.284120 +sd -13.286866 +ig -13.297060 +w_ -13.305133 +ko -13.334352 +od -13.372014 +ob -13.382208 +ee -13.389074 +ci -13.398850 +jo -13.429040 +ab -13.437703 +b_ -13.442890 +ja -13.446140 +ov -13.449119 +ey -13.453973 +cl -13.454721 +hc -13.455095 +gr -13.461753 +we -13.470522 +mp -13.474974 +ya -13.490616 +rg -13.502750 +av -13.527034 +ub -13.536902 +x_ -13.540571 +ow -13.544448 +rr -13.565303 +ct -13.566920 +iv -13.613464 +ly -13.627525 +ev -13.634925 +hn -13.658217 +fr -13.662748 +fo -13.672288 +pr -13.696307 +ex -13.702514 +wo -13.708301 +fe -13.767371 +fa -13.782554 +mb -13.791387 +ew -13.807184 +ii -13.825210 +ok -13.825210 +rk -13.832363 +sp -13.871162 +yn -13.876666 +og -13.889632 +cm -13.896858 +pu -13.946635 +fi -13.947161 +eb -13.952039 +pl -13.953889 +cr -13.954682 +bl -13.959850 +vn -14.008728 +ty -14.023082 +ah -14.047280 +f_ -14.061740 +hy -14.065163 +ts -14.066878 +az -14.084139 +rc -14.109981 +ik -14.120628 +eg -14.125533 +oh -14.135695 +z_ -14.136445 +cn -14.167844 +lv -14.173683 +ls -14.189319 +lt -14.191343 +nk -14.209065 +hl -14.212382 +tc -14.225250 +rm -14.239361 +ud -14.239845 +af -14.256544 +ff -14.299578 +_z -14.323122 +pp -14.333921 +cs -14.343589 +ug -14.364526 +if -14.379897 +tp -14.397950 +ib -14.399930 +tn -14.408420 +gl -14.408420 +za -14.410596 +hr -14.431615 +ht -14.473457 +zi -14.481058 +ku -14.494842 +ys -14.495611 +aw -14.505654 +rb -14.527919 +uk -14.538983 +ny -14.572898 +bn -14.589029 +tl -14.589645 +ds -14.634925 +ju -14.655634 +ze -14.673812 +mm -14.676209 +je -14.677082 +sk -14.680360 +ym -14.687817 +sl -14.690017 +ax -14.757184 +dt -14.757645 +ps -14.784434 +sy -14.784669 +yl -14.812208 +cc -14.869166 +ox -14.875163 +ez -14.907573 +ks -14.912960 +yu -14.922760 +tv -14.941776 +of -14.955211 +ky -14.961843 +oy -14.976810 +bs -14.978154 +q_ -14.985974 +ms -15.004751 +yr -15.012165 +lm -15.020726 +dy -15.023776 +nm -15.036880 +dd -15.069740 +dv -15.079227 +rp -15.083271 +yt -15.085876 +sm -15.089067 +dn -15.089939 +gt -15.135695 +my -15.202610 +fl -15.209854 +hm -15.214598 +hd -15.271790 +kr -15.288365 +ji -15.296054 +iz -15.301764 +sv -15.326200 +ek -15.344802 +yd -15.387734 +hp -15.434933 +yp -15.441962 +fu -15.472699 +aq -15.478393 +by -15.486020 +sr -15.498694 +ft -15.516158 +vd -15.517720 +ef -15.526739 +gn -15.530283 +cy -15.535418 +nz -15.538190 +ww -15.549733 +sn -15.563789 +sb -15.577983 +bb -15.592318 +yc -15.630692 +mt -15.636622 +ix -15.643003 +lf -15.643429 +lh -15.646417 +gd -15.673594 +gs -15.685839 +nr -15.731798 +nl -15.742264 +zu -15.754648 +pg -15.756953 +ws -15.768999 +db -15.777865 +zo -15.791977 +lk -15.798136 +rv -15.811969 +bt -15.851817 +mc -15.851817 +dw -15.860713 +kl -15.868667 +wn -15.886222 +cb -15.886726 +tm -15.891783 +bc -15.935497 +wh -15.939680 +dl -15.949137 +tz -15.951247 +nb -15.957065 +gm -15.970377 +cd -15.988680 +bh -16.023776 +tw -16.024886 +dp -16.027667 +uz -16.046716 +aa -16.055201 +lg -16.074045 +dg -16.090810 +tx -16.098390 +lc -16.103660 +eh -16.116626 +dc -16.135096 +nx -16.140502 +sw -16.145325 +rh -16.146534 +vt -16.149559 +xy -16.150771 +oz -16.155020 +iq -16.166618 +td -16.199471 +dh -16.220312 +py -16.227326 +lp -16.291369 +xh -16.303447 +kt -16.314269 +aj -16.404621 +rf -16.416961 +hs -16.444189 +sg -16.466650 +wt -16.491767 +gk -16.494072 +uv -16.506430 +gy -16.507982 +gg -16.519676 +bd -16.529100 +pc -16.537001 +dm -16.542558 +xt -16.558552 +nf -16.590261 +zh -16.593553 +tg -16.641298 +uf -16.654130 +hq -16.659295 +tb -16.669680 +mn -16.672288 +kn -16.675773 +jp -16.709305 +ij -16.761111 +fg -16.781615 +nj -16.790088 +cv -16.793869 +hh -16.795764 +hw -16.800511 +cp -16.896858 +kv -16.906037 +wr -16.917336 +nv -16.925610 +qr -16.928725 +np -16.959187 +ux -16.960249 +qd -16.991391 +ih -17.005573 +zy -17.019895 +rw -17.021003 +xv -17.061455 +zz -17.084428 +yw -17.085586 +ln -17.093721 +tq -17.113669 +qa -17.120776 +iy -17.154412 +pm -17.207645 +cq -17.224133 +vf -17.229245 +dk -17.240812 +xc -17.248575 +qg -17.264229 +j_ -17.265541 +wl -17.276081 +fp -17.285367 +rz -17.297396 +oj -17.300082 +kc -17.316308 +vs -17.317668 +uh -17.328598 +hb -17.338229 +bk -17.390237 +xp -17.417690 +xb -17.436779 +hx -17.453131 +gp -17.463635 +vl -17.466650 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/vol b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/vol new file mode 100644 index 00000000000..1073c849f5d --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/vol @@ -0,0 +1,415 @@ +#TOTAL_BIGRAMS 5551100 +#UNIQUE_BIGRAMS 411 +#UNSEEN_LOG_PROB -19.082414 +#EXPECTED_SCORE -6.608416 +s_ -4.148267 +_l -4.834776 +an -5.163029 +lo -5.186422 +ns -5.297391 +od -5.328926 +on -5.399080 +da -5.472012 +in -5.682535 +la -5.722651 +ab -5.742675 +_b -5.879770 +fa -5.901138 +_a -5.915561 +_p -5.955548 +al -5.993808 +u_ -6.081252 +as -6.125367 +do -6.136422 +bi -6.136714 +li -6.158455 +el -6.237885 +am -6.256819 +n_ -6.257316 +en -6.410917 +ad -6.420569 +a_ -6.481154 +me -6.543061 +mu -6.609419 +no -6.629452 +ne -6.642843 +ef -6.701169 +ik -6.724615 +po -6.748364 +_v -6.749287 +ni -6.754469 +ta -6.803529 +k_ -6.897632 +ma -6.917790 +d_ -6.921818 +is -7.003830 +na -7.016594 +ls -7.027437 +op -7.038601 +if -7.053196 +ye -7.060254 +_m -7.149535 +ul -7.180868 +ot -7.207049 +pi -7.238101 +bo -7.271843 +nu -7.282485 +ti -7.313279 +ba -7.362426 +va -7.484129 +so -7.500555 +_f -7.501826 +di -7.519933 +ol -7.521078 +o_ -7.559244 +be -7.627858 +ay -7.715218 +tu -7.716639 +ld -7.718334 +ut -7.755030 +_d -7.803907 +_u -7.806988 +f_ -7.844308 +_c -7.853535 +ur -7.856122 +er -7.856725 +r_ -7.887473 +um -7.944654 +ka -7.967891 +it -8.009679 +lu -8.086930 +pl -8.097426 +at -8.250157 +_s -8.260242 +id -8.266310 +om -8.306722 +t_ -8.308368 +su -8.336572 +le -8.346774 +m_ -8.395914 +ed -8.409547 +_t -8.409900 +fo -8.429659 +es -8.458441 +os -8.465039 +ap -8.465773 +bu -8.468441 +_y -8.482687 +em -8.490237 +se -8.497170 +de -8.500178 +im -8.505363 +_k -8.523515 +ca -8.535134 +pe -8.546556 +ia -8.554253 +of -8.555524 +fs -8.568588 +ke -8.609419 +uk -8.613383 +ps -8.630173 +_z -8.654892 +ao -8.665406 +za -8.679083 +si -8.696767 +iy -8.801411 +vo -8.993361 +l_ -9.006534 +ie -9.018884 +vi -9.114756 +et -9.130255 +ri -9.214827 +ea -9.238964 +ek -9.257137 +re -9.257614 +i_ -9.338926 +pa -9.372813 +au -9.375055 +us -9.424560 +ce -9.437116 +rf -9.454880 +ku -9.498579 +nd -9.534170 +ya -9.637192 +ag -9.697630 +ga -9.725742 +bl -9.770439 +yo -9.941329 +_n -10.004263 +e_ -10.987017 +sa -11.044044 +mo -11.153452 +nt -11.182755 +te -11.297779 +_h -11.420636 +to -11.523993 +il -11.614809 +_g -11.675571 +hi -11.793317 +_e -11.818441 +or -11.826913 +af -11.849753 +st -11.862278 +ai -11.910487 +ll -11.913491 +ro -11.993891 +_o -12.046790 +ar -12.080162 +ve -12.105134 +ra -12.162359 +pu -12.197328 +y_ -12.236924 +gr -12.297125 +mi -12.384752 +un -12.401527 +pr -12.418500 +fe -12.435675 +_j -12.478046 +fi -12.523228 +ko -12.576206 +_r -12.631203 +th -12.808152 +_w -12.853595 +ak -12.872961 +rt -12.953131 +ge -12.965550 +ng -12.969714 +ch -12.969714 +h_ -12.984382 +go -12.986490 +ki -13.055614 +he -13.062268 +ci -13.066720 +g_ -13.091459 +ok -13.102846 +ou -13.105134 +ig -13.151677 +ts -13.234417 +ei -13.275059 +ty -13.316879 +ha -13.415658 +io -13.467704 +ov -13.590561 +w_ -13.600211 +ew -13.603442 +av -13.626265 +og -13.652798 +eu -13.652798 +up -13.703903 +du -13.724862 +je -13.728385 +ds -13.742564 +ev -13.767718 +sp -13.778633 +we -13.892590 +rk -13.908487 +nc -13.940818 +ee -13.978077 +ru -13.982277 +p_ -14.033655 +eg -14.042398 +co -14.077913 +oa -14.100562 +ud -14.123571 +vu -14.128218 +zi -14.128218 +rd -14.137556 +ms -14.156415 +ac -14.185174 +cr -14.224433 +ob -14.264791 +ey -14.275059 +jo -14.295818 +uv -14.343646 +rr -14.381974 +fu -14.432799 +_i -14.455975 +gs -14.479530 +eb -14.485479 +iu -14.491453 +oo -14.491453 +ow -14.509524 +br -14.596987 +fr -14.603442 +lt -14.609926 +ic -14.616440 +iv -14.649455 +ks -14.649455 +ho -14.669633 +yu -14.676422 +ub -14.710855 +gh -14.731917 +rg -14.760486 +oi -14.834487 +ir -14.842100 +rs -14.880780 +oc -14.888642 +ck -14.896548 +tr -14.912489 +gt -14.961399 +eo -14.969714 +rl -14.986490 +wo -15.003463 +gu -15.012025 +kl -15.020638 +ht -15.038020 +wi -15.038020 +gl -15.055614 +kt -15.073425 +wa -15.073425 +sh -15.118940 +gi -15.165938 +nk -15.224433 +ja -15.244471 +az -15.254595 +ep -15.264791 +hl -15.275059 +nn -15.275059 +ji -15.316879 +kr -15.316879 +wn -15.327527 +rm -15.359948 +cl -15.359948 +sm -15.415658 +ju -15.427062 +dr -15.438558 +rn -15.450146 +tt -15.461828 +ry -15.558852 +mp -15.571452 +dg -15.596987 +ss -15.622983 +v_ -15.703903 +ff -15.746131 +ui -15.834487 +sc -15.834487 +sl -15.849753 +rc -15.849753 +mb -15.880780 +ct -15.896548 +ly -15.928609 +ny -15.978077 +b_ -15.994951 +oh -16.046790 +x_ -16.064492 +oe -16.082414 +tl -16.100562 +hu -16.100562 +sv -16.156415 +uf -16.175524 +my -16.254595 +ue -16.316879 +ae -16.316879 +ug -16.381974 +ex -16.497452 +wh -16.521699 +pt -16.521699 +ij -16.546361 +uo -16.571452 +dl -16.571452 +ph -16.596987 +nv -16.596987 +nf -16.596987 +ec -16.596987 +zu -16.622983 +rv -16.649455 +oy -16.649455 +ip -16.703903 +lb -16.703903 +ib -16.731917 +hn -16.760486 +lk -16.760486 +ln -16.789632 +ua -16.789632 +ox -16.849753 +xi -16.880780 +nw -16.880780 +lm -16.880780 +lp -16.944911 +dw -17.046790 +hm -17.082414 +lf -17.082414 +ym -17.118940 +sy -17.118940 +ft -17.156415 +aj -17.194889 +dt -17.194889 +lv -17.234417 +dn -17.234417 +uc -17.234417 +uz -17.275059 +z_ -17.275059 +ux -17.275059 +fl -17.275059 +gy -17.316879 +nr -17.316879 +rb -17.404342 +sb -17.404342 +kh -17.450146 +td -17.450146 +yt -17.497452 +mm -17.497452 +dy -17.546361 +dd -17.596987 +yi -17.596987 +xa -17.649455 +tw -17.649455 +gg -17.703903 +aw -17.703903 +wr -17.703903 +hr -17.760486 +df -17.819380 +ws -17.819380 +gf -17.944911 +xf -17.944911 +ej -17.944911 +ax -17.944911 +ix -17.944911 +ze -17.944911 +cc -18.012025 +wt -18.082414 +c_ -18.082414 +vs -18.156415 +nh -18.234417 +by -18.316879 +yr -18.316879 +yn -18.404342 +lg -18.404342 +iz -18.404342 +tg -18.404342 +ky -18.404342 +tf -18.497452 +hb -18.497452 +nz -18.497452 +zo -18.497452 +wp -18.497452 +pp -18.497452 +bs -18.497452 +ys -18.596987 +uh -18.596987 +sf -18.703903 +kw -18.703903 +md -18.819380 +kf -18.819380 +rh -18.819380 +hf -18.819380 +yl -18.819380 +dv -18.819380 +wy -18.819380 +eh -18.944911 +mf -18.944911 +yd -18.944911 +cu -18.944911 +gb -18.944911 +yf -18.944911 +rw -18.944911 +yc -18.944911 +lh -18.944911 +pb -19.082414 +sg -19.082414 +xu -19.082414 +tc -19.082414 +gn -19.082414 +aa -19.082414 +gd -19.082414 +db -19.082414 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/war b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/war new file mode 100644 index 00000000000..4042c6ebfc5 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/war @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 22734131 +#UNIQUE_BIGRAMS 619 +#UNSEEN_LOG_PROB -18.307516 +#EXPECTED_SCORE -7.055364 +a_ -4.578949 +n_ -4.853321 +ha -5.039216 +_h -5.217411 +an -5.307012 +s_ -5.480946 +_s -5.647410 +in -5.797466 +us -5.905000 +ga -5.956427 +on -6.009126 +_n -6.010917 +es -6.013920 +la -6.016468 +ng -6.021810 +ec -6.066571 +_g -6.135267 +to -6.188590 +ci -6.188748 +gi -6.188850 +ie -6.196755 +pe -6.214925 +ka -6.218257 +ul -6.234722 +y_ -6.246627 +sp -6.269219 +wa -6.291435 +hu -6.296105 +ag -6.309456 +ad -6.315059 +_u -6.321850 +sk -6.331640 +ay -6.334879 +nh -6.367004 +dt -6.368728 +gw -6.374751 +er -6.471389 +ra -6.625546 +te -7.038703 +op -7.252726 +_a -7.256292 +ar -7.312277 +pt -7.434652 +e_ -7.679314 +ri -7.756487 +_c -7.797335 +or -7.809968 +ta -7.811293 +_m -7.827721 +en -7.892879 +le -7.909475 +al -8.008901 +ah -8.017886 +ch -8.023556 +ya -8.023605 +ac -8.028485 +sy -8.031720 +ma -8.082334 +ne -8.083522 +co -8.091774 +st -8.103315 +is -8.138576 +li -8.166547 +da -8.215996 +se -8.288510 +_p -8.295571 +ct -8.309557 +_d -8.310302 +ns -8.320308 +ol -8.331507 +th -8.351204 +r_ -8.395671 +ro -8.404073 +ti -8.411271 +_b -8.461770 +as -8.494972 +id -8.501374 +at -8.507111 +el -8.507804 +no -8.512988 +ia -8.542734 +_i -8.554305 +ll -8.559204 +ho -8.563543 +ca -8.578361 +re -8.600137 +di -8.625051 +il -8.630700 +i_ -8.658227 +ic -8.658740 +ae -8.665243 +lo -8.692525 +_l -8.751172 +me -8.767728 +ea -8.779922 +ni -8.797321 +na -8.829149 +tr -8.863793 +he -8.875667 +_t -8.877516 +rt -8.877665 +_o -8.904449 +po -8.913937 +os -8.929045 +ip -8.972758 +sa -8.989724 +od -9.004999 +de -9.034479 +_f -9.093541 +d_ -9.094165 +_r -9.104467 +eo -9.104677 +ph -9.104712 +ry -9.224569 +hi -9.231800 +nt -9.239451 +_k -9.257593 +si -9.308832 +sc -9.337284 +am -9.338187 +_e -9.352552 +_w -9.372898 +t_ -9.389827 +l_ -9.463359 +ur -9.483660 +nd -9.505588 +o_ -9.569920 +ba -9.594926 +br -9.618327 +ig -9.630800 +be -9.691160 +om -9.694995 +mi -9.702007 +nn -9.705712 +ge -9.709108 +pi -9.772410 +so -9.785792 +et -9.786240 +oc -9.787866 +mo -9.791067 +do -9.798677 +hy -9.798790 +au -9.809911 +pa -9.810937 +tu -9.817049 +g_ -9.837340 +ce -9.842982 +lu -9.871173 +un -9.907767 +_v -9.945127 +eu -9.968333 +_j -9.973109 +it -9.989272 +ot -10.029032 +ep -10.034213 +vi -10.039746 +ii -10.060400 +h_ -10.080392 +ru -10.109962 +io -10.111225 +ei -10.134790 +em -10.165144 +if -10.171643 +rm -10.171863 +fr -10.184066 +yg -10.190577 +ve -10.198982 +m_ -10.220550 +nu -10.261015 +rd -10.275257 +va -10.312378 +rl -10.325184 +ir -10.334823 +ed -10.358122 +iu -10.389274 +cu -10.394304 +bi -10.402269 +pl -10.435629 +ap -10.441618 +im -10.449583 +ym -10.492181 +og -10.501903 +zo -10.505865 +av -10.533628 +ab -10.540511 +bo -10.555331 +ke -10.575332 +wi -10.598365 +k_ -10.619275 +ck -10.632210 +hn -10.640695 +um -10.651168 +tt -10.652495 +fe -10.654540 +ld -10.655563 +mu -10.658124 +dr -10.679092 +gr -10.684826 +su -10.691527 +go -10.699736 +nc -10.699842 +ob -10.708948 +cr -10.757667 +ou -10.790561 +rc -10.791573 +gu -10.792023 +rs -10.792360 +fo -10.800147 +rn -10.854451 +wo -10.854686 +mg -10.860220 +oa -10.867789 +fa -10.882569 +ss -10.899077 +oz -10.902960 +yc -10.933538 +ps -10.942876 +sh -10.945127 +fi -10.956053 +jo -10.972153 +rg -10.980078 +rr -10.986373 +ly -10.992178 +hr -11.016292 +ev -11.059114 +ud -11.059926 +ey -11.072991 +ov -11.074496 +yo -11.092258 +uc -11.104922 +yl -11.111787 +oi -11.121085 +ub -11.162814 +ui -11.172595 +mb -11.194290 +vo -11.200848 +ai -11.234550 +_z -11.253016 +mp -11.321200 +ko -11.336381 +ut -11.341971 +ki -11.369075 +lt -11.372604 +oe -11.380026 +oh -11.389018 +ax -11.404073 +oo -11.411833 +pu -11.434839 +bu -11.435366 +f_ -11.437300 +iv -11.440471 +ak -11.466993 +ue -11.486525 +ja -11.494560 +ys -11.506142 +eg -11.512431 +x_ -11.524906 +cl -11.571464 +ew -11.571657 +pr -11.605862 +ty -11.625578 +ug -11.629794 +xi -11.635638 +du -11.673485 +ts -11.706825 +ef -11.707037 +ms -11.720252 +w_ -11.744870 +gn -11.819054 +ee -11.819054 +lb -11.840304 +ow -11.856450 +yp -11.894841 +z_ -11.902838 +v_ -11.927345 +mm -11.948007 +we -11.948509 +lm -11.969224 +nk -11.976110 +dw -11.979950 +qu -11.987145 +nr -12.036944 +rh -12.049532 +ex -12.069032 +ze -12.086313 +sm -12.094893 +ik -12.101292 +yd -12.123207 +yn -12.147762 +ua -12.178613 +ff -12.185395 +ls -12.185395 +je -12.203240 +bs -12.238684 +my -12.259069 +gl -12.260937 +hl -12.269684 +eb -12.321363 +ib -12.329505 +bl -12.343289 +rp -12.382735 +ht -12.392938 +pp -12.400123 +ny -12.402870 +gh -12.408379 +_y -12.422593 +lk -12.424336 +lc -12.459289 +up -12.468251 +yr -12.481617 +cy -12.481617 +rb -12.492547 +lv -12.545055 +u_ -12.550374 +dy -12.554568 +cc -12.573397 +af -12.574945 +ye -12.603490 +aw -12.624575 +ao -12.644753 +ix -12.644753 +yt -12.678468 +hm -12.689325 +fu -12.720252 +kr -12.728411 +sf -12.743128 +ds -12.750106 +c_ -12.754485 +ok -12.770358 +za -12.798564 +lp -12.800373 +zi -12.804907 +rk -12.812648 +lf -12.819054 +sl -12.839839 +nb -12.868501 +tz -12.871351 +b_ -12.903567 +ku -12.919212 +dd -12.920195 +ez -12.931057 +xa -12.989724 +gs -13.006859 +zh -13.010520 +p_ -13.067669 +nz -13.069850 +by -13.080805 +rv -13.106320 +eh -13.119249 +fl -13.126041 +yi -13.182148 +sb -13.193993 +ws -13.225252 +vu -13.233785 +oy -13.270938 +dg -13.286072 +uv -13.309718 +xo -13.315529 +iz -13.330486 +_x -13.331139 +ek -13.337037 +ju -13.346260 +py -13.369578 +ox -13.432030 +kh -13.488530 +dl -13.506142 +cn -13.527714 +hw -13.536735 +db -13.543539 +bb -13.550374 +gy -13.558773 +ks -13.571850 +aj -13.618178 +mc -13.622973 +of -13.671828 +uo -13.685140 +kn -13.722395 +tc -13.730135 +ky -13.740520 +lh -13.805361 +az -13.816305 +tl -13.819054 +gm -13.838444 +nf -13.885687 +uz -13.911857 +uh -13.915775 +ji -13.925616 +tn -13.949512 +mn -13.954541 +_q -13.964651 +cq -13.975854 +uf -14.022615 +lg -14.034279 +yb -14.056814 +tm -14.069850 +xe -14.111927 +lz -14.161069 +yu -14.166893 +wy -14.192804 +sw -14.237458 +xy -14.255962 +hs -14.297527 +zy -14.314235 +aa -14.361541 +sz -14.364215 +sn -14.374961 +yh -14.429928 +yx -14.436948 +ux -14.451093 +iq -14.481254 +wl -14.485615 +wh -14.487072 +kl -14.534475 +wn -14.537490 +ij -14.567992 +uy -14.575719 +nw -14.580376 +uk -14.622973 +dm -14.648823 +cd -14.744870 +tw -14.769472 +dn -14.801732 +rf -14.805361 +ih -14.831026 +ln -14.845899 +wr -14.851517 +gg -14.889535 +zu -14.936519 +oj -14.958576 +df -14.991273 +cz -15.065491 +zs -15.139149 +sv -15.178613 +vr -15.236233 +wu -15.248532 +nl -15.273450 +kt -15.288609 +rz -15.296249 +jc -15.301365 +iy -15.311652 +nm -15.327221 +gt -15.407689 +bt -15.446835 +iw -15.446835 +ft -15.452515 +vs -15.461077 +fg -15.519493 +sq -15.543539 +hb -15.599153 +ej -15.621373 +tk -15.650454 +dh -15.656997 +yz -15.710436 +lj -15.730997 +sd -15.769472 +nj -15.776578 +uw -15.776578 +rj -15.853394 +uu -15.853394 +kb -15.857156 +eq -15.899198 +lw -15.946503 +sj -15.950516 +tj -16.003728 +hv -16.024729 +nv -16.067669 +hk -16.076413 +zz -16.116428 +rw -16.143736 +yw -16.199952 +wf -16.214355 +cs -16.233785 +j_ -16.243600 +vl -16.248532 +rq -16.253481 +uj -16.273450 +xp -16.298805 +kk -16.309074 +wm -16.319415 +yk -16.356207 +pm -16.383074 +xc -16.388508 +xt -16.393962 +gf -16.421548 +sr -16.438357 +hc -16.466813 +km -16.489989 +kw -16.507619 +jd -16.519493 +xu -16.519493 +cg -16.537490 +yv -16.549613 +fy -16.555713 +bd -16.555713 +fm -16.561840 +dc -16.605467 +hj -16.624575 +dz -16.663569 +dj -16.676805 +jn -16.744870 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/wol b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/wol new file mode 100644 index 00000000000..c3954c1d121 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/wol @@ -0,0 +1,480 @@ +#TOTAL_BIGRAMS 2144996 +#UNIQUE_BIGRAMS 476 +#UNSEEN_LOG_PROB -17.710615 +#EXPECTED_SCORE -7.290590 +_n -5.196493 +n_ -5.399946 +oo -5.474332 +aa -5.522676 +ee -5.631564 +_d -5.738713 +a_ -5.749672 +an -5.797502 +u_ -5.947735 +_m -5.990115 +e_ -6.069241 +en -6.106294 +al -6.126076 +_b -6.215360 +m_ -6.278752 +i_ -6.289497 +na -6.340745 +_s -6.350262 +y_ -6.368207 +_w -6.373105 +_y -6.387054 +am -6.432340 +l_ -6.505189 +_l -6.563156 +_j -6.586882 +ng -6.759914 +wa -6.795707 +_t -6.841639 +r_ -6.847591 +on -6.856370 +da -6.869444 +la -6.890676 +ar -6.930978 +ma -6.932620 +kk -6.950311 +le -7.030871 +ne -7.084359 +_a -7.113401 +ay -7.136968 +de -7.168357 +t_ -7.179234 +_k -7.179916 +ba -7.212963 +o_ -7.261467 +ga -7.278178 +nu -7.283617 +ta -7.306006 +_x -7.306219 +g_ -7.321307 +ax -7.321952 +_g -7.360560 +nd -7.373663 +ye -7.398527 +k_ -7.437636 +x_ -7.501284 +di -7.539439 +ka -7.562139 +sa -7.575163 +ge -7.585073 +ek -7.594271 +ya -7.633666 +gi -7.636340 +ll -7.659678 +xa -7.668135 +mo -7.678570 +ol -7.728192 +om -7.737205 +ii -7.738785 +te -7.746130 +in -7.749745 +at -7.779287 +ra -7.809749 +se -7.812014 +re -7.849839 +ja -7.868579 +er -7.909069 +be -7.915875 +_f -7.918150 +bo -7.929584 +pp -7.950561 +nn -7.956730 +mb -7.998776 +w_ -8.004983 +do -8.063157 +eg -8.073810 +s_ -8.101437 +fa -8.108102 +lo -8.108473 +aw -8.127909 +el -8.135265 +no -8.146275 +tu -8.146657 +su -8.177675 +gg -8.177675 +ko -8.199060 +ni -8.201434 +it -8.212165 +je -8.213961 +li -8.215560 +ir -8.222976 +em -8.247295 +mu -8.302498 +ak -8.311872 +ef -8.322168 +_r -8.351085 +oy -8.372548 +un -8.377460 +es -8.383063 +ri -8.422211 +p_ -8.441956 +bi -8.451814 +lu -8.455351 +ag -8.468632 +me -8.522769 +f_ -8.530706 +du -8.561885 +wo -8.606803 +uy -8.658047 +yo -8.677192 +or -8.701347 +ul -8.730476 +et -8.766050 +is -8.766636 +ur -8.783726 +ig -8.817314 +nt -8.817617 +xe -8.817921 +fe -8.825529 +ju -8.863871 +ku -8.874249 +so -8.878991 +b_ -8.882479 +ke -8.883432 +ep -8.886611 +_c -8.894272 +dd -8.910039 +ti -8.914576 +uu -8.945744 +ey -8.949729 +bu -8.950394 +ki -8.962087 +ad -8.989516 +ew -8.995340 +d_ -9.012605 +ik -9.047769 +ro -9.099222 +af -9.117038 +ox -9.185486 +tt -9.193734 +jo -9.212365 +we -9.239753 +as -9.240160 +gu -9.243417 +to -9.256110 +mi -9.351305 +iy -9.425213 +ab -9.433561 +_i -9.462213 +j_ -9.465063 +aj -9.466014 +_p -9.495811 +uk -9.515366 +yi -9.524254 +op -9.526732 +ru -9.526732 +go -9.561885 +wu -9.567998 +ok -9.587011 +um -9.588564 +ug -9.608902 +ji -9.635404 +xo -9.649920 +ex -9.672794 +ub -9.701067 +pa -9.823700 +nj -9.826140 +ut -9.853256 +ot -9.868894 +mm -9.949729 +yu -10.006020 +il -10.024115 +og -10.036070 +ob -10.048125 +ib -10.087100 +si -10.127909 +ed -10.139242 +ud -10.260229 +_e -10.276821 +xt -10.283512 +of -10.318298 +us -10.333839 +ap -10.339928 +ow -10.370765 +ci -10.401366 +ej -10.438219 +im -10.439152 +ce -10.530706 +os -10.535690 +fi -10.537688 +pe -10.547721 +pi -10.634869 +st -10.655333 +eb -10.681604 +c_ -10.715131 +nk -10.750613 +aq -10.807336 +bb -10.809749 +cc -10.826750 +co -10.866380 +iw -10.877725 +ah -10.880259 +fu -10.924019 +wi -10.970497 +ha -11.032544 +uf -11.043859 +po -11.053833 +ld -11.169906 +xu -11.171457 +fo -11.174563 +ac -11.177675 +h_ -11.385085 +jj -11.427064 +ca -11.522769 +fr -11.524749 +ns -11.546714 +ea -11.554785 +_h -11.577216 +if -11.597915 +xi -11.621033 +uw -11.625276 +q_ -11.625276 +_u -11.666221 +nc -11.674992 +rt -11.735627 +yy -11.756419 +qa -11.770449 +ww -11.847668 +he -11.892992 +tr -11.900687 +uj -11.911010 +rn -11.934511 +ve -11.945081 +bl -11.958402 +gl -12.001876 +cr -12.049550 +sk -12.075441 +ou -12.095906 +_o -12.095906 +io -12.152960 +mt -12.168357 +ie -12.190193 +sl -12.261054 +ic -12.291077 +hi -12.321737 +oj -12.325184 +yt -12.360118 +ho -12.392299 +ia -12.395919 +br -12.432631 +id -12.436354 +sr -12.489512 +ft -12.536688 +ec -12.593752 +iv -12.618916 +cu -12.653165 +_z -12.692694 +od -12.715131 +rk -12.724204 +xl -12.733336 +ch -12.847668 +rl -12.852634 +lt -12.892992 +ip -12.903261 +lm -12.903261 +th -12.903261 +rg -12.924019 +zi -12.939786 +sh -13.021316 +mn -13.038190 +ym -13.049550 +fl -13.055264 +qu -13.084176 +mp -13.107731 +rm -13.149900 +ny -13.187053 +pu -13.238128 +ai -13.244641 +ij -13.264359 +wl -13.311444 +tl -13.395919 +bh -13.403187 +uc -13.410492 +xn -13.425213 +ix -13.447581 +tc -13.485649 +uh -13.493385 +gs -13.508982 +nq -13.524749 +sp -13.532698 +qi -13.564938 +ph -13.564938 +eq -13.589600 +lx -13.597915 +ts -13.597915 +va -13.606279 +ml -13.674992 +ux -13.728763 +hu -13.756419 +yk -13.756419 +xy -13.784616 +av -13.784616 +ov -13.803725 +bd -13.823090 +ly -13.832871 +gn -13.852634 +oh -13.913602 +zo -13.955728 +ry -13.977261 +_v -13.988149 +wr -14.043859 +lb -14.043859 +sm -14.078347 +ss -14.090029 +gr -14.125653 +vi -14.137726 +oz -14.162179 +hm -14.174563 +kh -14.174563 +oe -14.187053 +pl -14.199654 +wf -14.212365 +lk -14.225189 +v_ -14.238128 +qo -14.238128 +wm -14.251184 +eo -14.304623 +tn -14.318298 +ln -14.417834 +up -14.432631 +dr -14.447581 +dh -14.447581 +hl -14.477955 +uq -14.493385 +ei -14.493385 +ty -14.524749 +_q -14.524749 +yn -14.540690 +gt -14.540690 +ae -14.556810 +rc -14.556810 +ue -14.556810 +rs -14.556810 +ui -14.606279 +jf -14.640226 +rd -14.674992 +yj -14.710615 +kt -14.728763 +pt -14.728763 +yr -14.747141 +rr -14.803725 +yp -14.823090 +tb -14.842719 +oq -14.842719 +ws -14.903261 +ff -14.903261 +nw -14.903261 +hy -14.924019 +ys -14.966454 +eh -14.988149 +yl -15.010176 +bv -15.078347 +kv -15.078347 +bt -15.101806 +nf -15.225189 +eu -15.225189 +rb -15.251184 +yd -15.251184 +za -15.304623 +lg -15.332104 +bn -15.360118 +ht -15.388687 +kr -15.388687 +vm -15.417834 +mx -15.508982 +oc -15.508982 +my -15.508982 +au -15.508982 +wn -15.540690 +ua -15.573112 +iq -15.606279 +nl -15.640226 +wt -15.710615 +ck -15.710615 +uo -15.710615 +jb -15.784616 +km -15.784616 +ev -15.784616 +ky -15.784616 +gh -15.823090 +ze -15.862619 +ps -15.862619 +mf -15.988149 +tk -15.988149 +kb -16.032544 +ls -16.032544 +sc -16.032544 +gw -16.125653 +cv -16.225189 +vo -16.225189 +hn -16.277656 +pr -16.277656 +dz -16.332104 +ez -16.332104 +py -16.332104 +dk -16.388687 +qe -16.388687 +fs -16.388687 +kp -16.388687 +xb -16.447581 +lh -16.447581 +cl -16.447581 +jd -16.508982 +lw -16.508982 +rf -16.508982 +ao -16.573112 +sy -16.573112 +dg -16.640226 +yw -16.640226 +vn -16.640226 +tv -16.640226 +wv -16.640226 +wb -16.640226 +ds -16.640226 +fn -16.640226 +gv -16.710615 +xk -16.710615 +gd -16.784616 +jk -16.784616 +dp -16.862619 +zu -16.862619 +iu -16.862619 +by -16.945081 +lf -16.945081 +oa -16.945081 +cf -16.945081 +uv -16.945081 +ks -17.032544 +kw -17.032544 +rx -17.032544 +lj -17.125653 +sf -17.125653 +yc -17.125653 +fk -17.125653 +uz -17.125653 +ms -17.225189 +zh -17.225189 +gk -17.225189 +vl -17.332104 +lv -17.332104 +jt -17.332104 +dn -17.332104 +lp -17.332104 +md -17.332104 +hr -17.332104 +js -17.332104 +cs -17.447581 +gb -17.447581 +az -17.573112 +tm -17.573112 +bw -17.573112 +mw -17.573112 +sb -17.573112 +bc -17.710615 +xp -17.710615 +iz -17.710615 +wh -17.710615 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/xho b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/xho new file mode 100644 index 00000000000..a3070ebf082 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/xho @@ -0,0 +1,463 @@ +#TOTAL_BIGRAMS 4085273 +#UNIQUE_BIGRAMS 459 +#UNSEEN_LOG_PROB -18.640073 +#EXPECTED_SCORE -7.207859 +a_ -4.631583 +o_ -4.961682 +ku -5.276924 +e_ -5.483026 +_k -5.577875 +i_ -5.647471 +_n -5.800456 +ng -5.885833 +ba -5.978785 +_u -5.997638 +an -6.029325 +_i -6.182872 +el -6.269576 +in -6.325320 +zi -6.345653 +en -6.347579 +la -6.415016 +wa -6.527927 +uk -6.597319 +_e -6.603075 +th -6.610682 +un -6.646215 +ok -6.656153 +lo -6.658042 +ub -6.703656 +le -6.705350 +ye -6.709483 +is -6.756742 +kw -6.807302 +ha -6.812968 +_a -6.851192 +ka -6.874953 +al -6.909306 +nd -6.954711 +ma -6.975048 +we -6.986377 +nt -7.024122 +ko -7.029048 +u_ -7.032234 +si -7.081700 +ab -7.104652 +na -7.116462 +ph -7.132031 +ny -7.174150 +am -7.192731 +on -7.192783 +ga -7.194282 +um -7.202633 +ak -7.231531 +ni -7.246522 +ya -7.255991 +se -7.283016 +ho -7.285218 +ul -7.297443 +li -7.298888 +ek -7.326736 +yo -7.377919 +_y -7.450619 +kh -7.459418 +he -7.462404 +lu -7.471840 +_l -7.485318 +ez -7.490389 +nz -7.510532 +_o -7.535605 +ne -7.575937 +hu -7.599783 +_z -7.607270 +hi -7.633887 +be -7.709927 +go -7.735363 +za -7.751862 +fu -7.812492 +es -7.837476 +nk -7.848422 +ay -7.870566 +ik -7.877857 +bo -7.879270 +sh -7.904686 +im -7.980969 +at -8.022238 +em -8.070028 +ze -8.071167 +mi -8.093371 +lw -8.101884 +ge -8.112303 +ol -8.121912 +iz -8.151028 +eb -8.156660 +_b -8.194541 +zo -8.221377 +az -8.228350 +il -8.253887 +ke -8.284281 +hl -8.288140 +wi -8.289907 +om -8.305129 +ts -8.325490 +ut -8.342125 +_w -8.344534 +et -8.352245 +_s -8.383864 +_m -8.399997 +ap -8.429280 +no -8.479193 +sa -8.508730 +it -8.517116 +mb -8.552212 +dl -8.565932 +di -8.574657 +so -8.651388 +ii -8.671118 +ey -8.707415 +aw -8.738603 +wo -8.774494 +uf -8.791920 +ib -8.799295 +me -8.819415 +as -8.860025 +tu -8.867593 +bi -8.933922 +da -8.947457 +us -8.991895 +nj -9.070787 +to -9.079931 +uz -9.136446 +mn -9.144618 +do -9.156459 +ki -9.197129 +uy -9.200658 +yi -9.219902 +iy -9.302897 +ce -9.348442 +bu -9.372818 +iw -9.386698 +af -9.391197 +bh -9.458175 +_p -9.519317 +de -9.554202 +ty -9.561655 +uq -9.563792 +qu -9.581540 +ip -9.587505 +up -9.597866 +xa -9.633046 +ci -9.636133 +if -9.643741 +wu -9.661935 +va -9.671982 +ob -9.678045 +qi -9.687041 +gq -9.700788 +nc -9.710223 +gu -9.747682 +ic -9.764784 +fa -9.775268 +ca -9.792389 +mv -9.794583 +xe -9.809082 +ee -9.817343 +mf -9.826933 +qa -9.836911 +qo -9.874207 +uh -9.876861 +ti -9.891209 +je -9.941021 +od -9.989380 +ig -9.991536 +ix -10.023524 +zw -10.034223 +sw -10.081652 +mt -10.125556 +te -10.132675 +qe -10.134261 +os -10.137042 +oo -10.153439 +mp -10.175731 +ad -10.176140 +m_ -10.178594 +ac -10.232805 +rh -10.283841 +mh -10.301114 +tl -10.301559 +su -10.312296 +dw -10.314093 +ed -10.333100 +gc -10.350515 +gi -10.353746 +kq -10.357448 +aq -10.385517 +mo -10.428185 +nx -10.439910 +ep -10.443840 +iq -10.465647 +ri -10.504107 +ms -10.549961 +ot -10.554733 +zu -10.588048 +uc -10.603899 +ja -10.626611 +mz -10.646285 +ve -10.651388 +ch -10.670255 +nu -10.691122 +hw -10.701082 +cw -10.701670 +pa -10.737396 +xh -10.758653 +ih -10.762329 +ax -10.768476 +ar -10.800239 +ah -10.814796 +ow -10.823729 +qh -10.863969 +gx -10.871889 +ag -10.874538 +id -10.891209 +vi -10.893895 +av -10.899280 +ta -10.914196 +nq -10.919658 +jo -10.932714 +fi -10.954273 +op -10.963411 +xo -10.965528 +tw -11.020953 +mg -11.107133 +vu -11.112596 +uv -11.120437 +ug -11.165150 +oq -11.197959 +fr -11.287809 +oy -11.328098 +fo -11.413179 +_t -11.419937 +ud -11.422842 +s_ -11.433547 +d_ -11.538885 +du -11.542041 +ef -11.566467 +cu -11.585876 +ex -11.619926 +_h -11.640073 +_c -11.665085 +re -11.668529 +eh -11.731981 +ov -11.742833 +eq -11.751330 +iv -11.758653 +ec -11.783336 +pi -11.822450 +_f -11.823729 +yu -11.873213 +t_ -11.889199 +ux -11.908075 +er -11.932714 +mk -11.934095 +_g -11.941021 +kc -11.949377 +of -11.953572 +ur -11.959186 +md -11.999105 +uw -12.029786 +ra -12.100914 +n_ -12.127530 +oz -12.183924 +yw -12.195472 +ml -12.203778 +l_ -12.232380 +ir -12.251195 +st -12.263296 +pe -12.270258 +po -12.273751 +_q -12.302005 +pr -12.386462 +ji -12.388354 +ro -12.432571 +nw -12.438439 +kx -12.494396 +_j -12.521132 +qw -12.531549 +_d -12.544149 +or -12.550490 +co -12.550490 +wc -12.608854 +_r -12.653662 +yh -12.667380 +xw -12.671982 +og -12.674289 +ox -12.674289 +gw -12.706972 +pu -12.730780 +eg -12.733182 +_x -12.764784 +io -12.782092 +xi -12.794583 +tr -12.807183 +c_ -12.869244 +ev -12.987586 +mm -13.004899 +mq -13.104020 +oh -13.132278 +uj -13.138634 +ew -13.230682 +vo -13.251195 +_v -13.279006 +y_ -13.296665 +kt -13.380800 +oc -13.380800 +mu -13.395947 +r_ -13.403580 +p_ -13.474161 +mc -13.478185 +xu -13.490326 +fe -13.498477 +gr -13.514918 +ej -13.591314 +aj -13.604449 +ij -13.662793 +my -13.718827 +ct -13.718827 +dc -13.752548 +mr -13.777126 +oj -13.777126 +ss -13.802130 +g_ -13.863969 +ju -13.917607 +ns -13.917607 +ea -13.928578 +py -14.007805 +ou -14.110252 +ll -14.129111 +k_ -14.167585 +sm -14.174099 +sc -14.275501 +kl -14.303790 +ie -14.303790 +ia -14.310949 +rt -14.347291 +ks -14.362088 +rs -14.407412 +pt -14.415107 +h_ -14.438439 +dp -14.631084 +ai -14.649118 +dy -14.649118 +aa -14.906719 +ru -14.950774 +jl -14.962001 +cl -14.973316 +ua -15.019487 +lt -15.019487 +kr -15.031264 +b_ -15.091636 +rn -15.116511 +z_ -15.193817 +pp -15.207114 +jw -15.220534 +tm -15.247756 +nv -15.261561 +nn -15.275501 +bl -15.422842 +mx -15.486268 +pl -15.535736 +rc -15.569684 +br -15.604449 +pg -15.622151 +eo -15.622151 +oe -15.752548 +rd -15.832718 +ds -15.874538 +cs -15.874538 +mj -15.895912 +x_ -15.895912 +gb -15.917607 +bs -15.939633 +rk -15.962001 +ls -15.984721 +rv -16.104020 +sk -16.129111 +rm -16.129111 +dr -16.129111 +oa -16.154646 +sg -16.180641 +hr -16.207114 +cc -16.261561 +au -16.289576 +ps -16.318145 +ck -16.347291 +v_ -16.438439 +ld -16.470148 +w_ -16.470148 +sb -16.535736 +by -16.535736 +ue -16.569684 +sy -16.604449 +pm -16.640073 +qm -16.676599 +tt -16.676599 +cy -16.676599 +gh -16.752548 +xt -16.832718 +tb -16.832718 +hn -16.832718 +ry -16.832718 +gy -16.917607 +oi -16.917607 +bb -16.962001 +ui -16.962001 +cd -17.007805 +rr -17.055110 +ff -17.055110 +fy -17.055110 +yn -17.104020 +ys -17.154646 +nf -17.207114 +ei -17.261561 +dm -17.318145 +rb -17.318145 +wn -17.438439 +cr -17.438439 +qf -17.502569 +f_ -17.502569 +sp -17.502569 +rl -17.569684 +rg -17.569684 +gd -17.569684 +nl -17.640073 +ae -17.640073 +hy -17.714074 +rp -17.714074 +lm -17.714074 +dt -17.714074 +gm -17.792076 +ht -17.874538 +zk -17.874538 +lb -17.962001 +vl -17.962001 +tq -17.962001 +nb -18.055110 +dh -18.055110 +dk -18.154646 +bj -18.154646 +fl -18.154646 +zl -18.154646 +ft -18.154646 +lr -18.261561 +kn -18.261561 +ao -18.377039 +dd -18.502569 +kp -18.502569 +sl -18.502569 +pk -18.502569 +tc -18.502569 +bw -18.640073 +db -18.640073 +nm -18.640073 +iu -18.640073 +ln -18.640073 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/yid b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/yid new file mode 100644 index 00000000000..5077823f2c1 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/yid @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 5125829 +#UNIQUE_BIGRAMS 637 +#UNSEEN_LOG_PROB -16.955627 +#EXPECTED_SCORE -7.325635 +ן_ -4.256840 +_א -4.633900 +ט_ -5.262193 +ער -5.403348 +ר_ -5.645413 +וו -5.728499 +יי -5.758856 +אי -5.785342 +_פ -5.873200 +וי -6.150122 +ע_ -6.162811 +ון -6.187706 +אר -6.203093 +או -6.235004 +גע -6.303512 +_ו -6.392312 +_ג -6.394986 +_ד -6.460920 +_מ -6.536398 +דע -6.551235 +ען -6.650918 +ין -6.652446 +_ז -6.668848 +_ה -6.691447 +ס_ -6.966018 +וע -6.971623 +יז -6.977499 +ענ -7.017526 +אל -7.074011 +פו -7.093482 +ז_ -7.137069 +זי -7.140004 +מע -7.141235 +הא -7.144736 +יט -7.151802 +ינ -7.157256 +ם_ -7.157618 +טע -7.161319 +אט -7.178830 +_ב -7.196473 +נע -7.223559 +אנ -7.248123 +י_ -7.250349 +פא -7.252794 +ך_ -7.309527 +_ק -7.330484 +רי -7.373055 +וא -7.441856 +אס -7.484971 +שט -7.506049 +ל_ -7.530246 +על -7.534258 +_נ -7.538124 +יע -7.549099 +_ש -7.568789 +לי -7.589944 +נט -7.634662 +יש -7.651427 +רא -7.714760 +מי -7.790380 +ני -7.790754 +רע -7.800321 +רו -7.826532 +יר -7.871633 +_י -7.916625 +לע -7.951523 +_צ -7.977251 +יס -7.982936 +נד -8.020739 +יל -8.058283 +עו -8.103472 +_ר -8.109600 +בי -8.123034 +טי -8.126412 +טא -8.130113 +רט -8.147885 +נג -8.175449 +יך -8.177077 +בע -8.201477 +רן -8.209453 +_ט -8.223433 +נא -8.231701 +ונ -8.239250 +אב -8.253266 +_ל -8.269589 +יב -8.276468 +סט -8.282239 +דא -8.307430 +מא -8.323660 +ג_ -8.338160 +בן -8.338525 +עט -8.381524 +עם -8.399090 +_ע -8.402324 +בא -8.413413 +_ס -8.433123 +פי -8.464594 +יג -8.477578 +פר -8.491692 +זע -8.529778 +צי -8.533945 +לט -8.554221 +צו -8.556445 +לא -8.561540 +קע -8.580701 +אפ -8.604824 +קא -8.612185 +יא -8.626686 +כע -8.629915 +די -8.641558 +טן -8.672805 +ור -8.687815 +ה_ -8.701342 +יד -8.734166 +אמ -8.750680 +רב -8.757729 +עס -8.795624 +פע -8.796875 +ף_ -8.846281 +ד_ -8.876387 +ים -8.881953 +יק -8.924399 +עג -8.959698 +ב_ -8.985716 +שע -8.988572 +ת_ -9.019497 +קו -9.088302 +ק_ -9.094136 +ומ -9.127277 +פט -9.180829 +אך -9.181483 +גר -9.181810 +קי -9.188692 +עק -9.203218 +הי -9.208536 +לו -9.229840 +אכ -9.260584 +לכ -9.263868 +דו -9.276380 +כט -9.277079 +זא -9.291116 +נו -9.295708 +אג -9.296062 +סי -9.307608 +אד -9.325013 +טו -9.335339 +רש -9.346107 +ש_ -9.351245 +יכ -9.356217 +אק -9.377213 +יף -9.411495 +סע -9.412453 +עמ -9.440340 +יפ -9.445237 +אז -9.447200 +גא -9.448183 +עב -9.471571 +עפ -9.474571 +בר -9.497988 +עד -9.502880 +יו -9.512715 +גן -9.542840 +טר -9.555279 +אן -9.575108 +שו -9.591083 +טש -9.619804 +שי -9.659542 +לן -9.674644 +גי -9.678329 +גט -9.690836 +_ח -9.696196 +הו -9.751650 +עש -9.758704 +פן -9.772915 +ות -9.779826 +צע -9.830947 +קט -9.834798 +זו -9.838143 +א_ -9.839689 +עה -9.853944 +ום -9.868341 +סא -9.929604 +דר -9.935931 +ץ_ -9.938967 +שן -9.973921 +שפ -9.999047 +רמ -10.017016 +מו -10.021689 +קר -10.070488 +אצ -10.072002 +רק -10.092137 +שר -10.099529 +_כ -10.099529 +ו_ -10.106029 +פל -10.107270 +עז -10.134852 +רג -10.146928 +בו -10.173985 +ול -10.214543 +ופ -10.222592 +וס -10.250778 +דן -10.267680 +יצ -10.278127 +קל -10.296770 +וט -10.309571 +טל -10.313864 +וב -10.322128 +נס -10.358617 +ימ -10.380586 +בל -10.389242 +עצ -10.391130 +של -10.396811 +נצ -10.409771 +כן -10.410920 +רך -10.431762 +עכ -10.439167 +ספ -10.459235 +רה -10.475974 +רנ -10.477578 +וד -10.498191 +סן -10.509635 +שא -10.517452 +אש -10.560158 +גל -10.591083 +נק -10.599356 +וק -10.654089 +הע -10.670968 +צט -10.685728 +רפ -10.717601 +לד -10.750195 +רד -10.758460 +לס -10.759923 +מש -10.771193 +קס -10.792001 +עי -10.795999 +_ת -10.804531 +רל -10.828898 +הר -10.842271 +מט -10.931802 +נה -10.945613 +וצ -10.947835 +צן -11.013230 +לב -11.068976 +רס -11.201228 +גו -11.208536 +דל -11.238825 +ח_ -11.239505 +וז -11.253867 +שנ -11.333704 +סג -11.339527 +זן -11.340987 +אה -11.341717 +פג -11.366770 +יה -11.388487 +זש -11.416679 +סו -11.426717 +כא -11.447790 +סל -11.448576 +נפ -11.451726 +בט -11.460424 +לק -11.475573 +וש -11.477177 +למ -11.520343 +מל -11.522825 +חי -11.529466 +מפ -11.623130 +חו -11.625796 +תו -11.637405 +כי -11.642795 +בנ -11.668218 +וג -11.698767 +מר -11.752136 +רכ -11.753107 +רצ -11.757972 +עא -11.779579 +כנ -11.833000 +ית -11.835055 +קן -11.844339 +סק -11.853684 +שה -11.890610 +צא -11.934003 +שמ -11.942840 +חס -11.970811 +גנ -11.985573 +מב -12.019059 +מה -12.020227 +רף -12.086006 +בה -12.099529 +נש -12.109445 +אם -12.116926 +יץ -12.121936 +זר -12.132007 +דז -12.149803 +כו -12.180829 +כד -12.182137 +נל -12.193957 +רץ -12.199242 +חמ -12.258687 +נז -12.279525 +כל -12.293587 +שב -12.304935 +מג -12.336613 +זנ -12.342448 +הם -12.364541 +נמ -12.386979 +עע -12.386979 +זט -12.400611 +כה -12.426717 +פש -12.442297 +וכ -12.451726 +רח -12.496564 +מז -12.514567 +טס -12.521170 +פ_ -12.524482 +נח -12.524482 +מד -12.541161 +מנ -12.542840 +לה -12.578547 +לח -12.583722 +נץ -12.595867 +טמ -12.597610 +בת -12.604605 +תל -12.604605 +מח -12.631142 +סד -12.641895 +הל -12.649109 +צפ -12.654543 +גס -12.678329 +כמ -12.706271 +רת -12.710038 +קב -12.713815 +מס -12.723300 +לפ -12.736685 +דש -12.748257 +עך -12.783542 +מק -12.797501 +מצ -12.819712 +וח -12.854726 +חל -12.865188 +לל -12.869394 +קצ -12.894891 +לץ -12.907811 +רז -12.907811 +מן -12.923032 +זײ -12.958437 +וך -12.978741 +לג -12.978741 +דה -12.999335 +אץ -13.041426 +קה -13.043801 +חה -13.070185 +הן -13.072608 +וה -13.121936 +וף -13.147247 +פנ -13.147247 +זל -13.157497 +כס -13.162649 +טג -13.183445 +דט -13.186066 +סף -13.193957 +יח -13.204546 +לם -13.209869 +חת -13.223265 +חר -13.236786 +המ -13.297832 +פס -13.300669 +חב -13.332252 +חש -13.332252 +סכ -13.340987 +סנ -13.343910 +חק -13.370491 +טה -13.382463 +לש -13.385472 +צב -13.400611 +צח -13.403658 +עץ -13.418989 +טפ -13.434486 +מת -13.453304 +תי -13.453304 +זם -13.481999 +טח -13.491692 +בס -13.514567 +הג -13.514567 +לף -13.534466 +_װ -13.551262 +כת -13.554644 +ײן -13.561433 +פח -13.568255 +לך -13.575108 +סת -13.595867 +גד -13.645498 +שכ -13.663645 +פה -13.682024 +צר -13.685728 +כג -13.700639 +בש -13.719498 +שק -13.727111 +אף -13.727111 +צל -13.750195 +קד -13.769718 +סח -13.777601 +ממ -13.781559 +הש -13.797501 +נת -13.809574 +קנ -13.829922 +פצ -13.838143 +חם -13.867289 +כש -13.897036 +דמ -13.905650 +כק -13.936207 +צד -13.945058 +וץ -13.958437 +תש -13.994733 +חכ -13.994733 +תפ -13.999335 +נב -14.031966 +חנ -14.036688 +דב -14.060535 +װא -14.079901 +טנ -14.134536 +װע -14.154928 +טב -14.165233 +חד -14.165233 +הכ -14.170413 +חז -14.175612 +כר -14.175612 +דכ -14.228658 +בכ -14.244960 +ײנ -14.244960 +סב -14.283729 +עת -14.283729 +פת -14.317810 +סר -14.329352 +סה -14.340987 +הק -14.346839 +רם -14.382463 +גמ -14.394536 +טק -14.431373 +בז -14.462805 +לנ -14.481999 +אח -14.494938 +סמ -14.501451 +בח -14.507994 +תב -14.534466 +שך -14.547887 +גב -14.568255 +תק -14.581995 +הנ -14.602853 +נף -14.624018 +תן -14.638302 +כפ -14.645498 +הת -14.659997 +הס -14.667302 +דס -14.682024 +במ -14.719498 +פמ -14.727111 +זכ -14.773654 +זק -14.781559 +תמ -14.805538 +ײט -14.829922 +קג -14.863089 +הח -14.905650 +תם -14.914314 +צה -14.923032 +סצ -14.931802 +זמ -14.940626 +קז -14.976471 +לז -14.985573 +דנ -14.985573 +רר -15.003952 +תר -15.050949 +שם -15.089682 +סש -15.089682 +בד -15.099529 +עװ -15.129483 +בק -15.139607 +תא -15.149803 +דת -15.149803 +כב -15.170413 +אױ -15.191322 +תנ -15.223265 +הד -15.289354 +הב -15.300669 +לצ -15.346839 +כצ -15.382463 +nd -15.418989 +קש -15.431373 +הט -15.443864 +אא -15.443864 +כם -15.481999 +דם -15.494938 +בב -15.521170 +מף -15.616929 +דק -15.631142 +את -15.645498 +_w -15.659997 +wi -15.659997 +in -15.659997 +do -15.659997 +ow -15.659997 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/yor b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/yor new file mode 100644 index 00000000000..23ab143cf8b --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/yor @@ -0,0 +1,367 @@ +#TOTAL_BIGRAMS 1351629 +#UNIQUE_BIGRAMS 363 +#UNSEEN_LOG_PROB -17.044340 +#EXPECTED_SCORE -6.793810 +n_ -4.054041 +_a -4.455578 +wo -4.916797 +on -4.921576 +i_ -4.968994 +a_ -5.066666 +aw -5.293423 +e_ -5.297951 +o_ -5.458391 +_i -5.491527 +_o -5.576530 +un -5.986280 +u_ -6.124955 +an -6.157805 +ti -6.166902 +at -6.177447 +gb -6.234250 +_n -6.359766 +ni -6.373152 +_l -6.563751 +in -6.569620 +_e -6.611589 +la -6.671366 +ri -6.675397 +ra -6.762757 +ba -6.799381 +aa -6.819253 +ar -6.957407 +or -7.074521 +_k -7.153771 +se -7.188381 +_w -7.207816 +lo -7.259541 +_g -7.285950 +_f -7.311155 +ka -7.381671 +_s -7.382027 +bo -7.405723 +ta -7.440528 +il -7.473346 +_t -7.475244 +le -7.485728 +pa -7.514714 +_m -7.544295 +is -7.563751 +ye -7.626698 +ko -7.632829 +be -7.640263 +si -7.683054 +yi -7.704935 +bi -7.704935 +el -7.733500 +fu -7.736911 +ay -7.749719 +na -7.761251 +er -7.778256 +_y -7.796175 +lu -7.797124 +ju -7.803310 +ir -7.860704 +ip -7.863685 +_b -7.863685 +ii -7.867169 +_p -7.930858 +nu -7.948679 +da -7.962457 +oj -7.970466 +re -7.977443 +ro -8.007891 +ig -8.037873 +ya -8.060776 +_d -8.100653 +it -8.102412 +iw -8.135947 +pe -8.204506 +de -8.205451 +ok -8.210501 +hu -8.242470 +mo -8.271530 +_j -8.278805 +ib -8.294805 +ma -8.307938 +po -8.318462 +op -8.324951 +as -8.328378 +so -8.328378 +ey -8.355740 +ak -8.372976 +to -8.394365 +og -8.397961 +en -8.403733 +ol -8.414255 +ru -8.417536 +wa -8.426689 +jo -8.442940 +oh -8.447777 +ab -8.458251 +oo -8.462762 +di -8.472208 +ee -8.481716 +om -8.487067 +ad -8.542502 +ja -8.599325 +ke -8.605548 +mi -8.620593 +tu -8.631982 +al -8.657615 +es -8.658909 +sa -8.663229 +du -8.672345 +je -8.726927 +ku -8.735091 +su -8.740559 +od -8.790256 +ed -8.791201 +id -8.816002 +ik -8.841726 +iy -8.866422 +of -8.900701 +fi -8.907861 +ag -8.942627 +ob -8.953698 +fa -8.958469 +os -8.973950 +ki -8.978789 +we -9.013121 +yo -9.046031 +ap -9.062487 +_r -9.100653 +ij -9.145285 +li -9.172742 +ow -9.186981 +aj -9.187603 +eh -9.279468 +et -9.279468 +ej -9.381849 +ji -9.420824 +ur -9.512958 +eg -9.537338 +nr -9.646879 +ns -9.665828 +ai -9.670170 +ha -9.701820 +ek -9.734181 +c_ -9.749719 +uj -9.751558 +am -9.804025 +ul -9.822269 +ho -9.823236 +pi -9.866422 +bb -9.878428 +bc -9.890534 +te -9.918151 +me -9.926437 +mu -9.979328 +bu -9.995580 +im -10.007617 +gu -10.008716 +_c -10.019754 +do -10.144681 +uk -10.156814 +oy -10.160475 +ia -10.169051 +if -10.285450 +t_ -10.294805 +af -10.345288 +nl -10.353643 +nn -10.366268 +ou -10.412071 +co -10.438490 +fo -10.457875 +ew -10.476004 +nk -10.503630 +uu -10.511399 +eb -10.594778 +ge -10.611380 +wi -10.624801 +fe -10.657184 +go -10.665828 +wu -10.704490 +ga -10.720609 +_h -10.747882 +ug -10.790729 +gi -10.802118 +ot -10.840747 +gh -10.860456 +ut -10.866422 +ht -10.886488 +ih -10.890534 +io -10.908887 +yr -10.912997 +py -10.917119 +us -10.980405 +ie -10.984725 +nt -11.042087 +hi -11.097141 +um -11.198850 +em -11.219063 +ft -11.262980 +st -11.265605 +s_ -11.284119 +nj -11.297490 +ic -11.327349 +nw -11.335601 +ud -11.366268 +ne -11.391853 +tw -11.406266 +ny -11.471450 +ng -11.471450 +up -11.486685 +y_ -11.523917 +nf -11.549284 +ub -11.558913 +nd -11.621434 +cr -11.631558 +ah -11.641754 +pu -11.652022 +uw -11.697383 +ep -11.729643 +kr -11.755243 +no -11.800214 +ea -11.866422 +ce -11.874415 +mb -11.961126 +l_ -12.062487 +tt -12.076249 +h_ -12.076249 +r_ -12.108880 +nm -12.132648 +mm -12.171511 +ve -12.221610 +uf -12.278805 +d_ -12.278805 +iv -12.394724 +ry -12.423753 +ll -12.435530 +tr -12.447405 +sh -12.453378 +ae -12.546089 +th -12.665828 +k_ -12.708056 +gl -12.766355 +eo -12.766355 +he -12.834886 +ef -12.948415 +gr -13.099481 +pt -13.118340 +br -13.127863 +ca -13.137449 +oi -13.216521 +fr -13.236985 +ua -13.247327 +ac -13.247327 +_u -13.247327 +np -13.278805 +ch -13.300179 +ui -13.343900 +av -13.400483 +iu -13.571852 +az -13.624801 +pp -13.652022 +eu -13.679767 +rt -13.693842 +uh -13.751558 +va -13.751558 +sc -13.751558 +mp -13.766355 +pr -13.766355 +rk -13.766355 +ck -13.781305 +w_ -13.858473 +ue -13.858473 +p_ -13.858473 +ty -13.890534 +bl -13.890534 +rc -13.906836 +oc -13.906836 +ec -13.973950 +ly -14.044340 +yu -14.062487 +au -14.080866 +m_ -14.321874 +vi -14.388988 +ov -14.435530 +sr -14.483625 +ws -14.533378 +wn -14.558913 +dr -14.558913 +nb -14.558913 +rl -14.611380 +zo -14.638347 +ss -14.638347 +ct -14.781305 +nc -14.781305 +rs -14.842706 +rg -14.906836 +kh -14.973950 +ph -15.044340 +pc -15.118340 +tc -15.118340 +ts -15.156814 +oa -15.236985 +ld -15.366268 +kk -15.412071 +km -15.459377 +rw -15.459377 +rn -15.459377 +pd -15.558913 +dp -15.558913 +uy -15.558913 +sl -15.558913 +ms -15.558913 +ci -15.558913 +za -15.611380 +hs -15.611380 +zi -15.611380 +g_ -15.611380 +sp -15.611380 +v_ -15.665828 +ze -15.665828 +pl -15.722412 +cy -15.722412 +lt -15.781305 +ky -15.781305 +ps -15.781305 +lw -15.842706 +ff -15.842706 +cs -15.842706 +cl -15.906836 +kl -15.906836 +hn -15.973950 +ys -15.973950 +kj -16.366268 +sq -16.366268 +ql -16.366268 +sg -16.459377 +gs -16.459377 +b_ -16.459377 +yb -16.459377 +ao -16.459377 +hm -16.459377 +td -16.558913 +cz -16.558913 +gp -16.558913 +sy -16.665828 +cc -16.665828 +cp -16.665828 +ls -16.665828 +gy -16.665828 +yz -16.665828 +zs -16.665828 +wh -16.665828 +yn -16.781305 +_z -16.781305 +sv -16.906836 +js -16.906836 +ev -16.906836 +yw -16.906836 +_v -16.906836 +gt -16.906836 +ks -17.044340 +yc -17.044340 +rm -17.044340 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/zho-simp b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/zho-simp new file mode 100644 index 00000000000..06ec07fda44 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/zho-simp @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 24485081 +#UNIQUE_BIGRAMS 34727 +#UNSEEN_LOG_PROB -15.941093 +#EXPECTED_SCORE -12.988798 +的_ -5.649647 +_的 -5.727226 +_一 -7.254670 +是_ -7.348164 +_是 -7.466655 +_不 -7.512579 +一_ -7.513063 +人_ -7.914564 +有_ -7.936899 +_有 -7.950011 +不_ -7.951979 +了_ -7.989477 +_在 -8.110364 +_人 -8.148327 +_我 -8.216146 +在_ -8.298733 +_这 -8.375455 +个_ -8.613808 +_了 -8.632206 +会_ -8.702558 +_大 -8.732947 +来_ -8.759666 +大_ -8.803248 +为_ -8.820886 +_中 -8.867515 +中_ -8.889757 +上_ -8.900107 +们_ -8.911496 +到_ -8.913597 +_以 -8.939573 +_会 -8.962699 +我_ -8.976434 +_为 -9.008849 +_可 -9.010610 +_个 -9.018717 +_就 -9.021746 +国_ -9.037450 +_也 -9.061584 +_他 -9.061678 +以_ -9.079769 +要_ -9.090811 +能_ -9.099770 +_国 -9.134617 +时_ -9.156306 +_上 -9.164364 +出_ -9.174678 +_到 -9.190324 +_要 -9.226892 +_来 -9.246657 +_年 -9.276163 +_出 -9.287238 +_自 -9.293732 +_能 -9.300961 +生_ -9.306734 +这_ -9.316356 +年_ -9.327896 +_们 -9.328805 +用_ -9.389909 +_时 -9.398314 +_发 -9.483986 +可_ -9.484028 +_之 -9.497850 +_生 -9.503013 +家_ -9.542673 +他_ -9.554099 +说_ -9.572061 +_对 -9.590385 +过_ -9.611663 +于_ -9.616095 +业_ -9.627981 +_成 -9.637054 +成_ -9.671523 +多_ -9.679038 +发_ -9.691895 +就_ -9.699763 +下_ -9.700547 +法_ -9.706393 +_所 -9.707033 +动_ -9.711571 +得_ -9.719497 +_公 -9.719646 +自_ -9.721336 +对_ -9.755153 +_都 -9.762913 +_而 -9.805724 +行_ -9.805777 +地_ -9.819607 +后_ -9.831637 +_你 -9.869443 +然_ -9.869663 +心_ -9.871373 +_还 -9.879009 +么_ -9.880508 +之_ -9.881897 +_地 -9.889925 +_用 -9.892499 +力_ -9.893115 +_和 -9.895470 +_多 -9.907303 +_经 -9.908039 +_过 -9.909512 +也_ -9.915303 +和_ -9.919405 +学_ -9.925810 +方_ -9.942585 +_没 -9.945661 +_得 -9.957095 +定_ -9.959029 +_学 -9.962082 +现_ -9.974121 +_最 -9.983754 +_现 -9.987218 +道_ -9.999289 +一个 -10.006362 +_方 -10.017923 +_家 -10.021776 +经_ -10.036119 +我们 -10.038534 +_那 -10.052170 +_开 -10.065114 +_看 -10.066125 +_下 -10.068717 +_如 -10.070046 +好_ -10.074677 +子_ -10.079706 +作_ -10.083025 +前_ -10.087378 +_业 -10.089366 +间_ -10.098187 +_与 -10.110837 +_说 -10.127020 +理_ -10.134418 +_法 -10.137800 +_当 -10.140990 +_主 -10.155365 +事_ -10.163519 +都_ -10.185856 +_因 -10.188741 +面_ -10.192253 +天_ -10.202700 +_只 -10.212524 +_产 -10.215323 +与_ -10.222134 +_进 -10.232517 +_民 -10.233935 +者_ -10.235000 +_于 -10.241334 +_台 -10.242547 +_行 -10.252431 +体_ -10.254661 +度_ -10.265644 +_然 -10.266297 +机_ -10.268694 +_分 -10.269348 +_无 -10.280737 +_高 -10.295101 +_全 -10.308638 +民_ -10.309012 +_其 -10.314929 +_心 -10.315755 +品_ -10.325550 +手_ -10.338004 +_很 -10.339683 +_同 -10.348260 +_好 -10.352722 +_但 -10.354572 +_动 -10.355961 +开_ -10.357816 +种_ -10.360524 +样_ -10.363005 +本_ -10.365957 +可以 -10.366346 +_实 -10.366657 +_电 -10.367202 +_本 -10.373285 +些_ -10.381200 +自己 -10.381436 +己_ -10.381436 +_作 -10.382694 +看_ -10.384032 +_提 -10.394462 +_身 -10.404809 +没_ -10.406408 +场_ -10.408569 +分_ -10.409610 +著_ -10.409770 +_政 -10.425810 +_后 -10.427107 +没有 -10.429137 +起_ -10.429218 +去_ -10.429950 +_前 -10.461920 +你_ -10.464915 +同_ -10.465415 +次_ -10.465915 +所_ -10.470756 +_事 -10.471425 +果_ -10.477629 +_日 -10.478638 +高_ -10.482426 +_机 -10.486816 +_十 -10.490880 +_小 -10.491559 +想_ -10.493681 +_手 -10.499043 +主_ -10.506138 +_天 -10.518790 +_第 -10.518963 +实_ -10.520260 +身_ -10.525201 +产_ -10.532513 +务_ -10.533386 +意_ -10.533736 +外_ -10.542497 +长_ -10.546721 +_相 -10.548926 +_更 -10.558224 +点_ -10.560981 +_表 -10.563565 +_工 -10.570985 +月_ -10.574485 +公_ -10.587119 +_关 -10.614755 +关_ -10.615679 +台_ -10.620587 +性_ -10.627468 +此_ -10.635975 +合_ -10.636163 +全_ -10.636632 +小_ -10.646798 +_面 -10.648594 +师_ -10.654565 +他们 -10.661993 +_体 -10.663425 +部_ -10.663616 +很_ -10.664381 +位_ -10.671763 +_理 -10.675324 +_美 -10.677059 +及_ -10.677349 +_想 -10.687030 +中国 -10.689752 +知_ -10.692675 +_重 -10.692870 +_月 -10.697657 +化_ -10.699420 +情_ -10.699420 +制_ -10.701969 +里_ -10.723227 +进_ -10.735632 +_两 -10.740664 +_已 -10.748853 +_应 -10.749664 +_知 -10.752508 +_并 -10.756783 +_定 -10.765783 +_新 -10.770201 +明_ -10.771642 +政_ -10.772776 +_她 -10.773291 +_么 -10.778767 +入_ -10.779181 +最_ -10.780735 +_加 -10.784160 +_三 -10.793751 +就是 -10.796368 +量_ -10.815460 +_制 -10.821739 +_让 -10.822379 +重_ -10.828259 +加_ -10.837286 +内_ -10.838688 +示_ -10.839335 +_起 -10.840091 +_力 -10.860432 +_保 -10.860432 +_意 -10.860980 +还_ -10.862186 +电_ -10.864710 +_正 -10.870325 +_将 -10.877401 +常_ -10.879287 +_老 -10.881175 +术_ -10.892107 +日_ -10.893563 +新_ -10.894348 +十_ -10.895358 +当_ -10.896705 +名_ -10.899629 +_感 -10.910589 +_使 -10.911383 +_合 -10.912177 +那_ -10.914223 +_等 -10.915588 +话_ -10.918892 +题_ -10.919805 +表_ -10.920604 +无_ -10.935182 +头_ -10.935298 +_真 -10.939110 +应_ -10.948162 +件_ -10.960555 +将_ -10.960555 +_问 -10.964199 +工_ -10.964317 +式_ -10.965848 +_及 -10.971043 +_道 -10.974002 +其_ -10.976968 +_平 -10.977087 +_特 -10.983873 +如_ -11.000194 +_接 -11.000435 +_专 -11.006604 +_教 -11.006847 +_文 -11.008060 +员_ -11.011583 +_内 -11.013409 +物_ -11.013653 +_品 -11.023922 +_比 -11.030069 +_原 -11.035748 +_部 -11.036862 +_子 -11.046550 +程_ -11.047423 +_回 -11.051169 +司_ -11.053171 +因为 -11.056304 +这个 -11.056681 +市_ -11.059067 +_著 -11.061836 +光_ -11.062340 +_设 -11.065998 +利_ -11.067009 +她_ -11.083025 +_长 -11.083409 +_者 -11.085968 +_市 -11.086480 +被_ -11.094575 +公司 -11.098058 +_目 -11.101809 +_舞 -11.103104 +计_ -11.106088 +_什 -11.106478 +_外 -11.108038 +期_ -11.109209 +什么 -11.114424 +_资 -11.123203 +不是 -11.141189 +_社 -11.141189 +_明 -11.146122 +问_ -11.153888 +_从 -11.155365 +_性 -11.157382 +_各 -11.161695 +_己 -11.163451 +展_ -11.176075 +_完 -11.176757 +代_ -11.182223 +受_ -11.202353 +统_ -11.208475 +_此 -11.210568 +_通 -11.210707 +由_ -11.212804 +台湾 -11.217426 +湾_ -11.217426 +提_ -11.219954 +_基 -11.227987 +三_ -11.228977 +的人 -11.232233 +色_ -11.235213 +权_ -11.235497 +d_ -11.237487 +_活 -11.240193 +美_ -11.240335 +_认 -11.241191 +_情 -11.242190 +_二 -11.249057 +_每 -11.255525 +已_ -11.256245 +_样 -11.265064 +命_ -11.265064 +_世 -11.267531 +_才 -11.271167 +是一 -11.272187 +_度 -11.272187 +_女 -11.272916 +正_ -11.275689 +_去 -11.276565 +了一 -11.277881 +_系 -11.279638 +_选 -11.283011 +已经 -11.286686 +气_ -11.290371 +感_ -11.290666 +_师 -11.294065 +舞_ -11.298065 +界_ -11.302523 +_数 -11.306249 +_结 -11.308339 +何_ -11.313578 +_服 -11.318987 +_几 -11.330170 +_利 -11.336403 +求_ -11.339454 +见_ -11.345421 +_建 -11.349566 +数_ -11.352490 +_计 -11.355729 +如果 -11.357429 +_规 -11.359595 +口_ -11.367824 +_化 -11.369071 +资_ -11.369226 +平_ -11.370786 +_间 -11.375005 +_果 -11.376884 +相_ -11.379550 +社会 -11.385686 +_里 -11.395494 +而_ -11.397718 +第_ -11.400901 +只_ -11.401857 +_需 -11.402654 +_技 -11.404250 +_网 -11.406008 +着_ -11.406488 +文_ -11.410652 +问题 -11.410973 +别_ -11.411776 +_或 -11.412418 +_信 -11.413864 +因_ -11.417566 +_医 -11.417889 +持_ -11.420471 +_场 -11.427432 +_今 -11.428081 +立_ -11.428568 +少_ -11.429868 +金_ -11.431332 +_安 -11.431820 +_由 -11.434590 +让_ -11.440473 +_变 -11.441129 +_直 -11.442440 +_常 -11.443096 +_点 -11.445723 +_打 -11.446381 +_名 -11.449014 +_金 -11.449508 +_再 -11.453799 +_光 -11.458932 +_代 -11.459263 +_被 -11.467749 +时间 -11.472262 +系_ -11.474943 +二_ -11.479310 +_权 -11.480825 +_改 -11.483522 +现在 -11.485548 +目_ -11.490117 +治_ -11.493851 +的一 -11.503740 +_次 -11.504083 +信_ -11.509913 +女_ -11.510601 +人的 -11.517494 +接_ -11.519222 +_做 -11.519741 +活_ -11.523205 +_未 -11.528243 +_给 -11.528765 +更_ -11.529636 +原_ -11.531554 +_传 -11.533648 +步_ -11.534697 +可能 -11.540126 +_企 -11.540301 +容_ -11.541530 +使用 -11.546457 +觉_ -11.546457 +le -11.547162 +_立 -11.549809 +特_ -11.551577 +造_ -11.553524 +教_ -11.554409 +元_ -11.564725 +_精 -11.565439 +e_ -11.566510 +_l -11.567404 +_受 -11.567404 +_位 -11.569373 +_期 -11.569731 +解_ -11.570268 +真_ -11.570806 +_种 -11.572420 +等_ -11.574215 +望_ -11.577813 +_总 -11.578534 +知道 -11.579615 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/zho-trad b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/zho-trad new file mode 100644 index 00000000000..4bcb586d769 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/zho-trad @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 82544350 +#UNIQUE_BIGRAMS 34621 +#UNSEEN_LOG_PROB -15.935056 +#EXPECTED_SCORE -13.040743 +的_ -5.937362 +_的 -5.990976 +_一 -7.450334 +國_ -7.721134 +一_ -7.737971 +_國 -7.779084 +是_ -7.835218 +_是 -7.959703 +_在 -7.967456 +人_ -8.005320 +_不 -8.009233 +_中 -8.102688 +_人 -8.177542 +_有 -8.215909 +有_ -8.221292 +在_ -8.230943 +中_ -8.344322 +_大 -8.382443 +大_ -8.489978 +不_ -8.526487 +年_ -8.535460 +會_ -8.564744 +_年 -8.632574 +為_ -8.707882 +了_ -8.723289 +_這 -8.789695 +_會 -8.829796 +_為 -8.977099 +_十 -9.007972 +出_ -9.027039 +上_ -9.027677 +個_ -9.029101 +_日 -9.145569 +_上 -9.153301 +_他 -9.155911 +來_ -9.186430 +_發 -9.190131 +_政 -9.193330 +_以 -9.203331 +到_ -9.218671 +_出 -9.255778 +_民 -9.255820 +家_ -9.279673 +時_ -9.285812 +十_ -9.288051 +_了 -9.290128 +民_ -9.296928 +中國 -9.309582 +_對 -9.310836 +_和 -9.335239 +和_ -9.359408 +_個 -9.392786 +要_ -9.398200 +以_ -9.402842 +_也 -9.407321 +_成 -9.409649 +發_ -9.433851 +行_ -9.464850 +_美 -9.467371 +生_ -9.504745 +_到 -9.507974 +政_ -9.512128 +_公 -9.521603 +_時 -9.525475 +_要 -9.534235 +_經 -9.536089 +_現 -9.542019 +地_ -9.544562 +_台 -9.551312 +現_ -9.556225 +對_ -9.557370 +成_ -9.561247 +日_ -9.579826 +_來 -9.590229 +_二 -9.590943 +_地 -9.593601 +_主 -9.605043 +於_ -9.626116 +業_ -9.629740 +們_ -9.644142 +能_ -9.660923 +_我 -9.674085 +多_ -9.676986 +_可 -9.685524 +後_ -9.690946 +方_ -9.697650 +_生 -9.710303 +下_ -9.724812 +_家 -9.738483 +經_ -9.742894 +_就 -9.755272 +_自 -9.777173 +_新 -9.778539 +前_ -9.785108 +作_ -9.789061 +_多 -9.791119 +動_ -9.812302 +_之 -9.828261 +_當 -9.829644 +_能 -9.838818 +_行 -9.840516 +過_ -9.841045 +長_ -9.843403 +月_ -9.854557 +學_ -9.860589 +_最 -9.865929 +_方 -9.867675 +場_ -9.885518 +_表 -9.896937 +_學 -9.909337 +_下 -9.912635 +_月 -9.918374 +力_ -9.924527 +主_ -9.925988 +市_ -9.938299 +台_ -9.952152 +_三 -9.952655 +_開 -9.962857 +內_ -9.988231 +_作 -9.995992 +他_ -9.998957 +_市 -10.005985 +法_ -10.006668 +本_ -10.013842 +_全 -10.020307 +這_ -10.021470 +美_ -10.024707 +期_ -10.032063 +_進 -10.042439 +部_ -10.050146 +事_ -10.050905 +者_ -10.063400 +_而 -10.068101 +_內 -10.075928 +新_ -10.092434 +_與 -10.106065 +_前 -10.107896 +_都 -10.112513 +_兩 -10.113500 +_本 -10.121811 +_業 -10.125321 +員_ -10.131326 +二_ -10.142030 +_們 -10.162116 +示_ -10.163658 +_分 -10.165644 +可_ -10.178712 +灣_ -10.198024 +開_ -10.198866 +_事 -10.199935 +_於 -10.205372 +_高 -10.206487 +外_ -10.211390 +_將 -10.211866 +_但 -10.213443 +_正 -10.217079 +次_ -10.221454 +_過 -10.223918 +自_ -10.230854 +分_ -10.235671 +_總 -10.237907 +台灣 -10.240505 +關_ -10.253525 +_部 -10.254720 +_加 -10.256109 +_工 -10.261398 +得_ -10.261892 +三_ -10.266857 +公_ -10.276125 +理_ -10.293283 +_文 -10.298996 +表_ -10.304224 +間_ -10.309338 +一個 -10.318348 +與_ -10.326696 +_長 -10.326831 +_同 -10.330247 +定_ -10.332544 +面_ -10.337714 +_關 -10.339434 +度_ -10.349159 +加_ -10.350847 +_法 -10.365805 +_重 -10.366590 +府_ -10.368067 +_因 -10.370125 +化_ -10.371490 +高_ -10.372208 +元_ -10.375989 +體_ -10.385169 +用_ -10.390602 +心_ -10.390907 +美國 -10.392175 +_還 -10.410732 +機_ -10.411993 +_後 -10.430543 +名_ -10.436586 +全_ -10.438573 +_其 -10.440175 +港_ -10.443798 +天_ -10.446356 +也_ -10.453666 +_提 -10.465603 +將_ -10.481283 +子_ -10.481483 +進_ -10.483608 +然_ -10.490808 +正_ -10.499612 +政府 -10.503946 +重_ -10.506511 +_電 -10.508190 +題_ -10.522130 +說_ -10.525063 +_沒 -10.529087 +_四 -10.531775 +展_ -10.536959 +_第 -10.536959 +_已 -10.539246 +_所 -10.550814 +_資 -10.552572 +_得 -10.562132 +就_ -10.563401 +_北 -10.569577 +都_ -10.569975 +_機 -10.571517 +_實 -10.577113 +_香 -10.580160 +合_ -10.583321 +同_ -10.584018 +工_ -10.585011 +文_ -10.586112 +資_ -10.587214 +_如 -10.592170 +及_ -10.593034 +黨_ -10.613479 +_今 -10.622847 +_建 -10.625992 +統_ -10.631610 +_五 -10.637250 +濟_ -10.654556 +_合 -10.657772 +香港 -10.661872 +務_ -10.663912 +_問 -10.667148 +實_ -10.668854 +經濟 -10.679191 +入_ -10.681457 +_動 -10.681543 +海_ -10.683726 +之_ -10.686604 +利_ -10.689112 +_九 -10.690873 +位_ -10.691798 +_外 -10.694084 +意_ -10.694171 +九_ -10.695388 +區_ -10.696808 +當_ -10.697447 +表示 -10.700555 +些_ -10.707641 +_相 -10.717760 +_海 -10.718201 +_小 -10.722536 +_認 -10.722743 +沒有 -10.723245 +點_ -10.732315 +最_ -10.737915 +道_ -10.759841 +_報 -10.765854 +情_ -10.772778 +_選 -10.778324 +_百 -10.779491 +_目 -10.780351 +好_ -10.786852 +我_ -10.792297 +_天 -10.792328 +_社 -10.797918 +報_ -10.800970 +此_ -10.801407 +_更 -10.801469 +_精 -10.808818 +代_ -10.808944 +_及 -10.823439 +四_ -10.824168 +_並 -10.834727 +議_ -10.835238 +容_ -10.837315 +五_ -10.845974 +_萬 -10.850679 +果_ -10.856533 +_特 -10.863028 +選_ -10.864690 +司_ -10.869227 +_網 -10.870339 +界_ -10.871157 +_面 -10.873122 +_體 -10.875944 +_世 -10.876503 +二十 -10.877029 +_力 -10.887652 +品_ -10.889375 +_指 -10.890834 +任_ -10.892660 +_者 -10.897787 +明_ -10.898821 +我們 -10.903366 +電_ -10.911255 +際_ -10.912332 +金_ -10.915772 +治_ -10.916988 +_很 -10.917157 +_理 -10.918476 +百_ -10.921354 +問_ -10.922948 +_華 -10.925903 +兩_ -10.938711 +_名 -10.947934 +起_ -10.948107 +_黨 -10.950249 +問題 -10.953400 +持_ -10.954474 +_港 -10.954821 +_只 -10.967224 +國家 -10.972974 +種_ -10.975823 +發展 -10.976668 +_利 -10.978183 +_意 -10.980830 +_保 -10.986138 +_交 -10.986705 +局_ -10.988692 +_基 -10.993566 +北_ -10.994493 +_通 -10.998421 +_金 -11.006165 +_比 -11.009332 +樣_ -11.012433 +總_ -11.018040 +_場 -11.021560 +軍_ -11.025891 +小_ -11.027823 +華_ -11.028407 +_無 -11.028772 +他們 -11.030343 +可以 -11.031075 +_明 -11.031294 +_員 -11.034590 +已_ -11.039144 +_期 -11.047998 +立_ -11.055009 +公司 -11.055343 +院_ -11.059814 +去_ -11.061830 +_情 -11.074627 +_用 -11.074928 +_等 -11.084727 +_灣 -11.095815 +_反 -11.097538 +受_ -11.097921 +_代 -11.101065 +京_ -11.101565 +_然 -11.107568 +至_ -11.108031 +_平 -11.109421 +_由 -11.111198 +_從 -11.113094 +_展 -11.113365 +_被 -11.114062 +_八 -11.115147 +_受 -11.120157 +_六 -11.124209 +_至 -11.124443 +_化 -11.132307 +_看 -11.139504 +_零 -11.143967 +_軍 -11.145312 +建_ -11.147967 +制_ -11.158875 +日本 -11.160674 +著_ -11.161715 +_聯 -11.162596 +_教 -11.164360 +_安 -11.171196 +目_ -11.174706 +沒_ -11.184436 +性_ -11.185372 +_各 -11.187408 +自己 -11.188428 +己_ -11.188428 +_七 -11.190019 +被_ -11.191572 +式_ -11.193412 +_投 -11.194887 +所_ -11.201869 +_立 -11.201910 +_馬 -11.204011 +內容 -11.207809 +_次 -11.208430 +_接 -11.218182 +麼_ -11.221058 +_共 -11.227120 +件_ -11.228000 +權_ -11.230602 +_回 -11.234471 +應_ -11.236325 +_東 -11.236578 +_改 -11.237168 +_好 -11.237295 +_此 -11.238012 +價_ -11.240844 +_參 -11.242198 +常_ -11.243214 +_定 -11.244740 +量_ -11.253333 +_首 -11.255083 +讀_ -11.268387 +社會 -11.269508 +國際 -11.270803 +亞_ -11.272445 +_南 -11.274825 +社_ -11.281118 +少_ -11.282685 +認為 -11.287089 +_強 -11.292910 +車_ -11.293963 +求_ -11.295148 +望_ -11.296335 +_心 -11.297434 +認_ -11.299195 +_去 -11.300737 +眾_ -11.303694 +提_ -11.304755 +市場 -11.305109 +_區 -11.306746 +_府 -11.306746 +彩_ -11.314782 +八_ -11.320000 +_商 -11.320715 +_任 -11.320894 +_原 -11.322013 +達_ -11.329819 +其_ -11.330539 +_結 -11.333288 +球_ -11.337126 +南_ -11.338528 +斯_ -11.339524 +第_ -11.340385 +商_ -11.344015 +程_ -11.347882 +_數 -11.350025 +_增 -11.351029 +論_ -11.351166 +_影 -11.356106 +東_ -11.366590 +_應 -11.367190 +導_ -11.371629 +零_ -11.373946 +_統 -11.376175 +因為 -11.377523 +看_ -11.378174 +門_ -11.379104 +網_ -11.387274 +計_ -11.394079 +平_ -11.396904 +世_ -11.397846 +工作 -11.403564 +_產 -11.405507 +_推 -11.407405 +手_ -11.409352 +_西 -11.409780 +_近 -11.410256 +_活 -11.412398 +萬_ -11.418269 +_閱 -11.418796 +_彩 -11.421813 +可能 -11.426328 +政治 -11.427290 +七_ -11.427675 +_每 -11.428205 +閱讀 -11.431291 +物_ -11.431291 +就是 -11.432063 +續_ -11.433513 +調_ -11.439131 +由_ -11.440539 +_領 -11.442484 +北京 -11.443700 +_英 -11.447405 +士_ -11.447796 +交_ -11.451217 +_專 -11.453176 +西_ -11.453421 +通_ -11.454549 +_度 -11.454843 +數_ -11.455776 +_調 -11.456905 +六_ -11.458970 +產_ -11.459216 +因_ -11.460250 +_收 -11.461580 diff --git a/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/zul b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/zul new file mode 100644 index 00000000000..9645f402c8b --- /dev/null +++ b/tika-eval/tika-eval-lite/src/main/resources/org/apache/tika/eval/textquality/bigram_profiles/zul @@ -0,0 +1,504 @@ +#TOTAL_BIGRAMS 7656004 +#UNIQUE_BIGRAMS 506 +#UNSEEN_LOG_PROB -19.546232 +#EXPECTED_SCORE -7.157574 +a_ -4.746829 +i_ -5.166827 +e_ -5.287570 +ku -5.421739 +o_ -5.609392 +_u -5.633249 +ng -5.636358 +_n -5.730378 +th -5.830452 +_k -5.842091 +an -5.896751 +la -6.150992 +en -6.155553 +in -6.200605 +ba -6.307514 +zi -6.312388 +_i -6.319714 +hi -6.394534 +le -6.409545 +_e -6.436858 +uk -6.442223 +el -6.497422 +kh -6.564718 +ma -6.612966 +wa -6.656177 +si -6.662157 +ka -6.670406 +ha -6.703567 +u_ -6.745191 +_a -6.781174 +ab -6.811925 +ni -6.813853 +um -6.893073 +ut -6.897493 +lo -6.904542 +al -6.924615 +ho -6.940729 +ok -6.969771 +ak -6.974290 +we -7.014071 +na -7.029522 +ul -7.029769 +is -7.045415 +am -7.075217 +nd -7.111838 +ph -7.129566 +ga -7.132419 +he -7.136125 +un -7.226334 +hu -7.307797 +on -7.373461 +lu -7.376965 +ez -7.395596 +be -7.400173 +_l -7.446161 +es -7.483444 +li -7.490306 +kw -7.514875 +ye -7.541099 +go -7.558115 +ne -7.558968 +se -7.571495 +iz -7.605038 +hl -7.622348 +at -7.629681 +om -7.675559 +ik -7.693078 +ek -7.697022 +nt -7.758289 +_y -7.763152 +ny -7.768484 +em -7.781900 +im -7.789759 +ge -7.796698 +ub -7.811522 +sh -7.849786 +nz -7.894194 +ya -7.948040 +az -7.956675 +ke -7.960424 +_o -7.967293 +no -7.976424 +_w -7.981559 +za -8.038784 +dl -8.039280 +_s -8.073592 +mb -8.095382 +ko -8.109625 +_b -8.109781 +mi -8.111708 +us -8.120072 +nk -8.147382 +ol -8.164094 +sa -8.271097 +bo -8.303713 +il -8.314531 +ze -8.342272 +da -8.376182 +yo -8.403232 +ap -8.460826 +as -8.481085 +ay -8.482365 +me -8.485874 +eb -8.497881 +zo -8.565236 +yi -8.574401 +_z -8.595457 +ob -8.596989 +fu -8.609815 +bu -8.640394 +nj -8.655512 +et -8.670406 +lw -8.704353 +uz -8.720399 +mu -8.741859 +so -8.772022 +_m -8.839391 +tu -8.843540 +gi -8.870010 +bi -8.894552 +it -8.967859 +je -8.967954 +ib -8.985135 +wo -9.004071 +fa -9.106609 +gu -9.183850 +od -9.212965 +aw -9.252990 +de -9.268247 +mp -9.292503 +to -9.301274 +uh -9.313691 +bh -9.337876 +zw -9.346805 +di -9.369186 +mo -9.459432 +up -9.491763 +ah -9.517635 +ki -9.530399 +if -9.633193 +ip -9.636339 +dw -9.636939 +os -9.651111 +af -9.661909 +uf -9.668641 +uy -9.695264 +_f -9.770457 +ca -9.790343 +su -9.834393 +do -9.858332 +mn -9.892850 +_p -9.895898 +ud -9.924730 +ns -9.963149 +ad -9.970882 +uq -9.980940 +nh -10.033097 +iw -10.054579 +ig -10.059397 +uv -10.066250 +zu -10.100596 +ti -10.105986 +ms -10.112438 +ts -10.169022 +ja -10.176417 +ot -10.183631 +qo -10.237665 +id -10.253450 +qa -10.275470 +ic -10.281086 +gc -10.325128 +va -10.325370 +iy -10.377059 +qe -10.388632 +ep -10.398264 +ed -10.401319 +hw -10.402084 +mt -10.414375 +ci -10.446358 +mh -10.446884 +nc -10.489594 +iq -10.513084 +ih -10.523864 +oy -10.556128 +eh -10.592327 +ta -10.624689 +ve -10.646271 +qu -10.660231 +ri -10.686076 +ce -10.702625 +te -10.708289 +pi -10.745656 +ug -10.774082 +ey -10.781360 +aq -10.797368 +pa -10.803081 +s_ -10.835081 +ow -10.854489 +sw -10.864291 +gq -10.871334 +fi -10.873807 +qh -10.890880 +qi -10.892670 +vu -10.908520 +nu -10.926746 +wi -10.937053 +wu -10.940753 +ch -10.971071 +nq -11.023062 +uc -11.024239 +du -11.028563 +mk -11.048380 +co -11.051976 +ag -11.140240 +_c -11.156493 +oh -11.167720 +fr -11.170758 +n_ -11.195293 +gw -11.199718 +ac -11.213972 +vi -11.317413 +er -11.363341 +eq -11.366821 +ef -11.370308 +mf -11.395977 +cw -11.403614 +xa -11.442420 +pe -11.463551 +mv -11.487699 +iv -11.500745 +z_ -11.501292 +nx -11.513360 +md -11.523864 +l_ -11.550748 +_h -11.581025 +ra -11.585651 +oz -11.593782 +ar -11.600203 +st -11.625581 +uj -11.687629 +op -11.743393 +eg -11.749868 +_t -11.772104 +mg -11.869570 +ec -11.873100 +t_ -11.878056 +ov -11.889450 +mc -11.905264 +or -11.934469 +av -11.938162 +cu -11.959017 +ml -11.965785 +mq -11.982464 +re -12.146207 +gx -12.167720 +c_ -12.192203 +io -12.223403 +fo -12.241539 +_d -12.245193 +uw -12.252531 +ro -12.254371 +_j -12.255292 +r_ -12.256213 +d_ -12.266389 +ji -12.273836 +of -12.311654 +p_ -12.327064 +tw -12.342639 +y_ -12.366323 +mz -12.371306 +k_ -12.374305 +ux -12.490950 +aj -12.506216 +oc -12.508411 +_g -12.519432 +xe -12.524974 +jo -12.554143 +xo -12.610772 +ll -12.633343 +ie -12.724777 +pu -12.738877 +ks -12.807464 +hh -12.816951 +po -12.862536 +og -12.882318 +xh -12.892312 +yw -12.911058 +ir -12.934469 +sz -13.024239 +fe -13.065644 +_q -13.067260 +_v -13.086800 +oq -13.114943 +ev -13.164257 +kk -13.195735 +tr -13.220702 +h_ -13.224304 +g_ -13.236983 +ai -13.244279 +rs -13.271970 +ou -13.281320 +m_ -13.294513 +ur -13.305918 +ea -13.315491 +pr -13.336779 +mm -13.360366 +ij -13.376307 +nw -13.386361 +rt -13.390402 +ox -13.446095 +ju -13.523864 +ax -13.548488 +ej -13.566664 +jw -13.568952 +ew -13.606065 +_r -13.610772 +fs -13.718413 +b_ -13.733734 +my -13.759636 +kt -13.764872 +yu -13.893746 +ia -13.896617 +ls -13.902376 +rd -13.902376 +xu -13.970315 +vo -13.973342 +ee -13.976376 +ty -13.982464 +fp -13.997795 +oj -14.025810 +ck -14.093373 +gr -14.116616 +ss -14.140240 +nl -14.188680 +qw -14.188680 +wn -14.235165 +iu -14.275703 +rk -14.325128 +kr -14.364334 +lb -14.408729 +mw -14.412833 +au -14.433532 +xi -14.437708 +pt -14.467281 +sp -14.467281 +rn -14.488782 +j_ -14.493121 +rg -14.515013 +ue -14.550748 +bs -14.573539 +dk -14.615495 +hr -14.620233 +nn -14.683285 +dj -14.693234 +sr -14.738877 +bl -14.770128 +ds -14.775403 +rl -14.807464 +oo -14.879475 +ru -14.925646 +eo -14.961270 +sb -14.985517 +gh -14.991643 +ix -15.010179 +rr -15.126693 +ct -15.181660 +ld -15.181660 +ws -15.217108 +kz -15.224304 +sm -15.224304 +sc -15.253450 +f_ -15.253450 +eu -15.260830 +v_ -15.290731 +zn -15.298305 +ry -15.305918 +ps -15.305918 +rb -15.313571 +lt -15.352460 +dp -15.384344 +rv -15.384344 +bc -15.441895 +x_ -15.441895 +nf -15.441895 +oa -15.441895 +xw -15.467281 +_x -15.519432 +br -15.519432 +ly -15.546232 +oe -15.564379 +ff -15.668488 +ua -15.668488 +cy -15.718413 +kl -15.718413 +sk -15.791345 +dv -15.834737 +hn -15.845792 +mx -15.879475 +ei -15.890880 +cr -15.925646 +xt -15.949297 +pl -15.949297 +rc -15.973342 +mj -15.973342 +w_ -15.985517 +cc -15.985517 +ht -15.985517 +fc -15.997795 +sl -16.047981 +yl -16.140240 +ex -16.153915 +cs -16.195735 +ae -16.195735 +tl -16.195735 +tt -16.238804 +rm -16.238804 +dr -16.253450 +bw -16.283198 +cl -16.329001 +jl -16.425217 +gl -16.441895 +gy -16.475843 +kg -16.475843 +tn -16.493121 +nb -16.678336 +hy -16.698235 +oi -16.738877 +aa -16.738877 +tz -16.780697 +pm -16.890880 +dt -16.890880 +rq -16.961270 +fn -17.010179 +ky -17.035270 +zb -17.035270 +np -17.253450 +sf -17.283198 +lf -17.283198 +hs -17.283198 +dh -17.313571 +vd -17.313571 +ii -17.344598 +db -17.408729 +by -17.510608 +mr -17.582758 +q_ -17.658707 +yp -17.698235 +ym -17.738877 +kn -17.780697 +wy -17.780697 +yk -17.780697 +tc -17.780697 +zz -17.823766 +ys -17.823766 +nv -17.868160 +cd -17.913964 +gt -17.961270 +nm -18.010179 +bm -18.113273 +rp -18.113273 +dy -18.113273 +yd -18.167720 +fd -18.224304 +qf -18.283198 +dd -18.283198 +pp -18.283198 +sy -18.344598 +bb -18.475843 +yn -18.475843 +cm -18.475843 +wc -18.546232 +gb -18.546232 +df -18.546232 +wh -18.546232 +ft -18.698235 +cp -18.698235 +wl -18.868160 +fw -18.868160 +gs -18.868160 +vl -18.868160 +tm -18.868160 +pj -18.868160 +gn -18.961270 +cn -18.961270 +yr -18.961270 +sq -18.961270 +vy -19.060805 +qd -19.060805 +dm -19.060805 +py -19.060805 +vr -19.167720 +dz -19.167720 +dc -19.167720 +rw -19.167720 +ui -19.283198 +gz -19.408729 +ln -19.408729 +hb -19.546232 +tv -19.546232 +yt -19.546232 diff --git a/tika-eval/tika-eval-lite/src/test/java/org/apache/tika/eval/textquality/BigramTextQualityScorerTest.java b/tika-eval/tika-eval-lite/src/test/java/org/apache/tika/eval/textquality/BigramTextQualityScorerTest.java new file mode 100644 index 00000000000..db446c48304 --- /dev/null +++ b/tika-eval/tika-eval-lite/src/test/java/org/apache/tika/eval/textquality/BigramTextQualityScorerTest.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.eval.textquality; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Locale; +import java.util.Map; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import org.apache.tika.textquality.TextQualityResult; +import org.apache.tika.textquality.TextQualityScorer; + +class BigramTextQualityScorerTest { + + // Arabic: "This is a test for the text quality assessment system + // which should work correctly in all cases" + private static final String ARABIC_TEXT = + "\u0647\u0630\u0627 \u0647\u0648 \u0627\u062e\u062a\u0628" + + "\u0627\u0631 \u0644\u0646\u0638\u0627\u0645 \u062a\u0642" + + "\u064a\u064a\u0645 \u062c\u0648\u062f\u0629 \u0627\u0644" + + "\u0646\u0635 \u0627\u0644\u0639\u0631\u0628\u064a " + + "\u0648\u0627\u0644\u0630\u064a \u064a\u062c\u0628 " + + "\u0623\u0646 \u064a\u0639\u0645\u0644 \u0628\u0634" + + "\u0643\u0644 \u0635\u062d\u064a\u062d \u0641\u064a " + + "\u062c\u0645\u064a\u0639 \u0627\u0644\u0623\u062d" + + "\u0648\u0627\u0644"; + + // Arabic (shorter): "This is a test... which should work correctly" + private static final String ARABIC_SHORT = + "\u0647\u0630\u0627 \u0647\u0648 \u0627\u062e\u062a\u0628" + + "\u0627\u0631 \u0644\u0646\u0638\u0627\u0645 \u062a\u0642" + + "\u064a\u064a\u0645 \u062c\u0648\u062f\u0629 \u0627\u0644" + + "\u0646\u0635 \u0627\u0644\u0639\u0631\u0628\u064a " + + "\u0648\u0627\u0644\u0630\u064a \u064a\u062c\u0628 " + + "\u0623\u0646 \u064a\u0639\u0645\u0644 \u0628\u0634" + + "\u0643\u0644 \u0635\u062d\u064a\u062d"; + + // Hebrew: "This is a test for the text quality assessment system + // that should work properly in all cases" + private static final String HEBREW_TEXT = + "\u05d6\u05d4\u05d5 \u05de\u05d1\u05d7\u05df " + + "\u05dc\u05de\u05e2\u05e8\u05db\u05ea " + + "\u05d4\u05e2\u05e8\u05db\u05ea " + + "\u05d0\u05d9\u05db\u05d5\u05ea " + + "\u05d4\u05d8\u05e7\u05e1\u05d8 " + + "\u05d4\u05e2\u05d1\u05e8\u05d9 " + + "\u05e9\u05d0\u05de\u05d5\u05e8 " + + "\u05dc\u05e2\u05d1\u05d5\u05d3 " + + "\u05db\u05e8\u05d0\u05d5\u05d9 \u05d1\u05db\u05dc " + + "\u05d4\u05de\u05e7\u05e8\u05d9\u05dd"; + + // Hebrew (shorter): "...that should work properly" + private static final String HEBREW_SHORT = + "\u05d6\u05d4\u05d5 \u05de\u05d1\u05d7\u05df " + + "\u05dc\u05de\u05e2\u05e8\u05db\u05ea " + + "\u05d4\u05e2\u05e8\u05db\u05ea " + + "\u05d0\u05d9\u05db\u05d5\u05ea " + + "\u05d4\u05d8\u05e7\u05e1\u05d8 " + + "\u05d4\u05e2\u05d1\u05e8\u05d9 " + + "\u05e9\u05d0\u05de\u05d5\u05e8 " + + "\u05dc\u05e2\u05d1\u05d5\u05d3 " + + "\u05db\u05e8\u05d0\u05d5\u05d9"; + + private static BigramTextQualityScorer scorer; + + @BeforeAll + static void setUp() { + scorer = new BigramTextQualityScorer(); + } + + @Test + void testEnglishDetection() { + TextQualityResult result = scorer.score( + "The quick brown fox jumps over the lazy dog. " + + "This is a test of the text quality scoring system."); + assertEquals("eng", result.getLanguage()); + assertTrue(result.getScore() > -15.0, + "English score should be reasonable: " + result.getScore()); + assertTrue(result.getBigramCount() > 0); + } + + @Test + void testArabicDetection() { + TextQualityResult result = scorer.score(ARABIC_TEXT); + assertEquals("ara", result.getLanguage()); + assertTrue(result.getScore() > -18.0, + "Arabic score should be reasonable: " + result.getScore()); + } + + @Test + void testHebrewDetection() { + TextQualityResult result = scorer.score(HEBREW_TEXT); + assertEquals("heb", result.getLanguage()); + assertTrue(result.getScore() > -18.0, + "Hebrew score should be reasonable: " + result.getScore()); + } + + @Test + void testFrenchDetection() { + TextQualityResult result = scorer.score( + "Ceci est un test du syst\u00e8me d'\u00e9valuation" + + " de la qualit\u00e9 du texte fran\u00e7ais. Cette" + + " phrase devrait \u00eatre suffisamment longue pour" + + " que le d\u00e9tecteur puisse identifier" + + " correctement la langue."); + assertEquals("fra", result.getLanguage()); + assertTrue(result.getScore() > -18.0, + "French score should be reasonable: " + result.getScore()); + } + + @Test + void testReversedArabicScoresLower() { + String reversed = new StringBuilder(ARABIC_SHORT).reverse() + .toString(); + TextQualityResult normalResult = scorer.score(ARABIC_SHORT); + TextQualityResult reversedResult = scorer.score(reversed); + + double delta = normalResult.getScore() - reversedResult.getScore(); + assertTrue(delta > 0.5, + String.format(Locale.ROOT, + "Normal (%.4f) vs reversed (%.4f): " + + "delta %.4f should be > 0.5 bits", + normalResult.getScore(), + reversedResult.getScore(), delta)); + } + + @Test + void testReversedHebrewScoresLower() { + String reversed = new StringBuilder(HEBREW_SHORT).reverse() + .toString(); + TextQualityResult normalResult = scorer.score(HEBREW_SHORT); + TextQualityResult reversedResult = scorer.score(reversed); + + double delta = normalResult.getScore() - reversedResult.getScore(); + assertTrue(delta > 0.5, + String.format(Locale.ROOT, + "Normal (%.4f) vs reversed (%.4f): " + + "delta %.4f should be > 0.5 bits", + normalResult.getScore(), + reversedResult.getScore(), delta)); + } + + @Test + void testGarbledTextScoresLow() { + String garbage = "xqzjk vwpfl mnbvc zxqwk jhrty plmnk"; + TextQualityResult result = scorer.score(garbage); + + assertTrue(result.getScore() < -8.0, + "Garbled text should score low: " + result.getScore()); + } + + @Test + void testEmptyTextReturnsNeutral() { + TextQualityResult result = scorer.score(""); + assertEquals(0.0, result.getScore(), 0.001); + assertEquals("unk", result.getLanguage()); + assertEquals(0, result.getBigramCount()); + } + + @Test + void testWhitespaceOnlyReturnsNeutral() { + TextQualityResult result = scorer.score(" \t\n "); + assertEquals(0.0, result.getScore(), 0.001); + assertEquals("unk", result.getLanguage()); + assertEquals(0, result.getBigramCount()); + } + + @Test + void testNumericOnlyReturnsNeutral() { + TextQualityResult result = scorer.score("12345 67890 2024"); + assertEquals(0, result.getBigramCount()); + } + + @Test + void testServiceLoaderDiscovery() { + TextQualityScorer discovered = TextQualityScorer.getDefault(); + assertTrue(discovered instanceof BigramTextQualityScorer, + "SPI should discover BigramTextQualityScorer"); + } + + @Test + void testRelativeComparisonForCharset() { + String correct = "The quick brown fox jumps over the lazy dog"; + // Caesar shift +3: simulates wrong glyph mapping (common in + // PDFs with bad ToUnicode). NFKD normalization can't fix this. + String garbled = "Wkh frpplwwhh uhfrpphqghg wkdw wkh djhqfb" + + " sxuvxh dowhuqdwlyh vwudwhjlhv"; + + TextQualityResult correctResult = scorer.score(correct); + TextQualityResult garbledResult = scorer.score(garbled); + + assertTrue(correctResult.getScore() > garbledResult.getScore(), + String.format(Locale.ROOT, + "Correct charset (%.4f) should score higher " + + "than garbled (%.4f)", + correctResult.getScore(), + garbledResult.getScore())); + } + + @Test + void testScoreNormalizedByLength() { + String shortText = "The quick brown fox"; + String longText = "The quick brown fox jumps over the lazy dog. " + + "This sentence contains more text to make it longer. " + + "The quality scorer should normalize by length."; + + TextQualityResult shortResult = scorer.score(shortText); + TextQualityResult longResult = scorer.score(longText); + + double diff = Math.abs(shortResult.getScore() + - longResult.getScore()); + assertTrue(diff < 2.0, + String.format(Locale.ROOT, + "Short (%.4f) and long (%.4f) English text " + + "should have similar per-bigram scores", + shortResult.getScore(), + longResult.getScore())); + } + + @Test + void testCJKBigrams() { + // CJK ideographs: 3 internal + 2 boundary bigrams + Map bigrams = + BigramTextQualityScorer.extractBigrams( + "\u4e2d\u56fd\u4eba\u6c11"); + assertEquals(5, bigrams.size()); + assertEquals(1, bigrams.get("_\u4e2d")); + assertEquals(1, bigrams.get("\u4e2d\u56fd")); + assertEquals(1, bigrams.get("\u56fd\u4eba")); + assertEquals(1, bigrams.get("\u4eba\u6c11")); + assertEquals(1, bigrams.get("\u6c11_")); + } + + @Test + void testConfidencePositiveForClearLanguage() { + TextQualityResult result = scorer.score( + "The quick brown fox jumps over the lazy dog. " + + "This is clearly English text with enough content " + + "to be confident about the language detection."); + assertTrue(result.getConfidence() > 0.0, + "Should have positive confidence for clear English"); + } +} From 40e47d1a8afeca7f3a6c40754a3a6323fa5c2014 Mon Sep 17 00:00:00 2001 From: tallison Date: Thu, 12 Feb 2026 17:29:57 -0500 Subject: [PATCH 2/2] TIKA-4659 -- tika-eval-lite --- tika-app/pom.xml | 5 + .../tika/detect/DefaultEncodingDetector.java | 29 ++ .../tika/detect/EncodingDetectorContext.java | 91 +++++ .../detect/TextQualityEncodingDetector.java | 283 ++++++++++++++ ....apache.tika.textquality.TextQualityScorer | 15 + .../tika-parsers-standard-package/pom.xml | 6 + .../TextQualityEncodingDetectorTest.java | 360 ++++++++++++++++++ .../testArabicMisleadingCharset.html | 12 + tika-server/tika-server-standard/pom.xml | 5 + 9 files changed, 806 insertions(+) create mode 100644 tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java create mode 100644 tika-core/src/main/java/org/apache/tika/detect/TextQualityEncodingDetector.java create mode 100644 tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/detect/TextQualityEncodingDetectorTest.java create mode 100644 tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/test-documents/testArabicMisleadingCharset.html diff --git a/tika-app/pom.xml b/tika-app/pom.xml index 4826b036e00..b7fd4500c97 100644 --- a/tika-app/pom.xml +++ b/tika-app/pom.xml @@ -55,6 +55,11 @@ tika-xmp ${project.version} + + ${project.groupId} + tika-eval-lite + ${project.version} + ${project.groupId} diff --git a/tika-core/src/main/java/org/apache/tika/detect/DefaultEncodingDetector.java b/tika-core/src/main/java/org/apache/tika/detect/DefaultEncodingDetector.java index 67cf26e27d2..bfb12f85d6b 100644 --- a/tika-core/src/main/java/org/apache/tika/detect/DefaultEncodingDetector.java +++ b/tika-core/src/main/java/org/apache/tika/detect/DefaultEncodingDetector.java @@ -16,10 +16,16 @@ */ package org.apache.tika.detect; +import java.io.IOException; +import java.nio.charset.Charset; import java.util.Collection; import javax.imageio.spi.ServiceRegistry; import org.apache.tika.config.ServiceLoader; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.textquality.TextQualityScorer; /** * A composite encoding detector based on all the {@link EncodingDetector} implementations @@ -31,23 +37,46 @@ * If you need to control the order of the Detectors, you should instead * construct your own {@link CompositeDetector} and pass in the list * of Detectors in the required order. + *

+ * When a real {@link TextQualityScorer} is on the classpath, this detector + * delegates to {@link TextQualityEncodingDetector} for collect-all-then-arbitrate + * behavior. Otherwise, it uses first-match-wins from {@link CompositeEncodingDetector}. * * @since Apache Tika 1.15 */ public class DefaultEncodingDetector extends CompositeEncodingDetector { + private final TextQualityEncodingDetector qualityDetector; + public DefaultEncodingDetector() { this(new ServiceLoader(DefaultEncodingDetector.class.getClassLoader())); } public DefaultEncodingDetector(ServiceLoader loader) { super(loader.loadServiceProviders(EncodingDetector.class)); + this.qualityDetector = initQualityDetector(); } public DefaultEncodingDetector(ServiceLoader loader, Collection> excludeEncodingDetectors) { super(loader.loadServiceProviders(EncodingDetector.class), excludeEncodingDetectors); + this.qualityDetector = initQualityDetector(); } + private TextQualityEncodingDetector initQualityDetector() { + if (!TextQualityScorer.getScorers().isEmpty()) { + return new TextQualityEncodingDetector(getDetectors()); + } + return null; + } + + @Override + public Charset detect(TikaInputStream tis, Metadata metadata, + ParseContext parseContext) throws IOException { + if (qualityDetector != null) { + return qualityDetector.detect(tis, metadata, parseContext); + } + return super.detect(tis, metadata, parseContext); + } } diff --git a/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java b/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java new file mode 100644 index 00000000000..7c13ed2b7a4 --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect; + +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * Context object used by {@link TextQualityEncodingDetector} to collect + * results from all child detectors before arbitrating. Stored temporarily + * in {@link org.apache.tika.parser.ParseContext} during detection and + * removed afterward to prevent contamination during recursive parsing. + * + * @since Apache Tika 3.2 + */ +public class EncodingDetectorContext { + + private final List results = new ArrayList<>(); + + /** + * Record a detection result from a child detector. + * + * @param charset the detected charset (must not be null) + * @param detectorName the simple class name of the detector + */ + public void addResult(Charset charset, String detectorName) { + results.add(new Result(charset, detectorName)); + } + + /** + * @return unmodifiable list of all results in detection order + */ + public List getResults() { + return Collections.unmodifiableList(results); + } + + /** + * @return unique charsets in detection order + */ + public Set getUniqueCharsets() { + Set charsets = new LinkedHashSet<>(); + for (Result r : results) { + charsets.add(r.getCharset()); + } + return charsets; + } + + /** + * A single detection result pairing a charset with the detector that found it. + */ + public static class Result { + private final Charset charset; + private final String detectorName; + + public Result(Charset charset, String detectorName) { + this.charset = charset; + this.detectorName = detectorName; + } + + public Charset getCharset() { + return charset; + } + + public String getDetectorName() { + return detectorName; + } + + @Override + public String toString() { + return detectorName + "=" + charset.name(); + } + } +} diff --git a/tika-core/src/main/java/org/apache/tika/detect/TextQualityEncodingDetector.java b/tika-core/src/main/java/org/apache/tika/detect/TextQualityEncodingDetector.java new file mode 100644 index 00000000000..09cea901bf7 --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/detect/TextQualityEncodingDetector.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect; + +import java.io.IOException; +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CodingErrorAction; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.textquality.TextQualityResult; +import org.apache.tika.textquality.TextQualityScorer; + +/** + * An encoding detector that runs all child detectors, + * collects their candidate charsets, and when they disagree, + * uses {@link TextQualityScorer} to pick the charset that produces + * the best-looking decoded text. + * + *

When all detectors agree (or only one returns a result), no + * scoring is needed. When {@code tika-eval-lite} is not on the + * classpath, the no-op scorer makes all scores equal and the + * detector falls back to first-match-wins (same as + * {@link CompositeEncodingDetector}).

+ * + *

This is used internally by {@link DefaultEncodingDetector} when + * a real {@link TextQualityScorer} is on the classpath.

+ * + * @since Apache Tika 3.2 + */ +public class TextQualityEncodingDetector implements EncodingDetector, Serializable { + + private static final long serialVersionUID = 1L; + + private static final Logger LOG = + LoggerFactory.getLogger(TextQualityEncodingDetector.class); + + private static final int DEFAULT_READ_LIMIT = 16384; + + private final List detectors; + + private int readLimit = DEFAULT_READ_LIMIT; + + public TextQualityEncodingDetector(List detectors) { + this.detectors = new ArrayList<>(detectors); + } + + public TextQualityEncodingDetector( + List detectors, + Collection> excludeEncodingDetectors) { + this.detectors = new ArrayList<>(); + for (EncodingDetector detector : detectors) { + if (!isExcluded(excludeEncodingDetectors, detector.getClass())) { + this.detectors.add(detector); + } + } + } + + private static boolean isExcluded( + Collection> excludes, + Class candidate) { + for (Class e : excludes) { + if (e.isAssignableFrom(candidate)) { + return true; + } + } + return false; + } + + public List getDetectors() { + return Collections.unmodifiableList(detectors); + } + + @Override + public Charset detect(TikaInputStream tis, Metadata metadata, + ParseContext parseContext) throws IOException { + // Collect results from all child detectors + EncodingDetectorContext context = new EncodingDetectorContext(); + parseContext.set(EncodingDetectorContext.class, context); + try { + for (EncodingDetector detector : getDetectors()) { + Charset detected = detector.detect(tis, metadata, parseContext); + if (detected != null) { + context.addResult(detected, detector.getClass().getSimpleName()); + } + } + } finally { + // Remove context to prevent contamination during recursive parsing + parseContext.set(EncodingDetectorContext.class, null); + } + + List results = context.getResults(); + if (results.isEmpty()) { + return null; + } + + Set uniqueCharsets = context.getUniqueCharsets(); + Charset winner; + String winnerDetector; + + if (uniqueCharsets.size() == 1) { + // Unanimous -- no arbitration needed + winner = results.get(0).getCharset(); + winnerDetector = results.get(0).getDetectorName(); + } else { + // Disagreement -- arbitrate via text quality scoring + EncodingDetectorContext.Result best = + arbitrate(tis, results, uniqueCharsets); + winner = best.getCharset(); + winnerDetector = best.getDetectorName(); + } + + metadata.set(TikaCoreProperties.DETECTED_ENCODING, winner.name()); + metadata.set(TikaCoreProperties.ENCODING_DETECTOR, winnerDetector); + return winner; + } + + /** + * When child detectors disagree, read raw bytes from the stream, + * decode with each candidate charset, strip tags, and score. + * Returns the result with the highest text quality score. + * Falls back to first result if scoring is unavailable (no-op scorer). + */ + private EncodingDetectorContext.Result arbitrate( + TikaInputStream tis, + List results, + Set uniqueCharsets) throws IOException { + + // Default to first result if we can't read bytes + EncodingDetectorContext.Result firstResult = results.get(0); + if (tis == null) { + return firstResult; + } + + byte[] bytes; + try { + tis.mark(readLimit); + bytes = new byte[readLimit]; + int totalRead = 0; + int bytesRead; + while (totalRead < readLimit && + (bytesRead = tis.read(bytes, totalRead, + readLimit - totalRead)) != -1) { + totalRead += bytesRead; + } + if (totalRead == 0) { + return firstResult; + } + if (totalRead < readLimit) { + byte[] trimmed = new byte[totalRead]; + System.arraycopy(bytes, 0, trimmed, 0, totalRead); + bytes = trimmed; + } + } finally { + tis.reset(); + } + + TextQualityScorer scorer = TextQualityScorer.getDefault(); + + double bestScore = Double.NEGATIVE_INFINITY; + EncodingDetectorContext.Result bestResult = firstResult; + boolean allEqual = true; + double firstScore = Double.NaN; + + for (Charset candidate : uniqueCharsets) { + String decoded = decode(bytes, candidate); + String stripped = stripTags(decoded); + TextQualityResult tqr = scorer.score(stripped); + // Use NEGATIVE_INFINITY for texts that produce no scorable bigrams + // (e.g., all replacement characters from invalid decoding). + // A score of 0.0 with 0 bigrams means "no text to score", which + // should not beat a real (negative) score from valid text. + double score = (tqr.getBigramCount() == 0) + ? Double.NEGATIVE_INFINITY : tqr.getScore(); + + if (Double.isNaN(firstScore)) { + firstScore = score; + } else if (score != firstScore) { + allEqual = false; + } + + // Find the first result that uses this charset, for detector name + EncodingDetectorContext.Result resultForCharset = null; + for (EncodingDetectorContext.Result r : results) { + if (r.getCharset().equals(candidate)) { + resultForCharset = r; + break; + } + } + + if (score > bestScore) { + bestScore = score; + bestResult = resultForCharset; + } + + LOG.debug("Charset {} (from {}): score={}, bigrams={}", + candidate.name(), + resultForCharset != null ? resultForCharset.getDetectorName() : "?", + score, tqr.getBigramCount()); + } + + // If all scores are equal (no-op scorer), fall back to first result + if (allEqual) { + return firstResult; + } + + LOG.debug("Arbitration winner: {} (from {}, score={})", + bestResult.getCharset().name(), + bestResult.getDetectorName(), bestScore); + return bestResult; + } + + /** + * Decode bytes using the given charset, replacing malformed/unmappable + * characters rather than throwing. + */ + static String decode(byte[] bytes, Charset charset) { + CharsetDecoder decoder = charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE); + CharBuffer cb = CharBuffer.allocate(bytes.length * 2); + decoder.decode(ByteBuffer.wrap(bytes), cb, true); + decoder.flush(cb); + cb.flip(); + return cb.toString(); + } + + /** + * Simple tag stripping: removes <...> sequences so that + * HTML/XML tag names and attributes don't pollute bigram scoring. + */ + static String stripTags(String text) { + StringBuilder sb = new StringBuilder(text.length()); + boolean inTag = false; + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + if (c == '<') { + inTag = true; + } else if (c == '>') { + inTag = false; + } else if (!inTag) { + sb.append(c); + } + } + return sb.toString(); + } + + public int getReadLimit() { + return readLimit; + } + + public void setReadLimit(int readLimit) { + this.readLimit = readLimit; + } +} diff --git a/tika-eval/tika-eval-lite/src/main/resources/META-INF/services/org.apache.tika.textquality.TextQualityScorer b/tika-eval/tika-eval-lite/src/main/resources/META-INF/services/org.apache.tika.textquality.TextQualityScorer index 1c6cb4a6f63..5c5d9f585c4 100644 --- a/tika-eval/tika-eval-lite/src/main/resources/META-INF/services/org.apache.tika.textquality.TextQualityScorer +++ b/tika-eval/tika-eval-lite/src/main/resources/META-INF/services/org.apache.tika.textquality.TextQualityScorer @@ -1 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + org.apache.tika.eval.textquality.BigramTextQualityScorer diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/pom.xml index d01ca152784..f29711423a3 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/pom.xml +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/pom.xml @@ -179,6 +179,12 @@ ${project.version} test + + ${project.groupId} + tika-eval-lite + ${project.version} + test + ${project.groupId} tika-parser-zip-commons diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/detect/TextQualityEncodingDetectorTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/detect/TextQualityEncodingDetectorTest.java new file mode 100644 index 00000000000..3db02a4bb4b --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/detect/TextQualityEncodingDetectorTest.java @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.textquality.TextQualityResult; +import org.apache.tika.textquality.TextQualityScorer; + +public class TextQualityEncodingDetectorTest extends TikaTest { + + /** + * Mock detector that always returns a fixed charset. + */ + private static class FixedEncodingDetector implements EncodingDetector { + private static final long serialVersionUID = 1L; + private final Charset charset; + + FixedEncodingDetector(Charset charset) { + this.charset = charset; + } + + @Override + public Charset detect(TikaInputStream tis, Metadata metadata, + ParseContext parseContext) { + return charset; + } + } + + /** + * Mock detector that always returns null. + */ + private static class NullEncodingDetector implements EncodingDetector { + private static final long serialVersionUID = 1L; + + @Override + public Charset detect(TikaInputStream tis, Metadata metadata, + ParseContext parseContext) { + return null; + } + } + + @Test + public void testUnanimousAgreement() throws Exception { + List detectors = Arrays.asList( + new FixedEncodingDetector(UTF_8), + new FixedEncodingDetector(UTF_8)); + TextQualityEncodingDetector detector = + new TextQualityEncodingDetector(detectors); + + byte[] data = "Hello, world!".getBytes(UTF_8); + try (TikaInputStream tis = TikaInputStream.get( + new ByteArrayInputStream(data))) { + Metadata metadata = new Metadata(); + ParseContext parseContext = new ParseContext(); + Charset result = detector.detect(tis, metadata, parseContext); + assertEquals(UTF_8, result); + assertEquals("UTF-8", + metadata.get(TikaCoreProperties.DETECTED_ENCODING)); + } + } + + @Test + public void testSingleDetector() throws Exception { + List detectors = + Collections.singletonList(new FixedEncodingDetector(UTF_8)); + TextQualityEncodingDetector detector = + new TextQualityEncodingDetector(detectors); + + byte[] data = "Test".getBytes(UTF_8); + try (TikaInputStream tis = TikaInputStream.get( + new ByteArrayInputStream(data))) { + Metadata metadata = new Metadata(); + ParseContext parseContext = new ParseContext(); + Charset result = detector.detect(tis, metadata, parseContext); + assertEquals(UTF_8, result); + } + } + + @Test + public void testNoDetectorsMatch() throws Exception { + List detectors = + Collections.singletonList(new NullEncodingDetector()); + TextQualityEncodingDetector detector = + new TextQualityEncodingDetector(detectors); + + byte[] data = "Test".getBytes(UTF_8); + try (TikaInputStream tis = TikaInputStream.get( + new ByteArrayInputStream(data))) { + Metadata metadata = new Metadata(); + ParseContext parseContext = new ParseContext(); + Charset result = detector.detect(tis, metadata, parseContext); + assertNull(result); + } + } + + @Test + public void testScorerIsAvailable() { + // Verify the real scorer is on the classpath, not the no-op + TextQualityScorer scorer = TextQualityScorer.getDefault(); + TextQualityResult result = scorer.score("This is a test of English text quality."); + // The real scorer returns non-zero scores; the no-op returns 0.0 + assertNotEquals(0.0, result.getScore(), + "Expected real TextQualityScorer, got no-op. " + + "Is tika-eval-lite on the test classpath?"); + } + + @Test + public void testDisagreementArbitration() throws Exception { + // With tika-eval-lite on test classpath, the real scorer should + // pick windows-1256 over UTF-8 for Arabic bytes + Charset windows1256 = Charset.forName("windows-1256"); + + // Several paragraphs of Arabic text to ensure enough bigrams for scoring + String arabicText = + "\u0641\u064a \u0642\u0631\u064a\u0629 \u0645\u0646 " + + "\u0627\u0644\u0642\u0631\u0649 \u0643\u0627\u0646 " + + "\u0647\u0646\u0627\u0643 \u0631\u062c\u0644 " + + "\u062d\u0643\u064a\u0645 \u064a\u0639\u0631\u0641 " + + "\u0643\u0644 \u0634\u064a\u0621 \u0639\u0646 " + + "\u0627\u0644\u062d\u064a\u0627\u0629 \u0648\u0643\u0627\u0646 " + + "\u064a\u0639\u0644\u0645 \u0627\u0644\u0646\u0627\u0633 " + + "\u0643\u064a\u0641 \u064a\u0639\u064a\u0634\u0648\u0646 " + + "\u0628\u0633\u0644\u0627\u0645 \u0648\u0627\u0646\u0633\u062c\u0627\u0645. " + + "\u0627\u0644\u0644\u063a\u0629 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 " + + "\u0647\u064a \u0648\u0627\u062d\u062f\u0629 \u0645\u0646 " + + "\u0623\u0643\u062b\u0631 \u0627\u0644\u0644\u063a\u0627\u062a " + + "\u0627\u0646\u062a\u0634\u0627\u0631\u0627 \u0641\u064a " + + "\u0627\u0644\u0639\u0627\u0644\u0645 \u0648\u064a\u062a\u062d\u062b\u0647\u0627 " + + "\u0623\u0643\u062b\u0631 \u0645\u0646 \u062b\u0644\u0627\u062b\u0645\u0627\u0626\u0629 " + + "\u0645\u0644\u064a\u0648\u0646 \u0625\u0646\u0633\u0627\u0646."; + byte[] arabicBytes = arabicText.getBytes(windows1256); + + List detectors = Arrays.asList( + new FixedEncodingDetector(UTF_8), + new FixedEncodingDetector(windows1256)); + TextQualityEncodingDetector detector = + new TextQualityEncodingDetector(detectors); + + try (TikaInputStream tis = TikaInputStream.get( + new ByteArrayInputStream(arabicBytes))) { + Metadata metadata = new Metadata(); + ParseContext parseContext = new ParseContext(); + Charset result = detector.detect(tis, metadata, parseContext); + // The real scorer should prefer windows-1256 since the bytes + // decode to valid Arabic with it but to replacement chars with UTF-8 + assertEquals(windows1256, result); + } + } + + @Test + public void testStreamResetAfterDetection() throws Exception { + List detectors = Arrays.asList( + new FixedEncodingDetector(UTF_8), + new FixedEncodingDetector(ISO_8859_1)); + TextQualityEncodingDetector detector = + new TextQualityEncodingDetector(detectors); + + byte[] data = "Hello, world!".getBytes(UTF_8); + try (TikaInputStream tis = TikaInputStream.get( + new ByteArrayInputStream(data))) { + Metadata metadata = new Metadata(); + ParseContext parseContext = new ParseContext(); + detector.detect(tis, metadata, parseContext); + + // Verify stream is back at the start + byte[] readBack = new byte[data.length]; + int bytesRead = tis.read(readBack); + assertEquals(data.length, bytesRead); + assertEquals("Hello, world!", new String(readBack, UTF_8)); + } + } + + @Test + public void testContextRemovedAfterDetection() throws Exception { + List detectors = + Collections.singletonList(new FixedEncodingDetector(UTF_8)); + TextQualityEncodingDetector detector = + new TextQualityEncodingDetector(detectors); + + byte[] data = "Test".getBytes(UTF_8); + try (TikaInputStream tis = TikaInputStream.get( + new ByteArrayInputStream(data))) { + ParseContext parseContext = new ParseContext(); + detector.detect(tis, new Metadata(), parseContext); + assertNull(parseContext.get(EncodingDetectorContext.class)); + } + } + + @Test + public void testNullStream() throws Exception { + List detectors = Arrays.asList( + new FixedEncodingDetector(UTF_8), + new FixedEncodingDetector(ISO_8859_1)); + TextQualityEncodingDetector detector = + new TextQualityEncodingDetector(detectors); + + Metadata metadata = new Metadata(); + ParseContext parseContext = new ParseContext(); + Charset result = detector.detect(null, metadata, parseContext); + assertEquals(UTF_8, result); + } + + @Test + public void testStripTags() { + assertEquals("Hello world", + TextQualityEncodingDetector.stripTags( + "Hello world")); + assertEquals("no tags here", + TextQualityEncodingDetector.stripTags("no tags here")); + assertEquals(" ", + TextQualityEncodingDetector.stripTags( + "


")); + assertEquals("", + TextQualityEncodingDetector.stripTags("")); + } + + @Test + public void testDecode() { + byte[] utf8Bytes = "caf\u00e9".getBytes(UTF_8); + String decoded = TextQualityEncodingDetector.decode(utf8Bytes, UTF_8); + assertEquals("caf\u00e9", decoded); + + byte[] asciiBytes = "hello".getBytes(US_ASCII); + assertEquals("hello", + TextQualityEncodingDetector.decode(asciiBytes, UTF_8)); + assertEquals("hello", + TextQualityEncodingDetector.decode(asciiBytes, ISO_8859_1)); + } + + @Test + public void testDetectorNameSetInMetadata() throws Exception { + List detectors = + Collections.singletonList(new FixedEncodingDetector(UTF_8)); + TextQualityEncodingDetector detector = + new TextQualityEncodingDetector(detectors); + + byte[] data = "Test".getBytes(UTF_8); + try (TikaInputStream tis = TikaInputStream.get( + new ByteArrayInputStream(data))) { + Metadata metadata = new Metadata(); + ParseContext parseContext = new ParseContext(); + detector.detect(tis, metadata, parseContext); + assertNotNull( + metadata.get(TikaCoreProperties.ENCODING_DETECTOR)); + assertEquals("FixedEncodingDetector", + metadata.get(TikaCoreProperties.ENCODING_DETECTOR)); + } + } + + @Test + public void testEmptyDetectorList() throws Exception { + TextQualityEncodingDetector detector = + new TextQualityEncodingDetector(Collections.emptyList()); + + byte[] data = "Test".getBytes(UTF_8); + try (TikaInputStream tis = TikaInputStream.get( + new ByteArrayInputStream(data))) { + Metadata metadata = new Metadata(); + ParseContext parseContext = new ParseContext(); + Charset result = detector.detect(tis, metadata, parseContext); + assertNull(result); + } + } + + @Test + public void testMixOfNullAndNonNullDetectors() throws Exception { + List detectors = Arrays.asList( + new NullEncodingDetector(), + new FixedEncodingDetector(UTF_8), + new NullEncodingDetector()); + TextQualityEncodingDetector detector = + new TextQualityEncodingDetector(detectors); + + byte[] data = "Test".getBytes(UTF_8); + try (TikaInputStream tis = TikaInputStream.get( + new ByteArrayInputStream(data))) { + Metadata metadata = new Metadata(); + ParseContext parseContext = new ParseContext(); + Charset result = detector.detect(tis, metadata, parseContext); + assertEquals(UTF_8, result); + } + } + + @Test + public void testReadLimitGetterSetter() { + TextQualityEncodingDetector detector = + new TextQualityEncodingDetector(Collections.emptyList()); + assertEquals(16384, detector.getReadLimit()); + detector.setReadLimit(4096); + assertEquals(4096, detector.getReadLimit()); + } + + @Test + public void testGetDetectors() { + List input = Arrays.asList( + new FixedEncodingDetector(UTF_8), + new FixedEncodingDetector(ISO_8859_1)); + TextQualityEncodingDetector detector = + new TextQualityEncodingDetector(input); + List returned = detector.getDetectors(); + assertEquals(2, returned.size()); + } + + @Test + public void testIsInstanceOfEncodingDetector() { + TextQualityEncodingDetector detector = + new TextQualityEncodingDetector(Collections.emptyList()); + assertTrue(detector instanceof EncodingDetector); + } + + @Test + public void testArabicMisleadingCharsetHtml() throws Exception { + // This HTML file is encoded in windows-1256 but declares charset=UTF-8 + // The text quality arbitration should override the misleading HTML meta + // and detect that the actual content is Arabic (windows-1256) + Metadata metadata = new Metadata(); + XMLResult result = getXML("testArabicMisleadingCharset.html", metadata); + // Verify encoding was detected as windows-1256, not the misleading UTF-8 + assertEquals("windows-1256", + metadata.get(TikaCoreProperties.DETECTED_ENCODING)); + // Verify extracted text contains readable Arabic, not mojibake + // \u0627\u0644\u0639\u0631\u0628\u064a\u0629 = "العربية" (Arabic) + assertContains("\u0627\u0644\u0639\u0631\u0628\u064a\u0629", result.xml); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/test-documents/testArabicMisleadingCharset.html b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/test-documents/testArabicMisleadingCharset.html new file mode 100644 index 00000000000..31b61fa0a9c --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/test-documents/testArabicMisleadingCharset.html @@ -0,0 +1,12 @@ + + + + + + +

. .

+

. .

+

. . .

+

. .

+ + diff --git a/tika-server/tika-server-standard/pom.xml b/tika-server/tika-server-standard/pom.xml index 82db4d689a4..8002d1c7d06 100644 --- a/tika-server/tika-server-standard/pom.xml +++ b/tika-server/tika-server-standard/pom.xml @@ -59,6 +59,11 @@ tika-xmp ${project.version} + + ${project.groupId} + tika-eval-lite + ${project.version} + org.apache.logging.log4j log4j-core