up

create_vocab.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+import os
+import json
+folder_path = "./vocabs"
+
+all_dict = {}
+
+def parse_file(filename):
+    dictionary = {  
+        "</s>": 2,
+        "<pad>": 0,
+        "<s>": 1,
+        "<unk>": 3,
+    }
+    value = 4
+
+    with open(filename, 'r') as file:
+        for line in file:
+            line = line.strip().split()
+            if line:
+                key = line[0]
+                dictionary[key] = value
+                value += 1
+
+    return dictionary
+
+for filename in os.listdir(folder_path):
+    filepath = os.path.join(folder_path, filename)
+    lang = filename.split(".")[0]
+    if os.path.isfile(filepath):
+        all_dict[lang] = parse_file(filepath)
+
+
+output_path = "vocab_1.json"  # Replace "output.json" with the desired output file path
+
+with open(output_path, 'w') as output_file:
+    json.dump(all_dict, output_file, indent=4, sort_keys=True)

download_vocabs.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+langs=(abi abk abp aca acd ace acf ach acn acr acu ade adh adj adx aeu afr agd agg agn agr agu agx aha ahk aia aka akb ake akp alj alp alt alz ame amf amh ami amk ann any aoz apb apr ara arl asa asg asm ast ata atb atg ati atq ava avn avu awa awb ayo ayr ayz azb azg azj-script_cyrillic azj-script_latin azz bak bam ban bao bas bav bba bbb bbc bbo bcc-script_arabic bcc-script_latin bcl bcw bdg bdh bdq bdu bdv beh bel bem ben bep bex bfa bfo bfy bfz bgc bgq bgr bgt bgw bha bht bhz bib bim bis biv bjr bjv bjw bjz bkd bkv blh blt blx blz bmq bmr bmu bmv bng bno bnp boa bod boj bom bor bos bov box bpr bps bqc bqi bqj bqp bre bru bsc bsq bss btd bts btt btx bud bul bus bvc bvz bwq bwu byr bzh bzi bzj caa cab cac-dialect_sanmateoixtatan cac-dialect_sansebastiancoatan cak-dialect_central cak-dialect_santamariadejesus cak-dialect_santodomingoxenacoj cak-dialect_southcentral cak-dialect_western cak-dialect_yepocapa cap car cas cat cax cbc cbi cbr cbs cbt cbu cbv cce cco cdj ceb ceg cek ces cfm cgc che chf chv chz cjo cjp cjs ckb cko ckt cla cle cly cme cmn-script_simplified cmo-script_khmer cmo-script_latin cmr cnh cni cnl cnt coe cof cok con cot cou cpa cpb cpu crh crk-script_latin crk-script_syllabics crn crq crs crt csk cso ctd ctg cto ctu cuc cui cuk cul cwa cwe cwt cya cym daa dah dan dar dbj dbq ddn ded des deu dga dgi dgk dgo dgr dhi did dig dik dip div djk dnj-dialect_blowowest dnj-dialect_gweetaawueast dnt dnw dop dos dsh dso dtp dts dug dwr dyi dyo dyu dzo eip eka ell emp enb eng enx epo ese ess est eus evn ewe eza fal fao far fas fij fin flr fmu fon fra frd fry ful gag-script_cyrillic gag-script_latin gai gam gau gbi gbk gbm gbo gde geb gej gil gjn gkn gld gle glg glk gmv gna gnd gng gof-script_latin gog gor gqr grc gri grn grt gso gub guc gud guh guj guk gum guo guq guu gux gvc gvl gwi gwr gym gyr had hag hak hap hat hau hay heb heh hif hig hil hin hlb hlt hne hnn hns hoc hoy hrv hsb hto hub hui hun hus-dialect_centralveracruz hus-dialect_westernpotosino huu huv hvn hwc hye hyw iba ibo icr idd ifa ifb ife ifk ifu ify ign ikk ilb ilo imo ina inb ind iou ipi iqw iri irk isl ita itl itv ixl-dialect_sangasparchajul ixl-dialect_sanjuancotzal ixl-dialect_santamarianebaj izr izz jac jam jav jbu jen jic jiv jmc jmd jpn jun juy jvn kaa kab kac kak kam kan kao kaq kat kay kaz kbo kbp kbq kbr kby kca kcg kdc kde kdh kdi kdj kdl kdn kdt kea kek ken keo ker key kez kfb kff-script_telugu kfw kfx khg khm khq kia kij kik kin kir kjb kje kjg kjh kki kkj kle klu klv klw kma kmd kml kmr-script_arabic kmr-script_cyrillic kmr-script_latin kmu knb kne knf knj knk kno kog kor kpq kps kpv kpy kpz kqe kqp kqr kqy krc kri krj krl krr krs kru ksb ksr kss ktb ktj kub kue kum kus kvn kvw kwd kwf kwi kxc kxf kxm kxv kyb kyc kyf kyg kyo kyq kyu kyz kzf lac laj lam lao las lat lav law lbj lbw lcp lee lef lem lew lex lgg lgl lhu lia lid lif lin lip lis lit lje ljp llg lln lme lnd lns lob lok lom lon loq lsi lsm ltz luc lug luo lwo lww lzz maa-dialect_sanantonio maa-dialect_sanjeronimo mad mag mah mai maj mak mal mam-dialect_central mam-dialect_northern mam-dialect_southern mam-dialect_western maq mar maw maz mbb mbc mbh mbj mbt mbu mbz mca mcb mcd mco mcp mcq mcu mda mdf mdv mdy med mee mej men meq met mev mfe mfh mfi mfk mfq mfy mfz mgd mge mgh mgo mhi mhr mhu mhx mhy mib mie mif mih mil mim min mio mip miq mit miy miz mjl mjv mkd mkl mkn mlg mlt mmg mnb mnf mnk mnw mnx moa mog mon mop mor mos mox moz mpg mpm mpp mpx mqb mqf mqj mqn mri mrw msy mtd mtj mto muh mup mur muv muy mvp mwq mwv mxb mxq mxt mxv mya myb myk myl myv myx myy mza mzi mzj mzk mzm mzw nab nag nan nas naw nca nch ncj ncl ncu ndj ndp ndv ndy ndz neb new nfa nfr nga ngl ngp ngu nhe nhi nhu nhw nhx nhy nia nij nim nin nko nlc nld nlg nlk nmz nnb nno nnq nnw noa nob nod nog not npi npl npy nso nst nsu ntm ntr nuj nus nuz nwb nxq nya nyf nyn nyo nyy nzi obo oci ojb-script_latin ojb-script_syllabics oku old omw onb ood orm ory oss ote otq ozm pab pad pag pam pan pao pap pau pbb pbc pbi pce pcm peg pez pib pil pir pis pjt pkb pls plw pmf pny poh-dialect_eastern poh-dialect_western poi pol por poy ppk pps prf prk prt pse pss ptu pui pus pwg pww pxm qub quc-dialect_central quc-dialect_east quc-dialect_north quf quh qul quw quy quz qvc qve qvh qvm qvn qvo qvs qvw qvz qwh qxh qxl qxn qxo qxr rah rai rap rav raw rej rel rgu rhg rif-script_arabic rif-script_latin ril rim rjs rkt rmc-script_cyrillic rmc-script_latin rmo rmy-script_cyrillic rmy-script_latin rng rnl roh-dialect_sursilv roh-dialect_vallader rol ron rop rro rub ruf rug run rus sab sag sah saj saq sas sat sba sbd sbl sbp sch sck sda sea seh ses sey sgb sgj sgw shi shk shn sho shp sid sig sil sja sjm sld slk slu slv sml smo sna snd sne snn snp snw som soy spa spp spy sqi sri srm srn srp-script_cyrillic srp-script_latin srx stn stp suc suk sun sur sus suv suz swe swh sxb sxn sya syl sza tac taj tam tao tap taq tat tav tbc tbg tbk tbl tby tbz tca tcc tcs tcz tdj ted tee tel tem teo ter tes tew tex tfr tgj tgk tgl tgo tgp tha thk thl tih tik tir tkr tlb tlj tly tmc tmf tna tng tnk tnn tnp tnr tnt tob toc toh tom tos tpi tpm tpp tpt trc tri trn trs tso tsz ttc tte ttq-script_tifinagh tue tuf tuk-script_arabic tuk-script_latin tuo tur tvw twb twe twu txa txq txu tye tzh-dialect_bachajon tzh-dialect_tenejapa tzj-dialect_eastern tzj-dialect_western tzo-dialect_chamula tzo-dialect_chenalho ubl ubu udm udu uig-script_arabic uig-script_cyrillic ukr umb unr upv ura urb urd-script_arabic urd-script_devanagari urd-script_latin urk urt ury usp uzb-script_cyrillic uzb-script_latin vag vid vie vif vmw vmy vot vun vut wal-script_ethiopic wal-script_latin wap war waw way wba wlo wlx wmw wob wol wsg wwa xal xdy xed xer xho xmm xnj xnr xog xon xrb xsb xsm xsr xsu xta xtd xte xtm xtn xua xuo yaa yad yal yam yao yas yat yaz yba ybb ycl ycn yea yka yli yor yre yua yue-script_traditional yuz yva zaa zab zac zad zae zai zam zao zaq zar zas zav zaw zca zga zim ziw zlm zmz zne zos zpc zpg zpi zpl zpm zpo zpt zpu zpz ztq zty zul zyb zyp zza)
+link_format="https://dl.fbaipublicfiles.com/mms/asr/dict/mms1b_all/"
+mkdir -p vocabs
+cd vocabs
+for lang in ${langs[@]}; do
+  link="${link_format}${lang}.txt"
+  echo "wget ${link}"
+  wget ${link}
+done

vocabs/abi.txt

@@ -0,0 +1,52 @@
+| 352102
+n 196148
+ɛ 156349
+ɩ 103554
+ʋ 91086
+a 81933
+ɔ 80967
+k 49477
+b 48993
+e 48340
+l 47460
+m 45338
+t 43740
+u 43344
+i 43317
+f 31899
+r 30943
+p 29382
+o 25618
+y 25092
+ε 21071
+ί 19453
+s 17456
+έ 17074
+w 16466
+j 14560
+d 13540
+- 12321
+̂ 12117
+́ 10746
+c 10568
+g 10385
+h 9408
+̌ 7508
+í 7190
+é 4217
+á 3379
+ê 2837
+â 1548
+î 1244
+ó 1217
+ě 735
+ú 722
+û 448
+ǐ 188
+' 145
+ǔ 91
+ǒ 54
+ǎ 45
+ń 13
+ḿ 8
+ô 7

vocabs/abk.txt

@@ -0,0 +1,42 @@
+а 173639
+| 112055
+ы 59500
+и 55920
+ә 42995
+р 37486
+у 27925
+н 27572
+ь 24682
+х 22888
+з 22033
+с 19821
+ш 19542
+л 19349
+г 16736
+е 16705
+м 16438
+т 15902
+о 14017
+д 13699
+ҳ 13346
+к 12788
+ҭ 10865
+б 10590
+ԥ 9441
+ц 9159
+ҟ 8261
+ҵ 7573
+қ 7534
+п 7042
+ӡ 5574
+ҩ 5439
+ж 5327
+ҿ 2995
+ӷ 1960
+ҽ 1793
+ч 1557
+- 1527
+џ 1358
+ҷ 1065
+ф 670
+в 566

vocabs/abp.txt

@@ -0,0 +1,32 @@
+a 414328
+| 351050
+n 220079
+i 127079
+o 106160
+y 92604
+h 78020
+k 77759
+t 75753
+e 69017
+l 68916
+m 66725
+g 59646
+p 55181
+b 47906
+d 35222
+u 28619
+w 22662
+s 12424
+- 10606
+j 6531
+r 4512
+ā 2626
+c 2234
+f 350
+z 232
+v 190
+q 124
+ō 120
+x 34
+' 32
+6 6

vocabs/aca.txt

@@ -0,0 +1,34 @@
+a 73218
+| 64220
+i 52211
+n 37226
+á 29629
+c 25509
+e 23900
+u 18700
+l 18157
+r 18011
+w 12285
+j 11317
+s 10423
+í 9994
+m 9640
+é 9473
+o 8973
+' 8505
+h 8203
+t 8043
+y 8013
+b 7103
+d 6504
+ú 4699
+q 3215
+ó 2998
+p 1217
+— 681
+g 562
+f 249
+z 156
+v 150
+x 17
+ñ 13

vocabs/acd.txt

@@ -0,0 +1,27 @@
+| 251555
+a 122073
+ɛ 94651
+n 71139
+ɔ 68542
+i 58307
+o 52977
+m 49602
+y 49594
+e 45882
+u 37771
+g 36641
+s 36216
+k 33589
+b 32116
+r 29425
+l 25768
+d 20220
+w 16922
+f 15463
+- 11313
+t 10711
+p 9100
+' 7296
+ŋ 4643
+h 245
+c 2

vocabs/ace.txt

@@ -0,0 +1,41 @@
+| 175419
+a 153113
+n 103726
+e 96571
+u 91933
+g 46960
+t 45174
+h 41663
+i 37425
+k 36085
+m 32258
+b 31359
+o 30534
+y 29234
+l 28404
+r 22092
+s 21756
+p 18987
+j 18472
+d 17012
+é 13634
+w 6207
+ô 5811
+ë 5792
+- 5707
+c 2451
+ö 845
+á 751
+ó 729
+f 565
+z 153
+' 24
+q 20
+ú 16
+` 10
+0 3
+6 3
+4 2
+3 2
+1 2
+2 1

vocabs/acf.txt

@@ -0,0 +1,32 @@
+| 181397
+a 81190
+n 68601
+é 49072
+i 48381
+o 46143
+s 30177
+t 29609
+k 29245
+y 27599
+p 27171
+l 25509
+w 24583
+m 24094
+è 22039
+u 21549
+d 14498
+- 12276
+e 11859
+b 11413
+v 9872
+j 9752
+ò 8592
+z 8012
+f 7257
+ʼ 4612
+h 3924
+g 3427
+c 3231
+r 748
+— 83
+' 9

vocabs/ach.txt

@@ -0,0 +1,27 @@
+| 145830
+o 72309
+a 59517
+i 54533
+e 48958
+k 37543
+n 36249
+m 32262
+w 23510
+t 23311
+u 23039
+y 20144
+l 19225
+c 18712
+d 15353
+b 15036
+g 14576
+r 12832
+p 12686
+ŋ 11583
+j 3372
+- 1075
+s 245
+' 54
+v 45
+f 35
+h 1

vocabs/acn.txt

@@ -0,0 +1,36 @@
+| 380189
+h 184965
+a 180533
+s 154590
+n 138757
+o 115898
+y 115160
+g 102846
+i 92429
+e 82883
+u 80461
+l 58440
+m 56345
+t 45429
+d 42937
+q 40307
+k 39484
+v 28602
+j 17716
+z 13507
+b 12438
+p 12238
+c 12037
+w 11080
+x 7668
+r 3735
+- 218
+f 54
+0 32
+4 24
+1 10
+6 8
+— 4
+2 4
+3 2
+5 2

vocabs/acr.txt

@@ -0,0 +1,36 @@
+| 198029
+a 118297
+i 100626
+' 78320
+k 62085
+r 54009
+u 47276
+e 46044
+j 36177
+n 35364
+o 34586
+c 33074
+l 32635
+h 31119
+t 27062
+q 22260
+w 20034
+x 18360
+m 18168
+b 17381
+s 15178
+y 12600
+p 10779
+z 6178
+d 4798
+— 1202
+ú 1035
+g 772
+á 698
+é 334
+ó 307
+f 271
+í 248
+v 229
+- 217
+ñ 7

vocabs/acu.txt

@@ -0,0 +1,34 @@
+a 203260
+| 139292
+i 118390
+n 94569
+u 88367
+t 74525
+r 54212
+m 52612
+k 51135
+s 48946
+e 32013
+h 25203
+j 22549
+c 20514
+w 17668
+y 15878
+p 13927
+g 7970
+o 4721
+í 2039
+— 1807
+ú 1589
+d 1368
+l 1284
+é 1258
+á 908
+b 790
+f 294
+v 195
+ó 159
+z 131
+q 86
+x 15
+ñ 2

vocabs/ade.txt

@@ -0,0 +1,39 @@
+| 205418
+a 83827
+â 73978
+n 50856
+e 41058
+b 35805
+g 35456
+æ 34818
+i 31233
+t 30262
+w 28085
+ô 27352
+y 26067
+k 24793
+o 24162
+r 23715
+l 20957
+u 20782
+d 20708
+m 17914
+f 16757
+s 13491
+û 11991
+p 8955
+à 7520
+- 6968
+è 3466
+ã 3008
+õ 2664
+ù 1690
+î 1646
+å 702
+ì 683
+ü 625
+ǹ 560
+ò 191
+h 145
+' 20
+c 3

vocabs/adh.txt

@@ -0,0 +1,28 @@
+| 135387
+o 89220
+i 60743
+a 59702
+e 44416
+n 39491
+k 30392
+m 29319
+w 28043
+r 25498
+y 25421
+h 18847
+t 18256
+d 17031
+j 15397
+g 14175
+u 12091
+l 11116
+p 10957
+c 10936
+b 8178
+ŋ 7767
+s 4896
+f 1418
+' 1047
+z 249
+v 124
+- 4

vocabs/adj.txt

@@ -0,0 +1,45 @@
+| 192049
+a 61817
+e 54180
+ɛ 48923
+n 37327
+m 36316
+l 34042
+k 34003
+i 29533
+ŋ 26202
+' 25888
+s 25206
+b 23899
+o 21830
+y 21042
+w 18837
+ɔ 17156
+u 16232
+r 15914
+g 13735
+c 11515
+t 9780
+d 8520
+j 7276
+f 6796
+p 2824
+- 2789
+ó 1150
+́ 1003
+í 754
+̀ 456
+á 404
+h 320
+é 310
+ì 307
+ò 189
+ú 156
+à 149
+ê 108
+v 99
+ù 74
+̂ 68
+è 65
+z 48
+ô 1

vocabs/adx.txt

@@ -0,0 +1,57 @@
+་ 443718
+ས 130009
+ི 103510
+ད 99016
+ག 85923
+ོ 82111
+ན 66712
+ང 62946
+བ 61911
+མ 60782
+ར 59062
+ུ 53253
+ེ 52335
+ྱ 49650
+འ 40876
+ལ 37238
+པ 35778
+ཁ 22137
+ཡ 21680
+ཀ 19665
+ྲ 16963
+ཞ 15011
+ཅ 14746
+ཕ 10498
+ྣ 10082
+ཆ 9404
+ྟ 9346
+ཐ 9333
+ཤ 8730
+ྐ 8453
+ཏ 7298
+ཉ 7003
+ྒ 6162
+ཚ 5979
+ཟ 5216
+ཱ 4029
+ླ 3567
+ྡ 3357
+ཙ 2392
+ྨ 2363
+ྤ 2258
+ཇ 2128
+ྗ 1896
+ཛ 1688
+ྔ 1591
+ཨ 1471
+ྩ 1199
+ཧ 1138
+ྙ 862
+ྷ 718
+ྦ 446
+ྫ 401
+ྕ 177
+ྭ 144
+ཝ 60
+ཊ 6
+ཪ 2

vocabs/aeu.txt

@@ -0,0 +1,24 @@
+| 508441
+a 433955
+l 275090
+i 258235
+q 252323
+n 164476
+e 138283
+h 136196
+o 121947
+v 77889
+g 64965
+- 57215
+u 52535
+b 49408
+m 48556
+k 44891
+t 41593
+d 38337
+y 34865
+j 33201
+s 32772
+p 21127
+c 4496
+w 34

vocabs/afr.txt

@@ -0,0 +1,73 @@
+| 22961
+e 19676
+i 9317
+a 8840
+n 8757
+s 7269
+r 7160
+o 6923
+t 6554
+d 6107
+l 4470
+g 3996
+k 3537
+m 2886
+v 2460
+u 2428
+w 2010
+p 1923
+h 1908
+b 1764
+y 1065
+f 963
+' 569
+j 397
+0 250
+c 214
+- 186
+1 183
+ë 171
+ê 105
+2 103
+9 81
+3 71
+4 71
+. 68
+5 68
+6 64
+‘ 61
+8 47
+z 44
+7 37
+x 25
+ï 23
+: 23
+, 15
+/ 15
+q 15
+á 10
+’ 9
+” 8
+; 6
+$ 5
+% 5
+ń 4
+– 4
+í 3
+¥ 3
+ó 2
+ϊ 2
++ 2
+& 2
+[ 2
+] 2
+² 1
+" 1
+ö 1
+ç 1
+ł 1
+£ 1
+° 1
+ͦ 1
+ú 1
+é 1

vocabs/agd.txt

@@ -0,0 +1,29 @@
+| 195967
+a 169398
+n 143027
+i 87368
+m 57758
+e 49200
+o 44200
+á 38115
+r 35952
+k 34439
+t 33538
+u 27132
+y 26087
+v 24838
+s 24076
+f 16309
+b 15332
+í 14732
+é 13229
+d 10514
+p 9224
+h 6570
+ú 4649
+ó 4445
+' 75
+- 20
+0 2
+4 1
+3 1

vocabs/agg.txt

@@ -0,0 +1,32 @@
+a 8591
+| 8306
+ɨ 6610
+o 5215
+n 4675
+m 4175
+h 3377
+r 3248
+u 2820
+d 2800
+i 2621
+b 2498
+e 2473
+s 2154
+ü 1776
+y 1589
+f 1557
+g 917
+ŋ 720
+k 688
+w 568
+p 534
+t 441
+- 348
+0 12
+1 7
+7 7
+2 4
+4 2
+5 1
+6 1
+3 1

vocabs/agn.txt

@@ -0,0 +1,39 @@
+a 192558
+| 184179
+n 132835
+g 99032
+i 79988
+o 67600
+t 54259
+m 42975
+d 29177
+s 26320
+l 24515
+e 24430
+p 24265
+r 21236
+k 16863
+b 16802
+y 13059
+w 5239
+- 5176
+u 2292
+j 2236
+c 1172
+h 530
+f 169
+z 95
+v 85
+q 76
+' 67
+ā 64
+ō 63
+0 21
+x 15
+1 5
+— 3
+6 3
+ē 2
+5 1
+2 1
+7 1

vocabs/agr.txt

@@ -0,0 +1,37 @@
+a 179800
+| 122441
+i 96532
+u 87623
+n 70910
+t 58912
+k 52801
+m 46128
+j 40764
+s 40264
+e 36076
+g 25925
+h 25231
+w 17964
+d 16028
+p 14637
+c 14455
+y 8434
+í 6430
+b 4672
+r 2979
+o 2797
+á 2191
+ú 2166
+l 1450
+— 1420
+é 395
+f 249
+' 205
+v 196
+z 152
+ó 110
+q 81
+x 10
+ñ 3
+2 1
+7 1

vocabs/agu.txt

@@ -0,0 +1,39 @@
+| 198660
+n 90006
+a 83046
+i 80900
+ꞌ 74469
+t 73083
+e 51358
+c 45019
+u 44513
+y 42877
+l 40738
+j 33921
+o 29739
+z 28643
+k 27138
+s 21294
+h 19422
+b 17860
+w 14666
+x 13972
+m 11533
+r 10732
+p 7927
+q 7100
+ẍ 3267
+ú 1024
+— 794
+d 729
+g 551
+í 548
+é 491
+' 336
+á 315
+ó 291
+- 278
+f 168
+v 62
+ñ 3
+6 3

vocabs/agx.txt

@@ -0,0 +1,33 @@
+а 10405
+| 10201
+и 5129
+у 4100
+н 3577
+е 2936
+р 2828
+с 2741
+г 2477
+ь 2184
+к 2184
+д 1966
+л 1934
+х 1862
+ӏ 1820
+т 1642
+в 1336
+ъ 1295
+й 1149
+ч 1103
+я 1058
+м 842
+б 767
+п 732
+ф 580
+з 524
+ш 479
+э 242
+ж 232
+– 223
+ц 135
+ю 39
+о 18

vocabs/aha.txt

@@ -0,0 +1,46 @@
+| 193764
+ɩ 92304
+n 80787
+a 76542
+ɛ 60015
+m 58517
+i 51853
+l 51494
+ɔ 49454
+y 44780
+b 33525
+ʋ 32088
+e 25477
+w 24789
+k 24460
+u 20289
+d 16017
+s 15015
+z 13639
+o 10069
+t 8503
+h 8079
+̃ 6607
+g 4763
+v 4075
+ɣ 3249
+p 2762
+f 2612
+á 2527
+ĩ 2153
+ã 1888
+́ 1132
+í 440
+é 403
+ũ 296
+0 165
+ó 93
+' 89
+ú 39
+1 22
+2 20
+5 14
+4 6
+6 2
+3 2
+7 2

vocabs/ahk.txt

@@ -0,0 +1,29 @@
+| 297016
+a 152620
+ˬ 103797
+e 101172
+h 75566
+u 61011
+- 54608
+w 41693
+m 39109
+i 32747
+̭ 32124
+n 31113
+y 25956
+' 22779
+l 22490
+g 22387
+d 20679
+s 19081
+k 19047
+t 18840
+o 18689
+ˆ 14034
+j 10583
+b 8729
+p 7505
+z 7414
+c 3744
+` 2
+f 1

vocabs/aia.txt

@@ -0,0 +1,28 @@
+a 488397
+| 359458
+i 248648
+ꞌ 171341
+n 130829
+o 117403
+r 106132
+u 103703
+m 75112
+h 70428
+e 65844
+s 45748
+t 43437
+g 38705
+d 32322
+w 22611
+b 20253
+k 7289
+- 5484
+j 5386
+p 3894
+l 3478
+v 560
+f 426
+— 6
+y 6
+6 6
+c 4

vocabs/aka.txt

@@ -0,0 +1,31 @@
+| 387538
+a 226094
+n 195927
+o 129155
+e 110131
+m 87542
+ɛ 80178
+i 72027
+s 67773
+ɔ 67186
+y 56463
+w 55100
+u 45686
+k 43474
+r 39598
+b 34180
+d 33248
+h 28072
+t 25236
+f 24693
+p 16680
+g 7613
+l 2941
+' 2173
+ʼ 1043
+- 44
+1 2
+7 2
+á 1
+3 1
+2 1

vocabs/akb.txt

@@ -0,0 +1,31 @@
+| 355075
+a 346962
+n 191272
+i 140394
+o 116633
+u 89683
+d 80447
+g 77437
+s 71592
+h 71361
+m 70993
+t 68893
+r 52658
+e 43718
+l 42110
+k 39643
+p 38022
+b 32222
+j 22797
+- 4221
+y 3273
+' 1304
+c 1048
+w 130
+z 120
+f 101
+v 34
+0 26
+4 16
+1 8
+6 2

vocabs/ake.txt

@@ -0,0 +1,31 @@
+| 312650
+a 205504
+' 146778
+n 143568
+e 114118
+p 111810
+ɨ 103522
+i 101548
+o 96568
+t 83052
+r 81458
+k 77822
+ʉ 73128
+s 67254
+u 66940
+m 48864
+y 40330
+w 24956
+0 494
+— 438
+1 360
+7 198
+2 162
+4 98
+5 94
+3 64
+6 44
+8 28
+9 26
+– 16
+- 2

vocabs/akp.txt

@@ -0,0 +1,34 @@
+| 312604
+a 172759
+i 114909
+ɔ 106117
+e 83907
+m 71998
+s 63780
+ɛ 55556
+o 54139
+r 48796
+n 48199
+k 47599
+u 40009
+b 38654
+t 35914
+g 27333
+l 26150
+̃ 23154
+ɖ 19774
+w 18825
+y 17685
+ɣ 17200
+p 15260
+d 12104
+ã 11543
+f 11434
+z 7290
+ũ 5420
+ĩ 1251
+- 1143
+h 644
+v 257
+6 6
+' 4

vocabs/alj.txt

@@ -0,0 +1,24 @@
+a 187253
+| 151525
+n 75055
+i 65115
+o 54783
+g 48274
+y 46215
+k 43560
+s 38030
+t 28172
+m 27904
+p 25517
+e 24662
+b 23780
+w 21129
+l 18817
+d 17345
+r 11700
+0 10
+4 3
+1 1
+5 1
+3 1
+ɩ 1

vocabs/alp.txt

@@ -0,0 +1,29 @@
+| 371947
+e 324527
+a 243744
+i 179125
+k 120589
+l 107571
+u 98265
+t 89715
+s 88391
+m 81747
+n 78127
+o 74194
+r 59805
+b 41481
+p 33113
+h 18065
+y 10663
+w 8764
+d 7140
+j 6276
+g 1672
+f 188
+c 184
+- 38
+' 22
+0 12
+1 4
+4 4
+2 2

vocabs/alt.txt

@@ -0,0 +1,45 @@
+| 98742
+а 65402
+е 42149
+р 38197
+ы 36146
+н 36053
+л 31079
+и 31043
+к 28947
+д 27723
+о 26383
+т 23065
+у 21517
+г 19115
+б 17251
+с 16220
+й 14205
+п 12571
+ј 11582
+ӱ 11254
+ҥ 10557
+м 9389
+ч 7983
+з 7127
+ӧ 6739
+ж 3972
+ш 3839
+э 3731
+– 1881
+- 1292
+в 776
+х 719
+ф 434
+я 361
+ь 160
+1 14
+2 13
+ц 9
+ю 4
+ÿ 4
+4 2
+r 1
+j 1
+6 1
+0 1

vocabs/alz.txt

@@ -0,0 +1,26 @@
+| 131613
+a 56427
+i 50778
+e 50191
+n 46112
+u 43140
+o 40701
+m 37828
+k 31529
+g 26626
+d 18942
+w 18887
+r 16584
+t 13507
+b 13174
+c 11686
+y 11066
+l 10881
+' 9605
+h 8325
+p 7402
+j 6751
+s 3499
+f 670
+v 347
+z 163

vocabs/ame.txt

@@ -0,0 +1,38 @@
+a 359877
+| 305568
+e 300657
+t 208426
+o 177576
+h 176025
+ñ 158577
+' 152791
+r 131197
+p 116307
+n 97885
+s 82508
+m 81026
+l 54744
+c 48943
+y 47812
+k 33133
+w 23616
+x 15719
+i 6433
+j 4788
+b 3353
+u 2833
+d 2611
+ú 2229
+— 2142
+ë 1792
+g 1382
+é 794
+í 498
+f 472
+v 358
+á 342
+ó 316
+z 226
+q 170
+- 28
+2 2

vocabs/amf.txt

@@ -0,0 +1,27 @@
+a 322659
+| 247100
+n 148855
+e 146771
+i 124527
+o 97340
+s 75160
+d 71880
+k 68061
+h 66967
+m 61138
+y 59545
+t 52063
+r 47073
+b 40919
+g 23982
+w 21700
+l 21157
+u 16962
+p 14338
+q 10202
+j 9272
+c 7948
+z 4978
+v 4499
+' 2800
+x 742

vocabs/amh.txt

@@ -0,0 +1,335 @@
+| 85715
+ን 22176
+በ 11256
+ት 10395
+የ 8670
+ው 10849
+ስ 9576
+ር 10440
+መ 7239
+ል 8332
+ም 12644
+ች 5926
+ይ 7267
+አ 9765
+ለ 9133
+ተ 6544
+እ 11473
+ነ 7507
+ያ 5215
+ና 5732
+ላ 5013
+ከ 5275
+ደ 5557
+ድ 4511
+ብ 4890
+ማ 3266
+ረ 2961
+ግ 4963
+ገ 4498
+ወ 4130
+ታ 3547
+ራ 2968
+ሰ 4459
+ጥ 1832
+ሚ 2132
+ክ 2491
+ባ 2662
+ዎ 955
+ቀ 2619
+ሉ 2613
+ሪ 485
+ጠ 1938
+ካ 1230
+ሳ 1903
+ሆ 2602
+ፍ 1648
+ቅ 1955
+ህ 4031
+ዳ 1704
+ዋ 1001
+ጋ 1775
+ቸ 2155
+ቶ 1460
+ኛ 1032
+ሮ 865
+ጣ 1328
+ፈ 1991
+ሊ 563
+ሎ 937
+ሁ 5544
+0 789
+ሩ 1186
+ቱ 1367
+ቃ 1150
+ዝ 833
+ኖ 767
+ሲ 556
+ዘ 1373
+ቢ 668
+፣ 639
+ዊ 308
+ዲ 1737
+ጊ 783
+1 585
+ጉ 376
+ሞ 1243
+ቦ 206
+ዜ 763
+ሌ 795
+ኑ 899
+ዛ 746
+ጀ 393
+ሽ 292
+ዙ 554
+ኮ 125
+ሜ 169
+ዓ 367
+ዚ 2878
+ቡ 645
+ፊ 513
+ዶ 329
+ሃ 405
+ጎ 231
+ቤ 578
+ቻ 516
+ኒ 59
+ዩ 346
+ዕ 512
+ጨ 313
+ቹ 389
+2 365
+ቁ 483
+ፋ 684
+ቆ 431
+ዱ 783
+ጅ 553
+ኪ 145
+ሬ 336
+ሱ 2664
+a 334
+ቲ 205
+ኤ 222
+ቋ 113
+ሙ 858
+ኔ 632
+ሶ 206
+ፓ 307
+ሻ 221
+ፕ 4
+ሀ 98
+ኝ 855
+ፖ 281
+ኢ 1000
+ጫ 120
+ዮ 392
+ቴ 169
+ሕ 1289
+ጭ 209
+ሥ 1162
+ጃ 115
+ሴ 354
+o 254
+ኩ 240
+ሄ 397
+. 245
+9 245
+ጓ 70
+n 233
+- 232
+ኦ 38
+5 223
+ሸ 222
+e 222
+ቂ 175
+ፎ 162
+4 209
+ኋ 769
+ጡ 417
+i 198
+3 196
+ኘ 205
+ፒ 189
+ጽ 568
+6 181
+r 178
+8 175
+t 165
+ፉ 291
+ጦ 275
+ኙ 114
+s 162
+l 158
+ጂ 233
+ዞ 128
+ቪ 152
+ቷ 56
+ፅ 145
+ጆ 269
+ኃ 3
+c 135
+ሠ 399
+u 129
+ኬ 36
+ዴ 210
+ፃ 81
+ኞ 269
+7 119
+ኳ 185
+m 111
+p 109
+ሂ 50
+ሏ 81
+h 97
+ፀ 67
+። 95
+ሐ 478
+ቫ 89
+ጸ 569
+ሺ 55
+ጤ 41
+, 80
+ፌ 59
+ሯ 62
+g 73
+፡ 72
+ጻ 337
+ዌ 13
+ዥ 44
+ጁ 154
+d 62
+ፐ 61
+b 61
+ቨ 59
+ሹ 61
+/ 57
+ሾ 96
+ቧ 27
+ቬ 52
+ሟ 33
+ቭ 51
+” 50
+ፔ 1
+ጪ 12
+k 46
+f 44
+ቄ 103
+ቺ 41
+y 37
+ፏ 8
+ሷ 96
+ጄ 64
+ዬ 127
+w 34
+v 34
+ሣ 381
+ሦ 122
+ጌ 816
+ዣ 18
+ቾ 17
+ሔ 1155
+ጩ 13
+ሑ 11
+ጧ 31
+ኗ 24
+j 26
+ጮ 84
+ዐ 599
+ቼ 119
+" 24
+: 24
+ኚ 2
+ዦ 25
+ዷ 23
+፤ 20
+ዢ 2
+ዑ 10
+ዉ 19
+ኅ 116
+ኣ 5
+ፆ 3
+ጾ 41
+x 16
+$ 16
+ኡ 1
+ጢ 364
+ቮ 13
+« 13
+» 13
+ፑ 3
+ጹ 127
+% 10
+ዟ 10
+z 9
+ዡ 30
+ጵ 43
+— 7
+ፁ 53
+ጿ 3
+ዪ 53
+? 6
+’ 6
+ሼ 18
++ 5
+፥ 5
+ዒ 5
+° 5
+ሿ 6
+ዖ 85
+ጐ 125
+õ 4
+– 4
+ኸ 186
+; 4
+ã 3
+¥ 3
+ኺ 1
+ጴ 147
+ጳ 242
+ኜ 26
+' 3
+ጇ 12
+ጺ 2
+‘ 2
+! 2
+² 2
+ቿ 22
+ኰ 47
+ፗ 2
+q 2
+& 2
+£ 2
+ኟ 8
+[ 1
+] 1
+፦ 1
+ኀ 395
+ቈ 221
+ጒ 187
+ኵ 138
+ሓ 103
+ቍ 102
+ቊ 81
+ሡ 63
+ጲ 40
+ጶ 31
+ኹ 28
+ኾ 24
+ሤ 24
+ጕ 21
+ኽ 20
+ኻ 16
+ዔ 14
+ጬ 9
+ጼ 8
+ኄ 5
+ዤ 5
+ኆ 4
+ሖ 3
+ዠ 3
+ጯ 2
+ጱ 2
+ጰ 2
+ሢ 1
+ፄ 1
+ሒ 1

vocabs/ami.txt

@@ -0,0 +1,25 @@
+a 404776
+| 359705
+o 165062
+n 152245
+i 151534
+k 90606
+t 81684
+m 67663
+y 57936
+s 53970
+r 51455
+l 45476
+w 41233
+c 40804
+p 39861
+g 37542
+d 28223
+' 26416
+h 22898
+e 19615
+f 16727
+u 1882
+x 32
+0 2
+- 2

vocabs/amk.txt

@@ -0,0 +1,34 @@
+| 368899
+a 368764
+i 259962
+n 181102
+e 156863
+o 135717
+r 106329
+m 87646
+t 85684
+u 82160
+w 73239
+f 54695
+s 53460
+k 39374
+d 35932
+g 31921
+h 29093
+b 26197
+y 24087
+p 16614
+l 13488
+j 540
+- 186
+0 176
+z 138
+' 60
+1 52
+2 40
+c 34
+4 24
+6 10
+— 6
+7 4
+3 4

vocabs/ann.txt

@@ -0,0 +1,39 @@
+| 192613
+e 120025
+i 108020
+a 68024
+n 64403
+k 64265
+u 44790
+m 42985
+ọ 32712
+b 31465
+o 27606
+t 24420
+̄ 23303
+y 21207
+g 19646
+r 18788
+w 18748
+s 18372
+j 15798
+l 12822
+p 11611
+è 7436
+ì 7279
+h 5855
+c 5700
+f 4734
+ò 3995
+î 2854
+d 2215
+- 1329
+ô 917
+ǹ 824
+ê 662
+â 245
+̀ 98
+à 38
+ù 11
+' 4
+û 1

vocabs/any.txt

@@ -0,0 +1,38 @@
+| 416653
+n 279079
+a 177605
+ɛ 117170
+ɔ 97248
+ɩ 88418
+m 87443
+l 69526
+ʋ 55780
+k 54205
+b 51059
+' 49814
+i 49712
+y 49223
+s 42636
+́ 40105
+u 39676
+e 36535
+w 25312
+t 23376
+f 16323
+d 15464
+z 13803
+h 13020
+o 11810
+g 10918
+p 8961
+- 8920
+j 8628
+á 4670
+c 4383
+̂ 3123
+â 2004
+v 1601
+ú 723
+í 719
+é 534
+ó 74

vocabs/aoz.txt

@@ -0,0 +1,24 @@
+| 24892
+a 18483
+n 17772
+e 9426
+i 9363
+u 5819
+s 5496
+o 5059
+m 4924
+t 4584
+' 4121
+l 3869
+k 3789
+h 2044
+f 1850
+b 1763
+p 944
+y 721
+g 642
+- 405
+j 290
+r 280
+d 138
+w 25

vocabs/apb.txt

@@ -0,0 +1,37 @@
+| 236567
+a 192718
+e 115555
+i 113203
+o 93557
+n 91965
+u 64010
+l 46471
+m 41526
+ꞌ 38468
+k 32979
+h 31400
+t 30128
+r 28780
+g 25369
+s 25105
+w 11136
+d 9503
+p 8629
+- 5473
+j 2484
+b 486
+1 234
+v 145
+2 136
+3 66
+f 61
+5 58
+4 58
+6 57
+8 51
+9 43
+0 36
+7 29
+— 16
+' 8
+z 3

vocabs/apr.txt

@@ -0,0 +1,30 @@
+| 296756
+a 182433
+n 125496
+o 118601
+i 112382
+e 99178
+t 74048
+l 58552
+g 57792
+k 45318
+m 42609
+u 38838
+y 37789
+d 34085
+p 29989
+b 26195
+r 21373
+s 17513
+w 6924
+0 91
+1 20
+4 16
+5 11
+6 8
+2 7
+7 6
+9 6
+3 5
+' 4
+8 1

vocabs/ara.txt

@@ -0,0 +1,117 @@
+| 753464
+ا 414915
+ل 366105
+ي 243376
+م 220617
+ن 218976
+و 200525
+ت 107267
+ر 115296
+ب 117382
+ة 49395
+ع 103441
+د 77178
+ف 85639
+س 85645
+ك 100464
+أ 111253
+ق 62850
+ه 126602
+ح 59914
+ج 42045
+ط 21998
+ى 30642
+ش 25690
+ص 21207
+خ 28556
+إ 35733
+ث 16532
+ض 20734
+ذ 40384
+ً 1092
+ز 11052
+غ 9208
+ء 14858
+ئ 11563
+ّ 614
+ظ 6501
+ُ 340
+0 14
+1 255
+ٍ 236
+آ 6722
+، 179
+ؤ 4278
+2 157
+ِ 131
+9 108
+4 1
+5 6
+3 79
+6 74
+َ 71
+8 70
+ٌ 65
+7 62
+- 168
+. 41
+a 4
+/ 30
+" 28
+n 7
+o 7
+e 6
+: 23
+s 1
+t 10
+٠ 21
+p 19
+g 5
+d 2
+c 1
+l 3
+, 14
+i 9
+١ 11
+m 2
+r 10
+v 10
+b 9
+% 9
+u 3
+h 1
+؛ 6
+٩ 6
+w 1
+— 1
+k 4
+ـ 4
+٣ 4
+x 3
+f 3
+٨ 2
+z 2
++ 2
+õ 2
+j 2
+٧ 2
+y 1
+چ 1
+٢ 1
+٦ 1
+٪ 1
+× 1
+٥ 1
+ٰ 62
+ی 60
+ۖ 34
+ک 21
+ۚ 19
+ۗ 6
+ۛ 2
+ھ 1
+' 344
+ڨ 1
+ۘ 1
+☭ 1
+– 4

vocabs/arl.txt

@@ -0,0 +1,33 @@
+a 609449
+i 403784
+| 334799
+u 300809
+n 238087
+j 166571
+r 145956
+o 140514
+e 113539
+c 102091
+s 100301
+t 93880
+q 72373
+y 64674
+p 57373
+h 48304
+m 41083
+l 3696
+d 3163
+b 2465
+— 1998
+é 1574
+g 815
+f 604
+í 530
+á 396
+ó 390
+v 388
+z 278
+' 104
+x 54
+ú 44
+k 10

vocabs/asa.txt

@@ -0,0 +1,27 @@
+| 210704
+a 176039
+i 120058
+e 105093
+n 87857
+u 72130
+k 71837
+m 58953
+o 48749
+v 47614
+h 44899
+w 38857
+t 36645
+r 31987
+s 28078
+y 24358
+g 23947
+l 18614
+d 12462
+b 12169
+j 11362
+z 8742
+c 7914
+f 6769
+p 2872
+' 1072
+- 102