jdtddlZddlZddlZddlmZddlmZddlmZddiZ de de fd Z d Z d e fd Z d!d e de de de de de de de de defdZdZdZde fdZ d"de de dedede de f d Zy)#N) defaultdict)SequenceMatcher)batch_error_ratehihindimanifest language_namec ddlm}dx}x}}d}|D]}|jd}||d} t| tr5| j j dr tj| } t| tr| }n't | jD cgc]} | g}} ||d||\} } } }\}}}}}}||z }||z }||z }||z }|r ||z|z|z nd}||||fS#tj$rYwxYwcc} w) uScore a manifest with voi_oiwer (lattice-based, orthography-aware WER). Each manifest entry must have "pred_text" and a reference in one of: - "reference_lists": the lattice — list of slots, each a list of accepted variants (a variant may span multiple words); - "text" as a JSON-encoded lattice (list of lists of str); - "text" as a plain string, converted to a trivial single-variant lattice (one slot per word). voi_oiwer applies its own indicnlp-based normalization internally, so no external normalizer should be applied beforehand. Returns (err_rate, total_ins, total_del, total_sub) with err_rate in [0, 1]. r)oiwerreference_liststextz[[ pred_text) hypothesisr input_languageg) voi_oiwerr get isinstancestrlstrip startswithjsonloadsJSONDecodeErrorlistsplit)rr r total_ins total_del total_subtotal_ref_wordsdatumr r word_score_h_r_opsinsdelesub ref_words_meta_stderr_rates /app/normalizer/eval_utils.py score_oiwerr.sQ ())I) IO%))$56  "=D$$)A)A$)G::d+D$%"&7:$ioo6G"HdD6"H"HIN[)+(J FB.sD# 5$ S T S 9$/%2IX I% 1_D]`H Y 9 44'++ #IsC1 D 1DDcgg}}t||D]8\}}|j}|j}td||}gg} } |jD]\} } } }}| dk(r)| j || | | j |||7dj || | }dj |||}||k(r#| j || j || j || | | j ||||j dj | |j dj | ;||fS)zAlign compound word boundaries between ref/pred pairs. When a mismatch region has identical characters ignoring whitespace, normalize both sides to the joined form. Nequal )ziprr get_opcodesextendjoinappend)refspredsnew_refs new_predsref_textrr) pred_wordssmnew_rwnew_pwtagi1i2j1j2rcpcs r-normalize_compound_pairsrHAsO biH"4/+)NN$ __& T9j 9R#%>>#3 5 CRRg~ i2./ jB/0WWYr"-.WWZ2./8MM"%MM"%MM)Br"23MM*R"34 5 ()&)*-+. Y  manifest_pathcg}t|dd5}|D]7}t|dkDstj|}|j |9 ddd|S#1swY|SxYw)ze Reads a manifest file (jsonl format) and returns a list of dictionaries containing samples. rutf-8encodingrN)openlenrrr7)rJdatafliner s r- read_manifestrUbsi D mS7 3#q #D4y1} 4( E" ## K # KsA(AA# referencestranscriptionsmodel_id dataset_path dataset_namer audio_lengthtranscription_timeaudio_filepaths extra_fieldsc |jdd}|jdd}|jdd}t|t|k7r$tdt|dt|d|;t|t|k7r$tdt|dt|d|;t|t|k7r$td t|dt|d|;t|t|k7r$td t|dt|d| xsi} | jD]C\} } t| t|k7std | d t| dt|d||nt|dgz}||nt|dgz}||nt|dgz}d } tj j | st j| tj j| d|d|d|d|d } t| dd5}tt|||||D]j\}\}}}}}|r|nd|||||d| jD cic] \}} || | c} }}|jtj|ddl ddd| Scc} }w#1swY| SxYw)a Writes a manifest file (jsonl format) and returns the path to the file. Args: references: Ground truth reference texts. transcriptions: Model predicted transcriptions. model_id: String identifier for the model. dataset_path: Path to the dataset. dataset_name: Name of the dataset. split: Dataset split name. audio_length: Length of each audio sample in seconds. transcription_time: Transcription time of each sample in seconds. audio_filepaths: List of file paths for each audio sample. extra_fields: Optional mapping of column name to a per-sample list, written alongside the standard fields. Returns: Path to the manifest file. /-z'The number of samples in `references` (z) must match `transcriptions` (z).Nz)The number of samples in `audio_length` (z) must match `references` (z/The number of samples in `transcription_time` (z,The number of samples in `audio_filepaths` (zThe number of samples in `z` (z ./results/MODEL_ _DATASET__.jsonlwrMrNsample_audio_filepathdurationtimer rF) ensure_ascii )replacerQ ValueErroritemsospathexistsmakedirsr6rP enumerater3writerdumps)rVrWrXrYrZrr[r\r]r^ field_namevaluesbasedirrJrSidxr transcriptrinamer s r-write_manifestr~osj<S)H''S1L''S1L :#n--5c*o5FG,,/,?+@ D  C $5Z$H7L8I7JK((+J'8 <  %#.@*AS_*T=cBT>U=VW((+J'8 <  "s?';s:'N:3;O:PQ((+J'8 <   %2L*002 F v;#j/ ),ZLCK=I,,/ O+>' " GGGLL6(9\N!L>5'QWXM mS7 3Bq "    B C     5C.'RUQV(*'  :F9K9K9MNv4$N E GGtzz%e<=R@ A/ BB2 O-B2 s/AK5K 0KKK parent_id chunk_indexc H |r t|dvr|Stt}|D]}||tj|d}g}t |D]}t ||d}|Dcgc] }|t  }}|tt t|k7rtd|d|d|j|||Dcgc]}|d c}||Dcgc]}|d  c}|dd d j fd |Dd |Scc}wcc}wcc}w)acCollapse per-chunk result rows into one row per parent session. Chunks are non-overlapping and cover the session in order, so predictions are concatenated by `chunk_index` and scored against the session reference. Durations and times are summed, leaving RTFx unaffected. Manifests without a `parent_id` field are returned unchanged. rc@td|Dr t|SdS)Nc3$K|]}|du ywN).0vs r- z?merge_chunked_manifest.._sum_or_none..s!@A!4-!@s)allsum)rys r- _sum_or_nonez,merge_chunked_manifest.._sum_or_nones!!@!@@s6{JdJrIc|tSr)CHUNK_INDEX_KEY)ds r-z(merge_chunked_manifest..s 1_;MrI)keyz#WARNING: chunk indices for session z are not contiguous from 0 (zE); some chunks may be missing, which will inflate the deletion count.rjrkr r2c3TK|]}|dxsdjxr!yw)rr1N)strip)rchunkpreds r-rz)merge_chunked_manifest..s7&!&{!3!9r @ @ BBB&s%(rh) CHUNK_PARENT_KEYrrr7sortedrrangerQprintr6) rsessionsr rmergedrchunksrindicesrs @r-merge_chunked_manifestrsO 'x{:4 H8'()0078KFH% +1MN7=>e5)>> d5W./ / 5i[A&&-Y/99   "+()P% *;)PQ$%HeFm%HIq &) XX&!'&   0 M+?*Q%Hs*D D D directory multilingualcsv_onlylanguagefamiliesc 56789:;<=|jtjr|dd}tt j|dd}tt |}|;|B|dk7r=t d||jdd }|Dcgc]}d|d|vsd |d |vr|}}t|d k(rtd |dtfdggd?gd?gd?d=d@ggd?dAgdB} dCdDdEdAdF5| jD]\\} } | D cic]} | dG| dH5| dIdf} } dJdKj5fdL| Dz}|jdM| dG| dH|| f^|g}|D]?\}}}} ||vs ||j|!|j| jA|Dcgc]t!7t=t>}.t=t>}/t=t@}0=jD]t\}+},|+j5djd jC}17|1xx|,dez cc<|,dh!|.|1xx|,dkz cc<|/|1xx|,dlz cc<n dx|.|1<|/|1<|0|1xxdmz cc<v|st t dbt dnt db7jD]\}+},|,|0|+z }t |+dd|dfdg!|.D]$}+|.|+ |.|+|/|+z })t |+do|)df&t dbdpj=j6dv=fdq 99fdr: dw79:;=fds }2|D]u\}}}} |||vr|jEdMr|tdMd}3n|jG}3|%t!6fdt| D}4|4sX|2|| ||3duf|6vsk|2|| ||3w7=fScc}wcc} wcc}wcc}wcc}wcc}wcc}wcc}"wcc}$w)xa Scores all result files in a directory and returns a composite score over all evaluated datasets. Args: directory: Path to the result directory, containing one or more jsonl files. model_id: Optional, model name to filter out result files based on model name. multilingual: If True, apply compound word boundary normalization before WER computation. Should only be enabled for non-English benchmarks. csv_only: If True, suppress all output except the CSV summary block. language: Language code used for normalization (e.g. 'en', 'de', 'fr'). When not 'en', ml_normalizer is used instead of the English normalizer. Languages in OIWER_LANGUAGES (e.g. 'hi') are scored with voi_oiwer over a reference lattice instead of plain WER. families: Optional list of family keys ("appen", "dataocean", "voicearena_private", "voicearena_private_hi", "public", "extra", "ml_de", "ml_fr", "ml_it", "ml_es", "ml_pt", "ml_nl") restricting which CSV summary blocks are printed. None prints all detected families. Returns: Composite score over all evaluated datasets and a dictionary of all results. Nz /**/*.jsonlT) recursiver1zFiltering models by id:r`rarbrcrzNo result files found in fpc0|jd}||d}|jd}|d|jddjd}|jd}|d|dz||dzdz}||d}|jddjd}||fS) NrbDATASET_r1rdrar`re)findrnrstrip removesuffix)r model_indexds_indexrX author_indexds_fp dataset_ids r-parse_filepathz%score_results..parse_filepathGsggh'  77:&ix=((26==cB}}S) M\*S08L1.s$* 4; )*$ /* sml_c38K|]}|dvyw)rNr)rsubstrrrs r-rz score_results..s Qv6^B/22Qsz matching families ) data_utilsrkrjenc*j|S)N)lang) ml_normalizer)trrs r-rzscore_results..sj&>&>qx&>&PrIr r)merge_compoundsr&delr(r,)r&rr(dz | )werr[inference_timertfxP********************************************************************************zResults per dataset:z: WER = rz0.2fz %rz , RTFx = |r[rrzComposite Results:z : RTFx = r2c|jD]F\}\}}||k(s jD]%\}}|j|vs||vs||ccSHyr)rpr) model_key col_labelcol_mapmetric ds_substrlabel_group result_key result_valresultss r-find_metric_inz%score_results..find_metric_inNsk*1--/ 2 &Iv !.5mmo2*J  '')Z7I.find_wer_inVsiGUCCrIc ~"|jDcgc]\}}| }}}t}|D cgc]} | |vr|j| r| }} t|j ddz t|z } g} |r6|D cgc]} | j dd} } |dzdj | z}|rd|dnd} ttdt| tdt#dk(ry#D]t"|D cgc] } %"| |}} |Dcgc]}|| }}|s/tt|t|z d }&&n"j}td |d |vt|#D]m"&&n"}|D cic] } | %"| |}} |D cgc]} || t|| nd }} | r8|D cgc] } $"| |d }} ||Dcgc]}| t|nd c}z }td|jD}|r8|jDcgc]"\}\}}|dk(r|j|x}|$}}}}|jDcgc]"\}\}}|dk(r|j|x}|$}}}}|jDcgc]}|| }}|r!tt|t|z d nd }|r!tt|t|z d nd }|r!tt|t|z d nd }t|d|d|d|ddj |z|dk(s|xsd jdrTt"'fd|D}t"'fd|D}|rt||z d nd } t| gd g| dz zz}!nd g| z}!tdj |g|!z|zptdycc}}wcc} wcc} wcc} wcc}wcc} wcc} wcc} wcc}wcc}}}wcc}}}wcc}w)Nrrrz RTFxz CSV Summary (z):z CSV Summary:rrz avg WER (z) = r1rc3*K|] \}}|du ywrr)r_lblgrps r-rz9score_results..print_csv_block..sOsS_Osrrrrc3xK|]1}D]*}j|vr||vr|d |d,3yw)r[Nrrrrkrrs r-rz9score_results..print_csv_block..s^'%")'$++-3%O#BK7C   N3'3'7:c3xK|]1}D]*}j|vr||vr|d |d,3yw)rNrrs r-rz9score_results..print_csv_block..s`&%")&$++-3%O#BK(89E   $45&5&r)rysetaddrQrrnr6rroundrrranyrprr)(headerr family_key family_nameper_dataset_rtfxlbl_grp csv_columnsseencn_prefix rtfx_columnstitlecolwer_valsravgrcsv_model_labelwer_cols rtfx_vals is_private_dsr  scripted_wersconversational_wersall_wers avg_overall avg_scriptedavg_conv family_audio family_timertfx_val prefix_colsr composite_werrroriginal_model_idrs( @r-print_csv_blockz&score_results..print_csv_blockYs-4NN,<=ysDs= =u"-PQa4i488A;qP Pv||C()A-K0@@ ALL1AIIfg6LLLc\CHH\$::F3>- }B/N  h e  h }  "* 8 LWXSK 3@XX'/A!1=AAAH H =qAC-8*&__. IeWD67 8 f &H LI%6%B!  FQ>A[C99H ''/sm&?HSM"RGH +#9c7FC iPq}SV"<PPOgnn>NOOJ,3==?!!'Zc3j(8<<3D.Da-Q! !,3==?'''Zc3..c9J4JA3W'#' (0'8J!AMAJJ?GE#h-#h-7;R %#m,s=/AA1E+#12S9L5MMqQ &'q Q|nAhZqQhhx() )j.>B-J-J5-Q#&')0'$L#&&)0&#KALl[8!IJKQH LT h]>QMYA  Q! ' Ks] O> PPPP PP#PP'P P"$P'6'P, 5'P3 2P::P:c3&K|]}|v ywrr)rrall_dataset_idss r-rz score_results..sSiY/9Ss)r)r)NNF)$endswithrqpathseprglobrrrnrQrorrpr6r7r5keysr normalizerrrrUrOIWER_LANGUAGESr.rHtuplerrrrrfloatintrr capitalize)>rrXrrrr result_filesrFAMILY_CONFIGSML_LANG_DATASETSrdatasetsrrrallowed_substrsrpresence_substr_header result_filermodel_id_of_filerr rkrj compute_rtfxrrrr normalizerV predictionsrL refs_splitp preds_splitrr[rrrkrmetricscomposite_audio_lengthcomposite_inference_time count_entriesrr7r has_publicrr9r5rrrr6rrs> ` ` @@@@@@@@@r- score_resultsrYs>"**%crN  YK{";tLML|,-L!B '2##C-# 8*A"$& )(D(J    <A4YK@AA $3 $"   J6Q7R5P3NJJD   4   J7R6QGG   & !  '.0M    $ " ,.A      r%>87,L*@*@#;,N-O%   6   G-#;#;"9   maNL&&&&k $,E%T]^*002Ph$ iqe $*;G*D)ET'JD&Q Q  * ?G* "   TF|qe_fgNOP=K ; 9J'X%".#**?;#**7<<>:  ;# QQQ   |  !+I;6I(T  &G#5 )- *DE'5k'B$*+34%f 443;<%E*%<<4y2S]  &4?/(34 0CIy4&11 P @HIu)E&M2IJIFNOU9U;%78OKO+C:{*[' K 6@@5+@J@5@A5+AKA [$OA./h%!E()yIJ-C!)IFC#Iq! x=L YNXT2A6D37 7L 7>D()ZL9 (,     _5 n  h $% hMMO DAq8AeHT?"5Gy$Yqy&677 'N   &M(/*51$M  1ggcl1o##%cah& V9 "3 '1^+< < ' $S )Q/?-@ @ )JN N "3 '*B3*Gca    h "# h!'') .DAqmA&&C QCxDz, - .( 2A%a(4-a03KA3NN9T$K01 2 hhhw||~.ODNSqqh9GJ4 OVW  Jh$>    '$SZ\2K%%'   "S7SSJGZt/1[I%J( ' !!K  V , &5<JOAAs6Y Y-Y Y#/ Y(Y-*Y2 Y79 Y<)NNNN)NFFrN)r<rrq collectionsrdifflibr kaldialignrr?rrr.rHrUdictr~rrrboolrYrrIr-r_s9 ##'  ' -5$-5s-5`B  (# llll l  l  lllll^,T,b P"P"P"P" P"  P"  P"rI