Ë
    ²²jh   ã                   óü   — d dl Z d dlZd dlZd dlmZmZmZ d dlmZ d dl	m
Z
mZ ddlmZmZmZmZ d„ Zi Z G d„ d	e
«      Zd
„ Z e«       Z	 ed¬«      Zd„ ZddddœiZddgZd„ Zd„ Zd„ Zdd„Zg d¢Zd„ Zd„ Zdd„Z y)é    N)ÚAudioÚIterableDatasetÚload_dataset)Úsnapshot_download)ÚBasicMultilingualTextNormalizerÚEnglishTextNormalizeré   )Úmerge_chunked_manifestÚnormalize_compound_pairsÚread_manifestÚwrite_manifestc                 óP   — | j                  «       dk(  ry| j                  «       dk7  S )Nzignore time segment in scoringFÚ )Ústrip)Úrefs    ú/app/normalizer/data_utils.pyÚis_target_text_in_ranger      s%   € Ø
‡y�yƒ{Ð6Ò6Øà�y‰y‹{˜bÑ Ð ó    c                   óB   ‡ — e Zd ZdZddefˆ fd„Zd„ Zd„ Zdˆ fd„	Zˆ xZ	S )	ÚMultilingualNormalizera  BasicMultilingualTextNormalizer with optional number normalization.

    Call with just text for standard normalization (backward-compatible).
    Pass lang= to also convert digits to words via num2words and remove
    language-specific filler words (see FILLER_WORDS).
    Úremove_diacriticsc                 óp  •— t         ‰| �  |«       i | _        t         ‰| �  }t        j                  «       D ]y  \  }}|D �ch c]
  } ||«      ’Œ }}|j                  d«       t        j                  ddj                  d„ t        |t        d¬«      D «       «      z   dz   «      | j                  |<   Œ{ y c c}w )Nr   z
(?<!\S)(?:ú|c              3   óF   K  — | ]  }t        j                  |«      –— Œ y ­w©N)ÚreÚescape)Ú.0Úws     r   ú	<genexpr>z2MultilingualNormalizer.__init__.<locals>.<genexpr>6   s   è ø€ Òa¨Aœ2Ÿ9™9 QŸ<Ñaùs   ‚!T)ÚkeyÚreversez)(?!\S))ÚsuperÚ__init__Ú_filler_patternsÚ__call__ÚFILLER_WORDSÚitemsÚdiscardr   ÚcompileÚjoinÚsortedÚlen)Úselfr   Úbase_normalizeÚlangÚwordsr   Únormalized_wordsÚ	__class__s          €r   r$   zMultilingualNormalizer.__init__&   s²   ø€ Ü‰ÑÐ*Ô+ð !#ˆÔÜ™Ñ)ˆÜ'×-Ñ-Ó/ò 	‰KˆD�%Ø;@ÖA°a¡¨qÕ 1ÐAÐÐAØ×$Ñ$ RÔ(Ü*,¯*©*ØØ—(‘(Ña´Ð8HÌcÐ[_Ô1`ÔaÓañbàñó+ˆD×!Ñ! $Ò'ñ	ùÚAs   Á B3c                 ó®   — | j                   j                  |«      }|€|S |j                  d|«      }t        j                  dd|«      j	                  «       S )Nr   z\s+ú )r%   ÚgetÚsubr   r   )r.   Útextr0   Úpatterns       r   Ú_remove_fillersz&MultilingualNormalizer._remove_fillers:   sN   € Ø×'Ñ'×+Ñ+¨DÓ1ˆØˆ?ØˆKØ�{‰{˜2˜tÓ$ˆÜ�v‰v�f˜c 4Ó(×.Ñ.Ó0Ð0r   c                 ój   ‡— t        j                  dd|«      }ˆfd„}t        j                  d||«      S )Nz(\d)\s+(\d{3})\bz\1\2c                 ó    •— 	 t        j                   t        | j                  «       «      ‰¬«      S # t        $ r | j                  «       cY S w xY w)N)r0   )Ú	num2wordsÚintÚgroupÚ	Exception)Úmr0   s    €r   Ú_replacez;MultilingualNormalizer._normalize_numbers.<locals>._replaceF   s@   ø€ ð!Ü ×*Ñ*¬3¨q¯w©w«y«>ÀÔEÐEøÜò !Ø—w‘w“yÒ ð!ús   ƒ-1 ±AÁAz\d+)r   r7   )r.   r8   r0   rB   s     ` r   Ú_normalize_numbersz)MultilingualNormalizer._normalize_numbersA   s0   ø€ ä�v‰vÐ)¨7°DÓ9ˆô	!ô �v‰v�f˜h¨Ó-Ð-r   c                 ór   •— t         ‰| �  |«      }|�$| j                  ||«      }| j                  ||«      }|S r   )r#   r&   r:   rC   )r.   Úsr0   r3   s      €r   r&   zMultilingualNormalizer.__call__N   sA   ø€ Ü‰GÑ˜QÓˆØÐØ×$Ñ$ Q¨Ó-ˆAØ×'Ñ'¨¨4Ó0ˆAØˆr   )Tr   )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úboolr$   r:   rC   r&   Ú__classcell__)r3   s   @r   r   r      s(   ø„ ññ¨$õ ò(1ò.÷ñ r   r   c                 ór   — d| v r| d   S d| v r| d   S d| v r| d   S d| v r| d   S d| v r| d   S t        d«      ‚)Nr8   ÚsentenceÚnormalized_textÚ
transcriptÚtranscriptionz´Expected transcript column of either 'text', 'sentence', 'normalized_text' or 'transcript'. Got sample of .join{sample.keys()}. Ensure a text column name is present in the dataset.)Ú
ValueError)Úsamples    r   Úget_textrS   V   st   € Ø�ÑØ�f‰~ÐØ	�vÑ	Ø�jÑ!Ð!Ø	˜fÑ	$ØÐ'Ñ(Ð(Ø	˜Ñ	Ø�lÑ#Ð#Ø	˜FÑ	"Ø�oÑ&Ð&äðYó
ð 	
r   F)r   c                 óD   — t        | «      | d<   t        | d   «      | d<   | S )NÚoriginal_textÚ	norm_text)rS   Ú
normalizer)Úbatchs    r   Ú	normalizerY   m   s)   € Ü% e›_€Eˆ/ÑÜ# E¨/Ñ$:Ó;€Eˆ+ÑØ€Lr   z0artificialanalysis/earnings22-cleaned-aa-chunkedz(ArtificialAnalysis/Earnings22-Cleaned-AAÚaudio)Úparent_dataset_pathÚ	audio_dirÚ	parent_idÚchunk_indexc                 ó@   — t        | «      j                  «       t        v S r   )ÚstrÚlowerÚCHUNKED_DATASETS)Údataset_paths    r   Úis_chunked_datasetrd   €   s   € Üˆ|Ó×"Ñ"Ó$Ô(8Ð8Ð8r   c                 ó8  ‡‡‡— t         t        | j                  «      j                  «          Št	        ‰d   | j
                  d¬«      }|D �ci c]  }|d   t        |«      “Œ c}Št	        | j                  | j
                  d¬«      }t        | j                  d‰d   › d�g¬«      Št        t        |d	   «      t        ‰«      z
  «      }|rt        d
‰d   › d|› d�«      ‚ˆˆˆfd„}|j                  |d¬«      }|j                  dt        «       «      }|S c c}w )zKLoad a chunked dataset, attaching each chunk's parent transcript as `text`.r[   T)ÚsplitÚtokenÚidÚdatasetr\   z/*)Úrepo_idÚ	repo_typeÚallow_patternsr]   zNo transcript found in z for parent id(s) z2. The chunked and parent datasets are out of sync.c                 ór   •— t         j                  j                  ‰‰d   | d   «      | d<   ‰| d      | d<   | S )Nr\   Ú	file_namerZ   r]   r8   )ÚosÚpathr+   )rR   Ú
audio_rootÚconfigÚparent_texts    €€€r   Úattach_audio_and_textz0load_chunked_data.<locals>.attach_audio_and_text�   sD   ø€ ÜŸ'™'Ÿ,™,Ø˜˜{Ñ+¨V°KÑ-@ó
ˆˆw‰ð % V¨KÑ%8Ñ9ˆˆv‰Øˆr   F)Úload_from_cache_filerZ   )rb   r`   rc   ra   r   rf   rS   r   r,   ÚsetrQ   ÚmapÚcast_columnr   )	ÚargsÚparent_datasetrR   ri   Úmissingrt   rq   rr   rs   s	         @@@r   Úload_chunked_datar|   „   s,  ú€ äœc $×"3Ñ"3Ó4×:Ñ:Ó<Ñ=€Fä!ØÐ$Ñ%¨T¯Z©Z¸tô€Nð AOÖO°f�6˜$‘<¤¨&Ó!1Ñ1ÒO€Kä˜4×,Ñ,°D·J±JÀdÔK€Gô #Ø×!Ñ!ØØ! +Ñ.Ð/¨rÐ2Ð3ô€Jô ”S˜ Ñ-Ó.´°[Ó1AÑAÓB€GÙÜØ% fÐ-BÑ&CÐ%Dð EØ�IÐOðQó
ð 	
ö
ð �k‰kÐ/ÀeˆkÓL€GØ×!Ñ! '¬5«7Ó3€Gà€Nùò9 Ps   ÁDc                 ó¶   — t        | j                  «      rt        | «      S t        | j                  | j                  | j
                  | j                  d¬«      }|S )NT)rf   Ú	streamingrg   )rd   rc   r|   r   ri   rf   r~   )ry   ri   s     r   Ú	load_datar   ª   sM   € Ü˜$×+Ñ+Ô,Ü  Ó&Ð&äØ×ÑØ�‰Ø�j‰jØ—.‘.Øô€Gð €Nr   c                 óÈ   — | j                  dt        |¬«      «      } t        | t        «      ri nddi} | j                  t
        fi |¤Ž} | j                  t        dg¬«      } | S )NrZ   )Úsampling_rateru   FrV   )Úinput_columns)rx   r   Ú
isinstancer   rw   rY   Úfilterr   )ri   r�   Ú
map_kwargss      r   Úprepare_datar†   ¹   sf   € à×!Ñ! '¬5¸}Ô+MÓN€Gô ˜¤/Ô2‰Ð9OÐQVÐ8Wð ð ˆg�k‰kœ)Ñ2 zÑ2€GØ�n‰nÔ4À[ÀMˆnÓR€Gà€Nr   )rh   rn   rn   c                 ó„   — | €y t        | «      j                  «       } | dk(  ry t        j                  j	                  | «      S )Nr   )r`   r   ro   rp   Úbasename)Úvalues    r   Ú_basename_or_nonerŠ   Î   s;   € Ø€}ØÜ�‹J×ÑÓ€EØ�‚{ØÜ�7‰7×Ñ˜EÓ"Ð"r   c                 ór   — | €y t         D ]  }	 || v rt        | |   «      }|�|c S Œ y # t        $ r Y  y w xY wr   )ÚAUDIO_FILEPATH_METADATA_KEYSrŠ   Ú	TypeError)rR   r!   rˆ   s      r   Ú"extract_audio_filepath_from_samplerŽ   ×   sX   € Ø€~Øä+ò ˆð	Ø�f‰}Ü,¨V°C©[Ó9�ØÐ'Ø#’Oøðð øô ò 	âð	ús   Ž)©	6µ6c                 óŒ  — |€^d| v rt        | d   «      }nKt        | «      dkD  r=t        t        | j                  «       «      «      }t	        |t
        «      rt        |«      }|€g S t        D ]M  }| j                  |«      }t	        |t
        «      sŒ%t        |«      |k(  sŒ4|D �cg c]  }t        |«      ‘Œ c}c S  d g|z  S c c}w )NrZ   r   )	r-   ÚnextÚiterÚvaluesrƒ   ÚlistrŒ   r6   rŠ   )rX   Ú
batch_sizeÚfirst_valuer!   r’   Úvs         r   Ú"extract_audio_filepaths_from_batchr—   ç   s¹   € ØÐØ�eÑÜ˜U 7™^Ó,‰JÜ�‹Z˜!Š^Üœt E§L¡L£NÓ3Ó4ˆKÜ˜+¤tÔ,Ü  Ó-�
àÐØˆ	ä+ò :ˆØ—‘˜3“ˆÜ�fœdÕ#¬¨F«°zÓ(AØ28Ö9¨QÔ% aÕ(Ò9Ò9ð:ð ˆ6�JÑÐùò :s   Â#C)i€>  r   )!ro   r   r=   Údatasetsr   r   r   Úhuggingface_hubr   rW   r   r   Ú
eval_utilsr
   r   r   r   r   r'   r   rS   Úml_normalizerrY   rb   ÚCHUNK_METADATA_KEYSrd   r|   r   r†   rŒ   rŠ   rŽ   r—   © r   r   ú<module>rž      s­   ðÛ 	Û 	ã ß 9Ñ 9Ý -ß M÷ó ò!ð €ô5Ð<ô 5òp
ñ$ #Ó$€
á&¸Ô?€òð 7ØIØñ9ðÐ ð # MÐ2Ð ò9ò#òLóò Ð ò#òô r   