a
    Bù’i±N  ã                   @  sf  d dl mZ d dlmZ d dlmZ ddlmZmZm	Z	 ddl
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ G dd„ dƒZG d	d
„ d
eƒZG dd„ deƒZG dd„ deƒZ G dd„ deƒZ!G dd„ deƒZ"G dd„ deƒZ#G dd„ deƒZ$G dd„ deƒZ%G dd„ deƒZ&edd�ddddœd d!„ƒZ'ed"d�d+d%d&dd&d'œd(d)„ƒZ(d*S ),é    )Úannotations)Ú	lru_cache)Ú	getLoggeré   )ÚCOMMON_SAFE_ASCII_CHARACTERSÚTRACEÚUNICODE_SECONDARY_RANGE_KEYWORD)Úis_accentuatedÚ	is_arabicÚis_arabic_isolated_formÚis_case_variableÚis_cjkÚis_emoticonÚ	is_hangulÚis_hiraganaÚis_katakanaÚis_latinÚis_punctuationÚis_separatorÚ	is_symbolÚis_thaiÚis_unprintableÚremove_accentÚunicode_rangeÚis_cjk_uncommonc                   @  sP   e Zd ZdZdddœdd„Zdddœdd	„Zdd
œdd„Zedd
œdd„ƒZdS )ÚMessDetectorPluginzy
    Base abstract class used for mess detection plugins.
    All detectors MUST extend and implement given methods.
    ÚstrÚbool©Ú	characterÚreturnc                 C  s   t ‚dS )z@
        Determine if given character should be fed in.
        N©ÚNotImplementedError©Úselfr   © r%   úR/home/httpd/docs/test/DocsMgr/lib/python3.9/site-packages/charset_normalizer/md.pyÚeligible'   s    zMessDetectorPlugin.eligibleÚNonec                 C  s   t ‚dS )z‰
        The main routine to be executed upon character.
        Insert the logic in witch the text would be considered chaotic.
        Nr!   r#   r%   r%   r&   Úfeed-   s    zMessDetectorPlugin.feed©r    c                 C  s   t ‚dS )zB
        Permit to reset the plugin to the initial state.
        Nr!   ©r$   r%   r%   r&   Úreset4   s    zMessDetectorPlugin.resetÚfloatc                 C  s   t ‚dS )z…
        Compute the chaos ratio based on what your feed() has seen.
        Must NOT be lower than 0.; No restriction gt 0.
        Nr!   r+   r%   r%   r&   Úratio:   s    zMessDetectorPlugin.ratioN)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r'   r)   r,   Úpropertyr.   r%   r%   r%   r&   r   !   s   r   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )Ú TooManySymbolOrPunctuationPluginr(   r*   c                 C  s"   d| _ d| _d| _d | _d| _d S )Nr   F)Ú_punctuation_countÚ_symbol_countÚ_character_countÚ_last_printable_charZ_frenzy_symbol_in_wordr+   r%   r%   r&   Ú__init__D   s
    z)TooManySymbolOrPunctuationPlugin.__init__r   r   r   c                 C  s   |  ¡ S ©N©Úisprintabler#   r%   r%   r&   r'   L   s    z)TooManySymbolOrPunctuationPlugin.eligiblec                 C  sp   |  j d7  _ || jkrf|tvrft|ƒr8|  jd7  _n.| ¡ du rft|ƒrft|ƒdu rf|  jd7  _|| _d S )Nr   Fé   )	r7   r8   r   r   r5   Úisdigitr   r   r6   r#   r%   r%   r&   r)   O   s    ÿþ
ÿþ
ýz%TooManySymbolOrPunctuationPlugin.feedc                 C  s   d| _ d| _d| _d S ©Nr   )r5   r7   r6   r+   r%   r%   r&   r,   a   s    z&TooManySymbolOrPunctuationPlugin.resetr-   c                 C  s0   | j dkrdS | j| j | j  }|dkr,|S dS )Nr   ç        ç333333Ó?)r7   r5   r6   )r$   Zratio_of_punctuationr%   r%   r&   r.   f   s    

þz&TooManySymbolOrPunctuationPlugin.ratioN©	r/   r0   r1   r9   r'   r)   r,   r3   r.   r%   r%   r%   r&   r4   C   s   r4   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )ÚTooManyAccentuatedPluginr(   r*   c                 C  s   d| _ d| _d S r?   ©r7   Ú_accentuated_countr+   r%   r%   r&   r9   s   s    z!TooManyAccentuatedPlugin.__init__r   r   r   c                 C  s   |  ¡ S r:   )Úisalphar#   r%   r%   r&   r'   w   s    z!TooManyAccentuatedPlugin.eligiblec                 C  s(   |  j d7  _ t|ƒr$|  jd7  _d S ©Nr   )r7   r	   rE   r#   r%   r%   r&   r)   z   s    zTooManyAccentuatedPlugin.feedc                 C  s   d| _ d| _d S r?   rD   r+   r%   r%   r&   r,   €   s    zTooManyAccentuatedPlugin.resetr-   c                 C  s*   | j dk rdS | j| j  }|dkr&|S dS )Né   r@   gffffffÖ?rD   )r$   Zratio_of_accentuationr%   r%   r&   r.   „   s    
zTooManyAccentuatedPlugin.ratioNrB   r%   r%   r%   r&   rC   r   s   rC   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )ÚUnprintablePluginr(   r*   c                 C  s   d| _ d| _d S r?   )Ú_unprintable_countr7   r+   r%   r%   r&   r9   Ž   s    zUnprintablePlugin.__init__r   r   r   c                 C  s   dS ©NTr%   r#   r%   r%   r&   r'   ’   s    zUnprintablePlugin.eligiblec                 C  s(   t |ƒr|  jd7  _|  jd7  _d S rG   )r   rJ   r7   r#   r%   r%   r&   r)   •   s    zUnprintablePlugin.feedc                 C  s
   d| _ d S r?   )rJ   r+   r%   r%   r&   r,   š   s    zUnprintablePlugin.resetr-   c                 C  s   | j dkrdS | jd | j  S )Nr   r@   rH   )r7   rJ   r+   r%   r%   r&   r.   �   s    
zUnprintablePlugin.ratioNrB   r%   r%   r%   r&   rI   �   s   rI   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )ÚSuspiciousDuplicateAccentPluginr(   r*   c                 C  s   d| _ d| _d | _d S r?   ©Ú_successive_countr7   Ú_last_latin_characterr+   r%   r%   r&   r9   ¦   s    z(SuspiciousDuplicateAccentPlugin.__init__r   r   r   c                 C  s   |  ¡ ot|ƒS r:   )rF   r   r#   r%   r%   r&   r'   ¬   s    z(SuspiciousDuplicateAccentPlugin.eligiblec                 C  st   |  j d7  _ | jd urjt|ƒrjt| jƒrj| ¡ rJ| j ¡ rJ|  jd7  _t|ƒt| jƒkrj|  jd7  _|| _d S rG   )r7   rO   r	   ÚisupperrN   r   r#   r%   r%   r&   r)   ¯   s    ÿþýz$SuspiciousDuplicateAccentPlugin.feedc                 C  s   d| _ d| _d | _d S r?   rM   r+   r%   r%   r&   r,   ½   s    z%SuspiciousDuplicateAccentPlugin.resetr-   c                 C  s   | j dkrdS | jd | j  S )Nr   r@   r=   )r7   rN   r+   r%   r%   r&   r.   Â   s    
z%SuspiciousDuplicateAccentPlugin.ratioNrB   r%   r%   r%   r&   rL   ¥   s   rL   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )ÚSuspiciousRanger(   r*   c                 C  s   d| _ d| _d | _d S r?   )Ú"_suspicious_successive_range_countr7   Ú_last_printable_seenr+   r%   r%   r&   r9   Ë   s    zSuspiciousRange.__init__r   r   r   c                 C  s   |  ¡ S r:   r;   r#   r%   r%   r&   r'   Ð   s    zSuspiciousRange.eligiblec                 C  sx   |  j d7  _ | ¡ s&t|ƒs&|tv r0d | _d S | jd u rD|| _d S t| jƒ}t|ƒ}t||ƒrn|  jd7  _|| _d S rG   )r7   Úisspacer   r   rS   r   Ú is_suspiciously_successive_rangerR   )r$   r   Úunicode_range_aÚunicode_range_br%   r%   r&   r)   Ó   s"    ÿþý


zSuspiciousRange.feedc                 C  s   d| _ d| _d | _d S r?   )r7   rR   rS   r+   r%   r%   r&   r,   ê   s    zSuspiciousRange.resetr-   c                 C  s"   | j dkrdS | jd | j  }|S )Né   r@   r=   )r7   rR   )r$   Zratio_of_suspicious_range_usager%   r%   r&   r.   ï   s    
þzSuspiciousRange.ratioNrB   r%   r%   r%   r&   rQ   Ê   s   rQ   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )ÚSuperWeirdWordPluginr(   r*   c                 C  s@   d| _ d| _d| _d| _d| _d| _d| _d| _d| _d| _	d S )Nr   FÚ )
Ú_word_countÚ_bad_word_countÚ_foreign_long_countÚ_is_current_word_badÚ_foreign_long_watchr7   Ú_bad_character_countÚ_bufferÚ_buffer_accent_countÚ_buffer_glyph_countr+   r%   r%   r&   r9   ü   s    zSuperWeirdWordPlugin.__init__r   r   r   c                 C  s   dS rK   r%   r#   r%   r%   r&   r'     s    zSuperWeirdWordPlugin.eligiblec                 C  s–  |  ¡ rÆ|  j|7  _t|ƒr,|  jd7  _| jdu rŒt|ƒdu sJt|ƒrŒt|ƒdu rŒt|ƒdu rŒt|ƒdu rŒt	|ƒdu rŒt
|ƒdu rŒd| _t|ƒs´t|ƒs´t|ƒs´t	|ƒs´t
|ƒrÂ|  jd7  _d S | jsÐd S | ¡ sêt|ƒsêt|ƒ�r\| j�r\|  jd7  _t| jƒ}|  j|7  _|dk�rª| j| dk�r:d| _npt| jd ƒ�rŠ| jd  ¡ �rŠtdd„ | jD ƒƒdu �rŠ|  jd7  _d| _n | jdk�rªd| _|  jd7  _|d	k�r| j�rd
d„ t| jtd|ƒƒD ƒ}d}|�røt|ƒ| dk�rød}|�s|  jd7  _d| _| j�rB|  jd7  _|  jt| jƒ7  _d| _d| _d| _d| _d| _n6|dv�r’| ¡ du �r’t|ƒ�r’d| _|  j|7  _d S )Nr   FTé   ç      à?éÿÿÿÿc                 s  s   | ]}|  ¡ V  qd S r:   ©rP   )Ú.0Ú_r%   r%   r&   Ú	<genexpr>8  ó    z,SuperWeirdWordPlugin.feed.<locals>.<genexpr>é   c                 S  s   g | ]\}}|  ¡ r|‘qS r%   rg   )rh   ÚcÚir%   r%   r&   Ú
<listcomp>@  s   þz-SuperWeirdWordPlugin.feed.<locals>.<listcomp>r   rA   rZ   >   ú|ú>ri   ú=ú-ú~ú<)rF   ra   r	   rb   r_   r   r   r   r   r   r   rc   rT   r   r   r[   Úlenr7   r^   rP   Úallr]   ÚzipÚranger\   r`   r>   r   )r$   r   Zbuffer_lengthZcamel_case_dstZprobable_camel_casedr%   r%   r&   r)     s¬    ÿ
þþ
ý
ü
û
ú
ù	ÿþýüûÿÿÿþ

ÿþýþÿ
þýzSuperWeirdWordPlugin.feedc                 C  s4   d| _ d| _d| _d| _d| _d| _d| _d| _d S )NrZ   Fr   )ra   r^   r_   r\   r[   r7   r`   r]   r+   r%   r%   r&   r,   _  s    zSuperWeirdWordPlugin.resetr-   c                 C  s$   | j dkr| jdkrdS | j| j S )Né
   r   r@   )r[   r]   r`   r7   r+   r%   r%   r&   r.   i  s    zSuperWeirdWordPlugin.ratioNrB   r%   r%   r%   r&   rY   û   s   Q
rY   c                   @  s^   e Zd ZdZddœdd„Zdddœd	d
„Zdddœdd„Zddœdd„Zeddœdd„ƒZ	dS )ÚCjkUncommonPluginz<
    Detect messy CJK text that probably means nothing.
    r(   r*   c                 C  s   d| _ d| _d S r?   ©r7   Ú_uncommon_countr+   r%   r%   r&   r9   v  s    zCjkUncommonPlugin.__init__r   r   r   c                 C  s   t |ƒS r:   )r   r#   r%   r%   r&   r'   z  s    zCjkUncommonPlugin.eligiblec                 C  s,   |  j d7  _ t|ƒr(|  jd7  _d S d S rG   )r7   r   r}   r#   r%   r%   r&   r)   }  s    zCjkUncommonPlugin.feedc                 C  s   d| _ d| _d S r?   r|   r+   r%   r%   r&   r,   „  s    zCjkUncommonPlugin.resetr-   c                 C  s.   | j dk rdS | j| j  }|dkr*|d S dS )NrH   r@   re   rz   r|   )r$   Zuncommon_form_usager%   r%   r&   r.   ˆ  s    
zCjkUncommonPlugin.ratioN)
r/   r0   r1   r2   r9   r'   r)   r,   r3   r.   r%   r%   r%   r&   r{   q  s   r{   c                   @  sZ   e Zd Zddœdd„Zdddœdd	„Zdddœd
d„Zddœdd„Zeddœdd„ƒZdS )ÚArchaicUpperLowerPluginr(   r*   c                 C  s.   d| _ d| _d| _d| _d| _d | _d| _d S )NFr   T)Ú_bufÚ_character_count_since_last_sepÚ_successive_upper_lower_countÚ#_successive_upper_lower_count_finalr7   Ú_last_alpha_seenÚ_current_ascii_onlyr+   r%   r%   r&   r9   •  s    z ArchaicUpperLowerPlugin.__init__r   r   r   c                 C  s   dS rK   r%   r#   r%   r%   r&   r'   ¢  s    z ArchaicUpperLowerPlugin.eligiblec                 C  s$  |  ¡ ot|ƒ}|du }|r†| jdkr†| jdkrV| ¡ du rV| jdu rV|  j| j7  _d| _d| _d | _d| _|  j	d7  _	d| _d S | jdu r¢| 
¡ du r¢d| _| jd urþ| ¡ r¾| j ¡ sÐ| ¡ rø| j ¡ rø| jdu rð|  jd7  _d| _qþd| _nd| _|  j	d7  _	|  jd7  _|| _d S )NFr   é@   r   Tr=   )rF   r   r€   r>   r„   r‚   r�   rƒ   r   r7   ÚisasciirP   Úislower)r$   r   Zis_concernedZ	chunk_sepr%   r%   r&   r)   ¥  sF    ÿ
þýÿ
ÿÿ
zArchaicUpperLowerPlugin.feedc                 C  s.   d| _ d| _d| _d| _d | _d| _d| _d S )Nr   FT)r7   r€   r�   r‚   rƒ   r   r„   r+   r%   r%   r&   r,   Ï  s    zArchaicUpperLowerPlugin.resetr-   c                 C  s   | j dkrdS | j| j  S )Nr   r@   )r7   r‚   r+   r%   r%   r&   r.   Ø  s    
zArchaicUpperLowerPlugin.ratioNrB   r%   r%   r%   r&   r~   ”  s   *	r~   c                   @  sZ   e Zd Zddœdd„Zddœdd„Zddd	œd
d„Zddd	œdd„Zeddœdd„ƒZdS )ÚArabicIsolatedFormPluginr(   r*   c                 C  s   d| _ d| _d S r?   ©r7   Ú_isolated_form_countr+   r%   r%   r&   r9   á  s    z!ArabicIsolatedFormPlugin.__init__c                 C  s   d| _ d| _d S r?   r‰   r+   r%   r%   r&   r,   å  s    zArabicIsolatedFormPlugin.resetr   r   r   c                 C  s   t |ƒS r:   )r
   r#   r%   r%   r&   r'   é  s    z!ArabicIsolatedFormPlugin.eligiblec                 C  s(   |  j d7  _ t|ƒr$|  jd7  _d S rG   )r7   r   rŠ   r#   r%   r%   r&   r)   ì  s    zArabicIsolatedFormPlugin.feedr-   c                 C  s   | j dk rdS | j| j  }|S )NrH   r@   r‰   )r$   Zisolated_form_usager%   r%   r&   r.   ò  s    
zArabicIsolatedFormPlugin.ratioN)	r/   r0   r1   r9   r,   r'   r)   r3   r.   r%   r%   r%   r&   rˆ   à  s   rˆ   é   )Úmaxsizez
str | Noner   )rV   rW   r    c                 C  s”  | du s|du rdS | |kr dS d| v r4d|v r4dS d| v sDd|v rHdS d| v sXd|v rld| v shd|v rldS |   d¡|  d¡ }}|D ]}|tv r”q†||v r† dS q†| dv |dv  }}|s¾|rÒd	| v sÎd	|v rÒdS |rÞ|rÞdS d
| v sðd
|v �r d	| v �sd	|v �rdS | dk�s|dk�r dS d	| v �sHd	|v �sH| dv �r�|dv �r�d| v �s\d|v �r`dS d| v �std|v �rxdS | dk�sŒ|dk�r�dS dS )za
    Determine if two Unicode range seen next to each other can be considered as suspicious.
    NTFZLatinZ	EmoticonsZ	Combiningú )ÚHiraganaÚKatakanaÚCJKÚHangulzBasic Latin)r�   rŽ   ÚPunctuationZForms)Úsplitr   )rV   rW   Zkeywords_range_aZkeywords_range_bÚelZrange_a_jp_charsZrange_b_jp_charsr%   r%   r&   rU   ü  sh    ÿÿþÿúÿÿÿþrU   i   çš™™™™™É?Fr   r-   )Údecoded_sequenceÚmaximum_thresholdÚdebugr    c              	   C  sX  dd„ t  ¡ D ƒ}t| ƒd }d}|dk r0d}n|dkr>d}nd	}t| d
 t|ƒƒD ]d\}}|D ]}	|	 |¡r`|	 |¡ q`|dkrŽ|| dksš||d krTtdd„ |D ƒƒ}||krT qºqT|�rNtdƒ}
|
 	t
d|› d|› d|› �¡ t| ƒdk�r(|
 	t
d| dd… › �¡ |
 	t
d| dd… › �¡ |D ] }|
 	t
|j› d|j› �¡ �q,t|dƒS )zw
    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
    c                 S  s   g | ]
}|ƒ ‘qS r%   r%   )rh   Zmd_classr%   r%   r&   ro   N  s   zmess_ratio.<locals>.<listcomp>r   r@   i   é    r‹   r…   é€   Ú
r   c                 s  s   | ]}|j V  qd S r:   )r.   )rh   Údtr%   r%   r&   rj   e  rk   zmess_ratio.<locals>.<genexpr>Zcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=é   zStarting with: NzEnding with: iðÿÿÿz: é   )r   Ú__subclasses__rv   rx   ry   r'   r)   Úsumr   Úlogr   Ú	__class__r.   Úround)r–   r—   r˜   Z	detectorsÚlengthZmean_mess_ratioZ!intermediary_mean_mess_ratio_calcr   ÚindexÚdetectorÚloggerrœ   r%   r%   r&   Ú
mess_ratioF  sR    ÿ
ÿ
ÿ
þÿÿþþr¨   N)r•   F))Ú
__future__r   Ú	functoolsr   Úloggingr   Zconstantr   r   r   Úutilsr	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r4   rC   rI   rL   rQ   rY   r{   r~   rˆ   rU   r¨   r%   r%   r%   r&   Ú<module>   s&   P"/%1v#LI ÿ