o
    ­}”e¨L  ã                
   @   sz  d dl mZ d dlmZ d dlmZmZ ddlmZm	Z	m
Z
 ddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ G dd„ dƒZG d	d
„ d
eƒZG dd„ deƒZG dd„ deƒZ G dd„ deƒZ!G dd„ deƒZ"G dd„ deƒZ#G dd„ deƒZ$G dd„ deƒZ%G dd„ deƒZ&edd�dee' dee' de(fd d!„ƒZ)ed"d�	$d+d%e'd&e*d'e(de*fd(d)„ƒZ+d*S ),é    )Ú	lru_cache)Ú	getLogger)ÚListÚOptionalé   )ÚCOMMON_SAFE_ASCII_CHARACTERSÚTRACEÚUNICODE_SECONDARY_RANGE_KEYWORD)Úis_accentuatedÚ	is_arabicÚis_arabic_isolated_formÚis_case_variableÚis_cjkÚis_emoticonÚ	is_hangulÚis_hiraganaÚis_katakanaÚis_latinÚis_punctuationÚis_separatorÚ	is_symbolÚis_thaiÚis_unprintableÚremove_accentÚunicode_rangec                   @   sP   e Zd ZdZdedefdd„Zdeddfdd„Zdd	d
„Ze	de
fdd„ƒZdS )ÚMessDetectorPluginzy
    Base abstract class used for mess detection plugins.
    All detectors MUST extend and implement given methods.
    Ú	characterÚreturnc                 C   ó   t ‚)z@
        Determine if given character should be fed in.
        ©ÚNotImplementedError©Úselfr   © r#   ú^/Users/abhinavbhinder/Documents/map/venv/lib/python3.10/site-packages/charset_normalizer/md.pyÚeligible%   ó   zMessDetectorPlugin.eligibleNc                 C   r   )z‰
        The main routine to be executed upon character.
        Insert the logic in witch the text would be considered chaotic.
        r   r!   r#   r#   r$   Úfeed+   s   zMessDetectorPlugin.feedc                 C   r   )zB
        Permit to reset the plugin to the initial state.
        r   ©r"   r#   r#   r$   Úreset2   r&   zMessDetectorPlugin.resetc                 C   r   )z…
        Compute the chaos ratio based on what your feed() has seen.
        Must NOT be lower than 0.; No restriction gt 0.
        r   r(   r#   r#   r$   Úratio8   s   zMessDetectorPlugin.ratio©r   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚstrÚboolr%   r'   r)   ÚpropertyÚfloatr*   r#   r#   r#   r$   r      s    
r   c                   @   óV   e Zd Zddd„Zdedefdd„Zdeddfdd	„Zdd
d„Ze	de
fdd„ƒZdS )Ú TooManySymbolOrPunctuationPluginr   Nc                 C   s"   d| _ d| _d| _d | _d| _d S )Nr   F)Ú_punctuation_countÚ_symbol_countÚ_character_countÚ_last_printable_charZ_frenzy_symbol_in_wordr(   r#   r#   r$   Ú__init__B   s
   
z)TooManySymbolOrPunctuationPlugin.__init__r   c                 C   ó   |  ¡ S ©N©Úisprintabler!   r#   r#   r$   r%   J   ó   z)TooManySymbolOrPunctuationPlugin.eligiblec                 C   sp   |  j d7  _ || jkr3|tvr3t|ƒr|  jd7  _n| ¡ du r3t|ƒr3t|ƒdu r3|  jd7  _|| _d S )Nr   Fé   )	r8   r9   r   r   r6   Úisdigitr   r   r7   r!   r#   r#   r$   r'   M   s   ÿþ
ÿþ
ý
z%TooManySymbolOrPunctuationPlugin.feedc                 C   s   d| _ d| _d| _d S ©Nr   )r6   r8   r7   r(   r#   r#   r$   r)   _   ó   
z&TooManySymbolOrPunctuationPlugin.resetc                 C   s0   | j dkrdS | j| j | j  }|dkr|S dS )Nr   ç        ç333333Ó?)r8   r6   r7   )r"   Zratio_of_punctuationr#   r#   r$   r*   d   s   

þz&TooManySymbolOrPunctuationPlugin.ratior+   ©r,   r-   r.   r:   r0   r1   r%   r'   r)   r2   r3   r*   r#   r#   r#   r$   r5   A   s    

r5   c                   @   r4   )ÚTooManyAccentuatedPluginr   Nc                 C   ó   d| _ d| _d S rB   ©r8   Ú_accentuated_countr(   r#   r#   r$   r:   q   ó   
z!TooManyAccentuatedPlugin.__init__r   c                 C   r;   r<   )Úisalphar!   r#   r#   r$   r%   u   r?   z!TooManyAccentuatedPlugin.eligiblec                 C   ó,   |  j d7  _ t|ƒr|  jd7  _d S d S ©Nr   )r8   r
   rJ   r!   r#   r#   r$   r'   x   ó   ÿzTooManyAccentuatedPlugin.feedc                 C   rH   rB   rI   r(   r#   r#   r$   r)   ~   rK   zTooManyAccentuatedPlugin.resetc                 C   s*   | j dk rdS | j| j  }|dkr|S dS )Né   rD   gffffffÖ?rI   )r"   Zratio_of_accentuationr#   r#   r$   r*   ‚   s   
zTooManyAccentuatedPlugin.ratior+   rF   r#   r#   r#   r$   rG   p   s    

rG   c                   @   r4   )ÚUnprintablePluginr   Nc                 C   rH   rB   )Ú_unprintable_countr8   r(   r#   r#   r$   r:   Œ   rK   zUnprintablePlugin.__init__r   c                 C   ó   dS ©NTr#   r!   r#   r#   r$   r%   �   ó   zUnprintablePlugin.eligiblec                 C   s(   t |ƒr|  jd7  _|  jd7  _d S rN   )r   rR   r8   r!   r#   r#   r$   r'   “   s   zUnprintablePlugin.feedc                 C   s
   d| _ d S rB   )rR   r(   r#   r#   r$   r)   ˜   s   
zUnprintablePlugin.resetc                 C   ó   | j dkrdS | jd | j  S )Nr   rD   rP   )r8   rR   r(   r#   r#   r$   r*   ›   ó   
zUnprintablePlugin.ratior+   rF   r#   r#   r#   r$   rQ   ‹   s    

rQ   c                   @   r4   )ÚSuspiciousDuplicateAccentPluginr   Nc                 C   ó   d| _ d| _d | _d S rB   ©Ú_successive_countr8   Ú_last_latin_characterr(   r#   r#   r$   r:   ¤   s   
z(SuspiciousDuplicateAccentPlugin.__init__r   c                 C   s   |  ¡ ot|ƒS r<   )rL   r   r!   r#   r#   r$   r%   ª   s   z(SuspiciousDuplicateAccentPlugin.eligiblec                 C   st   |  j d7  _ | jd ur5t|ƒr5t| jƒr5| ¡ r%| j ¡ r%|  jd7  _t|ƒt| jƒkr5|  jd7  _|| _d S rN   )r8   r\   r
   Úisupperr[   r   r!   r#   r#   r$   r'   ­   s   ÿþý
z$SuspiciousDuplicateAccentPlugin.feedc                 C   rY   rB   rZ   r(   r#   r#   r$   r)   »   rC   z%SuspiciousDuplicateAccentPlugin.resetc                 C   rV   )Nr   rD   r@   )r8   r[   r(   r#   r#   r$   r*   À   rW   z%SuspiciousDuplicateAccentPlugin.ratior+   rF   r#   r#   r#   r$   rX   £   s    

rX   c                   @   r4   )ÚSuspiciousRanger   Nc                 C   rY   rB   )Ú"_suspicious_successive_range_countr8   Ú_last_printable_seenr(   r#   r#   r$   r:   É   rC   zSuspiciousRange.__init__r   c                 C   r;   r<   r=   r!   r#   r#   r$   r%   Î   r?   zSuspiciousRange.eligiblec                 C   sx   |  j d7  _ | ¡ st|ƒs|tv rd | _d S | jd u r"|| _d S t| jƒ}t|ƒ}t||ƒr7|  jd7  _|| _d S rN   )r8   Úisspacer   r   r`   r   Ú is_suspiciously_successive_ranger_   )r"   r   Úunicode_range_aÚunicode_range_br#   r#   r$   r'   Ñ   s"   ÿþý



zSuspiciousRange.feedc                 C   rY   rB   )r8   r_   r`   r(   r#   r#   r$   r)   è   rC   zSuspiciousRange.resetc                 C   s"   | j dkrdS | jd | j  }|S )Né   rD   r@   )r8   r_   )r"   Zratio_of_suspicious_range_usager#   r#   r$   r*   í   s   
þzSuspiciousRange.ratior+   rF   r#   r#   r#   r$   r^   È   s    

r^   c                   @   r4   )ÚSuperWeirdWordPluginr   Nc                 C   s:   d| _ d| _d| _d| _d| _d| _d| _d| _d| _d S )Nr   FÚ )	Ú_word_countÚ_bad_word_countÚ_foreign_long_countÚ_is_current_word_badÚ_foreign_long_watchr8   Ú_bad_character_countÚ_bufferÚ_buffer_accent_countr(   r#   r#   r$   r:   ú   s   
zSuperWeirdWordPlugin.__init__r   c                 C   rS   rT   r#   r!   r#   r#   r$   r%     rU   zSuperWeirdWordPlugin.eligiblec                 C   s*  |  ¡ rH|  j|7  _t|ƒr|  jd7  _| jdu rFt|ƒdu s%t|ƒrFt|ƒdu rFt|ƒdu rFt|ƒdu rFt	|ƒdu rFt
|ƒdu rFd| _d S | jsMd S | ¡ sYt|ƒsYt|ƒrò| jrò|  jd7  _t| jƒ}|  j|7  _|dkr¡| j| dkr}d| _t| jd ƒr¡| jd  ¡ r¡tdd„ | jD ƒƒdu r¡|  jd7  _d| _|d	krÐ| jrÐd
d„ t| jtd|ƒƒD ƒ}d}|rÄt|ƒ| dkrÄd}|sÐ|  jd7  _d| _| jrç|  jd7  _|  jt| jƒ7  _d| _d| _d| _d| _d S |dv�r| ¡ du �rt|ƒ�rd| _|  j|7  _d S d S d S d S )Nr   FTé   gÃõ(\�ÂÕ?éÿÿÿÿc                 s   s   � | ]}|  ¡ V  qd S r<   ©r]   )Ú.0Ú_r#   r#   r$   Ú	<genexpr>-  s   € z,SuperWeirdWordPlugin.feed.<locals>.<genexpr>re   c                 S   s   g | ]
\}}|  ¡ r|‘qS r#   rr   )rs   ÚcÚir#   r#   r$   Ú
<listcomp>2  s    ýÿz-SuperWeirdWordPlugin.feed.<locals>.<listcomp>r   rE   rg   >   ú>ú|ú~ú-ú=rt   ú<)rL   rn   r
   ro   rl   r   r   r   r   r   r   ra   r   r   rh   Úlenr8   rk   r]   Úallrj   ÚzipÚrangeri   rm   rA   r   )r"   r   Zbuffer_lengthZcamel_case_dstZprobable_camel_casedr#   r#   r$   r'     s�   ÿ
þþ
ý
ü
û
ú
ù	ÿÿÿþ
ÿþýþ
ÿ
þýúzSuperWeirdWordPlugin.feedc                 C   s4   d| _ d| _d| _d| _d| _d| _d| _d| _d S )Nrg   Fr   )rn   rk   rl   ri   rh   r8   rm   rj   r(   r#   r#   r$   r)   P  s   
zSuperWeirdWordPlugin.resetc                 C   s$   | j dkr| jdkrdS | j| j S )Né
   r   rD   )rh   rj   rm   r8   r(   r#   r#   r$   r*   Z  s   zSuperWeirdWordPlugin.ratior+   rF   r#   r#   r#   r$   rf   ù   s    

E
rf   c                   @   sZ   e Zd ZdZddd„Zdedefdd„Zdeddfd	d
„Zddd„Z	e
defdd„ƒZdS )ÚCjkInvalidStopPluginu²   
    GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and
    can be easily detected. Searching for the overuse of 'ä¸…' and 'ä¸„'.
    r   Nc                 C   rH   rB   ©Ú_wrong_stop_countÚ_cjk_character_countr(   r#   r#   r$   r:   h  rK   zCjkInvalidStopPlugin.__init__r   c                 C   rS   rT   r#   r!   r#   r#   r$   r%   l  rU   zCjkInvalidStopPlugin.eligiblec                 C   s8   |dv r|  j d7  _ d S t|ƒr|  jd7  _d S d S )N>   u   ä¸…u   ä¸„r   )r†   r   r‡   r!   r#   r#   r$   r'   o  s   ÿzCjkInvalidStopPlugin.feedc                 C   rH   rB   r…   r(   r#   r#   r$   r)   v  rK   zCjkInvalidStopPlugin.resetc                 C   s   | j dk rdS | j| j  S )Né   rD   )r‡   r†   r(   r#   r#   r$   r*   z  s   
zCjkInvalidStopPlugin.ratior+   )r,   r-   r.   r/   r:   r0   r1   r%   r'   r)   r2   r3   r*   r#   r#   r#   r$   r„   b  s    

r„   c                   @   r4   )ÚArchaicUpperLowerPluginr   Nc                 C   s.   d| _ d| _d| _d| _d| _d | _d| _d S )NFr   T)Ú_bufÚ_character_count_since_last_sepÚ_successive_upper_lower_countÚ#_successive_upper_lower_count_finalr8   Ú_last_alpha_seenÚ_current_ascii_onlyr(   r#   r#   r$   r:   ‚  s   
z ArchaicUpperLowerPlugin.__init__r   c                 C   rS   rT   r#   r!   r#   r#   r$   r%   �  rU   z ArchaicUpperLowerPlugin.eligiblec                 C   s$  |  ¡ ot|ƒ}|du }|rC| jdkrC| jdkr+| ¡ du r+| jdu r+|  j| j7  _d| _d| _d | _d| _|  j	d7  _	d| _d S | jdu rQ| 
¡ du rQd| _| jd ur| ¡ r_| j ¡ sh| ¡ r|| j ¡ r|| jdu rx|  jd7  _d| _nd| _nd| _|  j	d7  _	|  jd7  _|| _d S )NFr   é@   r   Tr@   )rL   r   r‹   rA   r�   r�   rŒ   rŽ   rŠ   r8   Úisasciir]   Úislower)r"   r   Zis_concernedZ	chunk_sepr#   r#   r$   r'   ’  sF   ÿ
þýÿ
ÿÿ

zArchaicUpperLowerPlugin.feedc                 C   s.   d| _ d| _d| _d| _d | _d| _d| _d S )Nr   FT)r8   r‹   rŒ   r�   rŽ   rŠ   r�   r(   r#   r#   r$   r)   ¼  s   
zArchaicUpperLowerPlugin.resetc                 C   s   | j dkrdS | j| j  S )Nr   rD   )r8   r�   r(   r#   r#   r$   r*   Å  s   
zArchaicUpperLowerPlugin.ratior+   rF   r#   r#   r#   r$   r‰   �  s    

*	r‰   c                   @   sV   e Zd Zddd„Zddd„Zdedefdd	„Zdeddfd
d„Ze	de
fdd„ƒZdS )ÚArabicIsolatedFormPluginr   Nc                 C   rH   rB   ©r8   Ú_isolated_form_countr(   r#   r#   r$   r:   Î  rK   z!ArabicIsolatedFormPlugin.__init__c                 C   rH   rB   r”   r(   r#   r#   r$   r)   Ò  rK   zArabicIsolatedFormPlugin.resetr   c                 C   s   t |ƒS r<   )r   r!   r#   r#   r$   r%   Ö  r?   z!ArabicIsolatedFormPlugin.eligiblec                 C   rM   rN   )r8   r   r•   r!   r#   r#   r$   r'   Ù  rO   zArabicIsolatedFormPlugin.feedc                 C   s   | j dk rdS | j| j  }|S )NrP   rD   r”   )r"   Zisolated_form_usager#   r#   r$   r*   ß  s   
zArabicIsolatedFormPlugin.ratior+   )r,   r-   r.   r:   r)   r0   r1   r%   r'   r2   r3   r*   r#   r#   r#   r$   r“   Í  s    

r“   é   )Úmaxsizerc   rd   r   c                 C   sv  | du s|du r
dS | |krdS d| v rd|v rdS d| v s"d|v r$dS d| v s,d|v r6d| v s4d|v r6dS |   d¡|  d¡}}|D ]}|tv rJqC||v rQ dS qC| dv |dv }}|s_|rid	| v sgd	|v ridS |ro|rodS d
| v swd
|v r‹d	| v sd	|v r�dS | dks‰|dkr‹dS d	| v s›d	|v s›| dv r¹|dv r¹d| v s£d|v r¥dS d| v s­d|v r¯dS | dks·|dkr¹dS dS )za
    Determine if two Unicode range seen next to each other can be considered as suspicious.
    NTFZLatinZ	EmoticonsZ	Combiningú )ÚHiraganaÚKatakanaÚCJKZHangulzBasic Latin)rš   r™   ZPunctuationZForms)Úsplitr	   )rc   rd   Zkeywords_range_aZkeywords_range_bÚelZrange_a_jp_charsZrange_b_jp_charsr#   r#   r$   rb   é  sn   ÿÿÿþÿÿúÿÿÿþrb   i   çš™™™™™É?FÚdecoded_sequenceÚmaximum_thresholdÚdebugc              	   C   sR  dd„ t  ¡ D ƒ}t| ƒd }d}|dk rd}n	|dkrd}nd	}t| d
 t|ƒƒD ]2\}}|D ]}	|	 |¡r<|	 |¡ q0|dkrG|| dksM||d kr\tdd„ |D ƒƒ}||kr\ nq*|r¤tdƒ}
|
 	t
d|› d|› d|› �¡ t| ƒdkr’|
 	t
d| dd… › �¡ |
 	t
d| dd… › �¡ |D ]}|
 	t
|j› d|j› �¡ q”t|dƒS )zw
    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
    c                 S   s   g | ]}|ƒ ‘qS r#   r#   )rs   Zmd_classr#   r#   r$   rx   :  s    ÿzmess_ratio.<locals>.<listcomp>r   rD   i   é    r–   r�   é€   Ú
r   c                 s   s   � | ]}|j V  qd S r<   )r*   )rs   Údtr#   r#   r$   ru   Q  s   € zmess_ratio.<locals>.<genexpr>Zcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=rˆ   zStarting with: NzEnding with: iðÿÿÿz: é   )r   Ú__subclasses__r   r�   r‚   r%   r'   Úsumr   Úlogr   Ú	__class__r*   Úround)rŸ   r    r¡   Z	detectorsÚlengthZmean_mess_ratioZ!intermediary_mean_mess_ratio_calcr   ÚindexÚdetectorÚloggerr¥   r#   r#   r$   Ú
mess_ratio2  sV   ÿ

€ÿ
ÿ
þ€ÿÿþþ
r°   N)rž   F),Ú	functoolsr   Úloggingr   Útypingr   r   Zconstantr   r   r	   Úutilsr
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r5   rG   rQ   rX   r^   rf   r„   r‰   r“   r0   r1   rb   r3   r°   r#   r#   r#   r$   Ú<module>   sD    L"/%1iLÿÿþHÿÿÿÿþ