
    |jV                    n   U d dl mZ d dlZd dlmZ d dlmZ d dlmZ ddl	m
Z
mZmZmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZmZ ddlmZmZmZmZmZmZ  ej@                  d      Z! ejD                         Z#e#jI                   ejJ                  d              e&ed       Z'de(d<   	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ)	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ*	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ+	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ,y)    )annotationsN)	lru_cache)PathLike)BinaryIO   )coherence_ratioencoding_languagesmb_encoding_languagesmerge_coherence_ratios)IANA_SUPPORTEDIANA_SUPPORTED_SIMILARTOO_BIG_SEQUENCETOO_SMALL_SEQUENCETRACE)
mess_ratio)CharsetMatchCharsetMatches)any_specified_encodingcut_sequence_chunks	iana_nameidentify_sig_or_bomis_multi_byte_encodingshould_strip_sig_or_bomcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)sc                    t        |        S N)r   encodings    N/opt/daily-market/.venv/lib/python3.12/site-packages/charset_normalizer/api.py<lambda>r    .   s    -CH-M)M     )keyz	list[str]IANA_SUPPORTED_MB_FIRSTc
                R   t        | t        t        f      s#t        dj	                  t        |                   |rBt        j                  }
t        j                  t               t        j                  t               t        |       }|dk(  r_t        j                  d       |r.t        j                  t               t        j                  
       t        t!        | dddg d      g      S |Dt        j#                  t        d	d
j%                  |             |D cg c]  }t'        |d       }}ng }|Dt        j#                  t        dd
j%                  |             |D cg c]  }t'        |d       }}ng }|||z  k  r!t        j#                  t        d|||       d}|}|dkD  r||z  |k  rt)        ||z        }t        |       t*        k  }t        |       t,        k\  }|rt        j#                  t        d|       n|rt        j#                  t        d|       g }|rt/        |       nd}|,|j1                  |       t        j#                  t        d|       t3               }g }g }t3               } t5        d      t6              } t5        d      t8              }d}t3               }d}d}d}d}d}d}t               }t               } t;        |       \  }!}"|!7|j=                  d|!       t        j#                  t        dt        |"      |!       |j1                  d       d|vr|j1                  d       |t>        z   D ]  }#|r|#|vr|r|#|v r|#|v r|jA                  |#       d}$|!|#k(  }%|%xr tC        |#      }&|#dv r|%st        j#                  t        d|#       `|#dv r|%st        j#                  t        d|#       |#|v rt        j#                  t        d|#       	 tE        |#      }'|r[|'st3        tK        |#            }(nt3        tM        |#            }(|(jO                  |      st        j#                  t        d|#|(|       |r&|'s$||k\  rt        j#                  t        d|#||       3|r|'st        j#                  t        d|#       T|' xr | })	 |r5|'s3tQ        |&s| dt)        d       n| t        |"      t)        d       |#       nC|)sA|#d k(  r|%rtQ        | |#      }$|$r+|$d   d!k(  r#|$dd }$ntQ        |&s| n| t        |"      d |#      }$tW        |%sdn
t        |"      |t)        ||z              }+|'xr |$duxr t        |$      |k  },|,rt        j#                  t        d#|#       t)        t        |+      d$z        }-tY        |-d%      }-d}.d}/g }0g }1	 t[        | |#|+||%|&|"|'|$|)
      D ]`  }2|0j1                  |2       |1j1                   ||2||xr dt        |      cxk  xr d%k  nc              |1d&   |k\  r|.dz  }.|.|-k\  s|%s]|&r` n |/s$|r"|'s 	 | t)        d)      d j]                  |#d*+       |1rt_        |1      t        |1      z  nd}3|3|k\  s|.|-k\  r|j1                  |#       |#t`        v r|jc                  t`        |#          t        j#                  t        d-|#|.te        |3d.z  d/0             |	rS|#dd|d1d2fv rJ|/sH|$"	 tQ        |&s| n| t        |"      d |#      }$|rd}$t!        | |#||%g |$|4      }4|#|k(  r|4}n
|#dk(  r|4}n|4}|)r	 tQ        |&s| n| t        |"      d |#      }$t        j#                  t        d5|#te        |3d.z  d/0             |'stK        |#      }5ntM        |#      }5|5rt        j#                  t        d6|#|5       g }6|#dk7  r7|5rd7j%                  |5      nd}7|0D ]  }2 ||2||7      }8|6j1                  |8        tg        |6      }9|9rt        j#                  t        d8|9|#       t!        | |#|3|%|9|r|#|ddfv r|$nd|4      }:|j1                  |:       |r|'s
|3d9k  r|dz  }|#|ddfv ry|3d:k  rt|3dk(  r^t        j                  d;|:jh                         |r.t        j                  t               t        j                  
       t        |:g      c S | j1                  |:       t        |       r|||v rzd|v rvd|v rr| jk                         };|;J t        j                  d;|;jh                         |r.t        j                  t               t        j                  
       t        |;g      c S |sW|'sU|9rtY        d< |9D        d=      nd}<|<d>k\  r8d|v r4d|v r0d(}|jc                  |5       t        j#                  t        d?|#|3|<       |s\|'rZ|,rX|$Vt        |$      |d@z  k  rE|#dAvrAd|v r=d|v r9d(}t        j#                  t        dB|#|3t        |$      |t        |$      |z  d.z         |#|!k(  st        j                  dC|#       |r.t        j                  t               t        j                  
       t        ||#   g      c S  t        |      dk(  r|s|s|rt        j#                  t        dD       |r2t        j                  dE|jh                         |j1                  |       nr|r||r|r|jl                  |jl                  k7  s|'t        j                  dF       |j1                  |       n(|r&t        j                  dG       |j1                  |       |r<t        j                  dH|jk                         jh                  t        |      dz
         nt        j                  dI       |r.t        j                  t               t        j                  
       |S c c}w c c}w # tF        tH        f$ r t        j#                  t        d|#       Y w xY w# tR        tT        f$ rQ}*t        |*tT              s%t        j#                  t        d"|#tQ        |*             |j1                  |#       Y d}*~*	d}*~*ww xY w# tR        tT        f$ rr}*|)r<t        j#                  t        d"|#tQ        |*             |j1                  |#       Y d}*~*	\t        j#                  t        d'|#tQ        |*             |-}.d(}/Y d}*~*d}*~*ww xY w# tR        $ rA}*t        j#                  t        d,|#tQ        |*             |j1                  |#       Y d}*~*	d}*~*ww xY w# tR        tT        f$ r t        j#                  t        d3|#       Y 
w xY w# tR        tT        f$ rA}*t        j#                  t        d"|#tQ        |*             |j1                  |#       Y d}*~*
\d}*~*ww xY w)Jaf  
    Given a raw bytes sequence, return the best possibles charset usable to render str objects.
    If there is no results, it is a strong indicator that the source is binary/not text.
    By default, the process will extract 5 blocks of 512o each to assess the mess and coherence of a given sequence.
    And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will.

    The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page
    but never take it for granted. Can improve the performance.

    You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that
    purpose.

    This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32.
    By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain'
    toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging.
    Custom logging format and handler can be set manually.
    z3Expected object of type bytes or bytearray, got: {}r   z<Encoding detection on empty bytes, assuming utf_8 intention.utf_8g        F Nz`cp_isolation is set. use this flag for debugging purpose. limited list of encoding allowed : %s.z, zacp_exclusion is set. use this flag for debugging purpose. limited list of encoding excluded : %s.z^override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.r   z>Trying to detect encoding from a tiny portion of (%i) byte(s).zIUsing lazy str decoding because the payload is quite large, (%i) byte(s).z@Detected declarative mark in sequence. Priority +1 given for %s.)maxsize   zIDetected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.ascii>   utf_16utf_32z\Encoding %s won't be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.>   utf_7zREncoding %s won't be tested as-is because detection is unreliable without BOM/SIG.zY%s is deemed too similar to a code page that was already considered unsuited. Continuing!z2Encoding %s does not provide an IncrementalDecoderzbSkipping %s: definitive match already found, this encoding targets different languages (%s vs %s).zXSkipping %s: already accumulated %d same-family results after definitive match (cap=%d).zCSkipping single-byte %s: multi-byte definitive match already found.g    Ar   r,   u   ﻿z9Code page %s does not fit given bytes sequence at ALL. %szpCode page %s is a multi byte encoding table and it appear that at least one character was encoded using n-bytes.      zaLazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %sTg     j@strict)errorsz^LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %szc%s was excluded because of initial chaos probing. Gave up %i time(s). Computed mean chaos is %f %%.d      )ndigitsr*   r+   z>%s does not decode the whole payload: fallback entry withheld.)preemptive_declarationz=%s passed initial chaos probing. Mean measured chaos is %f %%z&%s should target any language(s) of %s,z We detected language %s using %sg{Gz?皙?z.Encoding detection: %s is most likely the one.c              3  &   K   | ]	  \  }}|  y wr    ).0_vs      r   	<genexpr>zfrom_bytes.<locals>.<genexpr>  s     441aQ4s   )defaultg      ?zyDefinitive match found: %s (chaos=%.3f, coherence=%.2f). Encodings targeting different language families will be skipped.g\(\?>	   r,   r%   r*   r+   	utf_16_be	utf_16_le	utf_32_be	utf_32_le	utf_8_sigzjMulti-byte definitive match: %s (chaos=%.3f, decoded=%d/%d=%.1f%%). Single-byte encodings will be skipped.zoEncoding detection: %s is most likely the one as we detected a BOM or SIG within the beginning of the sequence.zONothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.z7Encoding detection: %s will be used as a fallback matchz:Encoding detection: utf_8 will be used as a fallback matchz:Encoding detection: ascii will be used as a fallback matchz]Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.z=Encoding detection: Unable to determine any suitable charset.)7
isinstance	bytearraybytes	TypeErrorformattypeloggerlevel
addHandlerexplain_handlersetLevelr   lendebugremoveHandlerr   r   logjoinr   intr   r   r   appendsetr   r   r   r   insertr#   addr   r   ModuleNotFoundErrorImportErrorr	   r
   intersectionstrUnicodeDecodeErrorLookupErrorrangemaxr   decodesumr   updateroundr   r   bestfingerprint)=	sequencessteps
chunk_size	thresholdcp_isolationcp_exclusionpreemptive_behaviourexplainlanguage_thresholdenable_fallbackprevious_logger_levellengthcpis_too_small_sequenceis_too_large_sequenceprioritized_encodingsspecified_encodingtestedtested_but_hard_failuretested_but_soft_failuresoft_failure_skipcached_mess_ratiocached_coherence_ratiodefinitive_match_founddefinitive_target_languages post_definitive_sb_success_countPOST_DEFINITIVE_SB_CAPmb_definitive_match_foundfallback_asciifallback_u8fallback_specifiedresultsearly_stop_resultssig_encodingsig_payloadencoding_ianadecoded_payloadbom_or_sig_availablestrip_sig_or_bomis_multi_byte_decoderenc_languagesdeferred_decodinger_multi_byte_bonusmax_chunk_gave_upearly_stop_countlazy_str_hard_failure	md_chunks	md_ratioschunkmean_mess_ratiofallback_entrytarget_languages	cd_ratioslg_inclusionchunk_languagescd_ratios_mergedcurrent_matchprobable_resultbest_coherences=                                                                r   
from_bytesr   2   s   < i)U!34AHHY
 	
 %+\\/*i.F{ST  1OO12|IwUBPRSTUU

5IIl#		
 8DD	"e,DD

6IIl#		
 8DD	"e,DD*u$%

l	
 
qyVe^j0%(
"%i.3E"E"%i.4D"D

L	

 


W	
 (* .By)t  %$$%78

N	
 uF)+)+"%% 0	$/
;4Yt4_E $),/E -.$"# ',*.N'+K.2,.G)7)9 3I >L+$$Q5

W		
   )++$$W-.1HH S<M=M\9F"

=!&*%1]%B!5 "
:Q;
 009MJJn
 I%.BJJd
  --JJk
 	*@*O! "( #$6}$E F #$9-$H I --.IJ

x!!/  #)04JJJJj0& 
 %-BJJU
  &%C.C*C 	*	$-B  0 "+CI.&s;'7#d)D* ' !G+0D&)!!.'O '?1+=+I*9!"*=&) $4 &!*3{+;+=!>!.'O& )As;/?
 " .t+.O$v- 	 JJ-	 "%SWq[!1 115 ! %!		5	),$ %!    '  %!?A\):$?a$? R=I-$)$$(99(1A9r &%)
#d)+&--mH-M ENY#i.!@SVi'+;?P+P#**=9 66!(()?)NOJJ0 o+Q7  !W&8(HMN- #*!*- (8 !*%.s;/?/A%B%2+ -*.!-!(#+=" !$66)7&"g-%3N"0K"%  0 "&s;'7'9:*#$ 	

K/C'3		
 %*<]*K4]CJJ8 	 	 G# /?)*D  # 2"8& #   12 2)<JJ2 	 %  .$);Wg(NN  
 #5
" 	}% #)$&,1, 0'7CC#% #%D!** ((9OO$9:%}o66%%m4 "##+/AV/K6!6!0557O"...LL@(( $$_5 56!?"344 &.C $ 4#34cB 
 $F):w&?P)-&+223CD

 P!#" *% +O$v}4
 6!6!(,%JJ|O$O$v-3 L(LL1
 $$_5 56!7=#9":;;gS<j 7|q.,>JJa
 LLI"++ NN-.^3"++~/I/II'LLUVNN;'LLUVNN>*kLLN##L1	
 	TU_--.Nq E Ex $[1 	JJD
 	H #K0 		a-

O!F	 $**=9		N 
 	) ! 

O!F	 (..}=JJsA	  1$(!/	)D & 

t!F	 (..}=X /< !

!\)
 !!P '4 

O!F	 (..}=s   n6%n;(o "A:o1#A0qqq!st%u *o.-o.1q Aqqs#8s!)ss	t"!6tt"%*uuv&%6v!!v&c
                F    t        | j                         |||||||||	
      S )z
    Same thing than the function from_bytes but using a file pointer that is already ready.
    Will not close the file pointer.
    )r   read)
fprh   ri   rj   rk   rl   rm   rn   ro   rp   s
             r   from_fpr   '  s5      
	 r!   c
                n    t        | d      5 }
t        |
|||||||||	
      cddd       S # 1 sw Y   yxY w)z
    Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode.
    Can raise IOError.
    rbN)openr   )pathrh   ri   rj   rk   rl   rm   rn   ro   rp   r   s              r   	from_pathr   E  sK      
dD	 
R 

 
 
s   +4c
                    t        | t        t        f      rt        | |||||||||	
      }
|
 S t        | t        t
        f      rt        | |||||||||	
      }
|
 S t        | |||||||||	
      }
|
 S )a)  
    Detect if the given input (file, bytes, or path) points to a binary file. aka. not a string.
    Based on the same main heuristic algorithms and default kwargs at the sole exception that fallbacks match
    are disabled to be stricter around ASCII-compatible but unlikely to be a string.
    )	rh   ri   rj   rk   rl   rm   rn   ro   rp   )rD   r\   r   r   rF   rE   r   r   )fp_or_path_or_payloadrh   ri   rj   rk   rl   rm   rn   ro   rp   guessess              r   	is_binaryr   d  s    " '#x9!!%%!51+
Z ;C 
	

 !!%%!51+
4 ; !!%%!51+
 ;r!   )	      皙?NNTFr7   T)rg   zbytes | bytearrayrh   rT   ri   rT   rj   floatrk   list[str] | Nonerl   r   rm   boolrn   r   ro   r   rp   r   returnr   )r   r   rh   rT   ri   rT   rj   r   rk   r   rl   r   rm   r   rn   r   ro   r   rp   r   r   r   )r   zstr | bytes | PathLikerh   rT   ri   rT   rj   r   rk   r   rl   r   rm   r   rn   r   ro   r   rp   r   r   r   )	r   r   r   NNTFr7   F)r   z!PathLike | str | BinaryIO | bytesrh   rT   ri   rT   rj   r   rk   r   rl   r   rm   r   rn   r   ro   r   rp   r   r   r   )-
__future__r   logging	functoolsr   osr   typingr   cdr   r	   r
   r   constantr   r   r   r   r   mdr   modelsr   r   utilsr   r   r   r   r   r   	getLoggerrJ   StreamHandlerrM   setFormatter	Formattersortedr#   __annotations__r   r   r   r   r9   r!   r   <module>r      s   "        0  
		/	0'''')   GAB &,M&   %)%)!% # r rr r 	r
 #r #r r r r r rn %)%)!% #   	
 # #     @ %)%)!% # 

 

 
 	

 #
 #
 
 
 
 
 
B %)%)!% #!?<?? ? 	?
 #? #? ? ? ? ? 
?r!   