a
    ÷£Be[<  ã                   @   sl   d dl Z d dlZd dlZd dlmZ d dlmZmZmZ G dd„ deƒZ	G dd„ deƒZ
G dd	„ d	eƒZdS )
é    N)ÚBeautifulSoup)ÚEntitySubstitutionÚEncodingDetectorÚUnicodeDammitc                   @   sb   e Zd ZdZdd„ Zej dg d¢¡dd„ ƒZdd	„ Z	d
d„ Z
dd„ Zdd„ Zdd„ Zdd„ ZdS )ÚTestUnicodeDammitz"Standalone tests of UnicodeDammit.c                 C   s   d}t |ƒ}|j|ksJ ‚d S )Nu   I'm already Unicode! â˜ƒ)r   Úunicode_markup)ÚselfÚmarkupÚdammit© r   úX/home/yuhw/wc/wire-cell-python/wcpy/lib/python3.9/site-packages/bs4/tests/test_dammit.pyÚtest_unicode_input   s    z$TestUnicodeDammit.test_unicode_inputz smart_quotes_to,expect_converted))Nu   â€˜â€™â€œâ€�)Úxmlz &#x2018;&#x2019;&#x201C;&#x201D;)Úhtmlz&lsquo;&rsquo;&ldquo;&rdquo;)Úasciiz''""c                 C   s,   d}t |dg|d�j}|d |¡ks(J ‚dS )zbVerify the functionality of the smart_quotes_to argument
        to the UnicodeDammit constructor.s   <foo>‘’“”</foo>úwindows-1252)Úknown_definite_encodingsÚsmart_quotes_toz<foo>{}</foo>N)r   r   Úformat)r   r   Zexpect_convertedr	   Z	convertedr   r   r   Útest_smart_quotes_to   s    þz&TestUnicodeDammit.test_smart_quotes_toc                 C   s0   d}t |ƒ}|j ¡ dksJ ‚|jdks,J ‚d S )Ns   SacrÃ© bleu! â˜ƒúutf-8u   SacrÃ© bleu! â˜ƒ©r   Úoriginal_encodingÚlowerr   )r   Úutf8r
   r   r   r   Útest_detect_utf8&   s    z"TestUnicodeDammit.test_detect_utf8c                 C   s4   d}t |dgƒ}|j ¡ dks"J ‚|jdks0J ‚d S )Nó   íåìùú
iso-8859-8u   ×�×•×œ×©r   ©r   Úhebrewr
   r   r   r   Útest_convert_hebrew,   s    z%TestUnicodeDammit.test_convert_hebrewc                 C   s6   d}t |ƒ}|j ¡ dksJ ‚|j d¡|ks2J ‚d S )Ns   ã‚±ãƒ¼ã‚¿ã‚¤ Watchr   )r   r   r   r   Úencode)r   Úutf_8r
   r   r   r   Ú/test_dont_see_smart_quotes_where_there_are_none2   s    zATestUnicodeDammit.test_dont_see_smart_quotes_where_there_are_nonec                 C   s,   d  d¡}t|dgƒ}|j ¡ dks(J ‚d S )Nõ   RÃ¤ksmÃ¶rgÃ¥sr   r   ©r!   r   r   r   ©r   Ú	utf8_datar
   r   r   r   Ú test_ignore_inappropriate_codecs8   s    
z2TestUnicodeDammit.test_ignore_inappropriate_codecsc                 C   s6   d  d¡}dD ]"}t||gƒ}|j ¡ dksJ ‚qd S )Nr$   r   )z.utf8z...z
utF---16.!r%   )r   r'   Zbad_encodingr
   r   r   r   Útest_ignore_invalid_codecs=   s    
z,TestUnicodeDammit.test_ignore_invalid_codecsc                 C   sL   d  d¡}t|dgd�}|j ¡ dks*J ‚t|ddgd�}|jd ksHJ ‚d S )Nr$   r   )Zexclude_encodingsr   r%   r&   r   r   r   Útest_exclude_encodingsC   s    
ÿz(TestUnicodeDammit.test_exclude_encodingsN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚpytestÚmarkÚparametrizer   r   r    r#   r(   r)   r*   r   r   r   r   r      s   þ

r   c                   @   sT   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
dd„ ZdS )ÚTestEncodingDetectorc                 C   s"   t dƒ}t|jƒ}d|v sJ ‚d S )Ns'   <?xml version="1.0" encoding="UTF-Û" ?>u   utf-ï¿½)r   ÚlistÚ	encodings)r   Zdetectedr4   r   r   r   ÚPtest_encoding_detector_replaces_junk_in_encoding_name_with_replacement_characterS   s
    ÿ
zeTestEncodingDetector.test_encoding_detector_replaces_junk_in_encoding_name_with_replacement_characterc                 C   s(   dD ]}t |dd�}d|jksJ ‚qd S )N)s&   <html><meta charset="euc-jp" /></html>s&   <html><meta charset='euc-jp' /></html>s$   <html><meta charset=euc-jp /></html>s#   <html><meta charset=euc-jp/></html>T©Úis_htmlzeuc-jp)r   r   ©r   Údatar
   r   r   r   Ú test_detect_html5_style_meta_tagY   s    z5TestEncodingDetector.test_detect_html5_style_meta_tagc              	   C   s”   d}t jj}t tj¡ z`dd„ }|t j_t|ƒ}d|jks@J ‚d|jv sNJ ‚t	|dƒ}|jsbJ ‚W t tj
¡ |t j_nt tj
¡ |t j_0 d S )NsT   ï»¿<?xml version="1.0" encoding="UTF-8"?>
<html><b>Ø¨ØªØ±</b>
<i>ÈÒÑ�ÊÑíä</i></html>c                 S   s   d S ©Nr   )Ústrr   r   r   Únoopy   s    zETestEncodingDetector.test_last_ditch_entity_replacement.<locals>.noopTu   ï¿½zhtml.parser)Úbs4r
   Zchardet_dammitÚloggingÚdisableÚWARNINGr   Zcontains_replacement_charactersr   r   ÚNOTSET)r   ÚdocÚchardetr=   r
   Zsoupr   r   r   Ú"test_last_ditch_entity_replacementc   s    

ÿz7TestEncodingDetector.test_last_ditch_entity_replacementc                 C   s,   d}t |ƒ}d|jksJ ‚d|jks(J ‚d S )Nó   ÿþ< a > á é < / a > u   <a>Ã¡Ã©</a>úutf-16le)r   r   r   r8   r   r   r   Útest_byte_order_mark_removed†   s    z1TestEncodingDetector.test_byte_order_mark_removedc                 C   s¢   d}t |ƒ}t |dgd�}d|jks(J ‚t |dgd�}d|jksDJ ‚dgdd„ |jD ƒks^J ‚d	}t |dgd
gd�}d
|jks‚J ‚dd
gdd„ |jD ƒksžJ ‚d S )NrF   zutf-16)r   r   )Úuser_encodingsrG   c                 S   s   g | ]}|d  ‘qS ©r   r   ©Ú.0Úxr   r   r   Ú
<listcomp>Ÿ   ó    zRTestEncodingDetector.test_known_definite_versus_user_encodings.<locals>.<listcomp>r   r   )r   rI   c                 S   s   g | ]}|d  ‘qS rJ   r   rK   r   r   r   rN   ª   rO   ©r   r   Útried_encodings)r   r9   r
   ÚbeforeÚafterr   r   r   r   Ú)test_known_definite_versus_user_encodings�   s    ÿz>TestEncodingDetector.test_known_definite_versus_user_encodingsc                 C   sH   d}t |dgdgdgd�}d|jks(J ‚g d¢dd„ |jD ƒksDJ ‚d S )	Nr   ú	shift-jisr   r   )r   Zoverride_encodingsrI   )rU   r   r   c                 S   s   g | ]}|d  ‘qS rJ   r   rK   r   r   r   rN   »   rO   zKTestEncodingDetector.test_deprecated_override_encodings.<locals>.<listcomp>rP   r   r   r   r   Ú"test_deprecated_override_encodings¬   s    üÿz7TestEncodingDetector.test_deprecated_override_encodingsc                 C   st   d  d¡}d  d¡}|| | }t t¡� | d¡ W d   ƒ n1 sJ0    Y  t |¡}d| d¡kspJ ‚d S )Nu	   â˜ƒâ˜ƒâ˜ƒr   u   â€œHi, I like Windows!â€�Úwindows_1252u+   â˜ƒâ˜ƒâ˜ƒâ€œHi, I like Windows!â€�â˜ƒâ˜ƒâ˜ƒ)r!   r/   ZraisesÚUnicodeDecodeErrorÚdecoder   Ú	detwingle)r   r   rW   rC   Úfixedr   r   r   Útest_detwingle¾   s    
ÿþ(
z#TestEncodingDetector.test_detwinglec                 C   s<   dD ]2}|  d¡}| d¡s J ‚t |¡}||ksJ ‚qd S )N)u   Å“u   â‚“u   Ã°Â�Â�Â“r   ó   “)r!   Úendswithr   rZ   )r   Ztricky_unicode_charÚinputÚoutputr   r   r   Ú+test_detwingle_ignores_multibyte_charactersÕ   s
    

z@TestEncodingDetector.test_detwingle_ignores_multibyte_charactersc                 C   s  d}|  d¡}d}|  d¡}tj}||dd�d u s6J ‚d||dd�ksJJ ‚d||dd�ks^J ‚d||ƒksnJ ‚d||ƒks~J ‚d	d
 }||| ƒd u sšJ ‚||| ƒd u s®J ‚||| ddd�dksÈJ ‚||dd�dksÜJ ‚|d	| dd�dksôJ ‚|d| dd�d u �sJ ‚d S )Nz0<html><head><meta charset="utf-8"></head></html>r   z,<?xml version="1.0" encoding="ISO-8859-1" ?>Fr6   r   Tz
iso-8859-1ó    iˆ  )r7   Úsearch_entire_document)rc   ó   a)r!   r   Zfind_declared_encoding)r   Zhtml_unicodeZ
html_bytesZxml_unicodeZ	xml_bytesÚmZspacerr   r   r   Útest_find_declared_encodingä   s(    

ÿÿz0TestEncodingDetector.test_find_declared_encodingN)r+   r,   r-   r5   r:   rE   rH   rT   rV   r\   ra   rf   r   r   r   r   r2   Q   s   
#r2   c                   @   s’   e Zd ZdZdd„ Zej dddg¡dd„ ƒZd	d
„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd „ Zd!S )"ÚTestEntitySubstitutionz1Standalone tests of the EntitySubstitution class.c                 C   s
   t | _d S r;   )r   Úsub©r   r   r   r   Úsetup_method  s    z#TestEntitySubstitution.setup_methodzoriginal,substituted)u   fooâˆ€â˜ƒÃµbaru   foo&forall;â˜ƒ&otilde;bar)u   â€˜â€™fooâ€œâ€�z&lsquo;&rsquo;foo&ldquo;&rdquo;c                 C   s   | j  |¡|ksJ ‚d S r;   ©rh   Zsubstitute_html)r   ÚoriginalZsubstitutedr   r   r   Útest_substitute_html  s    z+TestEntitySubstitution.test_substitute_htmlc                 C   s:   dD ]0\}}d}|| }|| }| j  |¡|ksJ ‚qd S )N)
)z&models;u   âŠ§)z&Nfr;u   ð�”‘)z&ngeqq;u   â‰§Ì¸)z&not;õ   Â¬)z&Not;u   â«¬z||)Úfjro   )z&gt;ú>)z&lt;ú<)z&amp;ú&z3 %s 4rk   )r   ÚentityÚuÚtemplateÚrawZwith_entitiesr   r   r   Útest_html5_entity%  s
    z(TestEntitySubstitution.test_html5_entityc                 C   s<   d}d}| j  |¡|ksJ ‚d}d}| j  |¡|ks8J ‚d S )Nu   fjords âŠ” penguinszfjords &sqcup; penguinsu   fjords âŠ”ï¸€ penguinszfjords &sqcups; penguinsrk   )r   r9   r	   r   r   r   Ú)test_html5_entity_with_variation_selectorC  s    z@TestEntitySubstitution.test_html5_entity_with_variation_selectorc                 C   s   d}| j  |d¡|ksJ ‚d S )NúWelcome to "my bar"F©rh   Zsubstitute_xml©r   Úsr   r   r   ÚItest_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_falseO  s    z`TestEntitySubstitution.test_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_falsec                 C   s0   | j  dd¡dksJ ‚| j  dd¡dks,J ‚d S )NZWelcomeTz	"Welcome"z	Bob's Barz"Bob's Bar"rz   ri   r   r   r   Ú6test_xml_attribute_quoting_normally_uses_double_quotesS  s    zMTestEntitySubstitution.test_xml_attribute_quoting_normally_uses_double_quotesc                 C   s   d}| j  |d¡dksJ ‚d S )Nry   Tz'Welcome to "my bar"'rz   r{   r   r   r   ÚOtest_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotesW  s    zfTestEntitySubstitution.test_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotesc                 C   s   d}| j  |d¡dksJ ‚d S )NúWelcome to "Bob's Bar"Tz""Welcome to &quot;Bob's Bar&quot;"rz   r{   r   r   r   Úbtest_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotes[  s    zyTestEntitySubstitution.test_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotesc                 C   s   d}| j  |¡|ksJ ‚d S )Nr€   rz   )r   Úquotedr   r   r   Ú<test_xml_quotes_arent_escaped_when_value_is_not_being_quoted_  s    zSTestEntitySubstitution.test_xml_quotes_arent_escaped_when_value_is_not_being_quotedc                 C   s   | j  d¡dksJ ‚d S )Nzfoo<bar>zfoo&lt;bar&gt;rz   ri   r   r   r   Ú'test_xml_quoting_handles_angle_bracketsc  s    z>TestEntitySubstitution.test_xml_quoting_handles_angle_bracketsc                 C   s   | j  d¡dksJ ‚d S )NzAT&TzAT&amp;Trz   ri   r   r   r   Ú#test_xml_quoting_handles_ampersandsf  s    z:TestEntitySubstitution.test_xml_quoting_handles_ampersandsc                 C   s   | j  d¡dksJ ‚d S )Nú&Aacute;T&Tz&amp;Aacute;T&amp;Trz   ri   r   r   r   ÚEtest_xml_quoting_including_ampersands_when_they_are_part_of_an_entityi  s    z\TestEntitySubstitution.test_xml_quoting_including_ampersands_when_they_are_part_of_an_entityc                 C   s   | j  d¡dksJ ‚d S )Nr†   z&Aacute;T&amp;T)rh   Z"substitute_xml_containing_entitiesri   r   r   r   ÚDtest_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entityl  s    z[TestEntitySubstitution.test_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entityc                 C   s   d}| j  |¡|ksJ ‚dS )z:There's no need to do this except inside attribute values.zBob's "bar"Nrk   )r   Útextr   r   r   Ú test_quotes_not_html_substitutedo  s    z7TestEntitySubstitution.test_quotes_not_html_substitutedN)r+   r,   r-   r.   rj   r/   r0   r1   rm   rw   rx   r}   r~   r   r�   rƒ   r„   r…   r‡   rˆ   rŠ   r   r   r   r   rg     s*   øþ
rg   )r/   r?   r>   r   Z
bs4.dammitr   r   r   Úobjectr   r2   rg   r   r   r   r   Ú<module>   s   E ?