3
tûœ]§:  ã               @   sÔ   d Z ddlZddlmZ ddlmZ ddlZddlmZm	Z	 ddl
mZ ddlmZ ddlmZ G d	d
„ d
ƒZdd„ ZG dd„ dƒZddiZdd„ ZG dd„ deeƒZdd„ Zddd„Zd dd„ZedkrÐeƒ  dS )!zGa similarities / code duplication command line tool and pylint checker
é    N)Údefaultdict)Úgroupby)ÚBaseCheckerÚtable_lines_from_stats)ÚIRawChecker)ÚTable)Údecoding_streamc               @   sL   e Zd ZdZddd„Zddd„Zd	d
„ Zdd„ Zdd„ Zdd„ Z	dd„ Z
dS )ÚSimilarz,finds copy-pasted lines of code in a projecté   Fc             C   s"   || _ || _|| _|| _g | _d S )N)Ú	min_linesÚignore_commentsÚignore_docstringsÚignore_importsÚlinesets)Úselfr   r   r   r   © r   ú9/tmp/pip-build-fn42xldr/pylint/pylint/checkers/similar.pyÚ__init__#   s
    zSimilar.__init__Nc             C   sZ   |dkr|j }nt||ƒj }y$| jjt||ƒ | j| j| jƒƒ W n tk
rT   Y nX dS )z(append a file to search for similaritiesN)	Ú	readlinesr   r   ÚappendÚLineSetr   r   r   ÚUnicodeDecodeError)r   ZstreamidÚstreamÚencodingr   r   r   r   Úappend_stream0   s    zSimilar.append_streamc             C   s   | j | jƒ ƒ dS )z<start looking for similarities and display results on stdoutN)Ú_display_simsÚ_compute_sims)r   r   r   r   ÚrunC   s    zSimilar.runc             C   sÐ   t tƒ}x|| jƒ D ]p\}}}}}|| }xX|D ]:}||f|ksJ||f|kr.|j||fƒ |j||fƒ P q.W |j||f||fhƒ qW g }	x0|jƒ D ]$\}}
x|
D ]}|	j||fƒ q¢W q”W |	jƒ  |	jƒ  |	S )z&compute similarities in appended files)r   ÚlistÚ
_iter_simsÚaddr   ÚitemsÚsortÚreverse)r   Zno_duplicatesÚnumÚlineset1Zidx1Úlineset2Zidx2Z	duplicateÚcouplesÚsimsZ	ensemblesr   r   r   r   G   s     

zSimilar._compute_simsc       	      C   sÖ   d}xž|D ]–\}}t ƒ  t |dt|ƒdƒ t|ƒ}d }}x"|D ]\}}t d|j|f ƒ q@W |rŒx(|j||| … D ]}t d|jƒ ƒ qvW ||t|ƒd  7 }q
W tdd	„ | jD ƒƒ}t d
|||d | f ƒ dS )z'display computed similarities on stdoutr   zsimilar lines inÚfilesNz==%s:%sz  é   c             S   s   g | ]}t |ƒ‘qS r   )Úlen)Ú.0Úlinesetr   r   r   ú
<listcomp>i   s    z)Similar._display_sims.<locals>.<listcomp>z)TOTAL lines=%s duplicates=%s percent=%.2fg      Y@)Úprintr+   ÚsortedÚnameÚ_real_linesÚrstripÚsumr   )	r   r(   Znb_lignes_dupliqueesr$   r'   r-   ÚidxÚlineZnb_total_lignesr   r   r   r   [   s$    zSimilar._display_simsc             c   sî   |j }|j }|j}d}| j}xÌ|t|ƒk rèd}d}	xª||| ƒD ]š}
d}x�tt||ƒ||
ƒƒƒD ]L\}	\\}}\}}||kr ||kr”|	||||
fV  t||	ƒ}P |r`|d7 }q`W |	d7 }	||krÐ|	||||
fV  t||	ƒ}q@W ||7 }qW dS )z+find similarities in the two given linesetsr   r*   N)Úenumerate_strippedÚfindr   r+   Ú	enumerateÚzipÚmax)r   r%   r&   Zlines1Zlines2r8   Zindex1r   Úskipr$   Zindex2Z	non_blankÚ_Zline1Zline2r   r   r   Ú_find_commons   s0    (
zSimilar._find_commonc             c   s\   xVt | jdd… ƒD ]@\}}x6| j|d d… D ] }x| j||ƒD ]
}|V  qBW q0W qW dS )zWiterate on similarities among all files, by making a cartesian
        product
        Nr*   éÿÿÿÿ)r9   r   r>   )r   r5   r-   r&   Úsimr   r   r   r   ‘   s    zSimilar._iter_sims)r
   FFF)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   r>   r   r   r   r   r   r	       s      

r	   c                sø   |rBt jdj| ƒƒ}dd„ |jD ƒ}dd„ t|dd„ d�D ƒ}d	}g }d
}	x¨t| dd�D ]˜\}
‰ ˆ jƒ ‰ |r¶|	 r t‡ fdd„dD ƒƒr ˆ d
d… }	ˆ dd
… ‰ |	r¶ˆ j|	ƒr²d
}	d‰ |rÎ|j	|
|ƒ}|rÎd‰ |ræˆ j
ddƒd jƒ ‰ |jˆ ƒ qXW |S )z\return lines with leading/trailing whitespace and any ignored code
    features removed
    Ú c             s   s&   | ]}|j t|tjtjfƒfV  qd S )N)ÚlinenoÚ
isinstanceÚastroidÚImportÚ
ImportFrom)r,   Únoder   r   r   ú	<genexpr>¢   s   z!stripped_lines.<locals>.<genexpr>c             S   s$   i | ]\}}t d d„ |D ƒƒ|“qS )c             s   s   | ]\}}|V  qd S )Nr   )r,   r=   Z	is_importr   r   r   rL   ¦   s    z,stripped_lines.<locals>.<dictcomp>.<genexpr>)Úall)r,   rF   Znode_is_import_groupr   r   r   ú
<dictcomp>¥   s   z"stripped_lines.<locals>.<dictcomp>c             S   s   | d S )Nr   r   )Úxr   r   r   Ú<lambda>¨   s    z stripped_lines.<locals>.<lambda>)ÚkeyFNr*   )Ústartc             3   s   | ]}ˆ j |ƒV  qd S )N)Ú
startswith)r,   Úi)r6   r   r   rL   ³   s    ú"""ú'''úr"""úr'''é   ú#r   )rU   rV   rW   rX   )rH   ÚparseÚjoinÚbodyr   r9   ÚstripÚanyÚendswithÚgetÚsplitr   )Úlinesr   r   r   ÚtreeZnode_is_import_by_linenoZline_begins_importZcurrent_line_is_importZstrippedlinesZ	docstringrF   r   )r6   r   Ústripped_lines›   s<    

re   c               @   s\   e Zd ZdZddd„Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	ddd„Z
dd„ Zdd„ ZdS )r   z7Holds and indexes all the lines of a single source fileFc             C   s*   || _ || _t||||ƒ| _| jƒ | _d S )N)r1   r2   re   Ú_stripped_linesÚ	_mk_indexÚ_index)r   r1   rc   r   r   r   r   r   r   r   Ê   s
    zLineSet.__init__c             C   s
   d| j  S )Nz<Lineset for %s>)r1   )r   r   r   r   Ú__str__Ù   s    zLineSet.__str__c             C   s
   t | jƒS )N)r+   r2   )r   r   r   r   Ú__len__Ü   s    zLineSet.__len__c             C   s
   | j | S )N)rf   )r   Úindexr   r   r   Ú__getitem__ß   s    zLineSet.__getitem__c             C   s   | j |j k S )N)r1   )r   Úotherr   r   r   Ú__lt__â   s    zLineSet.__lt__c             C   s   t | ƒS )N)Úid)r   r   r   r   Ú__hash__å   s    zLineSet.__hash__r   c             c   sB   |}|r| j |d… }n| j }x|D ]}||fV  |d7 }q$W dS )zgreturn an iterator on stripped lines, starting from a given index
        if specified, else 0
        Nr*   )rf   )r   Zstart_atr5   rc   r6   r   r   r   r7   è   s    

zLineSet.enumerate_strippedc             C   s   | j j|f ƒS )z7return positions of the given stripped line in this set)rh   ra   )r   Zstripped_liner   r   r   r8   ö   s    zLineSet.findc             C   s6   t tƒ}x(t| jƒD ]\}}|r|| j|ƒ qW |S )zcreate the index for this set)r   r   r9   rf   r   )r   rk   Zline_nor6   r   r   r   rg   ú   s
    zLineSet._mk_indexN)FFF)r   )rA   rB   rC   rD   r   ri   rj   rl   rn   rp   r7   r8   rg   r   r   r   r   r   Ç   s     
	
r   ÚR0801úSimilar lines in %s files
%súduplicate-codeú�Indicates that a set of similar lines has been detected among multiple file. This usually means that the code should be refactored to avoid this duplication.c             C   s6   ddddg}|t ||dƒ7 }| jt|dddd	�ƒ d
S )z/make a layout with some stats about duplicationrE   ÚnowÚpreviousÚ
differenceÚnb_duplicated_linesÚpercent_duplicated_linesr
   r*   )ÚchildrenÚcolsZrheadersZcheadersN)rx   ry   )r   r   r   )ÚsectÚstatsZ	old_statsrc   r   r   r   Úreport_similarities  s    r~   c            	   @   sš   e Zd ZdZefZdZeZddddddœfd	d
ddddœfdd
ddddœfddddddœffZ	dde
ffZd dd„Zd!dd„Zdd„ Zdd„ Zdd„ ZdS )"ÚSimilarCheckerz£checks for similarities and duplicated code. This computation may be
    memory / CPU intensive, so you should disable it if you experiment some
    problems.
    Zsimilaritieszmin-similarity-linesr
   Úintz<int>z%Minimum lines number of a similarity.)ÚdefaultÚtypeÚmetavarÚhelpzignore-commentsTZynz<y or n>z,Ignore comments when computing similarities.zignore-docstringsz.Ignore docstrings when computing similarities.zignore-importsFz+Ignore imports when computing similarities.ZRP0801ZDuplicationNc             C   s(   t j| |ƒ tj| dddd� d | _d S )Nr
   T)r   r   r   )r   r   r	   r}   )r   Úlinterr   r   r   r   N  s    zSimilarChecker.__init__c             C   sd   t j| ||||ƒ |dkr&| jj| _n:|dkr:| jj| _n&|dkrN| jj| _n|dkr`| jj| _dS )z�method called to set an option (registered in the options list)

        overridden to report options setting to Similar
        zmin-similarity-lineszignore-commentszignore-docstringszignore-importsN)r   Ú
set_optionÚconfigZmin_similarity_linesr   r   r   r   )r   ÚoptnameÚvalueÚactionÚoptdictr   r   r   r†   U  s    zSimilarChecker.set_optionc             C   s   g | _ | jjddd�| _dS )z<init the checkers: reset linesets and statistics informationr   )rx   ry   N)r   r…   Z	add_statsr}   )r   r   r   r   Úopend  s    zSimilarChecker.openc             C   s,   |j ƒ �}| j| jj||jƒ W dQ R X dS )z�process a module

        the module's content is accessible via the stream object

        stream must implement the readlines method
        N)r   r   r…   Zcurrent_nameÚfile_encoding)r   rK   r   r   r   r   Úprocess_modulek  s    
zSimilarChecker.process_modulec       
      C   sì   t dd„ | jD ƒƒ}d}| j}x¬| jƒ D ] \}}g }d }}x$|D ]\}}|jd|j|f ƒ qBW |jƒ  |r˜x(|j||| … D ]}	|j|	jƒ ƒ q‚W | j	dt
|ƒdj|ƒfd� ||t
|ƒd	  7 }q(W ||d
< |oâ|d | |d< dS )zAcompute and display similarities on closing (i.e. end of parsing)c             s   s   | ]}t |ƒV  qd S )N)r+   )r,   r-   r   r   r   rL   w  s    z'SimilarChecker.close.<locals>.<genexpr>r   Nz==%s:%srq   Ú
)Úargsr*   rx   g      Y@ry   )r4   r   r}   r   r   r1   r"   r2   r3   Zadd_messager+   r\   )
r   ÚtotalZ
duplicatedr}   r$   r'   Úmsgr-   r5   r6   r   r   r   Úcloseu  s     zSimilarChecker.close)N)NN)rA   rB   rC   rD   r   Z__implements__r1   ÚMSGSZmsgsÚoptionsr~   Zreportsr   r†   rŒ   rŽ   r“   r   r   r   r   r     s<   


r   c             C   s   | j t| ƒƒ dS )z.required method to auto register this checker N)Zregister_checkerr   )r…   r   r   r   Úregister‹  s    r–   c             C   s$   t dƒ t ƒ  t dƒ tj| ƒ dS )z&display command line usage informationz*finds copy pasted blocks in a set of filesz~Usage: symilar [-d|--duplicates min_duplicated_lines] [-i|--ignore-comments] [--ignore-docstrings] [--ignore-imports] file1...N)r/   ÚsysÚexit)Ústatusr   r   r   Úusage�  s
    rš   c             C   s  | dkrt jdd… } ddlm} d}d}d}d}d}d}|| ||ƒ\}}	xZ|D ]R\}
}|
dkrjt|ƒ}qP|
dkrztƒ  qP|
dkrˆd}qP|
dkr–d}qP|
dkrPd}qPW |	s²tdƒ t||||ƒ}x,|	D ]$}t|ƒ�}|j||ƒ W dQ R X qÆW |jƒ  t j	dƒ dS )z$standalone command line access pointNr*   r   )ÚgetoptZhdir„   úduplicates=úignore-commentsúignore-importsúignore-docstringsr
   Fú-dú--duplicatesú-hú--helpú-iú--ignore-commentsTú--ignore-docstringsú--ignore-imports)r„   rœ   r�   rž   rŸ   )r    r¡   )r¢   r£   )r¤   r¥   )r¦   )r§   )
r—   Úargvr›   r€   rš   r	   rŒ   r   r   r˜   )r¨   r›   Zs_optsZl_optsr   r   r   r   Úoptsr�   ÚoptÚvalr@   Úfilenamer   r   r   r   ÚRun›  sB        


r­   Ú__main__)rr   rs   rt   )r   )N)rD   r—   Úcollectionsr   Ú	itertoolsr   rH   Zpylint.checkersr   r   Zpylint.interfacesr   Zpylint.reporters.ureports.nodesr   Zpylint.utilsr   r	   re   r   r”   r~   r   r–   rš   r­   rA   r   r   r   r   Ú<module>   s,   {,=  
s

(