3
O4W^×b  ã               @   sn  d dl mZmZmZmZ d dlZd dlZd dlmZm	Z	 d dl
mZ d dlZd dljZd dlmZmZ G dd„ deƒZdd	„ Zee_eed
ƒefi ƒZG dd„ deejƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZ G dd„ deƒZ!ejdd�Z"e Z#ejej$ej%d�Z&ejdd�Z'ejej$ej(d�Z)dS ) é    )Úabsolute_importÚdivisionÚprint_functionÚunicode_literalsN)ÚABCMetaÚabstractmethod)Úpartial)ÚListÚOptionalc               @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )Ú_PartialWrapperc             C   s
   || _ d S )N)Úp)Úselfr   © r   ú</tmp/pip-build-e66j97b2/torch/torch/quantization/observer.pyÚ__init__   s    z_PartialWrapper.__init__c             O   s   | j ||ŽS )N)r   )r   ÚargsÚkeywordsr   r   r   Ú__call__   s    z_PartialWrapper.__call__c             C   s
   | j jƒ S )N)r   Ú__repr__)r   r   r   r   r      s    z_PartialWrapper.__repr__N)Ú__name__Ú
__module__Ú__qualname__r   r   r   r   r   r   r   r      s   r   c             K   s   t t| f|Žƒ}|S )zÚ
    Wrapper around functools.partial that allows chaining.

    Often you want to assign it to a class as a class method:

        Foo.with_args = classmethod(_with_args)
        Foo.with_args(x=1).with_args(y=2)
    )r   r   )Úcls_or_selfÚkwargsÚrr   r   r   Ú
_with_args   s    	r   ÚABCc                   s@   e Zd ZdZ‡ fdd„Zedd„ ƒZedd„ ƒZee	ƒZ
‡  ZS )ÚObservera_  
    Observer base Module. Any observer implementation should derive from this class.

    Concrete observers should follow the same API. In forward, they will update
    the statistics of the observed Tensor. And they should provide a
    `calculate_qparams` function that computes the quantization parameters given
    the collected statistics.
    c                s   t t| ƒjƒ  || _d S )N)Úsuperr   r   Údtype)r   r   )Ú	__class__r   r   r   3   s    zObserver.__init__c             C   s   d S )Nr   )r   Úxr   r   r   Úforward7   s    zObserver.forwardc             K   s   d S )Nr   )r   r   r   r   r   Úcalculate_qparams;   s    zObserver.calculate_qparams)r   r   r   Ú__doc__r   r   r"   r#   Úclassmethodr   Ú	with_argsÚ__classcell__r   r   )r    r   r   *   s
   r   c                   s<   e Zd ZdZejejdf‡ fdd„	Zdd„ Zdd„ Z	‡  Z
S )	Ú_ObserverBasez3
    Common base for all qint/quint8 observers
    Fc                sn   t t| ƒj|d� || _|| _tjtjƒj| _| jtj	tj
tjtjfksPtdƒ‚| jtjtjfksjtdƒ‚d S )N)r   z©Default Observer only works for per_tensor_affine,                 per_tensor_symmetric, per_channel_affine and                 per_channel_symmetric quantization schemez:Default Observer only works for qint8 and quint8 data type)r   r(   r   ÚqschemeÚreduce_rangeÚtorchZfinfoÚfloat32ÚepsÚper_tensor_affineÚper_tensor_symmetricZper_channel_affineÚper_channel_symmetricÚAssertionErrorr   Úqint8Úquint8)r   r   r)   r*   )r    r   r   r   G   s    z_ObserverBase.__init__c             C   sê   |dks|dkr2t jdƒ tjdgƒtjdgƒfS x<tt|ƒƒD ],}|| || ks@tdj|| || ƒƒ‚q@W tj|j	ƒ tj
d�}tj|j	ƒ tjd�}xHtt|ƒƒD ]8}| j|| || ƒ}t|d ƒ||< t|d ƒ||< q¦W ||fS )zw
        Given min and max value tensors, this function calculates per channel
        quantization parameters
        Nz~must run observer before calling calculate_qparams.                                    Returning default scale and zero point g      ð?r   z!min {} should be less than max {})r   é   )ÚwarningsÚwarnr+   ÚtensorÚrangeÚlenr1   ÚformatÚemptyÚsizer,   Zint64Ú_calculate_qparamsÚfloatÚint)r   Úmin_valsÚmax_valsÚiZscalesZzero_pointsZqparamr   r   r   Ú_calculate_per_channel_qparams\   s    z,_ObserverBase._calculate_per_channel_qparamsc             C   s~  |dks|dkr2t jdƒ tjdgƒtjdgƒfS ||ksJtdj||ƒƒ‚| jtjkrp| jrfd\}}qˆd\}}n| jr€d\}}nd\}}t	|ƒt	|ƒ }}t
d|ƒ}td|ƒ}||krÀd}d}n¦| jtjksÚ| jtjk�rt| |ƒ}||| d  }t|| jƒ}| jtjk�rdnd}nL|| t	|| ƒ }t|| jƒ}|t|| ƒ }t||ƒ}t
||ƒ}t|ƒ}tj|gƒtj|gƒfS )z\
        Given min and max values, this function calculates quantization parameters
        Nz~must run observer before calling calculate_qparams.                                    Returning default scale and zero point g      ð?r   z!min {} should be less than max {}é@   é?   é€   é   éÿ   g        é   éÀÿÿÿ)rJ   rE   é€ÿÿÿ)rK   rG   )r   rG   )r   rH   )r5   r6   r+   r7   r1   r:   r   r2   r*   r>   ÚminÚmaxr)   r/   r0   r-   Úroundr?   )r   Úmin_valÚmax_valZqminZqmaxZscaleZ
zero_pointr   r   r   r=   z   s>    







z _ObserverBase._calculate_qparams)r   r   r   r$   r+   r3   r.   r   rC   r=   r'   r   r   )r    r   r(   B   s   r(   c                   sv   e Zd ZdZeej eej dœZ‡ fdd„Zdd„ Z	ej
jdd„ ƒZej
jd	d
„ ƒZ‡ fdd„Z‡ fdd„Z‡  ZS )ÚMinMaxObservera'  Default Observer Module
    A default implementation of the observer module, only works for
    `per_tensor_affine` quantization scheme.  The module will record the
    running average of max and min value of the observed Tensor and
    calculate_qparams will calculate scale and zero_point
    )rO   rP   c                sH   t t| ƒjf |Ž d | _d | _| jtjkrD| jrD| j	tj
krDtdƒ‚d S )Nz9Cannot reduce range for symmetric quantization for quint8)r   rQ   r   rO   rP   r)   r+   r/   r*   r   r3   ÚNotImplementedError)r   r   )r    r   r   r   º   s    zMinMaxObserver.__init__c             C   sn   |j ƒ }| j}| j}|d ks$|d kr:tj|ƒ}tj|ƒ}n$tjtj|ƒ|ƒ}tjtj|ƒ|ƒ}|| _|| _|S )N)ÚdetachrO   rP   r+   rL   rM   )r   Úx_origr!   rO   rP   r   r   r   r"   Î   s    
zMinMaxObserver.forwardc             C   s   | j | j| jƒS )N)r=   rO   rP   )r   r   r   r   r#   Ü   s    z MinMaxObserver.calculate_qparamsc             C   s   dj | j| jƒS )Nzmin_val={}, max_val={})r:   rO   rP   )r   r   r   r   Ú
extra_reprà   s    zMinMaxObserver.extra_reprc                s4   t t| ƒj|||ƒ | j||d < | j||d < d S )NrO   rP   )r   rQ   Ú_save_to_state_dictrO   rP   )r   ÚdestinationÚprefixÚ	keep_vars)r    r   r   rV   ä   s    z"MinMaxObserver._save_to_state_dictc                s@   |j |d ƒ| _|j |d ƒ| _tt| ƒj|||d|||ƒ d S )NrO   rP   F)ÚpoprO   rP   r   rQ   Ú_load_from_state_dict)r   Ú
state_dictrX   Úlocal_metadataÚstrictÚmissing_keysÚunexpected_keysÚ
error_msgs)r    r   r   r[   é   s    z$MinMaxObserver._load_from_state_dict)r   r   r   r$   r
   r+   ÚTensorÚ__annotations__r   r"   ÚjitÚexportr#   rU   rV   r[   r'   r   r   )r    r   rQ   ­   s   rQ   c                   s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )ÚMovingAverageMinMaxObserverç{®Gáz„?c                s   || _ tt| ƒjf |Ž d S )N)Úaveraging_constantr   rf   r   )r   rh   r   )r    r   r   r   ò   s    z$MovingAverageMinMaxObserver.__init__c             C   sz   |j ƒ }| j}| j}|d ks$|d kr:tj|ƒ}tj|ƒ}n0|| jtj|ƒ|   }|| jtj|ƒ|   }|| _|| _|S )N)rS   rO   rP   r+   rL   rM   rh   )r   rT   r!   rO   rP   r   r   r   r"   ö   s    
z#MovingAverageMinMaxObserver.forward)rg   )r   r   r   r   r"   r'   r   r   )r    r   rf   ñ   s   rf   c                   sF   e Zd ZdZd‡ fdd„	Zdd„ Zdd„ Zd	d
„ Z‡ fdd„Z‡  Z	S )ÚPerChannelMinMaxObserverzáPer Channel Observer Module
    The module will record the running average of max and min value for each
    channel of the observed Tensor and calculate_qparams will calculate
    scales and zero_points for each channel
    r   c                sZ   t t| ƒjf |Ž || _| jdd ƒ | jdd ƒ | jtjkrV| jrV| j	tj
krVtdƒ‚d S )Nr@   rA   z9Cannot reduce range for symmetric quantization for quint8)r   ri   r   Úch_axisÚregister_bufferr)   r+   r0   r*   r   r3   rR   )r   rj   r   )r    r   r   r     s    z!PerChannelMinMaxObserver.__init__c             C   sÎ   |j ƒ }| j}| j}|jƒ }ttt|ƒƒƒ}d|| j< | j|d< |jt	|ƒƒ}t
j|dd�}|d ksl|d krŽt
j|dƒd }t
j|dƒd }n0t
jt
j|dƒd |ƒ}t
jt
j|dƒd |ƒ}|| _|| _|S )Nr   r4   )Ú	start_dim)rS   r@   rA   r<   Úlistr8   r9   rj   ÚpermuteÚtupler+   ÚflattenrL   rM   )r   rT   r!   r@   rA   Úx_dimÚnew_axis_listÚyr   r   r   r"     s"    

z PerChannelMinMaxObserver.forwardc             C   s   | j | j| jƒS )N)rC   r@   rA   )r   r   r   r   r#   /  s    z*PerChannelMinMaxObserver.calculate_qparamsc             C   s   dj | j| jƒS )Nzmin_val={}, max_val={})r:   r@   rA   )r   r   r   r   rU   2  s    z#PerChannelMinMaxObserver.extra_reprc                s@   |j |d ƒ| _|j |d ƒ| _tt| ƒj|||d|||ƒ d S )Nr@   rA   F)rZ   r@   rA   r   ri   r[   )r   r\   rX   r]   r^   r_   r`   ra   )r    r   r   r[   5  s    z.PerChannelMinMaxObserver._load_from_state_dict)r   )
r   r   r   r$   r   r"   r#   rU   r[   r'   r   r   )r    r   ri     s   ri   c                   s*   e Zd ZdZd‡ fdd„	Zdd„ Z‡  ZS )Ú%MovingAveragePerChannelMinMaxObserverzáPer Channel Observer Module
    The module will record the running average of max and min value for each
    channel of the observed Tensor and calculate_qparams will calculate
    scales and zero_points for each channel
    ç{®Gáz„?c                s   || _ tt| ƒjf |Ž d S )N)rh   r   rt   r   )r   rh   r   )r    r   r   r   E  s    z.MovingAveragePerChannelMinMaxObserver.__init__c             C   sÚ   |j ƒ }| j}| j}|jƒ }ttt|ƒƒƒ}d|| j< | j|d< |jt	|ƒƒ}t
j|dd�}|d ksl|d krŽt
j|dƒd }t
j|dƒd }n<|| jt
j|dƒd |   }|| jt
j|dƒd |   }|| _|| _|S )Nr   r4   )rl   )rS   r@   rA   r<   rm   r8   r9   rj   rn   ro   r+   rp   rL   rM   rh   )r   rT   r!   r@   rA   rq   rr   rs   r   r   r   r"   I  s"    

z-MovingAveragePerChannelMinMaxObserver.forward)ru   )r   r   r   r$   r   r"   r'   r   r   )r    r   rt   >  s   rt   c                   s„   e Zd ZdZeej eej dœZd‡ fdd„	Ze	dd„ ƒZ
dd	„ Zd
d„ Zdd„ Zdd„ Zdd„ Z‡ fdd„Z‡ fdd„Z‡  ZS )ÚHistogramObserverz˜
    The module records the running histogram of tensor values along with
    min/max values. calculate_qparams will calculate scale and zero_point
    )rO   rP   é   c                s<   t t| ƒjf |Ž || _| jdtj| jƒƒ d | _d | _d S )NÚ	histogram)	r   rv   r   Úbinsrk   r+   ÚzerosrO   rP   )r   ry   r   )r    r   r   r   i  s
    zHistogramObserver.__init__c             C   s@   |dkst dƒ‚d}|dkr8|| | | |  |   d }|| S )z×
        Compute the norm of the values uniformaly distributed between
        delta_begin and delta_end.

        norm = density * (integral_{begin, end} x^2)
             = density * (end^3 - begin^3) / 3
        ÚL2z%Only L2 norms are currently supportedg        é   )r1   )Údelta_beginÚ	delta_endÚdensityÚ	norm_typeÚnormr   r   r   Ú	_get_normq  s    	
zHistogramObserver._get_normc             C   sŽ  dt j| jƒj }| jjƒ | jjƒ  | j }d}||| d  | }|dkrPdS �x6t| jƒD �]&}|| | }	|	| }
t	|d t
dtj|	| ƒƒƒ}t	|d t
dtj|
| ƒƒƒ}|| |d  }| j| | }||k�r|	| }|
| }|| j||||ƒ }q^|	| }|d }|| j||||ƒ }||| d | j| d |d ||ƒ  }|| |d  }| d }|
| }|| j||||ƒ }q^W |S )z‚
        Compute the quantization error if we use start_bin to end_bin as the
        min and max to do the quantization.
        rI   g        r4   )r+   Ziinfor   ÚbitsrP   ÚitemrO   ry   r8   rL   rM   ÚmathÚfloorrx   r‚   )r   Únext_start_binÚnext_end_binr€   Z	dst_nbinsÚ	bin_widthr�   Údst_bin_widthZsrc_binÚsrc_bin_beginÚsrc_bin_endZdst_bin_of_beginZdst_bin_of_endZdst_bin_of_begin_centerr   r}   r~   Zdst_bin_of_end_centerr   r   r   Ú_compute_quantization_errorƒ  s:    

z-HistogramObserver._compute_quantization_errorc             C   sl  | j jƒ d | jkstdƒ‚| j| j | j }t| j ƒ}tj| j dd�}d}d}d}d}| jd }t	dƒ}	xØ||k �rB|| }
|| }|}|}x$||k r²|| |
| k r²|d }q�W x$||krØ|| || krØ|d }q¶W |}|}|| || krü|}|
}n|}|}||k�r||k�rql| j
||d	ƒ}||	k�r4P |}	|}|}qlW | j||  }| j||d   }||fS )
a=  
        An approximation for L2 error minimization for selecting min/max.
        By selecting new min/max, we filter out outliers in input distribution.
        This follows the implementation of NormMinimization::NonlinearQuantizationParamsSearch in
        caffe2/quantization/server/norm_minimization.cc
        r   zbins mistmatch)Zdimgñhãˆµøä>g        g      ð?r4   Úinfr{   )rx   r<   ry   r1   rP   rO   Úsumr+   Zcumsumr>   r�   )r   r‰   ÚtotalZcSumZstepsizeÚalphaÚbetaZ	start_binZend_binZnorm_minZ
next_alphaZ	next_betaÚlr   r‡   rˆ   r�   Únew_minÚnew_maxr   r   r   Ú_non_linear_param_search¸  sJ    


z*HistogramObserver._non_linear_param_searchc             C   s2  |j ƒ d }|j ƒ d }|| | }	|| | }
xüt|ƒD ]ð}|| jƒ }|dkrTq:||
|  }||
 }d}|	r€t|| |	 ƒ}||	|  }||	 }d}|	r¶tt|| |	 ƒ|d ƒ}||d ksÊtdƒ‚d}|
dksÞ|	dkrä|}ntt|| |
 | ƒ|ƒ}||  |7  < ||k r:||  || 7  < q:W d S )Nr   r4   rI   z)1 src_bin is mapped to at most 2 dst_bins)r<   r8   r„   r?   rL   r1   rN   )r   Zdst_histogramZdst_minZdst_maxZsrc_histogramZsrc_minZsrc_maxZbins_dstZbins_srcrŠ   Zsrc_bin_widthrB   Zsrc_bin_countr‹   rŒ   Zdst_binZdst_bin_beginZdst_bin_endZdst_bin2Zdst_bin_cntr   r   r   Ú_combine_histogramsò  s<    z%HistogramObserver._combine_histogramsc       
      C   s  t jƒ ��
 | j}| j}|d ks(|d kr`t j|ƒ}t j|ƒ}|| _|| _t j|| j||d�| _n°t j|ƒ}t j|ƒ}t j|| j||d�}t j	| jƒ}t j|| jƒ}t j|| jƒ}	| j
||jƒ |	jƒ | j| jjƒ | jjƒ ƒ | j
||jƒ |	jƒ ||jƒ |jƒ ƒ || _|| _|	| _W d Q R X |S )N)rL   rM   )r+   Zno_gradrO   rP   rL   rM   Zhistcry   rx   Z
zeros_liker—   r„   )
r   r!   rO   rP   r”   r•   Znew_histogramZcombined_histogramZcombined_minZcombined_maxr   r   r   r"   %  sB    




zHistogramObserver.forwardc             C   sn   | j d ks| jd kr6tjdƒ tjdgƒtjdgƒfS | jt| jƒksNt	dƒ‚| j
ƒ \}}| j|jƒ |jƒ ƒS )Nz~must run observer before calling calculate_qparams.                                    Returning default scale and zero point g      ð?r   ziThe number of bins in histogram should be equal to the number of bins supplied while making this observer)rO   rP   r5   r6   r+   r7   ry   r9   rx   r1   r–   r=   r„   )r   r”   r•   r   r   r   r#   L  s    z#HistogramObserver.calculate_qparamsc                s4   t t| ƒj|||ƒ | j||d < | j||d < d S )NrO   rP   )r   rv   rV   rO   rP   )r   rW   rX   rY   )r    r   r   rV   \  s    z%HistogramObserver._save_to_state_dictc                s@   |j |d ƒ| _|j |d ƒ| _tt| ƒj|||d|||ƒ d S )NrO   rP   F)rZ   rO   rP   r   rv   r[   )r   r\   rX   r]   r^   r_   r`   ra   )r    r   r   r[   a  s    z'HistogramObserver._load_from_state_dict)rw   )r   r   r   r$   r
   r+   rb   rc   r   Ústaticmethodr‚   r�   r–   r—   r"   r#   rV   r[   r'   r   r   )r    r   rv   ^  s   5:3'rv   c                   sZ   e Zd ZdZdeeej  iZ‡ fdd„Z	dd„ Z
ejjdd„ ƒZejjd	d
„ ƒZ‡  ZS )ÚRecordingObserverzU
    The module is mainly for debug and records the tensor values during runtime
    Ú
tensor_valc                s   t t| ƒjf |Ž g | _d S )N)r   r™   r   rš   )r   r   )r    r   r   r   n  s    zRecordingObserver.__init__c             C   s   | j j|jƒ ƒ |S )N)rš   ÚappendÚclone)r   r!   r   r   r   r"   r  s    zRecordingObserver.forwardc             C   s   t dƒ‚d S )Nz<calculate_qparams should not be called for RecordingObserver)Ú	Exception)r   r   r   r   r#   v  s    z#RecordingObserver.calculate_qparamsc             C   s   | j S )N)rš   )r   r   r   r   Úget_tensor_valuez  s    z"RecordingObserver.get_tensor_value)r   r   r   r$   r	   r
   r+   rb   rc   r   r"   rd   re   r#   rž   r'   r   r   )r    r   r™   h  s   r™   c                   s6   e Zd ZdZejf‡ fdd„	Zdd„ Zdd„ Z‡  Z	S )ÚNoopObserverzÛ
    Observer that doesn't do anything and just passes its configuration to the
    quantized module's ``.from_float()`.

    Primarily used for quantization to float16 which doesn't require determining
    ranges.
    c                s(   |t jkrtdƒ‚tt| ƒj|d� d S )NzAOnly float16 quantization can be used without calibration process)r   )r+   Úfloat16Ú
ValueErrorr   rŸ   r   )r   r   )r    r   r   r   ‡  s    
zNoopObserver.__init__c             C   s   |S )Nr   )r   r!   r   r   r   r"   Œ  s    zNoopObserver.forwardc             C   s   t dƒ‚d S )Nz7calculate_qparams should not be called for NoopObserver)r�   )r   r   r   r   r#   �  s    zNoopObserver.calculate_qparams)
r   r   r   r$   r+   r    r   r"   r#   r'   r   r   )r    r   rŸ     s   rŸ   T)r*   )r   r)   )*Ú
__future__r   r   r   r   r…   r5   Úabcr   r   Ú	functoolsr   r+   Ztorch.nnÚnnZtorch._jit_internalr	   r
   Úobjectr   r   r&   Ústrr   ÚModuler   r(   rQ   rf   ri   rt   rv   r™   rŸ   Zdefault_observerZdefault_debug_observerr2   r/   Zdefault_weight_observerZdefault_histogram_observerr0   Z#default_per_channel_weight_observerr   r   r   r   Ú<module>   s6   
kD9   