3
I4W^nG  ã               @   sŠ   d dl Z d dlmZmZ d dlZ d dlmZ d dlZdd„ Zdd„ Z	dd	d
„Z
ddd„Zddd„Zdd„ Zdd„ Zddd„Zd dd„ZdS )!é    N)Úcontainer_abcsÚistuple)Úproductc             C   sT   t | tjƒr.| jd k	rP| jjƒ  | jjjƒ  n"t | tjƒrPx| D ]}t	|ƒ q@W d S )N)
Ú
isinstanceÚtorchÚTensorÚgradZdetach_ÚdataÚzero_r   ÚIterableÚzero_gradients)ÚxÚelem© r   ú9/tmp/pip-build-e66j97b2/torch/torch/autograd/gradcheck.pyr      s    


r   c                sŒ   t | tjƒr8| jƒ sd S | js"d S tj| jƒ ˆ | jd�S t | tj	ƒr„t | t
ƒ r„ttdd„ ‡ fdd„| D ƒƒƒ}|sxd S t| ƒ|ƒS d S d S )N)Údtypec             S   s   | d k	S )Nr   )r   r   r   r   Ú<lambda>   s    zmake_jacobian.<locals>.<lambda>c             3   s   | ]}t |ˆ ƒV  qd S )N)Úmake_jacobian)Ú.0r   )Únum_outr   r   ú	<genexpr>   s    z make_jacobian.<locals>.<genexpr>)r   r   r   Úis_floating_pointÚrequires_gradÚzerosZnelementr   r   r   ÚstrÚlistÚfilterÚtype)Úinputr   Z	jacobiansr   )r   r   r      s    r   Fc             c   sd   t | tjƒr | js| r`| V  n@t | tjƒr`t | tƒ r`x&| D ]}xt||ƒD ]
}|V  qNW q>W d S )N)r   r   r   r   r   r   r   Úiter_tensors)r   Zonly_requiring_gradr   Úresultr   r   r   r   #   s    
r   çü©ñÒMbP?c                sì  |dkr|}| |ƒj ƒ }t||ƒ}dd„ t|dƒD ƒ}dd„ t|ƒD ƒ}�xœt||ƒD �]Œ\}}	|j�r€dd„ }
|jƒ }t|jƒ ƒ}|jƒ j	ƒ }|j
ƒ j}|
|ƒ‰xÚt|ƒD ]Î}|| }xÀtdd„ |jƒ d	d… D ƒŽ D ]ž}|| jƒ t|ƒ ‰ t‡ ‡fd
d„tt|ƒƒD ƒƒ}|| jƒ }|| ||< | |ƒjƒ }|| ||< | |ƒjƒ }|||< || d|  }|jƒ jdƒ|	|< qØW q¬W qV|jtjk�rNt|ƒd	k�rªtdt|ƒƒ‚|j}xšttdd„ |jƒ D ƒŽ ƒD ]|\}}|jƒ }|| jƒ }|| ||< |jƒ }| |gƒ}|| ||< |jƒ }| |gƒ}|| d|  }|jƒ jdƒ|	|< �qÌW qV|j}xŽttdd„ |jƒ D ƒŽ ƒD ]p\}}|| jƒ }|| ||< | |ƒjƒ }|| ||< | |ƒjƒ }|||< || d|  }|jƒ jdƒ|	|< �qpW qVW |S )zì
    input: input to `fn`
    target: the Tensors wrt whom Jacobians are calculated (default=`input`)

    Note that `target` may not even be part of `input` to `fn`, so please be
    **very careful** in this to not clone `target`.
    Nc             S   s   g | ]}|‘qS r   r   )r   Útr   r   r   ú
<listcomp>=   s    z*get_numerical_jacobian.<locals>.<listcomp>Tc             S   s   g | ]}|‘qS r   r   )r   r"   r   r   r   r#   >   s    c             S   sD   t | ƒ}d}dg| }x(tt|ƒƒD ]}|||< || | 9 }q$W |S )Né   r   )ÚlenÚreversedÚrange)ÚsizeZdimÚtmpZstrideÚir   r   r   Ú
get_strideE   s    
z*get_numerical_jacobian.<locals>.get_stridec             S   s   g | ]}t |ƒ‘qS r   )r'   )r   Úmr   r   r   r#   V   s    r$   c             3   s   | ]}ˆ | ˆ|  V  qd S )Nr   )r   Úk)ÚindicesÚx_strider   r   r   X   s    z)get_numerical_jacobian.<locals>.<genexpr>é   zCgradcheck currently only supports functions with 1 input, but got: c             S   s   g | ]}t |ƒ‘qS r   )r'   )r   r,   r   r   r   r#   f   s    c             S   s   g | ]}t |ƒ‘qS r   )r'   )r   r,   r   r   r   r#   x   s    éÿÿÿÿr1   r1   )Únumelr   r   ÚzipÚ	is_sparseZ_nnzr   r(   Z_indicesr"   Z_valuesr	   r'   r   ÚtolistÚsumr%   ÚitemÚcloneÚdetachZreshapeÚlayoutr   Ú_mkldnnÚ
ValueErrorÚ	enumerateÚto_denseZ	to_mkldnn)Úfnr   ÚtargetÚepsZoutput_sizeÚjacobianZ	x_tensorsZ	j_tensorsZx_tensorZd_tensorr+   Zx_nnzZx_sizeZ	x_indicesZx_valuesr*   Zx_valueZx_idxZd_idxÚorigZoutaZoutbÚrZx_tensor_denseZx_tensor_mklr   )r.   r/   r   Úget_numerical_jacobian-   sl    
	
$ 
$

$rE   ç        c          	   C   sÀ  |j rtdƒ‚|jtjkr"tdƒ‚tt| dƒƒ}t| |jƒ ƒ}t| |jƒ ƒ}tj	|ƒ}|j
dƒ}d}d}	�xt|jƒ ƒD ]ø}
|jƒ  d||
< xâ||fD ]Ö}tjj|||ddd�}xºt|||ƒD ]ª\}}}|d k	ræ|jƒ |jƒ kræd}	q¾|jƒ dkr¾|d k�r|d d …|
f jƒ  q¾|jtjk�s(|jƒ n|}|d d …|
f jƒ |jƒ k�sNt‚|jƒ j
d	ƒ|d d …|
f< q¾W q–W qxW x@t||ƒD ]2\}}|jƒ dk�r€|| jƒ jƒ |k�r€d}�q€W |||	fS )
NzjSparse output is not supported at gradcheck yet. Please call to_dense() on the output of fn for gradcheck.zjMKLDNN output is not supported at gradcheck yet. Please call to_dense() on the output of fn for gradcheck.Tr$   )Zretain_graphÚallow_unusedFr   r1   r1   )r4   r<   r:   r   r;   r   r   r   r2   Ú
zeros_likeÚviewr'   r
   Úautogradr   r3   r(   Ústridedr>   ÚAssertionErrorÚ
contiguousÚabsÚmax)r   ÚoutputÚ
nondet_tolÚdiff_input_listrB   Zjacobian_reentrantZgrad_outputZflat_grad_outputÚ	reentrantÚcorrect_grad_sizesr*   Z
jacobian_cÚgrads_inputZ
jacobian_xZd_xr   Z	d_x_denseZjacobian_reentrant_xr   r   r   Úget_analytical_jacobian…   s<    



"&$
rV   c             C   s(   t | ƒr| S t| tƒrt| ƒS | fS d S )N)r   r   r   Útuple)r   r   r   r   Ú	_as_tuple®   s
    
rX   c             C   s   t dd„ t| ƒD ƒƒS )Nc             s   s   | ]}|j r|V  qd S )N)r   )r   Úor   r   r   r   ¸   s    z*_differentiable_outputs.<locals>.<genexpr>)rW   rX   )r   r   r   r   Ú_differentiable_outputs·   s    rZ   ç�íµ ÷Æ°>çñhãˆµøä>Tc                s`  ‡fdd„}t |ƒ}	tdd„ |	D ƒƒr4| r4|dƒS d}
d}xF|	D ]>}t|tjƒrB|jrt|jtjkrntj	dƒ d}
nd}|j
ƒ  qBW |
sœtd	ƒ‚|rœtd
ƒ‚ˆ |	Ž }t|ƒ}|�sxXt|ƒD ]L\‰}‡ ‡fdd„}t||	|d�}x&|D ]}ttj|ƒƒdkræ|dƒS qæW q¼W dS xØt|ƒD ]Ì\‰}|j�s.�q‡ ‡fdd„}t|	||d�\}}}t||	|d�}|�sl|dƒS xbtt||ƒƒD ]P\}\}}|jƒ dk�s¤|jƒ dk�r|tj||||ƒ�s||dˆ|||f ƒS �q|W |�s|dj|ƒƒS �qW tˆ |	Ž ƒ}tdd„ |D ƒƒ�r\tt|	dƒƒ}|�s&tdƒ‚tjj||dd„ |D ƒdd�}�xt||ƒD �]\}‰|dk�rj�qRt|tjƒ�r
|jtjk�r
|jˆjk�r¸|dt|jƒ d tˆjƒ d ƒS |jtjk�rú|jƒ ˆjƒ k�rà|dƒS |j ƒ ˆj ƒ k�rú|dƒS |j!ƒ }ˆj!ƒ ‰|j"dƒj#ƒ �s"|d ƒS |j$ƒ ˆj$ƒ k�r<|d!ƒS |j%ƒ ˆj%ƒ k�rR|d"ƒS �qRW dS )#aÆ  Check gradients computed via small finite differences against analytical
    gradients w.r.t. tensors in :attr:`inputs` that are of floating point type
    and with ``requires_grad=True``.

    The check between numerical and analytical gradients uses :func:`~torch.allclose`.

    .. note::
        The default values are designed for :attr:`input` of double precision.
        This check will likely fail if :attr:`input` is of less precision, e.g.,
        ``FloatTensor``.

    .. warning::
       If any checked tensor in :attr:`input` has overlapping memory, i.e.,
       different indices pointing to the same memory address (e.g., from
       :func:`torch.expand`), this check will likely fail because the numerical
       gradients computed by point perturbation at such indices will change
       values at all other indices that share the same memory address.

    Args:
        func (function): a Python function that takes Tensor inputs and returns
            a Tensor or a tuple of Tensors
        inputs (tuple of Tensor or Tensor): inputs to the function
        eps (float, optional): perturbation for finite differences
        atol (float, optional): absolute tolerance
        rtol (float, optional): relative tolerance
        raise_exception (bool, optional): indicating whether to raise an exception if
            the check fails. The exception gives more information about the
            exact nature of the failure. This is helpful when debugging gradchecks.
        check_sparse_nnz (bool, optional): if True, gradcheck allows for SparseTensor input,
            and for any SparseTensor at input, gradcheck will perform check at nnz positions only.
        nondet_tol (float, optional): tolerance for non-determinism. When running
            identical inputs through the differentiation, the results must either match
            exactly (default, 0.0) or be within this tolerance.

    Returns:
        True if all differences satisfy allclose condition
    c                s   ˆ rt | ƒ‚dS )NF)ÚRuntimeError)Úmsg)Úraise_exceptionr   r   Ú	fail_testá   s    zgradcheck.<locals>.fail_testc             s   s    | ]}t |tjƒr|jV  qd S )N)r   r   r   r4   )r   r"   r   r   r   r   ç   s    zgradcheck.<locals>.<genexpr>zTgradcheck expects all tensor inputs are dense when check_sparse_nnz is set to False.Fz·At least one of the inputs that requires gradient is not of double precision floating point. This check will likely fail if all the inputs are not of double precision floating point. Tzngradcheck expects at least one input tensor to require gradient, but none of the them have requires_grad=True.zzgradcheck expects if at least one input tensor is required gradient, then all other inputs should have requires_grad=True.c                s   t ˆ | Ž ƒˆ S )N)rX   )r   )Úfuncr*   r   r   r?     s    zgradcheck.<locals>.fn)rA   r   z3Numerical gradient for function expected to be zeroc                s   t ˆ | Ž ƒˆ S )N)rX   )r   )ra   r*   r   r   r?     s    )rQ   z&Analytical gradient has incorrect sizezUJacobian mismatch for output %d with respect to input %d,
numerical:%s
analytical:%s
zÚBackward is not reentrant, i.e., running backward with same input and grad_output multiple times gives different values, although analytical gradient matches numerical gradient. The tolerance for nondeterminism was {}.c             S   s   g | ]
}|j ‘qS r   )r   )r   rY   r   r   r   r#   +  s    zgradcheck.<locals>.<listcomp>z(no Tensors requiring grad found in inputc             S   s   g | ]}t j|ƒ‘qS r   )r   rH   )r   rY   r   r   r   r#   /  s    )rG   Nzgrad is incorrect layout (z is not ú)z3grad is sparse tensor, but has incorrect sparse_dimz2grad is sparse tensor, but has incorrect dense_dimz&backward not multiplied by grad_outputzgrad is incorrect typezgrad is incorrect size)&rX   Úanyr   r   r   r   r   Zfloat64ÚwarningsÚwarnZretain_gradr<   rZ   r=   rE   r%   ZnonzerorV   r3   r2   ZallcloseÚformatr   r   r]   rJ   r   r:   rK   r   Z
sparse_cooZ
sparse_dimZ	dense_dimr>   ÚeqÚallr   r(   )ra   ÚinputsrA   ÚatolÚrtolr_   Zcheck_sparse_nnzrQ   r`   Útupled_inputsZany_input_requiring_gradZsome_input_not_requiring_gradÚinpZfunc_outrP   rY   r?   Z	numericalÚnZ
analyticalrS   rT   ÚjÚarR   rU   Zgir   )ra   r*   r_   r   Ú	gradcheck»   s”    &


$rq   c	          	      sx   t |ƒ}	|dkr@‡fdd„‰t ˆ |	Ž ƒ}
t‡fdd„|
D ƒƒ}nt |ƒ}t|ƒ‰‡ ‡fdd„}t||	| |||||d�S )	aÚ	  Check gradients of gradients computed via small finite differences
    against analytical gradients w.r.t. tensors in :attr:`inputs` and
    :attr:`grad_outputs` that are of floating point type and with
    ``requires_grad=True``.

    This function checks that backpropagating through the gradients computed
    to the given :attr:`grad_outputs` are correct.

    The check between numerical and analytical gradients uses :func:`~torch.allclose`.

    .. note::
        The default values are designed for :attr:`input` and
        :attr:`grad_outputs` of double precision. This check will likely fail if
        they are of less precision, e.g., ``FloatTensor``.

    .. warning::
       If any checked tensor in :attr:`input` and :attr:`grad_outputs` has
       overlapping memory, i.e., different indices pointing to the same memory
       address (e.g., from :func:`torch.expand`), this check will likely fail
       because the numerical gradients computed by point perturbation at such
       indices will change values at all other indices that share the same
       memory address.

    Args:
        func (function): a Python function that takes Tensor inputs and returns
            a Tensor or a tuple of Tensors
        inputs (tuple of Tensor or Tensor): inputs to the function
        grad_outputs (tuple of Tensor or Tensor, optional): The gradients with
            respect to the function's outputs.
        eps (float, optional): perturbation for finite differences
        atol (float, optional): absolute tolerance
        rtol (float, optional): relative tolerance
        gen_non_contig_grad_outputs (bool, optional): if :attr:`grad_outputs` is
            ``None`` and :attr:`gen_non_contig_grad_outputs` is ``True``, the
            randomly generated gradient outputs are made to be noncontiguous
        raise_exception (bool, optional): indicating whether to raise an exception if
            the check fails. The exception gives more information about the
            exact nature of the failure. This is helpful when debugging gradchecks.
        nondet_tol (float, optional): tolerance for non-determinism. When running
            identical inputs through the differentiation, the results must either match
            exactly (default, 0.0) or be within this tolerance. Note that a small amount
            of nondeterminism in the gradient will lead to larger inaccuracies in
            the second derivative.

    Returns:
        True if all differences satisfy allclose condition
    Nc                s4   t jj| jƒ r| n| jƒ ƒ}ˆ r,t jj|ƒ}|jƒ S )N)r   ZtestingÚ
randn_liker   ÚdoubleZmake_non_contiguousZrequires_grad_)r   Úy)Úgen_non_contig_grad_outputsr   r   rr     s    z!gradgradcheck.<locals>.randn_likec             3   s   | ]}ˆ |ƒV  qd S )Nr   )r   r   )rr   r   r   r   …  s    z gradgradcheck.<locals>.<genexpr>c                 sR   | d ˆ … }| ˆ d … }t ˆ |Ž ƒ}tdd„ |D ƒƒ}tjj|||dd�}|S )Nc             s   s$   | ]}t |tjƒr|jr|V  qd S )N)r   r   r   r   )r   r   r   r   r   r   �  s    z2gradgradcheck.<locals>.new_func.<locals>.<genexpr>T)Zcreate_graph)rZ   rW   r   rJ   r   )ÚargsZ
input_argsÚgrad_outputsÚoutputsZgrad_inputs)ra   Únum_outputsr   r   Únew_func‹  s    zgradgradcheck.<locals>.new_func)rQ   )rX   rW   r%   rq   )ra   ri   rw   rA   rj   rk   ru   r_   rQ   rl   rx   Ztupled_grad_outputsrz   r   )ra   ru   ry   rr   r   ÚgradgradcheckH  s    2r{   )F)Nr!   )rF   )r[   r\   r!   TFrF   )Nr[   r\   r!   FTrF   )r   Z
torch._sixr   r   Ztorch.testingÚ	itertoolsr   rd   r   r   r   rE   rV   rX   rZ   rq   r{   r   r   r   r   Ú<module>   s    



X
)	
   