3
O4W^¹j  ã               @   s$  d dl Z d dlZd dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z& ddl'm(Z(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7 i Z8i Z9dd„ Z:eG dd„ de;ƒƒZ<d d!„ Z=d"d#„ Z>d$d%„ Z?d&d'„ Z@d(d)„ ZAd*ZBe:eeƒd+d,„ ƒZCe:e
e
ƒd-d.„ ƒZDe:eeƒd/d0„ ƒZEe:eeƒd1d2„ ƒZFe:eeƒd3d4„ ƒZGe:eeƒd5d6„ ƒZHe:eeƒd7d8„ ƒZIe:eeƒd9d:„ ƒZJe:eeƒd;d<„ ƒZKe:eeƒd=d>„ ƒZLe:eeƒd?d@„ ƒZMe:e"e"ƒdAdB„ ƒZNe:e$e$ƒdCdD„ ƒZOe:e(e$ƒdEdF„ ƒZPe:e$e(ƒdGdH„ ƒZQe:e(e(ƒdIdJ„ ƒZRe:e+e+ƒdKdL„ ƒZSe:e-e-ƒdMdN„ ƒZTe:e/e/ƒdOdP„ ƒZUe:e1e1ƒdQdR„ ƒZVe:e3e3ƒdSdT„ ƒZWe:e5e5ƒdUdV„ ƒZXe:ee1ƒdWdX„ ƒZYe:e
e/ƒdYdZ„ ƒZZe:e
eƒd[d\„ ƒZ[e:e
eƒd]d^„ ƒZ\e:e
e+ƒd_d`„ ƒZ]e:e
e5ƒdadb„ ƒZ^e:ee
ƒe:ee/ƒe:ee5ƒdcdd„ ƒƒƒZ_e:eeƒdedf„ ƒZ`e:eeƒdgdh„ ƒZae:ee+ƒdidj„ ƒZbe:ee
ƒe:ee/ƒe:ee5ƒdkdl„ ƒƒƒZce:eeƒdmdn„ ƒZde:eeƒdodp„ ƒZee:ee+ƒdqdr„ ƒZfe:ee
ƒe:eeƒe:eeƒe:ee/ƒe:ee5ƒdsdt„ ƒƒƒƒƒZge:ee+ƒdudv„ ƒZhe:e"e
ƒe:e"eƒe:e"eƒe:e"e/ƒe:e"e5ƒdwdx„ ƒƒƒƒƒZie:e"e+ƒdydz„ ƒZje:e+e
ƒe:e+eƒe:e+eƒe:e+e/ƒe:e+e5ƒd{d|„ ƒƒƒƒƒZke:e+eƒd}d~„ ƒZle:e/e
ƒe:e/e5ƒdd€„ ƒƒZme:e/eƒd�d‚„ ƒZne:e/eƒdƒd„„ ƒZoe:e/e+ƒd…d†„ ƒZpe:e1eƒe:e1eƒd‡dˆ„ ƒƒZqe:e5e
ƒd‰dŠ„ ƒZre:e5eƒd‹dŒ„ ƒZse:e5eƒd�dŽ„ ƒZte:e5eƒd�d�„ ƒZue:e5e+ƒd‘d’„ ƒZve:e5e/ƒd“d”„ ƒZwe:e e ƒd•d–„ ƒZxdS )—é    N)Útotal_ordering)Úinfé   )Ú	Bernoulli)ÚBeta)ÚBinomial)ÚCategorical)Ú	Dirichlet)ÚDistribution)ÚExponential)ÚExponentialFamily)ÚGamma)Ú	Geometric)ÚGumbel)Ú
HalfNormal)ÚIndependent)ÚLaplace)ÚLowRankMultivariateNormalÚ_batch_lowrank_logdetÚ_batch_lowrank_mahalanobis)ÚMultivariateNormalÚ_batch_mahalanobis)ÚNormal)ÚOneHotCategorical)ÚPareto)ÚPoisson)ÚTransformedDistribution)ÚUniform)Ú_sum_rightmostc                sZ   t ˆ tƒ r$tˆ tƒr$tdjˆ ƒƒ‚t ˆtƒ rHtˆtƒrHtdjˆƒƒ‚‡ ‡fdd„}|S )a[  
    Decorator to register a pairwise function with :meth:`kl_divergence`.
    Usage::

        @register_kl(Normal, Normal)
        def kl_normal_normal(p, q):
            # insert implementation here

    Lookup returns the most specific (type,type) match ordered by subclass. If
    the match is ambiguous, a `RuntimeWarning` is raised. For example to
    resolve the ambiguous situation::

        @register_kl(BaseP, DerivedQ)
        def kl_version1(p, q): ...
        @register_kl(DerivedP, BaseQ)
        def kl_version2(p, q): ...

    you should register a third most-specific implementation, e.g.::

        register_kl(DerivedP, DerivedQ)(kl_version1)  # Break the tie.

    Args:
        type_p (type): A subclass of :class:`~torch.distributions.Distribution`.
        type_q (type): A subclass of :class:`~torch.distributions.Distribution`.
    z8Expected type_p to be a Distribution subclass but got {}z8Expected type_q to be a Distribution subclass but got {}c                s   | t ˆ ˆf< tjƒ  | S )N)Ú_KL_REGISTRYÚ_KL_MEMOIZEÚclear)Úfun)Útype_pÚtype_q© ú7/tmp/pip-build-e66j97b2/torch/torch/distributions/kl.pyÚ	decoratorD   s    zregister_kl.<locals>.decorator)Ú
isinstanceÚtypeÚ
issubclassr
   Ú	TypeErrorÚformat)r#   r$   r'   r%   )r#   r$   r&   Úregister_kl%   s    r-   c               @   s*   e Zd ZdgZdd„ Zdd„ Zdd„ ZdS )	Ú_MatchÚtypesc             G   s
   || _ d S )N)r/   )Úselfr/   r%   r%   r&   Ú__init__P   s    z_Match.__init__c             C   s   | j |j kS )N)r/   )r0   Úotherr%   r%   r&   Ú__eq__S   s    z_Match.__eq__c             C   s8   x2t | j|jƒD ] \}}t||ƒs&dS ||k	rP qW dS )NFT)Úzipr/   r*   )r0   r2   ÚxÚyr%   r%   r&   Ú__le__V   s    
z_Match.__le__N)Ú__name__Ú
__module__Ú__qualname__Ú	__slots__r1   r3   r7   r%   r%   r%   r&   r.   L   s   r.   c       	         s�   ‡ ‡fdd„t D ƒ}|stS tdd„ |D ƒƒj\}}tdd„ |D ƒƒj\}}t ||f }t ||f }||k	rŒtjdjˆ jˆj|j|jƒtƒ |S )zP
    Find the most specific approximate match, assuming single inheritance.
    c                s,   g | ]$\}}t ˆ |ƒrt ˆ|ƒr||f‘qS r%   )r*   )Ú.0Zsuper_pZsuper_q)r#   r$   r%   r&   ú
<listcomp>c   s    z _dispatch_kl.<locals>.<listcomp>c             s   s   | ]}t |Ž V  qd S )N)r.   )r<   Úmr%   r%   r&   ú	<genexpr>h   s    z_dispatch_kl.<locals>.<genexpr>c             s   s   | ]}t t|ƒŽ V  qd S )N)r.   Úreversed)r<   r>   r%   r%   r&   r?   i   s    z;Ambiguous kl_divergence({}, {}). Please register_kl({}, {}))	r   ÚNotImplementedÚminr/   ÚwarningsÚwarnr,   r8   ÚRuntimeWarning)	r#   r$   ÚmatchesZleft_pZleft_qZright_qZright_pZleft_funZ	right_funr%   )r#   r$   r&   Ú_dispatch_kl_   s    rG   c             C   s   t j| tƒS )zI
    Helper function for obtaining infinite KL Divergence throughout
    )ÚtorchZ	full_liker   )Útensorr%   r%   r&   Ú_infinite_likes   s    rJ   c             C   s   | | j ƒ  S )z2
    Utility function for calculating x log x
    )Úlog)rI   r%   r%   r&   Ú_x_log_xz   s    rL   c             C   sD   | j dƒ}| j dƒ}| jd|| ƒjdƒjdƒ}|j| jdd… ƒS )	zp
    Utility function for calculating the trace of XX^{T} with X having arbitrary trailing batch dimensions
    r   é   NéÿÿÿÿéþÿÿÿrN   rN   rO   )ÚsizeZreshapeÚpowÚsumÚshape)ZbmatÚnr>   Z
flat_tracer%   r%   r&   Ú_batch_trace_XXT�   s    

rU   c             C   sj   yt t| ƒt|ƒf }W n: tk
rR   tt| ƒt|ƒƒ}|t t| ƒt|ƒf< Y nX |tkr`t‚|| |ƒS )a"  
    Compute Kullback-Leibler divergence :math:`KL(p \| q)` between two distributions.

    .. math::

        KL(p \| q) = \int p(x) \log\frac {p(x)} {q(x)} \,dx

    Args:
        p (Distribution): A :class:`~torch.distributions.Distribution` object.
        q (Distribution): A :class:`~torch.distributions.Distribution` object.

    Returns:
        Tensor: A batch of KL divergences of shape `batch_shape`.

    Raises:
        NotImplementedError: If the distribution types have not been registered via
            :meth:`register_kl`.
    )r    r)   ÚKeyErrorrG   rA   ÚNotImplementedError)ÚpÚqr"   r%   r%   r&   Úkl_divergence‹   s    rZ   g¶oüŒxâ?c             C   sx   | j | j |j  jƒ  }t||j dk< d|| j dk< d| j  d| j  d|j   jƒ  }t||j dk< d|| j dk< || S )Nr   r   )ÚprobsrK   r   )rX   rY   Út1Út2r%   r%   r&   Ú_kl_bernoulli_bernoulli±   s    "r^   c       	      C   s¦   | j | j }|j |j }|j jƒ |jjƒ  |jƒ  }| j jƒ | jjƒ  |jƒ  }| j |j  tj| j ƒ }| j|j tj| jƒ }|| tj|ƒ }|| | | | S )N)Úconcentration1Úconcentration0ÚlgammarH   Údigamma)	rX   rY   Zsum_params_pZsum_params_qr\   r]   Út3Út4Út5r%   r%   r&   Ú_kl_beta_beta¼   s    rf   c             C   sh   | j |j k jƒ rtdƒ‚| j | j| j|j  | j jƒ  |j jƒ   }| j |j k}t|| ƒ||< |S )NzKKL between Binomials where q.total_count > p.total_count is not implemented)Ztotal_countÚanyrW   r[   ÚlogitsÚlog1prJ   )rX   rY   ÚklZinf_idxsr%   r%   r&   Ú_kl_binomial_binomialÈ   s    0rk   c             C   sD   | j | j|j  }t||j dkj|ƒ< d|| j dkj|ƒ< |jdƒS )Nr   r   rN   )r[   rh   r   Z	expand_asrR   )rX   rY   Útr%   r%   r&   Ú_kl_categorical_categoricalÔ   s    rm   c             C   s|   | j jdƒ}|j jdƒ}|jƒ |jƒ  }| j jƒ |j jƒ  jdƒ}| j |j  }| j jƒ |jƒ jdƒ }|| || jdƒ S )Nr   rN   rN   rN   rN   rN   )ÚconcentrationrR   ra   rb   Ú	unsqueeze)rX   rY   Zsum_p_concentrationZsum_q_concentrationr\   r]   rc   rd   r%   r%   r&   Ú_kl_dirichlet_dirichletÜ   s    rp   c             C   s"   |j | j  }|jƒ  }|| d S )Nr   )ÚraterK   )rX   rY   Z
rate_ratior\   r%   r%   r&   Ú_kl_exponential_exponentialè   s    
rr   c             C   s    t | ƒt |ƒkstdƒ‚dd„ | jD ƒ}|j}| j|Ž }tjj|jƒ |dd�}|j|Ž |jƒ  }x:t	|||ƒD ]*\}}}	|| |	 }
|t
|
t|jƒƒ8 }qnW |S )Nz‡The cross KL-divergence between different exponential families cannot                             be computed using Bregman divergencesc             S   s   g | ]}|j ƒ jƒ ‘qS r%   )ÚdetachZrequires_grad_)r<   Únpr%   r%   r&   r=   ô   s    z+_kl_expfamily_expfamily.<locals>.<listcomp>T)Zcreate_graph)r)   rW   Z_natural_paramsZ_log_normalizerrH   ZautogradZgradrR   Úcloner4   r   ÚlenÚevent_shape)rX   rY   Z	p_nparamsZ	q_nparamsZ	lg_normalZ	gradientsÚresultZpnpZqnpÚgZtermr%   r%   r&   Ú_kl_expfamily_expfamilyï   s    
rz   c             C   sn   |j | j|j jƒ  }tj|j ƒtj| j ƒ }| j |j  tj| j ƒ }|j| j | j | j  }|| | | S )N)rn   rq   rK   rH   ra   rb   )rX   rY   r\   r]   rc   rd   r%   r%   r&   Ú_kl_gamma_gammaÿ   s
    r{   c             C   sl   | j |j  }|j|j  }| j|j  }|jƒ  | | }|t }tj|d| jƒ  | ƒ}|| | dt  S )Nr   )ÚscaleÚlocrK   Ú_euler_gammarH   Úexpra   )rX   rY   Zct1Zct2Zct3r\   r]   rc   r%   r%   r&   Ú_kl_gumbel_gumbel  s    r€   c             C   s$   | j ƒ  tj|j ƒ| j  |j S )N)ÚentropyrH   ri   r[   rh   )rX   rY   r%   r%   r&   Ú_kl_geometric_geometric  s    r‚   c             C   s   t | j|jƒS )N)Ú_kl_normal_normalÚ	base_dist)rX   rY   r%   r%   r&   Ú_kl_halfnormal_halfnormal  s    r…   c             C   sV   | j |j  }| j|j jƒ }|jƒ  }||j  }|tj| | j  ƒ }|| | d S )Nr   )r|   r}   ÚabsrK   rH   r   )rX   rY   Úscale_ratioZloc_abs_diffr\   r]   rc   r%   r%   r&   Ú_kl_laplace_laplace  s    

rˆ   c             C   s  | j |j krtdƒ‚t|j|j|jƒt| j| j| jƒ }t|j|j|j| j |jƒ}|jjdd	ƒ|jj	d
ƒ }t
j||jdd�d }| j|j jdƒ}t| j|jjƒ j	dƒ ƒ}t|| jjƒ j	dƒ ƒ}t|j| jƒƒ}	|| | |	 }
d||
 | | j d   S )NzKL-divergence between two Low Rank Multivariate Normals with                          different event shapes cannot be computedr   rM   F)Úupperr   g      à?rN   rO   rO   rN   rN   rO   )rw   Ú
ValueErrorr   Ú_unbroadcasted_cov_factorÚ_unbroadcasted_cov_diagÚ_capacitance_trilr   r}   Ú	transposero   rH   Útriangular_solverR   rU   ÚrsqrtÚsqrtÚmatmul)rX   rY   Úterm1Úterm3Ú	qWt_qDinvÚAÚterm21Úterm22Zterm23Zterm24Úterm2r%   r%   r&   Ú7_kl_lowrankmultivariatenormal_lowrankmultivariatenormal(  s(    




rš   c       	      C   sÜ   | j |j krtdƒ‚t|j|j|jƒd| jjd	d
d�jƒ j	dƒ  }t
|j|j|j| j |jƒ}|jjddƒ|jjdƒ }tj||jdd�d }t| j|jjƒ jdƒ ƒ}t|j| jƒƒ}|| }d|| | | j d   S )Nz�KL-divergence between two (Low Rank) Multivariate Normals with                          different event shapes cannot be computedrM   r   )Údim1Údim2F)r‰   r   g      à?rO   rN   rN   rN   rO   rO   rN   )rw   rŠ   r   r‹   rŒ   r�   Ú_unbroadcasted_scale_trilÚdiagonalrK   rR   r   r}   rŽ   ro   rH   r�   rU   r�   r’   )	rX   rY   r“   r”   r•   r–   r—   r˜   r™   r%   r%   r&   Ú0_kl_multivariatenormal_lowrankmultivariatenormalD  s"    
 

rŸ   c             C   s(  | j |j krtdƒ‚d|jjd	d
d�jƒ jdƒ t| j| j| j	ƒ }t
|j|j| j ƒ}tjj|jjd d… | jjd d… ƒ}| j d }|jj|||f ƒ}| jj||| jjdƒf ƒ}tj| jjƒ ƒj|||f ƒ}ttj||dd�d ƒ}	ttj||dd�d ƒ}
|	|
 }d|| | | j d   S )Nz�KL-divergence between two (Low Rank) Multivariate Normals with                          different event shapes cannot be computedrM   r   )r›   rœ   r   F)r‰   g      à?rO   rN   rN   rO   rO   rN   )rw   rŠ   r�   rž   rK   rR   r   r‹   rŒ   r�   r   r}   rH   Ú_CÚ_infer_sizerS   ÚexpandZ
cov_factorrP   Z
diag_embedr‘   rU   r�   )rX   rY   r“   r”   Úcombined_batch_shaperT   Úq_scale_trilZp_cov_factorZ
p_cov_diagr—   r˜   r™   r%   r%   r&   Ú0_kl_lowrankmultivariatenormal_multivariatenormal]  s&    


r¥   c       	      C   sà   | j |j krtdƒ‚|jjd	d
d�jƒ jdƒ| jjddd�jƒ jdƒ }tjj|jj	d d… | jj	d d… ƒ}| j d }|jj
|||f ƒ}| jj
|||f ƒ}ttj||dd�d ƒ}t|j|j| j ƒ}|d|| |   S )NzvKL-divergence between two Multivariate Normals with                          different event shapes cannot be computedrM   r   )r›   rœ   r   F)r‰   g      à?rO   rN   rN   rO   rN   rN   rO   rO   )rw   rŠ   r�   rž   rK   rR   rH   r    r¡   rS   r¢   rU   r�   r   r}   )	rX   rY   Z
half_term1r£   rT   r¤   Zp_scale_trilr™   r”   r%   r%   r&   Ú)_kl_multivariatenormal_multivariatenormalw  s    
r¦   c             C   sB   | j |j  jdƒ}| j|j |j  jdƒ}d|| d |jƒ   S )NrM   g      à?r   )r|   rQ   r}   rK   )rX   rY   Z	var_ratior\   r%   r%   r&   rƒ   Š  s    rƒ   c             C   s   t | j|jƒS )N)rm   Z_categorical)rX   rY   r%   r%   r&   Ú'_kl_onehotcategorical_onehotcategorical‘  s    r§   c             C   sX   | j |j  }|j| j }|j|jƒ  }|jƒ  }|| | d }t|| jj|jjk < |S )Nr   )r|   ÚalpharK   r   ÚsupportÚlower_bound)rX   rY   r‡   Zalpha_ratior\   r]   rx   r%   r%   r&   Ú_kl_pareto_pareto–  s    
r«   c             C   s&   | j | j jƒ |j jƒ   | j |j   S )N)rq   rK   )rX   rY   r%   r%   r&   Ú_kl_poisson_poisson¢  s    r¬   c             C   sB   | j |j krt‚| j|jkr t‚t| jƒ}t| j|jƒ}t||ƒS )N)Z
transformsrW   rw   rv   rZ   r„   r   )rX   rY   Zextra_event_dimZbase_kl_divergencer%   r%   r&   Ú_kl_transformed_transformed§  s    
r­   c             C   s<   |j |j | j | j  jƒ }t||j| jk|j | j k B < |S )N)ÚhighÚlowrK   r   )rX   rY   rx   r%   r%   r&   Ú_kl_uniform_uniform³  s    r°   c             C   s    | j ƒ  | j|jjƒ  |j  S )N)r�   r[   rq   rK   )rX   rY   r%   r%   r&   Ú_kl_bernoulli_poisson»  s    r±   c             C   s
   t | jƒS )N)rJ   r_   )rX   rY   r%   r%   r&   Ú_kl_beta_infinityÀ  s    r²   c             C   s,   | j ƒ  |jjƒ  |j| j| j| j    S )N)r�   rq   rK   r_   r`   )rX   rY   r%   r%   r&   Ú_kl_beta_exponentialÅ  s    r³   c             C   sp   | j ƒ  }|jjƒ |j|jjƒ   }|jd | jjƒ | j| j jƒ   }|j| j | j| j  }|| | | S )Nr   )r�   rn   ra   rq   rK   r_   rb   r`   )rX   rY   r\   r]   rc   rd   r%   r%   r&   Ú_kl_beta_gammaÊ  s
    
$r´   c       	      C   sš   | j | j | j  }|jjdƒ}| jƒ  }d|d tj jƒ  }|d|  | j | j d  |jdƒ d }|j| }|jjdƒd }|| || | |  S )NrM   g      à?r   )	r_   r`   r|   rQ   r�   ÚmathÚpirK   r}   )	rX   rY   ZE_betaÚ
var_normalr\   r]   rc   rd   re   r%   r%   r&   Ú_kl_beta_normalÕ  s    
*
r¸   c             C   s>   | j ƒ  |j|j jƒ  }t||j| jjk|j| jjk B < |S )N)r�   r®   r¯   rK   r   r©   rª   Úupper_bound)rX   rY   rx   r%   r%   r&   Ú_kl_beta_uniformá  s     rº   c             C   s
   t | jƒS )N)rJ   rq   )rX   rY   r%   r%   r&   Ú_kl_exponential_infinityè  s    r»   c             C   sB   |j | j  }|j tj|ƒ }|| |jjƒ  |jt  dt  S )Nr   )rq   rn   rH   rK   ra   r~   )rX   rY   Zratior\   r%   r%   r&   Ú_kl_exponential_gammaï  s    r¼   c             C   sR   | j |j }|j|j }|jƒ d }tj|ƒ| |d  }|jƒ }|| | | S )Nr   )rq   r|   r}   rK   rH   r   Ú
reciprocal)rX   rY   Úscale_rate_prodÚloc_scale_ratior\   r]   rc   r%   r%   r&   Ú_kl_exponential_gumbelö  s    rÀ   c             C   sp   |j jdƒ}| jjdƒ}dtj|| d tj ƒ }|jƒ }|j| j }|jjdƒd }|d || | |  S )NrM   g      à?r   )	r|   rQ   rq   rH   rK   rµ   r¶   r½   r}   )rX   rY   r·   Zrate_sqrr\   r]   rc   rd   r%   r%   r&   Ú_kl_exponential_normal  s    rÁ   c             C   s
   t | jƒS )N)rJ   rn   )rX   rY   r%   r%   r&   Ú_kl_gamma_infinity  s    rÂ   c             C   s&   | j ƒ  |jjƒ  |j| j | j  S )N)r�   rq   rK   rn   )rX   rY   r%   r%   r&   Ú_kl_gamma_exponential  s    rÃ   c             C   s~   | j |j }|j|j }| jd | jjƒ  | jjƒ  | j }|jƒ | j|  }tj|ƒd|j	ƒ  j
| j ƒ | }|| | S )Nr   )rq   r|   r}   rn   rb   ra   rK   rH   r   r½   rQ   )rX   rY   Zbeta_scale_prodr¿   r\   r]   rc   r%   r%   r&   Ú_kl_gamma_gumbel  s    $$rÄ   c             C   s¨   |j jdƒ}| jjdƒ}dtj|| d tj ƒ | j | jjƒ  }d| jjdƒ| j  | }|j	| j | j }d|j	jdƒ }|| jd | jj
ƒ   || | |  S )NrM   g      à?r   )r|   rQ   rq   rH   rK   rµ   r¶   rn   ra   r}   rb   )rX   rY   r·   Zbeta_sqrr\   r]   rc   rd   r%   r%   r&   Ú_kl_gamma_normal%  s    ,rÅ   c             C   s
   t | jƒS )N)rJ   r}   )rX   rY   r%   r%   r&   Ú_kl_gumbel_infinity0  s    rÆ   c             C   sx   | j |j  }|tjdtj ƒ jƒ }tj| d jdƒd }| j| j t  |j |j  jdƒd }| | | td  S )NrM   g      à?é   r   )r|   rµ   r‘   r¶   rK   rQ   r}   r~   )rX   rY   Zparam_ratior\   r]   rc   r%   r%   r&   Ú_kl_gumbel_normal;  s
    &rÈ   c             C   s
   t | jƒS )N)rJ   r}   )rX   rY   r%   r%   r&   Ú_kl_laplace_infinityD  s    rÉ   c             C   s~   |j jdƒ}| j jdƒ| }dtjd| tj ƒ }d| jjdƒ }| j|j }d|jjdƒ }| | || | |  d S )NrM   g      à?r   )r|   rQ   rH   rK   rµ   r¶   r}   )rX   rY   r·   Zscale_sqr_var_ratior\   r]   rc   rd   r%   r%   r&   Ú_kl_laplace_normalM  s    rÊ   c             C   s
   t | jƒS )N)rJ   r}   )rX   rY   r%   r%   r&   Ú_kl_normal_infinityX  s    rË   c             C   s|   | j |j }| j|j jdƒ}|j |j }|jƒ d }|| }tj| d|  | ƒ}| | | ddtjdtj ƒ   S )NrM   g      à?r   )r}   r|   rQ   rK   rH   r   rµ   r¶   )rX   rY   Zmean_scale_ratioZvar_scale_sqr_ratior¿   r\   r]   rc   r%   r%   r&   Ú_kl_normal_gumbela  s    rÌ   c             C   s
   t | jƒS )N)rJ   r|   )rX   rY   r%   r%   r&   Ú_kl_pareto_infinityn  s    rÍ   c             C   sZ   | j |j }| j| jƒ }| jjƒ }| j| | jd  }|| | d }t|| jdk< |S )Nr   )r|   rq   r¨   rK   r½   r   )rX   rY   r¾   r\   r]   rc   rx   r%   r%   r&   Ú_kl_pareto_exponentialt  s    
rÎ   c             C   sŒ   | j jƒ | jjƒ  }| jjƒ | }|jjƒ |j|jjƒ   }d|j | }|j| j | j  | jd  }|| | | d }t|| jdk< |S )Nr   )r|   rK   r¨   r½   rn   ra   rq   r   )rX   rY   Úcommon_termr\   r]   rc   rd   rx   r%   r%   r&   Ú_kl_pareto_gamma  s    rÐ   c       	      C   sª   d|j jdƒ }| j | jd  }tjdtj ƒ|j  | j | j  jƒ }| jjƒ }| j|jdƒ | jd  }| j| |j jdƒ}|| || |  d }t	|| jdk< |S )NrM   r   )
r|   rQ   r¨   rµ   r‘   r¶   rK   r½   r}   r   )	rX   rY   r·   rÏ   r\   r]   rc   rd   rx   r%   r%   r&   Ú_kl_pareto_normal�  s    &
rÑ   c             C   s
   t | jƒS )N)rJ   rq   )rX   rY   r%   r%   r&   Ú_kl_poisson_infinityš  s    rÒ   c             C   sÂ   | j | j }tj|ƒ}|jd t| j ƒt| jƒ |  | }|jd td| j  ƒtd| j ƒ |  | }|jjƒ |jjƒ  |j|j jƒ  }|| | | }t|| j |j	j
k| j|j	jk B < |S )Nr   )r®   r¯   rH   rK   r_   rL   r`   ra   r   r©   r¹   rª   )rX   rY   rÏ   r\   r]   rc   rd   rx   r%   r%   r&   Ú_kl_uniform_beta   s    
&.$ rÓ   c             C   sB   |j | j| j  d | j| j |j  jƒ  }t|| j|jjk < |S )NrM   )rq   r®   r¯   rK   r   r©   rª   )rX   rY   rx   r%   r%   r&   Ú_kl_uniform_exponetial¬  s    ,rÔ   c             C   s’   | j | j }|jƒ }|jjƒ |j|jjƒ   }d|j t| j ƒt| jƒ |  | }|j| j | j  d }| | | | }t|| j|jj	k < |S )Nr   rM   )
r®   r¯   rK   rn   ra   rq   rL   r   r©   rª   )rX   rY   rÏ   r\   r]   rc   rd   rx   r%   r%   r&   Ú_kl_uniform_gamma³  s    &rÕ   c             C   sn   |j | j| j  }| j|j |j  }| j|j |j  }|jƒ d||   }|tj| ƒtj| ƒ  }|| S )Ng      à?)r|   r®   r¯   r}   rK   rH   r   )rX   rY   rÏ   Zhigh_loc_diffZlow_loc_diffr\   r]   r%   r%   r&   Ú_kl_uniform_gumbel¿  s    rÖ   c             C   st   | j | j }tjtjd ƒ|j | jƒ }|jdƒd }| j | j d|j  d jdƒ}|d||  |jjdƒ  S )NrM   é   g      à?)	r®   r¯   rµ   r‘   r¶   r|   rK   rQ   r}   )rX   rY   rÏ   r\   r]   rc   r%   r%   r&   Ú_kl_uniform_normalË  s
     rØ   c             C   sl   | j | j }|j|jj|jƒ | jƒ }t| j ƒt| jƒ | | }||jd  | }t|| j|jj	k < |S )Nr   )
r®   r¯   r¨   r|   rQ   rK   rL   r   r©   rª   )rX   rY   Zsupport_uniformr\   r]   rx   r%   r%   r&   Ú_kl_uniform_paretoÔ  s    rÙ   c             C   s*   | j |j krt‚t| j|jƒ}t|| j ƒS )N)Zreinterpreted_batch_ndimsrW   rZ   r„   r   )rX   rY   rx   r%   r%   r&   Ú_kl_independent_independentÞ  s    rÚ   )yrµ   rC   Ú	functoolsr   rH   Z
torch._sixr   Z	bernoullir   Úbetar   Zbinomialr   Zcategoricalr   Z	dirichletr	   Údistributionr
   Zexponentialr   Z
exp_familyr   Úgammar   Z	geometricr   Zgumbelr   Zhalf_normalr   Zindependentr   Zlaplacer   Zlowrank_multivariate_normalr   r   r   Zmultivariate_normalr   r   Znormalr   Zone_hot_categoricalr   Zparetor   Zpoissonr   Ztransformed_distributionr   Úuniformr   Úutilsr   r   r    r-   Úobjectr.   rG   rJ   rL   rU   rZ   r~   r^   rf   rk   rm   rp   rr   rz   r{   r€   r‚   r…   rˆ   rš   rŸ   r¥   r¦   rƒ   r§   r«   r¬   r­   r°   r±   r²   r³   r´   r¸   rº   r»   r¼   rÀ   rÁ   rÂ   rÃ   rÄ   rÅ   rÆ   rÈ   rÉ   rÊ   rË   rÌ   rÍ   rÎ   rÐ   rÑ   rÒ   rÓ   rÔ   rÕ   rÖ   rØ   rÙ   rÚ   r%   r%   r%   r&   Ú<module>   sÜ   '
!			
