3
I4W^‰  ã               @   s`   d dl mZmZmZmZ d dlZd dlmZm	Z	 d dl
mZ d dlmZmZ G dd„ deƒZdS )é    )Úabsolute_importÚdivisionÚprint_functionÚunicode_literalsN)ÚcoreÚschema)Ú
ModelLayer)ÚBoundedGradientProjectionÚ
LogBarrierc                   sf   e Zd Zd‡ fdd„	Zd	d
„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Z‡  ZS )ÚAdaptiveWeightÚadaptive_weightNFÚlog_stdÚlog_barrierçš™™™™™¹?c
                s`  t tˆ ƒj|||f|
Ž tjtjˆ jdƒƒˆ _ˆ j	j
ƒ ˆ _tˆ jƒˆ _|ˆ _|d k	rht|ƒˆ jks€t‚n‡ fdd„tˆ jƒD ƒ}t|ƒdks”tdƒ‚tj|ƒjtjƒˆ _t|ƒjƒ ˆ _t|ƒjƒ ˆ _t|	ƒˆ _|ˆ _tˆ ˆ jd ƒˆ _tˆ ˆ jd ƒˆ _tˆ ˆ jd ƒˆ _ˆ jƒ  ˆ j�r\‡ fd	d„tˆ jƒD ƒˆ _ x&tˆ jƒD ]}ˆ j!j"ˆ j | ƒ �q@W d S )
Nr   c                s   g | ]}d ˆ j  ‘qS )g      ð?)Únum)Ú.0Ú_)Úself© úE/tmp/pip-build-e66j97b2/torch/caffe2/python/layers/adaptive_weight.pyú
<listcomp>(   s    z+AdaptiveWeight.__init__.<locals>.<listcomp>r   z initial weights must be positiveÚ_initZ_weightZ_regc                s   g | ]}ˆ j d | ƒ‘qS )zadaptive_weight_%d)Úget_next_blob_reference)r   Úi)r   r   r   r   7   s   )#Úsuperr   Ú__init__r   ZScalarÚnpZfloat32r   Úoutput_schemaÚinput_recordZfield_blobsÚdataÚlenr   Ú	optimizerÚAssertionErrorÚrangeÚminÚarrayZastypeÚweightsÚstrÚlowerÚestimation_methodÚpos_optim_methodÚfloatÚ
reg_lambdaÚenable_diagnoseÚgetattrZ	init_funcÚweight_funcÚreg_funcÚweight_iÚmodelZadd_ad_hoc_plot_blob)r   r2   r   Únamer!   r&   r-   r)   r*   r,   Úkwargsr   )Ú	__class__)r   r   r      s0    

zAdaptiveWeight.__init__c                s€   ‡ fdd„t | jƒD ƒ}x>t | jƒD ]0}ˆ j| j| g|| ˆ jd| ƒgdgd� q$W ˆ jdƒ}ˆ j||ˆ jdƒgdd	� |S )
Nc                s   g | ]}ˆ j d | ƒ‘qS )zreshaped_data_%d)ÚNextScopedBlob)r   r   )Únetr   r   r   >   s    z.AdaptiveWeight.concat_data.<locals>.<listcomp>znew_shape_%dé   )ÚshapeZconcated_dataZconcated_new_shaper   )Zaxis)r#   r   ZReshaper   r6   ZConcat)r   r7   Zreshapedr   Zconcatedr   )r7   r   Úconcat_data=   s    

zAdaptiveWeight.concat_datac             C   s@   t jd| j ƒ}d|tjjdœf}| jd| jg|| jd�| _	dS )	z}
        mu = 2 log sigma, sigma = standard variance
        per task objective:
        min 1 / 2 / e^mu X + mu / 2
        g      ð?g       @ÚGivenTensorFill)ÚvaluesÚdtypeÚmu)Ú
param_namer9   Úinitializerr!   Ng      à?)
r   Úlogr&   r   ÚDataTypeÚFLOATÚcreate_paramr   r!   r>   )r   r<   r@   r   r   r   Úlog_std_initL   s    zAdaptiveWeight.log_std_initc             C   sB   |j dƒ}|j| j|ƒ |j dƒ}|j||ƒ |j||dd� dS )z-
        min 1 / 2 / e^mu X + mu / 2
        Úmu_negÚ
mu_neg_expg      à?)ÚscaleN)r6   ZNegativer>   ZExpÚScale)r   Úxr7   ÚweightrF   rG   r   r   r   Úlog_std_weight^   s
    

zAdaptiveWeight.log_std_weightc             C   s   |j | j|dd� d S )Ng      à?)rH   )rI   r>   )r   r7   Úregr   r   r   Úlog_std_regh   s    zAdaptiveWeight.log_std_regc             C   s|   d| j  }d|tjjdœf}| jdkr4t| jd�}n(| jdkrLtddd	�}ntd
j	| jƒƒ‚| j
d| jg|| j|d�| _dS )zg
        k = 1 / variance
        per task objective:
        min 1 / 2 * k  X - 1 / 2 * log k
        g       @r;   )r<   r=   r   )r,   Zpos_grad_projr   T)ZlbZ	left_openz*unknown positivity optimization method: {}Úk)r?   r9   r@   r!   ÚregularizerN)r&   r   rB   rC   r*   r
   r,   r	   Ú	TypeErrorÚformatrD   r   r!   rO   )r   r<   r@   rP   r   r   r   Úinv_var_initk   s     



zAdaptiveWeight.inv_var_initc             C   s   |j | j|dd� d S )Ng      à?)rH   )rI   rO   )r   rJ   r7   rK   r   r   r   Úinv_var_weightˆ   s    zAdaptiveWeight.inv_var_weightc             C   s,   |j dƒ}|j| j|ƒ |j||dd� d S )NÚlog_kg      à?)rH   g      à¿)r6   ÚLogrO   rI   )r   r7   rM   rU   r   r   r   Úinv_var_reg‹   s    
zAdaptiveWeight.inv_var_regc       	      C   s¸   | j |ƒ}|jdƒ}|jdƒ}|jdƒ}|jdƒ}| j|||ƒ | j||ƒ |j||g|ƒ |j||g|ƒ |j|| jƒ ƒ |r´x2t| j	ƒD ]$}|j
|| j| |g|d gd� qŒW d S )NrK   rM   Ú
weighted_xÚweighted_x_add_regr8   )ZstartsZends)r:   r6   r/   r0   ZMulÚAddZSumElementsr   r#   r   ÚSlicer1   )	r   r7   r-   rJ   rK   rM   rX   rY   r   r   r   r   Ú_add_ops_impl�   s    




zAdaptiveWeight._add_ops_implc             C   s   | j || jƒ d S )N)r\   r-   )r   r7   r   r   r   Úadd_opsŸ   s    zAdaptiveWeight.add_ops)r   NNFr   r   r   )Ú__name__Ú
__module__Ú__qualname__r   r:   rE   rL   rN   rS   rT   rW   r\   r]   Ú__classcell__r   r   )r5   r   r      s          "
r   )Ú
__future__r   r   r   r   Znumpyr   Zcaffe2.pythonr   r   Zcaffe2.python.layers.layersr   Zcaffe2.python.regularizerr	   r
   r   r   r   r   r   Ú<module>   s
   