3
I4W^"  ã               @   sb   d dl mZ d dlmZ d dlZd dlmZ d dlmZ G dd„ de	ƒZ
e
ƒ ZG dd	„ d	e	ƒZdS )
é    )Údefaultdict)Úcontainer_abcsN)Údeepcopy)Úchainc               @   s   e Zd ZdZdd„ ZdS )Ú_RequiredParameterzCSingleton class representing a required parameter for an Optimizer.c             C   s   dS )Nz<required parameter>© )Úselfr   r   ú6/tmp/pip-build-e66j97b2/torch/torch/optim/optimizer.pyÚ__repr__   s    z_RequiredParameter.__repr__N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r
   r   r   r   r	   r   	   s   r   c               @   sX   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
d„ Zdd„ Z	dd„ Z
dd„ Zdd„ ZdS )Ú	Optimizera]  Base class for all optimizers.

    .. warning::
        Parameters need to be specified as collections that have a deterministic
        ordering that is consistent between runs. Examples of objects that don't
        satisfy those properties are sets and iterators over values of dictionaries.

    Arguments:
        params (iterable): an iterable of :class:`torch.Tensor` s or
            :class:`dict` s. Specifies what Tensors should be optimized.
        defaults: (dict): a dict containing default values of optimization
            options (used when a parameter group doesn't specify them).
    c             C   s�   t jjdƒ || _t|t jƒr0tdt j|ƒ ƒ‚tt	ƒ| _
g | _t|ƒ}t|ƒdkr\tdƒ‚t|d t	ƒstd|ig}x|D ]}| j|ƒ qzW d S )Nzpython.optimizerzZparams argument given to the optimizer should be an iterable of Tensors or dicts, but got r   z%optimizer got an empty parameter listÚparams)ÚtorchZ_CZ_log_api_usage_onceÚdefaultsÚ
isinstanceÚTensorÚ	TypeErrorÚtypenamer   ÚdictÚstateÚparam_groupsÚlistÚlenÚ
ValueErrorÚadd_param_group)r   r   r   r   Úparam_groupr   r   r	   Ú__init__    s    


zOptimizer.__init__c             C   s   | j | j| jdœS )N)r   r   r   )r   r   r   )r   r   r   r	   Ú__getstate__5   s    zOptimizer.__getstate__c             C   s   | j j|ƒ d S )N)Ú__dict__Úupdate)r   r   r   r   r	   Ú__setstate__<   s    zOptimizer.__setstate__c             C   sx   | j jd }x^t| jƒD ]P\}}|d7 }|dj|ƒ7 }x0t|jƒ ƒD ] }|dkrD|dj||| ƒ7 }qDW qW |d7 }|S )Nz (Ú
zParameter Group {0}
r   z    {0}: {1}
ú))Ú	__class__r   Ú	enumerater   ÚformatÚsortedÚkeys)r   Úformat_stringÚiÚgroupÚkeyr   r   r	   r
   ?   s    zOptimizer.__repr__c                s:   dd„ ‰ ‡ fdd„| j D ƒ}dd„ | jjƒ D ƒ}||dœS )a  Returns the state of the optimizer as a :class:`dict`.

        It contains two entries:

        * state - a dict holding current optimization state. Its content
            differs between optimizer classes.
        * param_groups - a dict containing all parameter groups
        c             S   s,   dd„ | j ƒ D ƒ}dd„ | d D ƒ|d< |S )Nc             S   s   i | ]\}}|d kr||“qS )r   r   )Ú.0ÚkÚvr   r   r	   ú
<dictcomp>U   s    z<Optimizer.state_dict.<locals>.pack_group.<locals>.<dictcomp>c             S   s   g | ]}t |ƒ‘qS r   )Úid)r/   Úpr   r   r	   ú
<listcomp>V   s    z<Optimizer.state_dict.<locals>.pack_group.<locals>.<listcomp>r   )Úitems)r-   Úpackedr   r   r	   Ú
pack_groupT   s    z(Optimizer.state_dict.<locals>.pack_groupc                s   g | ]}ˆ |ƒ‘qS r   r   )r/   Úg)r8   r   r	   r5   X   s    z(Optimizer.state_dict.<locals>.<listcomp>c             S   s*   i | ]"\}}|t |tjƒr"t|ƒn|“qS r   )r   r   r   r3   )r/   r0   r1   r   r   r	   r2   Z   s   z(Optimizer.state_dict.<locals>.<dictcomp>)r   r   )r   r   r6   )r   r   Zpacked_stater   )r8   r	   Ú
state_dictJ   s    
zOptimizer.state_dictc                s$  t |ƒ}| j}|d }t|ƒt|ƒkr.tdƒ‚dd„ |D ƒ}dd„ |D ƒ}tdd„ t||ƒD ƒƒrjtdƒ‚dd	„ ttd
d„ |D ƒŽ tdd„ |D ƒŽ ƒD ƒ}‡ fdd„‰ ttƒ}x@|d j	ƒ D ]0\}}	||krä|| }
ˆ |
|	ƒ||
< q¼|	||< q¼W dd„ ‰‡fdd„t||ƒD ƒ}| j
||dœƒ dS )z·Loads the optimizer state.

        Arguments:
            state_dict (dict): optimizer state. Should be an object returned
                from a call to :meth:`state_dict`.
        r   z<loaded state dict has a different number of parameter groupsc             s   s   | ]}t |d  ƒV  qdS )r   N)r   )r/   r9   r   r   r	   ú	<genexpr>q   s    z,Optimizer.load_state_dict.<locals>.<genexpr>c             s   s   | ]}t |d  ƒV  qdS )r   N)r   )r/   r9   r   r   r	   r;   r   s    c             s   s   | ]\}}||kV  qd S )Nr   )r/   Zp_lenZs_lenr   r   r	   r;   s   s    z]loaded state dict contains a parameter group that doesn't match the size of optimizer's groupc             S   s   i | ]\}}||“qS r   r   )r/   Zold_idr4   r   r   r	   r2   x   s    z-Optimizer.load_state_dict.<locals>.<dictcomp>c             s   s   | ]}|d  V  qdS )r   Nr   )r/   r9   r   r   r	   r;   y   s    c             s   s   | ]}|d  V  qdS )r   Nr   )r/   r9   r   r   r	   r;   z   s    c                s‚   t |tjƒr0ˆ jƒ r |jˆ jƒ}|jˆ jƒ}|S t |tƒrR‡‡ fdd„|jƒ D ƒS t |t	j
ƒrzt|ƒ‡‡ fdd„|D ƒƒS |S dS )zBMake a deep copy of value, casting all tensors to device of param.c                s   i | ]\}}ˆ ˆ|ƒ|“qS r   r   )r/   r0   r1   )ÚcastÚparamr   r	   r2   †   s    z;Optimizer.load_state_dict.<locals>.cast.<locals>.<dictcomp>c             3   s   | ]}ˆ ˆ|ƒV  qd S )Nr   )r/   r1   )r<   r=   r   r	   r;   ˆ   s    z:Optimizer.load_state_dict.<locals>.cast.<locals>.<genexpr>N)r   r   r   Zis_floating_pointZtoZdtypeZdevicer   r6   r   ÚIterableÚtype)r=   Úvalue)r<   )r=   r	   r<   |   s    
z'Optimizer.load_state_dict.<locals>.castr   c             S   s   | d |d< |S )Nr   r   )r-   Z	new_groupr   r   r	   Úupdate_group˜   s    z/Optimizer.load_state_dict.<locals>.update_groupc                s   g | ]\}}ˆ ||ƒ‘qS r   r   )r/   r9   Zng)rA   r   r	   r5   œ   s    z-Optimizer.load_state_dict.<locals>.<listcomp>)r   r   N)r   r   r   r   ÚanyÚzipr   r   r   r6   r#   )r   r:   ÚgroupsZsaved_groupsZ
param_lensZ
saved_lensZid_mapr   r0   r1   r=   r   r   )r<   rA   r	   Úload_state_dicta   s,    zOptimizer.load_state_dictc             C   sD   x>| j D ]4}x.|d D ]"}|jdk	r|jjƒ  |jjƒ  qW qW dS )z>Clears the gradients of all optimized :class:`torch.Tensor` s.r   N)r   ZgradZdetach_Zzero_)r   r-   r4   r   r   r	   Ú	zero_gradŸ   s
    

zOptimizer.zero_gradc             C   s   t ‚dS )zÝPerforms a single optimization step (parameter update).

        Arguments:
            closure (callable): A closure that reevaluates the model and
                returns the loss. Optional for most optimizers.
        N)ÚNotImplementedError)r   Zclosurer   r   r	   Ústep§   s    zOptimizer.stepc             C   s&  t |tƒstdƒ‚|d }t |tjƒr2|g|d< n t |tƒrFtdƒ‚nt|ƒ|d< x<|d D ]0}t |tjƒs~tdtj|ƒ ƒ‚|j	s\t
dƒ‚q\W x@| jjƒ D ]2\}}|tkrÂ||krÂt
d| ƒ‚qœ|j||ƒ qœW tƒ }x | jD ]}|jt|d ƒƒ qàW |jt|d ƒƒ�st
dƒ‚| jj|ƒ dS )	a�  Add a param group to the :class:`Optimizer` s `param_groups`.

        This can be useful when fine tuning a pre-trained network as frozen layers can be made
        trainable and added to the :class:`Optimizer` as training progresses.

        Arguments:
            param_group (dict): Specifies what Tensors should be optimized along with group
            specific optimization options.
        zparam group must be a dictr   zšoptimizer parameters need to be organized in ordered collections, but the ordering of tensors in sets will change between runs. Please use a list instead.z>optimizer can only optimize Tensors, but one of the params is z can't optimize a non-leaf TensorzJparameter group didn't specify a value of required optimization parameter z7some parameters appear in more than one parameter groupN)r   r   ÚAssertionErrorr   r   Úsetr   r   r   Zis_leafr   r   r6   ÚrequiredÚ
setdefaultr   r"   Ú
isdisjointÚappend)r   r   r   r=   ÚnameÚdefaultZ	param_setr-   r   r   r	   r   °   s0    



zOptimizer.add_param_groupN)r   r   r   r   r   r    r#   r
   r:   rE   rF   rH   r   r   r   r   r	   r      s   >	r   )Úcollectionsr   Z
torch._sixr   r   Úcopyr   Ú	itertoolsr   Úobjectr   rK   r   r   r   r   r	   Ú<module>   s   