a
    ×ýddÛ/  ã                   @   sä   d Z ddlZddlZddlmZ ddlmZmZ ddlZddl	m
Z
 ddlZddlmZ ddlmZmZ ddlZe d¡Zd	d
„ Zdd„ Zedddddd�ZdZdd„ ZG dd„ dƒZdd„ Zd#dd„Zdd„ Zdd „ Zd!d"„ ZdS )$zí
I/O for cluster files.

A cluster file is an archive (zip or tar, compressed or not) holding
an ICluster graph as a JSON object or as a set of Numpy arrays.  As a
special case, a bare .json file can be read for a single cluster
graph.

é    N)ÚPath)Ú
namedtupleÚdefaultdict)Ú
fromarrays)ÚCounter)ÚarioÚjsiozwirecell.imgc                    sl   t ƒ }‡ fdd„ˆ  |¡D ƒ}|D ]D}‡ fdd„ˆ  |¡D ƒ}|D ]"}| ‡ fdd„ˆ  |¡D ƒ¡ qBq"|S )zA
    Return list of channel nodes reachable from slice node.
    c                    s"   g | ]}ˆ j | d  dkr|‘qS )ÚcodeÚb©Únodes©Ú.0Ún©Úgr© ú2/home/yuhw/wc/wire-cell-python/wirecell/img/tap.pyÚ
<listcomp>   ó    z"slice_channels.<locals>.<listcomp>c                    s"   g | ]}ˆ j | d  dkr|‘qS )r	   Úmr   r   r   r   r   r   !   r   c                    s"   g | ]}ˆ j | d  dkr|‘qS )r	   Úcr   r   r   r   r   r   #   r   )ÚsetÚ	neighborsÚupdate)r   ÚsnodeZcnodesÚbnodesZbnodeZmnodesZmnoder   r   r   Úslice_channels   s    "r   c              	   C   sŽ   t j| d�}|d D ],}|d }|j|f||d dœ|d ¤Ž q|d D ]}| |d |d	 ¡ qJt d
| › d| ¡ › d| ¡ › d�¡ |S )z4
    Return networkx graph from dict base data 
    ©Únamer   ÚidentÚtype)Údescr	   ÚdataÚedgesÚtailÚheadzload z with z nodes and z edges)ÚnxÚGraphÚadd_nodeÚadd_edgeÚlogÚdebugÚnumber_of_nodesÚnumber_of_edges)r   Údatr   ZvtxZvdescÚedger   r   r   Úmake_nxgraph&   s    "&r1   ÚactivityZwireÚblobÚsliceZmeasure©ÚaÚwr
   Úsr   )ZawÚbsZbwÚbbÚamÚbmÚasc                 C   s°   t ƒ }tddƒ}tddƒ}| ¡ D ]ˆ\}}| d¡rT|d }t| }||d�||< q"| d¡r˜|d	d
… }	t|	d  }
t|	d  }||d�||
|	|f< q"t d|› d�¡ q"|S )a=  Return graph data represented in a manner similar to
      torch_geometric.data.HeteroData.

      Note: see pg2nx() to convert the graph returned by this method
      to one that is equivalent to the graph returned by
      make_nxgraph().

      The provided "dat" argument must be a dictionary providing
      arrays matching the WCT Cluster Arrays schema (see
      ClusterArrays.org file) such as read directly from a cluster
      array archive file.  They keys of this dictionary are expected
      to be of the forms:

      <nodecode>nodes and <edgecode>edges

      Where <nodecode> is a letter representing a node type and
      <edgecode> is a pair of node codes in alphabetical order

      The returned data structure matches the schema of
      pytorch-geometric's HeteroData class.

      The node type names are taken to be the full names corresponding
      to the node type codes.  For example:

      >>> pghd = make_pggraph(name, dat)

      >>> channel_features = pghd["channel"].x

      ICluster graph edges only provide connectivity information and
      thus edge types are only distinquished by the types of the edge
      endpoint nodes.  HeterData requires an edge type identifer and
      the two-letter edgecode is used.  For example,

      >>> blob_slice_edges = pghd["blob","bs","slice"].edge_index
        ÚNodeÚxÚEdgeÚ
edge_indexr   r   )r?   r$   Né   é   )rA   zWarning: unexpected array: "ú")Údictr   ÚitemsÚendswithÚ
node_typesr+   Úwarning)r   r/   Zpghdr>   r@   ÚkeyÚarrÚncÚntÚecZnt1Znt2r   r   r   Úmake_pggraph8   s"    &



rO   c                   @   sž  e Zd Zedefdefdefdefdefdefgdefdefdefdefdefdefd	efd
efdefdefdefdefgdefdefdefdefdefdefdefdefdefdefdefdefdefdefdefgdd„ edƒD ƒ defdefdefdefdefdefdefgdefdefdefdefdefgd�Zdd „ Zd!d"„ Z	d#d$„ Z
d%d&„ Zd'd(„ Zd)d*„ Zd+d,„ Zd-d.„ Zd/d0„ Zd1d2„ Zd3d4„ Zd5d6„ Zd7S )8ÚPgFillerr"   r    ÚvalÚuncÚindexÚwpidÚsegZchidZtailxZtailyZtailzZheadxZheadyZheadzZfaceidÚsliceidÚstartÚspanÚmin1Úmax1Úmin2Úmax2Zmin3Zmax3Zncornersc                 C   s   g | ]}d |› �t f‘qS )r   )Úint)r   Úir   r   r   r   ˆ   r   zPgFiller.<listcomp>é   Zframeidr5   c                 C   sR   || _ || _t ¡ D ]$\}}|  |¡}t| d|› �ƒƒ  qtD ]}|  |¡ q>d S )NZadd_)r   ÚpgrH   rF   Úget_ndatÚgetattrÚ
edge_typesÚ	add_edges)Úselfr   r`   rL   ÚntypeÚndatrN   r   r   r   Ú__init__”   s    
zPgFiller.__init__c                 C   s   t | }| j| jS )N)rH   r`   r?   )re   rL   rf   r   r   r   ra   ž   s    zPgFiller.get_ndatc                 C   s(   t | }| j| j}t|j| j| d�S ©N©Údtype©rH   r`   r?   r   ÚTÚdtypes)re   rL   rf   rg   r   r   r   Úget_arr¢   s    zPgFiller.get_arrc                 C   s0   t | }| j| j}t|j| j| d�}||fS ri   rl   )re   rL   rf   rg   rK   r   r   r   Úget_both§   s    zPgFiller.get_bothc                 C   s2   t t| jjdd�ƒ ¡ ƒ}t |› d|› �¡ d S )Nr	   )r#   z: )r   rE   r   r   Úvaluesr+   r,   )re   ÚcontextÚnode_counterr   r   r   r,   ®   s    zPgFiller.debugc           
      K   s‚   |d  }|d< |d |d< t |d ƒD ]D\}}t|d�}| ¡ D ]\}}	|	| ||< qB| jj|fi |¤Ž q(|  d|› �¡ |S )z8
        Common creation of nodes (except s/c).
        r"   r    )r	   zadd )Ú	enumeraterE   rF   r   r)   r,   )
re   rL   rK   r#   ÚdescsÚirowr"   ÚparamsÚkÚvr   r   r   Ú
add_common³   s    
zPgFiller.add_commonc           
   	   C   st   |   d¡}|d }|d }|d }|d }|d }|d }t|ƒD ],\}}	| jj|	d|	|| || || d	� qBd
S )z;
        Activity gets coalesced back to channels.
        r6   r"   r    rS   rT   rQ   rR   r   )r	   r"   rS   r    rT   N)ro   rt   r   r)   )
re   rK   ru   ZidentsZindexsZwpidsÚvalsZuncsrv   r"   r   r   r   Úadd_activityÂ   s    
üzPgFiller.add_activityc                    sô   |   d¡\}‰ ‡ fdd„d ¡ D ƒ}| jdˆ fi |¤Ž}|d d …dd …f  d¡}|d }t | dd	¡d
¡}| d¡}t ||f¡}t|ƒD ]`\}}tj|| dd…  dd¡t	d�‰‡fdd„t
dƒD ƒ| jj| d< || | jj| d< qŽd S )Nr
   c                    s   i | ]}|ˆ | “qS r   r   ©r   rx   ©rK   r   r   Ú
<dictcomp>Ù   r   z%PgFiller.add_blob.<locals>.<dictcomp>z!val unc faceid sliceid start spané   )éÿÿÿÿé   rB   rW   r�   rC   r‚   )r�   r‚   rC   é   é   é   rB   rj   c                    s(   g | ] }t ˆ | d  ˆ | d d�‘qS )r   rC   )ZbegÚend)rE   )r   Úind)r
   r   r   r   ã   r   z%PgFiller.add_blob.<locals>.<listcomp>ÚboundsÚcorners)rp   Úsplitrz   ÚreshapeÚnumpyÚtileÚdstackrt   Úarrayr]   Úranger   r   )re   rg   r#   ru   r‰   Zstartsrv   r"   r   )rK   r
   r   Úadd_blob×   s    
""zPgFiller.add_blobc                    s8   |   d¡‰ ‡ fdd„d ¡ D ƒ}| jdˆ fi |¤Ž d S )Nr   c                    s   i | ]}|ˆ | “qS r   r   r}   r~   r   r   r   è   r   z(PgFiller.add_measure.<locals>.<dictcomp>zval unc wpid)ro   rŠ   rz   )re   r#   r   r~   r   Úadd_measureæ   s    
zPgFiller.add_measurec                    sl   |   d¡\}‰ t|j| jd d�‰ ‡ fdd„d ¡ D ƒ}dd„ tˆ d jƒD ƒ|d	< | jdˆ fi |¤Ž d S )
Nr8   rj   c                    s   i | ]}|ˆ | “qS r   r   r}   r~   r   r   r   î   r   z&PgFiller.add_slice.<locals>.<dictcomp>zframeid start spanc                 S   s   g | ]
}t ƒ ‘qS r   )Úlist)r   Ú_r   r   r   r   ï   r   z&PgFiller.add_slice.<locals>.<listcomp>r"   Úsignal)rp   r   rm   rn   rŠ   r�   Úsizerz   )re   rg   r#   r   r~   r   Ú	add_sliceë   s
    zPgFiller.add_slicec                    sv   |   d¡‰ ‡ fdd„d ¡ D ƒ}| jdˆ fi |¤Ž}t|ƒD ]4\‰}dD ]&‰‡ ‡‡fdd„dD ƒ| jj| ˆ< qHq<d S )Nr7   c                    s   i | ]}|ˆ | “qS r   r   r}   r~   r   r   r   ô   r   z%PgFiller.add_wire.<locals>.<dictcomp>zindex seg chid wpid)r%   r&   c                    s"   i | ]}|ˆ ˆ› |› � ˆ “qS r   r   )r   r   ©rK   Úeprv   r   r   r   ø   r   Úxyz)ro   rŠ   rz   rt   r   r   )re   r#   ru   r"   r   r˜   r   Úadd_wireò   s    
zPgFiller.add_wirec                 C   s  |\}}t | |t | f}| j| j}| j ¡ }|jd dksBJ ‚|  |¡}|  |¡}tƒ }	|D ]ž\}
}}||f}||	v r|q`|	 |¡ |d | }|d | }|dkrì| jj	| }| jj	| }|d  
t|d |d | |d | d	�¡ q`| jj|||
d
� q`| j ¡ | }dS )z�
        Add all edges with given two-letter edge code.

        Edges to activity (a-nodes) are changed to edges to channel (c-nodes).
        rC   r…   r"   r=   r•   r    rQ   rR   )r    rQ   rR   )r"   N)rH   r`   rA   r   r.   Úshapero   r   Úaddr   ÚappendrE   r*   )re   rN   ÚtcÚhcZekeyZedatZnedgesZtarrÚharrÚuniqZedescÚtrowZhrowrJ   ZtvtxZhvtxZtdatZhdatr   r   r   rd   ú   s0    




ÿzPgFiller.add_edgesN)Ú__name__Ú
__module__Ú__qualname__rE   r]   Úfloatr�   rn   rh   ra   ro   rp   r,   rz   r|   r‘   r’   r—   r›   rd   r   r   r   r   rP   s   sV   ýüøø
üýè
rP   c                 C   s   t tj| d�|ƒjS )a±  Convert a pg graph to an nx graph.

    The pg graph should be as returned by make_pggraph().

    The nx graph will be returned by make_nxgraph().

    In particular, cluster array activity nodes will be created to
    cluster graph channel nodes and slice-activity edges will be
    converted to entries in a "signal" attribute on slice nodes.

    nx node values are those of the original cluster graph vertex
    descriptor.
    r   )rP   r'   r(   r   )r   r`   r   r   r   Úpg2nx  s    r¨   r   c                 k   sZ   t | ƒ}tj| |fi |¤Ž}t|tƒs,tg}t|ƒD ] \}}t|j› d|› �|ƒV  q4dS )z5
    Load a file assuming it is a JSON-like file
    r”   N)r   r   ÚloadÚ
isinstancer“   rt   r1   Ústem)ÚfilenameÚpathÚkwdsÚgotÚcountÚoner   r   r   Ú	load_jsio1  s    
r²   c           
      C   s¾   t |  ¡ ƒ}t ƒ }|rº|d }| j| }d|v rF| |¡ | d¡ q| dd¡\}}}|dkrntd|› �ƒ‚d|› d�}|g}	| d¡ |r®|d  |¡r®|	 | d¡¡ qŠ| |	¡ q|S )zG
    Group sequence of keys into single .json or list of .npy keys
    r   ú.jsonr”   rB   Úclusterzfailed to parse key: Zcluster_)r“   ÚkeysÚmember_namesrž   ÚpoprŠ   Ú
ValueErrorÚ
startswith)
Úarfrµ   ÚretrJ   ÚmemberÚclr   ÚkindÚpreÚsubr   r   r   Ú
group_keys<  s&    



rÁ   c                 +   sÄ   t | ƒ}t | d¡‰ tˆ ƒD ]¢}t|tƒrtˆ | }t|tƒsB|g}t|ƒD ]&\}}t|j	› d|› d|› �|ƒV  qJq‡ fdd„|D ƒ}|d  
dd¡\}}}	|j	› d|› �}
t|
t|
|ƒƒV  qdS )z9
    Load a file assuming it is an archive of files.
    Fr”   c                    s"   i | ]}|  d d¡d ˆ | “qS )r”   rB   r�   )rŠ   r}   ©rº   r   r   r   f  r   zload_ario.<locals>.<dictcomp>r   rB   N)r   r   r©   rÁ   rª   Ústrr“   rt   r1   r«   rŠ   r¨   rO   )r¬   r®   r­   rJ   r/   r°   r±   r½   r   r¾   r   r   rÂ   r   Ú	load_arioW  s    

"rÄ   c                 k   s\   |   d¡r2t d¡ t| fi |¤ŽD ]
}|V  q$n&t d¡ t| fi |¤ŽD ]
}|V  qLdS )zB
    Yield a sequence of graphs loaded from file like object.
    )r³   zjson.gzz	.json.bz2z.jsonnetzloading cluster file as JSONzloading cluster file as arioN)rG   r+   r,   r²   rÄ   )r¬   r®   r±   r   r   r   r©   k  s    



r©   )r   ) Ú__doc__ÚsysÚjsonÚpathlibr   Úcollectionsr   r   rŒ   Znumpy.core.recordsr   Únetworkxr'   r   Úwirecell.utilr   r   ÚloggingÚ	getLoggerr+   r   r1   rE   rH   rc   rO   rP   r¨   r²   rÁ   rÄ   r©   r   r   r   r   Ú<module>   s.   	
; +
