o
    ;ήc.                     @   sh   d dl mZ d dlmZ d dlmZ d dlmZ d dlm	Z	 ee
ZG dd deZG dd	 d	eZd
S )    )	getLogger)Dict)Fusion)helper)	OnnxModelc                       4   e Zd Zdef fddZdedefddZ  ZS )FusionLayerNormalizationmodelc                    s   t  |dd d S )NLayerNormalization
ReduceMeansuper__init__selfr	   	__class__ P/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/fusion_layernorm.pyr      s   z!FusionLayerNormalization.__init__input_name_to_nodesoutput_name_to_nodec                 C   s  | j ||}t|dkst|dkrdS |jd }|d jdks*|d jd |kr,dS t|dkrD|d jdksB|d jd |krDdS d}|D ]}| j j|d|dd}|durZ nqH|du radS | j |g d	g d
fg dg dfg|\}}	}
|dk rdS |	d }||vrdS |	d }| j |\}}|du s|dks|dkrt	d|  dS |	d }| j 
|ddksdS ||jd  d }|jdkrdS ||jd  d }|jdkrdS |g}|| ||	dd  ||||g | j ||j||std dS |jd| j |jd |  }| j |dds dS |jd| j |jd |  }| j |dds:dS | j| tjd|jd ||g|jd g| j jdddd}|jtdt|g | j| | j| j|j< dS )a  
        Fuse Layer Normalization subgraph into one node LayerNormalization:
              +----------------------+
              |                      |
              |                      v
          [Root] --> ReduceMean -->  Sub  --> Pow --> ReduceMean --> Add --> Sqrt --> Div --> Mul --> Add
                     (axis=2 or -1)  |      (Y=2)   (axis=2 or -1)  (E-6 or E-12 or 0)    ^
                                     |                                               |
                                     +-----------------------------------------------+

         It also handles cases of duplicated sub nodes exported from older version of PyTorch:
              +----------------------+
              |                      v
              |           +-------> Sub-----------------------------------------------+
              |           |                                                           |
              |           |                                                           v
          [Root] --> ReduceMean -->  Sub  --> Pow --> ReduceMean --> Add --> Sqrt --> Div  --> Mul --> Add
              |                      ^
              |                      |
              +----------------------+
        r      NSub   DivF)	recursive)SqrtAddr   Powr   )r   r   r   r   r   )r   r   r   r   Castr   )r   r   r   r   r   r   g-C6?zepsilon value is not expeced:    g       @Mulr   z4It is not safe to fuse LayerNormalization node. Skipzlayernorm weightzlayernorm biasr
   	LayerNormname_prefixinputsoutputsnameepsilon)r	   get_childrenleninputop_typefind_first_child_by_typematch_parent_pathsget_constant_inputloggerwarningfind_constant_inputoutputextendis_safe_to_fuse_nodesdebuginput_index$is_constant_with_specified_dimensionnodes_to_remover   	make_nodecreate_node_name	attributemake_attributefloatnodes_to_addappendthis_graph_namenode_name_to_graph_namer)   )r   noder   r   children
root_inputdiv_nodechildpath_idparent_nodes_sub_nodesecond_add_nodei
add_weightpow_nodemul_nodelast_add_nodesubgraph_nodesweight_input
bias_inputnormalize_noder   r   r   fuse   s   
  





zFusionLayerNormalization.fuse__name__
__module____qualname__r   r   r   rX   __classcell__r   r   r   r   r          r   c                       r   )FusionLayerNormalizationTFr	   c                    s   t  |ddd d S )Nr
   r   TFr   r   r   r   r   r      s   z#FusionLayerNormalizationTF.__init__r   r   c                 C   s(  g }| j |g dg dfg dg dfg|\}}}|du r!dS t|dks)J |d dv r;|d	 dv r;|d
 dv sBtd dS |dd \}}}	}
}}|dd \}}}}d}t|dkrk|d }|jdkskJ | j |dd|}|du rtd dS | j |d|}|du r|n| j |d|}|du rtd dS | j |\}}|du s|dks|dkr|du rtd dS |du r|j	d |j	vs|j	d |j	vrtd dS |dur|j	d |j	vs|j	d |j	vrtd dS |j	d |j	d	 krtd dS ||||	|
|||||||g}|dur7| j |dd|}|du r/td dS |
|||g | j ||j| j  | j  sOtd dS | j
| |	j	d	 }|j	d }tjd|j	d ||g|jd g| j jdddd}|j
tdt|g | j| | j| j|j< dS )aU  
         Layer Norm from Tensorflow model(using keras2onnx or tf2onnx):
          +------------------------------------+
          |                                    |
          |                                    |
        (Cast_1)                               |
          |                                    |
          |                                    v                                           (B)                             (B)             (A)
         Add --> (Cast_1) --> ReduceMean -->  Sub  --> Mul --> ReduceMean --> (Cast_3) --> Add --> Sqrt --> Reciprocol --> Mul --> Mul --> Sub --> Add
          |                       |                                                                                         |       ^              ^
          |                       |                                                                                         |       |              |
          |                       +--------------------------------------------------(Cast_2)-------------------------------|-------+              |
          |                                                                                                                 v                      |
          +---------------------------------------------------------------------------------------------------------------> Mul--------------------+
        )
r   r"   r"   
Reciprocalr   r   r   r"   r   r   )
r   r   Nr   r   r   Nr   r   N)r   r"   r"   ra   r   r   r   r   r"   r   r   )r   r   Nr   r   r   r   Nr   r   NNr!   r   )r   r   r   r   z=return indice is exepected in [0, 1], but got {return_indice}      r   r"   zmul_node_3 not foundzroot node is nonegh㈵>zepsilon is not matchedz;reduce_mean_node_1 and mul_node_3 shall link from root nodez%mul_node_2 shall have two same inputszcast_node_2 not foundz$not safe to fuse layer normalizationr
   r#   r$   r&   r*   )r	   r0   r,   r2   r8   r.   match_parent
get_parentr1   r-   r6   r7   r5   r   r   r;   r   r<   r=   r>   r?   r@   rA   rB   rC   rD   r)   )r   rE   r   r   return_indicerL   rK   
sub_node_0
mul_node_0
mul_node_1reciprocol_node	sqrt_node
add_node_0reduce_mean_node_0
mul_node_2
sub_node_1reduce_mean_node_1cast_node_3
mul_node_3node_before_reduce	root_noderO   r*   rT   cast_node_2rU   rV   
fused_noder   r   r   rX      s   !
&$

	

 










zFusionLayerNormalizationTF.fuserY   r   r   r   r   r_      r^   r_   N)loggingr   typingr   fusion_baser   onnxr   
onnx_modelr   rZ   r2   r   r_   r   r   r   r   <module>   s   v