o
    ;ήcI                     @   sx   d dl mZ d dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZ d dlmZ eeZG dd deZG d	d
 d
eZdS )    )	getLoggerN)Fusion)FusionUtils)TensorProtohelpernumpy_helper)	OnnxModelc                       sJ   e Zd ZdZdedef fddZdd Zdd	 Zd
d Z	dd Z
  ZS )FusionGptAttentionPastBasez3Base class for GPT Attention Fusion with past statemodel	num_headsc                    s,   t  |ddd || _t|| _i | _d S )N	AttentionLayerNormalizationz	with past)super__init__r   r   utilscasted_attention_maskselfr
   r   	__class__ T/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/fusion_gpt_attention.pyr      s   

z#FusionGptAttentionPastBase.__init__c           
      C   s   | j |d|}|jdkrtd d S | j |ddks$td d S |jd }| j |d|}|jdkr9|}n| j |ddgddg}|d u rPtd d S |d }| j |ddksdtd	 d S |jd }	||	krttd
 d S |S )Nr   Gatherz,match_past_pattern_1: expect Gather for past   z9match_past_pattern_1: expect indices=1 for Gather of past	Transposez7match_past_pattern_1: failed match Transpose and Gatherz;match_past_pattern_1: expect indices=0 for Gather k of pastz,match_past_pattern_1: expect past to be same)r
   
get_parentop_typeloggerdebugfind_constant_inputinputmatch_parent_path)
r   concat_kconcat_voutput_name_to_nodegatherpastparentgather_past_kpast_k_nodespast_kr   r   r   match_past_pattern_1   s0   








z/FusionGptAttentionPastBase.match_past_pattern_1c           
      C   sd  | j |d|}|jdkrtd d S | j |d|}|jdkr(td d S | j  }|dk rQt|ddgs@td d S t|d	d
d
gsPtd d S n!| j	|d
dgsatd d S | j	|d
d
d
gsrtd d S tj|ddddstd d S |j
d }| j |ddgddg}|d u rtd d S |d j
d }	||	krtd d S |S )Nr   Squeezez:match_past_pattern_2: expect Squeeze as parent of concat_vSplitz0match_past_pattern_2: expect Split for past path   axesz:match_past_pattern_2: axes != [0] for Squeeze in past pathsplitr   z<match_past_pattern_2: split != [1, 1] for Split in past pathaxis)default_valuezKmatch_past_pattern_2: attribute axis of Split are not expected in past pathz7match_past_pattern_2: failed to match past_k_nodes pathr   z,match_past_pattern_2: expect past to be same)r
   r   r   r   r   get_opset_versionr   check_node_attributer   check_node_input_valuer!   r"   info)
r   r#   r$   r%   squeezer1   opset_versionr'   r*   r+   r   r   r   match_past_pattern_2J   sH   












z/FusionGptAttentionPastBase.match_past_pattern_2c                 C   sZ   | j j|d|dd}|std d S | j j|d|dd}|s&td d S |jd }|S )N	UnsqueezeF)	recursivezexpect unsqueeze for presentConcatzexpect concat for presentr   )r
   find_first_child_by_typer   r7   output)r   r$   input_name_to_nodesunsqueeze_present_vconcat_presentpresentr   r   r   match_present   s   


z(FusionGptAttentionPastBase.match_presentc                 C   s`   || j v r| j | }|S | j|r!| j|\}}|| j |< |S | j|\}}|| j |< |S N)r   r
   find_graph_inputr   cast_graph_input_to_int32cast_input_to_int32)r   
input_nameattention_mask_input_namecasted	cast_noder   r   r   cast_attention_mask   s   



z.FusionGptAttentionPastBase.cast_attention_mask)__name__
__module____qualname____doc__r   intr   r,   r:   rD   rM   __classcell__r   r   r   r   r	      s    1Cr	   c                       s:   e Zd ZdZdedef fddZdd Zdd	 Z  Z	S )
FusionGptAttentionzP
    Fuse GPT-2 Attention with past state subgraph into one Attention node.
    r
   r   c                    s   t  || d S rE   )r   r   r   r   r   r   r      s   zFusionGptAttention.__init__c
                 C   s   | j d}
tjd|||||g|
d |g|
d}d|_|jtd| jtd|	r+dnd	g tjd
|
d |j	d g|
d g|
d d}tjd|
d |j	d g|g|
d d}| j
|||g | j| j|j< | j| j|j< | j| j|j< d S )NGptAttentionr   _output)inputsoutputsnamezcom.microsoftr   unidirectionalr   r   MatMul_matmul_output_matmulAdd   _add)r
   create_node_namer   	make_nodedomain	attributeextendmake_attributer   r!   nodes_to_addthis_graph_namenode_name_to_graph_namerY   )r   	fc_weightfc_biasgemm_qkvr'   rC   r!   r?   maskis_unidirectionalattention_node_nameattention_nodematmul_nodeadd_noder   r   r   create_attention_node   s:   
z(FusionGptAttention.create_attention_nodec           6      C   s  d }d }g }| j j|g dg d||d}|d u rd S |\}}	}
}}}}|jd|d   }| j |g dg d}|d u rDtd d S |\}}}}| j |g d	g d
|}|d u r| j |g dg d|}|d u rstd d S |d jd }| j |d \}}|d j| }n|d jd }|d jd }|d }||jvrtd d S d}d }d }d }| j |g dg d}|d ur|\} }!}"}#}$| j |!g dg d}%|%d u rtd d S |%d }&|%d }|#|&krtd d S n| j |g dg dfg dg dfg|\}}}|d u rtd d S |d }'|d }#|d }$|dkrZ|d }(| j |(g d g d!fg d"g d#fg d$g d%fg|\}}}|d u rZtd& d S | j |'g d'g d(|}%|%d u rstd) d S |%d }| j |%d d|})|)jd*kr|)}&|#|&krtd d S n|)jd+kr|)}ntd) t	
| j |jd }*t|*jd,kr|*jd d d-kr|*jd |*jd kstd. d S t|*t|*rd/}nt|*tt|*std0 d S | j |$g d1g d2}+|+d u rtd3 d S |+\},}-}.||.krtd4 d S | j |$g dg d}/|/d u rO| j |$g d5g d6}/|/d u rGtd7 d S |/\}}0}1}2}3n|/\}0}1}2}3||3kratd8 d S |rp|0|krptd9 d S d:}4|d ur|d jd }5| |5}4| |0||p| |0||}|d u rtd; d S | j |std< | ||}|d u rtd= d S | j |std> d S | |||
|||jd |	jd |4|	 d| _d S )?N)r^   ReshapeGemmrt   rt   r   r[   )r   Nr   r   r   r   r   )r%   return_indicer   r   )r=   r   rt   r.   )r   r   r   r   z&fuse_attention: failed to match v path)rt   ru   rt   r   )r   r   r   r   )r^   r[   r   )r   Nr   z'fuse_attention: failed to match fc pathr_   r   z4Add and LayerNormalization shall have one same inputT)SoftmaxSubMulDivr[   )r   r   r   r   r   )
ry   rx   Slicer{   r;   rx   r-   r{   Shaperz   )
r   r   r   r   r   r   r   r   r   r   z8fuse_attention: failed to match unidirectional mask path   z-fuse_attention: skip since div_qk != div_mask)rw   Whererz   r[   )r   r   r   r   )rw   r^   r~   rz   r[   )r   r   Nr   r   z(fuse_attention: failed to match qk nodes)ry   rx   Castr;   r;   rt   )Nr   r   r   r   r   )ry   rx   r;   r;   rt   )Nr   r   r   r   )ry   rx   r;   r;   )Nr   r   r   z9fuse_attention: failed to match input attention mask path)r   r{   r{   r;   rx   r-   r{   r|   )r   r   r   r   r   r   r   r   z)fuse_attention: failed to match mask pathrz   r=      )r   r   z4fuse_attention: skip since mask shape is not 1x1xWxWFzDfuse_attention: skip since mask is neither lower triangular nor ones)r   rt   r.   )r   r   r   z&fuse_attention: failed to match q pathz.fuse_attention: skip since split_fc != split_q)r   r=   r   rt   r.   )r   r   r   r   r   z&fuse_attention: failed to match k pathz.fuse_attention: skip since split_fc != split_kz8fuse_attention: skip since concat_k != concat_k_to_match z)fuse_attention: failed to match past pathzpast is not graph input.z,fuse_attention: failed to match present pathz!expect present to be graph output)r
   r"   r!   r   r   get_constant_inputmatch_parent_pathsr   r   r   to_arrayget_initializerlenshapenpallclose	ones_liketrilrM   r,   r:   r7   rF   rD   find_graph_outputrs   r?   prune_graph)6r   normalize_noder@   r%   r'   rC   rv   	qkv_nodesadd_qkvreshape_qkvrl   	reshape_1	reshape_2transpose_qkv
matmul_qkvanother_inputv_nodesr$   transpose_v	reshape_vsplit_fcfc_nodesrj   i_rk   layernorm_before_attentionrn   
slice_maskinput_mask_nodesconcat_k_to_matchqk_nodes
softmax_qksub_qkmul_qkdiv_qk	matmul_qk
mask_nodesdiv_maskwhere_qkadd_qkdiv_or_concat	mask_dataq_nodestranspose_q	reshape_qsplit_qk_nodesr#   transpose_k	reshape_ksplit_krJ   rI   r   r   r   fuse   s  	








































zFusionGptAttention.fuse)
rN   rO   rP   rQ   r   rR   r   rs   r   rS   r   r   r   r   rT      s
    -rT   )loggingr   numpyr   fusion_baser   fusion_utilsr   onnxr   r   r   
onnx_modelr   rN   r   r	   rT   r   r   r   r   <module>   s    