o
    ;ήc+                     @   s|   d dl mZ d dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZmZmZ d dlmZ eeZdd	 ZG d
d deZdS )    )	getLoggerN)Fusion)FusionGptAttentionPastBase)FusionUtils)TensorProtohelpernumpy_helper)	OnnxModelc                 C   s   t | | dkS )Ngư>)abs)valueexpected_value r   ]/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/fusion_gpt_attention_megatron.pyis_close   s   r   c                       sB   e Zd ZdZdedef fddZdd Zdd	 Zd
d Z	  Z
S )FusionGptAttentionMegatronz^
    Fuse GPT-2 Attention with past state subgraph from Megatron into one Attention node.
    model	num_headsc                    s   t  || d S )N)super__init__)selfr   r   	__class__r   r   r      s   z#FusionGptAttentionMegatron.__init__c                 C   s   | j d}| |}	|jd }
|jd |jd krdnd}tjd||jd |j| |	|g|
|g|d}d|_|j	t
d| jt
ddg |g}| j	| |D ]	}| j| j|j< qS| j| d	| _d S )
NGptAttentionr      	Attention)inputsoutputsnamezcom.microsoftr   unidirectionalT)r   create_node_namecast_attention_maskoutputinputr   	make_nodedomain	attributeextendmake_attributer   nodes_to_addthis_graph_namenode_name_to_graph_namer   nodes_to_removeappendprune_graph)r   matmul_before_splitadd_before_splitpastpresentr"   reshape_qkvmaskattention_node_name
int32_maskr!   iattention_noder(   noder   r   r   fuse_attention_node   s6   




z.FusionGptAttentionMegatron.fuse_attention_nodec                 C   sP  | j |g dg d}|d u rtd d S |\}}}}	|jd |jd kr.td d S | j|dds=td d S | j|dd	sLtd
 d S | j |	jd s\t	d d S | j|ddgsltd d S | j|ddgs|td dS | j|ddgstd dS | j|	ddgstd d S | j|	ddgstd d S | j |g dg d}
|
d u s|
d |krtd d S | j |	g dg d}|d u s|d |krtd d S | j |	g dg d}|d u s|d |krtd d S | j |	g dg d}|d u s|d |kr#td d S |	jd S ) N)MulSubSlicer<   )r   r   r   r   z8fuse_attention: failed to match unidirectional mask pathr   r   zCfuse_attention failed: mul_qk.input[1] != last_slice_mask.output[0]g     @z?fuse_attention failed: mul_mask input 1 is not constant 10000.0g      ?z;fuse_attention failed: sub_mask input 0 is not constant 1.0z+expect slick_mask input 0 to be graph inputzKfuse_attention failed: last_slice_mask input 1 (starts) is not constant [0]   zIfuse_attention failed: last_slice_mask input 3 (axes) is not constant [3]F   zJfuse_attention failed: last_slice_mask input 4 (steps) is not constant [1]   zDfuse_attention failed: slice_mask input 3 (axes) is not constant [2]zEfuse_attention failed: slice_mask input 4 (steps) is not constant [1])	UnsqueezeGatherShapeMatMul)r?   r   r   r   z/fuse_attention: failed to match last slice pathz0fuse_attention: failed to match first slice path)r@   r;   rA   rB   rC   )r   r   r   r   r   z3fuse_attention: failed to match last slice sub path)r@   r;   rA   rB   LayerNormalization)r   r   r   r   r   z5fuse_attention: failed to match last slice sub path 1)
r   match_parent_pathloggerdebugr"   r!   utilscheck_node_input_valuefind_graph_inputinfo)r   sub_qkmul_qk	matmul_qklayernorm_before_attention
mask_nodesmul_masksub_masklast_slice_mask
slice_masklast_slice_pathfirst_slice_pathfirst_slice_subfirst_slice_sub_1r   r   r   
match_maskJ   s~   














z%FusionGptAttentionMegatron.match_maskc           -   	   C   s  d }d }| j j|g dg d|d}|d u rd S |\}}}	}
}}|jd }| j |g dg d}|d u r<td d S |\}}}}}}}||jd krStd d S | j |g d	g d
}|d u rjtd d S |\}}}}| j |ddkrtd d S | ||||}| j |g dg d
}|d u rtd d S |\}}}} || krtd d S | j |g dg d}!|!d u rtd d S |!\}"}#}$}%}&}'||'krtd d S | j |&\}(})t|)t	j
rt|)jdgkr|)d dkr|)d dkr|)d dkr|)d dkstd d S |)d }*|*| jkr.td|* d| j  |*| _|)d }+| j |"\}(})tt	t	|+},t|)|,sWtd|) d|,  d S | j |\}(})t|)|,srtd|) d|,  d S | |$||}|d u rtd d S | j |std  | ||}|d u rtd! d S | j |std" d S | |||||jd |
| d S )#N)Addr[   rC   Reshape	TransposerC   )r   r   Nr   r   r   )output_name_to_noder   )Concatr]   r\   Splitr[   rC   rE   )r   r   r   r   r   Nr   z&fuse_attention: failed to match v pathzAfuse_attention: skip_input != layernorm_before_attention.input[0])Softmaxr;   r:   rC   )r   r   r   r   z'fuse_attention: failed to match qk pathaxisr=   z+fuse_attention failed: softmax_qk axis != 3)Divr]   r\   r`   z&fuse_attention: failed to match q pathz-fuse_attention: skip since split_v != split_q)rc   r]   r_   r]   r\   r`   )r   r   r   r   r   r   z&fuse_attention: failed to match k pathz-fuse_attention: skip since split_v != split_kr>   r   r?   z:fuse_attention: reshape constant input is not [0, 0, N, H]zDetected num_heads=z. Ignore user specified value zfuse_attention: div_k value=z
 expected=zfuse_attention: div_q value=z!fuse_attention: match past failedz(fuse_attention: past is not graph input.z$fuse_attention: match present failedz1fuse_attention: expect present to be graph output)r   rF   r"   rG   rH   get_node_attributerZ   get_constant_input
isinstancenpndarraylistshaper   rL   floatsqrtr   match_past_pattern_2rK   match_presentfind_graph_outputr9   r!   )-r   normalize_nodeinput_name_to_nodesr^   r0   r1   	qkv_nodesadd_skipadd_after_attentionmatmul_after_attentionr2   transpose_qkv
matmul_qkv
skip_inputv_nodesconcat_vtranspose_v	reshape_vsplit_vr/   r.   rP   qk_nodes
softmax_qkrM   rN   rO   attention_maskq_nodesdiv_qtranspose_q	reshape_qsplit_qk_nodesdiv_k_concat_ktranspose_k	reshape_ksplit_kr6   r   r   hidden_size_per_headr   r   r   r   fuse   s   
	
	


















zFusionGptAttentionMegatron.fuse)__name__
__module____qualname____doc__r	   intr   r9   rZ   r   __classcell__r   r   r   r   r      s    -Or   )loggingr   numpyrg   fusion_baser   fusion_gpt_attentionr   fusion_utilsr   onnxr   r   r   
onnx_modelr	   r   rG   r   r   r   r   r   r   <module>   s   