o
    ;ήc                     @   sJ  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	m
Z
 d dlZd dlZd dlmZmZmZmZmZ d dlmZmZ ejejejedd d dlmZ d dlmZ e e Z!d	Z"G d
d dZ#G dd deZ$G dd deZ%eddfeddfeddfe$ddfe%ddfdZ&G dd deZ'G dd deZ(dS )    N)Path)DictListUnion)
Gpt2Helper
Gpt2InputsMyGPT2LMHeadModelMyGPT2LMHeadModel_NoPaddingMyGPT2Model)
GPT2ConfigGPT2LMHeadModelz..)
TypeHelper)torch_onnx_exportg     c                   @   s   e Zd ZedddZdS )Gpt2HelperFactorydefaultc                 C   s   t ttd}||  }|S )N)r   beam_search_stepconfigurable_one_step_search)r   Gpt2BeamSearchHelper)helper_typehelpersw r   b/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/models/gpt2/gpt2_beamsearch_helper.pycreate_helper   s   zGpt2HelperFactory.create_helperN)r   )__name__
__module____qualname__staticmethodr   r   r   r   r   r      s    r   c                       s,   e Zd ZdZ fddZ fddZ  ZS )GPT2LMHeadModel_BeamSearchStepzpHere we wrap a class for Onnx model conversion for GPT2LMHeadModel with past state and one
    step beam search.c                    s    t  | || j_|| j_d S N)super__init__config
batch_size	beam_size)selfr"   r#   r$   	__class__r   r   r!   -   s   z'GPT2LMHeadModel_BeamSearchStep.__init__c	              
      st  | | jjd|d} fddttD t j| d|d||dd}
t	|
| jj
\}}|d d df  | jjd|d}tj|dd}tj|| jjdddd\}}| }||dtj  |d	 |d
 ||d| jj |d| }| | jjd}|j| jjdddd\}}tj|| jjdd}| | jjdd|}| | jjd|d}|d|ddd|d}|d|}||tj|tjd| jj @ }tj||dgdd }| | jjd|d}|d|ddd|d}tj||dgdd }|||||| | jj| jj d| | jj| jj dfS )Nc                    s    g | ]}|  d  d qS )   r   index_select.0ibeam_select_idxpastr   r   
<listcomp>?   s     z:GPT2LMHeadModel_BeamSearchStep.forward.<locals>.<listcomp>Fposition_idsattention_maskpast_key_valuesreturn_dictdimTr9   largestsorted).r   r   truncrounding_moder)   dtype)viewr"   r#   sizerangelenr    forwardr
   post_processn_layertorchlog_softmaxtopkr$   masked_fill_	unsqueezenumpyinfeos_token_iddivgatherrepeatne	ones_likeintcat
contiguous)r%   	input_idsr4   r5   r0   input_log_probsinput_unfinished_sentsprev_step_resultsprev_step_scoresr1   resultlogits_flatpresent_flatnext_token_logitsnext_token_log_probsnext_token_idsfinished_sentsoutput_log_probsselected_index_flatselected_input_seqoutput_unfinished_sentscurrent_step_resultscurrent_step_scoresr&   r/   r   rF   2   sl   $
	

z&GPT2LMHeadModel_BeamSearchStep.forward)r   r   r   __doc__r!   rF   __classcell__r   r   r&   r   r   )   s    r   c                       sX   e Zd ZdZ								d fdd	Zed	d
 ZedddZ fddZ  Z	S ))GPT2LMHeadModel_ConfigurableOneStepSearchzHere we wrap a class for Onnx model conversion for GPT2LMHeadModel with past state and one
    step beam search with configuration support.F      ?Nr)   r   c                    s`   t  | || j_|| j_|| j_|| j_|| j_|| j_|| j_	|	| j_
|
| j_|| j_d S r   )r    r!   r"   r#   r$   
ignore_eostemperaturerepetition_penaltyexcluded_token_idslength_penalty	do_sampledo_sample_top_pdo_sample_top_k)r%   r"   r#   r$   ro   rp   rq   rr   rs   rt   ru   rv   r&   r   r   r!      s   z2GPT2LMHeadModel_ConfigurableOneStepSearch.__init__c                 C   s   | j dg|  dd  R  S )Nr(      )rB   rC   )tensorr   r   r   collapse_first_two_dims   s   zAGPT2LMHeadModel_ConfigurableOneStepSearch.collapse_first_two_dimsc                 C   s   t j| dd\}}t j| dd}||k}t jt |dddf |dddf gdd}|dkrJt j|dd|f t |d|df gdd}||t | 	d||S )	z6Set tail event (out of top_p) to a big negative numberT)
descendingr(   r8   .Nr)   r   )
rI   sortcumsumexprW   
zeros_likerU   rL   BIG_NEGscatter)	log_probstop_ptop_ksorted_log_probssorted_indicescumulative_probssorted_indices_to_remover   r   r   top_k_top_p_filtering   s$   z?GPT2LMHeadModel_ConfigurableOneStepSearch.top_k_top_p_filteringc              	      s  | | jjd|d}|d}| |d| djdd d}| jjr1|| jjk	 }	nt
|j	 |j}	|	dd jdd }
|r|d d}|d d |d f }|
d d |d f }
| d| djdd d  t fdd	|D }t j| d|d|
|	|dd
}t|| jj\}}|| d d |df }|dt
| jjg|jt  ||d||d d df  | jjdkr| d|}t
!|dk|| jj || jj }|"d|| | jj#dkr |d| jjf }t
!|dk|| jj# || jj# }||d| jjf< | jj$dkr-|| jj$ }| jj%d urA|d| jj%|jt t
j&|dd}| jj'r~|d}| j(| d|| jj)| jj*d}t
j+|, | jj-dd}| | jj|d}| d|}nt
j.|| jj-dddd\}}|d| }| | jjd}|j.| jj-dddd\}}t
j/|| jj-dd}| | jjd d|}| | jjd|d0 }| d|d1|j|df }| d|}||2| jj@ }t
j3||dgdd0 }| | jjd|d}| d|d1|j|df }t
j3||dgdd0 }| d	 dd jdd 4| d|}| | jj| jj- d||||| | jj| jj- dfS )Nr(   r)   r   F)as_tuple)minc                    s   g | ]}| d  qS )r)   r*   r-   p,unfinished_index_relative_to_last_unfinishedr   r   r2      s    zEGPT2LMHeadModel_ConfigurableOneStepSearch.forward.<locals>.<listcomp>r3   rw   rn   .r8   )r   r   )num_samplesreplacementTr:   r=   r>   )5rB   r"   r#   rC   ry   r+   nonzeroro   rP   floatrI   onesshapetodevicer|   clamplongtupler    rF   r
   rG   rH   	new_zerosindex_fill_
LongTensorr   masked_scatter_rM   	expand_asrq   rR   wherescatter_rs   rp   rr   rJ   rt   r   rv   ru   multinomialr}   r$   rK   rQ   rX   expandrT   rW   
reshape_as)r%   rY   r0   rZ   r[   r]   r1   input_num_seq_per_sampleinput_ids_unfinished_flatr5   r4   last_seq_lenr^   r_   r`   ra   _penrb   
vocab_size_next_token_log_probsrc   re   rf   rg   r\   rh   ri   rj   !index_relative_to_last_unfinishedr&   r   r   rF      s   	

"









	z1GPT2LMHeadModel_ConfigurableOneStepSearch.forward)Frn   rn   Nrn   Fr)   r   )rn   r   )
r   r   r   rk   r!   r   ry   r   rF   rl   r   r   r&   r   rm      s     
rm   logitsTF
last_state)r   GPT2LMHeadModel_NoPadding	GPT2Modelr   rm   c                       s>   e Zd Z					d	 fdd	ZdefddZdd Z  ZS )
Gpt2BeamSearchInputsNc
           
         sX   t  j||||d || _|	| _|d u r!tdt|g | _n|| _|| _	|| _
d S )N)r1   r)   )r    r!   r\   r]   rI   zerosrE   r   r0   rZ   r[   )
r%   rY   r1   r4   r5   r0   rZ   r[   r\   r]   r&   r   r   r!   u  s   
zGpt2BeamSearchInputs.__init__returnc              	   C   sD   dd | j | j| j| j| j| j| j| jfD }| jr |	| j |S )Nc                 S   s   g | ]}|d ur|qS r   r   )r-   vr   r   r   r2     s    
z0Gpt2BeamSearchInputs.to_list.<locals>.<listcomp>)
rY   r4   r5   r0   rZ   r[   r\   r]   r1   extend)r%   
input_listr   r   r   to_list  s   zGpt2BeamSearchInputs.to_listc                 C   sj   dd | j D }| jd ur| jjtjdn| j}t| j|| j|| j| j	jtjd| j
| j| jjtjd	S )Nc                 S   s   g | ]	}|j tjd qS )r@   )r   rI   float32r   r   r   r   r2     s    z0Gpt2BeamSearchInputs.to_fp32.<locals>.<listcomp>r@   )r1   r5   r   rI   r   r   rY   r4   r0   rZ   r[   r\   r]   )r%   r1   r5   r   r   r   to_fp32  s   zGpt2BeamSearchInputs.to_fp32NNNNN)r   r   r   r!   r   r   r   rl   r   r   r&   r   r   t  s    r   c                    @   s  e Zd ZdZedddejejejfdededededed	ed
edejde	de	de	dej
dej
dej
defddZe		d<dedededededededededeeee f fddZed=dd Ze		!	!d>d"d#Ze				d?d$ed%e	d&e	de	de	f
d'd(Zed@d)ed*efd+d,Ze	-	-	-	-	-dAd.d/Ze			dBd)ed0eeejf d1eeee f d*ed2e	d3e	fd4d5Ze		6	6	7				dCd8d9ZedDd:d;Zd-S )Er   zEA helper class for Gpt2 model conversion, inference and verification.FTr#   past_sequence_lengthsequence_lengthnum_attention_headshidden_size	num_layerr   r   float16has_position_idshas_attention_maskinput_ids_dtypeposition_ids_dtypeattention_mask_dtyper   c                 C   s   t j| |||||||||	|
|||d}|rtjntj}tjd| g|d }tj| dg||d}tj| dgtj|d}|	rMtj	d|d | |ftj
|d}nd}tj| dg||d}t|j|j|j|j|||||	S )z*Create random inputs for GPT2 beam search.)r   r   r   r)   )r   rA   r   r   )lowhighrC   rA   r   N)r   get_dummy_inputsrI   r   r   r   r   r   boolrandintint64r   rY   r1   r4   r5   )r#   r   r   r   r   r   r   r   r   r   r   r   r   r   gpt2_dummy_inputs
float_typer0   rZ   r[   r\   r]   r   r   r   r     sR   z%Gpt2BeamSearchHelper.get_dummy_inputsr   r   context_lenr$   stepr"   model_classnum_seqc	                 C   s\  |j }	|j}
|j}|j}t| d }|dkr| |g}n| | || | d g}|dkrS|dkr=d| |	|| t|
|	 g}n%|dkrC|}d| | |	|| t|
|	 g}nd| |	|| | t|
|	 g}||i}t|D ]
}||dt| < qjd| | g|d< | |g|d< | |g|d< |dkr| | || | d g|d	< | | || d g|d
< td| |S )zAReturns a dictionary with output name as key, and shape as value.r)   r   r   rw   present_output_selected_indicesre   rh   ri   rj   output_shapes)	r   r   num_hidden_layersr   MODEL_CLASSESrV   rD   strprint)r#   r   r   r   r$   r   r"   r   r   r   r   r   r   output_namelast_state_shapepresent_state_shaper   r.   r   r   r   get_output_shapes  s^   


	



z&Gpt2BeamSearchHelper.get_output_shapesc                 C   s   |rt jnt j}i }|  D ]?\}}|dks|dks|dkr-t jt|t j|d||< q|dkr@t jt|t j|d||< qt jt|||d||< q|S )zpReturns a dictionary of output name as key, and 1D tensor as value. The tensor has enough space for given shape.r   ri   r   r   rh   )	rI   r   r   itemsemptyrN   prodr   r   )r   r   
is_float16	data_typeoutput_buffersnamer   r   r   r   get_output_buffers@  s   z'Gpt2BeamSearchHelper.get_output_buffersMbP?c              	   C   s   |dkrd}t |d }nd}t |d }tj|| | |   ||d}td|  |}t|D ])}	tj|d|	  | d |	   ||d}td	|	 d
|	 d|  |o\|}q4|sot| |}
t	d|
d |S )zZReturns True if torch and ORT outputs are close for given thresholds, and False otherwise.r      r      )rtolatolz9PyTorch and OnnxRuntime output 0 (last_state) are close: r)   zPyTorch and OnnxRuntime layer z state (present_z) are close:z@PyTorch and OnnxRuntime results are not all close: max_abs_diff=z.5f)
rE   rN   allclosecpuloggerdebugrD   r   diff_outputsinfo)torch_outputsort_outputsr   r   r   
results_id
num_layersis_closeis_all_closelayermax_abs_diffr   r   r   compare_outputsO  s4   	

z$Gpt2BeamSearchHelper.compare_outputsonnx_model_pathverboseuse_external_data_formatc                 C   s  t | ttfs	J | j}|j}tjddd|j|j||j	|d||d}	|	
 }
t  | |
 }W d   n1 s9w   Y  dd t|D }dd t|D }d	g| }|r^|g d
7 }n|g d7 }dddd|d dddi}|D ]	}ddd||< qt|D ]	}ddd||< qdg}|rddd|d< |d |rddd|d< |d ddi|d< |d ddd|d< |d ddd|d< |d |rddd|d< |d ddd|d< |d || ddd}t | trddd|d	< nddd|d	< t|D ]}||dt| < qddi|d< ddd|d < ddd|d!< d"|v r4ddd|d"< ddi|d#< td$|	jj d%|	jd j d&|d j d'|d d j  t|jjd(d(d) t| t|
||||d*d(||d+
 dS ),z1Export GPT-2 model with past state to ONNX model.r)   rw   Fr#   r   r   r   r   r   r   r   r   r   r   Nc                 S      g | ]}d | qS )past_r   r,   r   r   r   r2         z4Gpt2BeamSearchHelper.export_onnx.<locals>.<listcomp>c                 S   r   )r   r   r,   r   r   r   r2     r   r   )r   re   rh   ri   rj   )r   re   rh   rj   rY   r#   seq_len)r   r)   r   past_seq_len)r)      cur_seq_lenr4   total_seq_lenr5   r0   r$   rZ   r[   r\   r]   zbatch_size * beam_sizer   r   re   rh   ri   rj   zShapes: input_ids=z past=z output=z	 present=T)parentsexist_ok   )	argsfinput_namesoutput_namesdynamic_axesopset_versiondo_constant_foldingr   r   )
isinstancer   rm   r"   rH   r   r   r   r   r   r   rI   no_gradrD   appendr   r   r   r   rY   r   r1   r   parentmkdirr   r   )modelr   r   r   r   r   r   r"   r   dummy_inputsr   outputs
past_namespresent_namesr  r  r   r  present_axesr.   r   r   r   export_onnxx  s   
















6
z Gpt2BeamSearchHelper.export_onnxinputs
total_runsc                 C   s  t d dt|j  i}|jdur"t|j  |d< |jdur3t|j  |d< |jdurDt|j  |d< |j	durUt|j	  |d< |j
durft|j
  |d< |jdurwt|j  |d	< |jdurt|j  |d
< |jdurt|jD ]\}}t|  |d| < q| d|}|dkr|S g }t|D ]}t }	| d|}|t |	  qt|d t| }
t dt|
d ||
fS )zcRun inference of ONNX model, and returns average latency in ms when total_runs > 0 besides outputs.zstart onnxruntime_inferencerY   Nr4   r5   r0   rZ   r[   r\   r]   r   r     z"OnnxRuntime Inference time = {} ms.2f)r   r   rN   ascontiguousarrayrY   r   r4   r5   r0   rZ   r[   r\   r]   r1   	enumeraterunrD   timer  sumrE   format)ort_sessionr  r  
ort_inputsr.   past_ir   latency_startaverage_latencyr   r   r   onnxruntime_inference  s<   








z*Gpt2BeamSearchHelper.onnxruntime_inferenceNc              
   C   s   t j| ||||||d}|||	|
|d}t| }| D ]"\}}|dur?| s+J |||jjd|| t	|
 |  q|S )z)Returnas IO binding object for a session.)r1   r   r   )r0   rZ   r[   r\   r]   Nr   )r   prepare_io_bindingr   get_io_numpy_type_mapr   is_contiguous
bind_inputr   typelistrC   data_ptr)r&  rY   r4   r5   r1   r   r   r0   rZ   r[   r\   r]   
io_bindingother_inputsname_to_np_typer   rx   r   r   r   r.  #  s:   

	z'Gpt2BeamSearchHelper.prepare_io_bindingr   r   return_numpyinclude_copy_output_latencyc                 C   s   t d t| |j|j|j|j|||j|j	|j
|j|j}| | t| |||}|dkr1|S g }	t|D ]}
t }| | |rLt| |||}
|	t |  q7t|	d t|	 }t dt|d ||fS )zUInference with IO binding. Returns outputs, and optional latency when total_runs > 0.z*start onnxruntime_inference_with_binded_ior   r  z2OnnxRuntime with IO binding inference time = {} msr  )r   r   r   r.  rY   r4   r5   r1   r0   rZ   r[   r\   r]   run_with_iobinding"get_outputs_from_io_binding_bufferrD   r#  r  r$  rE   r%  )r&  r  r   r   r  r8  r9  r5  r   r)  r*  r+  r,  r   r   r   $onnxruntime_inference_with_binded_ioV  sB   


z9Gpt2BeamSearchHelper.onnxruntime_inference_with_binded_ioMb@?d   c                 C   s  |j }td| d| d| d| d| d| d d}d	}d
}d	}d}|r:t|||||d||}t|||}d}t|D ]j}td|}td| || }td|}t	d| d| d t
||||j|j|j|j|||	|
}t||}|rt| |}nt|||||d||}t| |||}tj|||||d}|r|d7 }q@td| d|  |d| krtdt|d | dd ||kS )zKGenerate random inputs and compare the results of PyTorch and Onnx Runtime.zRunning parity test (rtol=z, atol=z, test_cases=z, use_io_binding=z model_class=z is_float16=z) ...r)      rw   Nr   z#Running parity test for batch_size=z past_sequence_length=z...)r   r   r   zParity Test Cases=z	; Passed=gffffff?zParity is good: passed rate=r>  z.0f%)r"   r   r   r   r   r   rD   randomr   r   r   r   r   rH   r   pytorch_inferencer-  r<  r   rV   )r&  r  r   r   r   r   total_test_casesuse_io_bindingr   r   r   r"   max_batch_sizemax_past_seq_lenmax_seq_lenr$   r   max_output_shapespassed_test_casesr*  r   r   r#   r  r  r   r   r   r   r   r   test_parity  s   (


 z Gpt2BeamSearchHelper.test_parityc                 C   s:   t jddd|j|j|j|j|d||d }tj	| |S )zJIT trace for TorchScript.r)   Fr   )
r   r   r   r   rH   r   r   rI   jittrace)r  r"   r   r   r   r   r   r   r   torchscript  s    z Gpt2BeamSearchHelper.torchscript)r   r   )F)r   r   r   )FFTT)r   r   )r   TF)Fr=  r=  r>  Tr   TT)TT)r   r   r   rk   r   rI   r   rV   r   r   rA   r   r   r   r   r   r   r   r   r   r  r-  r.  Tensorr<  rJ  rM  r   r   r   r   r     s    
	
?		
L( '	27[r   ))loggingosrA  sysr#  pathlibr   typingr   r   r   rN   rI   gpt2_helperr   r   r   r	   r
   transformersr   r   pathr  joindirname__file__io_binding_helperr   torch_onnx_export_helperr   	getLoggerr   r   r   r   r   rm   r   r   r   r   r   r   r   <module>   sD    
\ `<