o
    ;ήcA                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZmZ d dl	m
Z
mZ ejejejedd d dlmZ e eZG dd dZG dd	 d	eZdS )
    N)Gpt2BeamSearchHelperGpt2BeamSearchInputs)
Gpt2Metric
Gpt2Testerz..)	Precisionc                   @   s   e Zd ZedddZdS )Gpt2TesterFactorydefaultc                 C   s   t ttd}||  }|S )N)r   beam_search_stepconfigurable_one_step_search)r   Gpt2BeamSearchTester)tester_typetestersw r   b/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/models/gpt2/gpt2_beamsearch_tester.pycreate_tester   s   zGpt2TesterFactory.create_testerN)r   )__name__
__module____qualname__staticmethodr   r   r   r   r   r      s    r   c                
       sn   e Zd Z			d fdd	ZdefddZdd	 Zeej	d
dddddddf	ddZ
e		dddZ  ZS )r   F   c                    s   t  j||||	|
|||||d
 |jd | _|| _|| _||| _|r'tj	ntj
}|||| _||| _|d urB||nd | _|||| _d | _d S )N)num_attention_headshidden_size	num_layerdeviceis_fp16top_ktop_k_required_order)super__init__shapeinput_lengthn_layer	beam_sizetobeam_select_idxtorchfloat16float32typeinput_log_probsinput_unfinished_sentsprev_step_resultsprev_step_scores
last_state)self	input_idsposition_idsattention_maskr&   r+   r,   r-   r.   r   r   r   r$   r   r   r   r   
float_type	__class__r   r   r    %   s,   
zGpt2BeamSearchTester.__init__returnc              
   C   s*   t | j| j| j| j| j| j| j| j| j		S )N)
r   r1   pastr2   r3   r&   r+   r,   r-   r.   )r0   r   r   r   
get_inputsS   s   zGpt2BeamSearchTester.get_inputsc                 C   s  t |d tjrt|d |n	|d    | _	| j	
| j| j d|| _| jdurd}t |d tjrEt|d |n
|d   || _t| j| d gd| j| j d|| _| jd| j| j kr| j| j| j d| _t| jt| j| j dg| jgd|| _nd}t ||d  tjrt||d  |n||d    || _t ||d  tjrt||d  |n||d    || _t || tjrt|| |n
||   || _t |d tjrt|d |n
|d   || _| jd | _| j	| _g | _t |d t rCt!|d | _dS t"| j#D ]*}t ||d  tjr^t||d  n	||d    }| j$|| qHdS )z7
        Update the inputs for next inference.
        r   r   N      )%
isinstancenumpyndarrayr'   
from_numpyr%   clonedetachcpur/   view
batch_sizer$   r1   r2   r-   tensorr"   	unsqueezerepeatr3   sizecatonestype_asr&   r+   r,   r.   top_1_tokenstop_k_tokensr8   tuplelistranger#   append)r0   outputstepr   input_unfinished_sents_idipast_ir   r   r   update`   sn   
zGpt2BeamSearchTester.updateGPT2LMHeadModel_BeamSearchStepT   r   .c           6      C   sJ  t d| d| d| dt| d|	 d |jj}|jj}|jj}|jj}|jj}d}|tj	k}|
 | tjdd	d	d
dd|j|d}tj|||d}d}|tjkrWdnd}t|||}t|||}t|d ||}t|D ]\}}|	dkr||	kr n|d dkrt |  |d }d|v r|d nd}d|v r|d nd}d|v r|d nd} d|v r|d nd}!|d }"|dkr|d }#nd}#d|v r|d }$n
t|jd dg}$t|||| |!|"|#|$|||||||| }%t|||| |!|"|#|$|||||||| }&t|||| |!|"|#|$|||||d|| }'|'j}(||( ||( t|%j d })t  t|D ]}*t d|*  |dkrZ|}+t|%j d },t|%jd  d }-n"t|&j d! " }+t|%jd  d }-t|%j d |- },t#$ }.t%||'& }/|'|-t#$ |.  |'(|/|*| tj)| |%& dd \}0}1|'|-|1d!  |%(|0|*| |dkr|}+nt|&j d! " }+tj|(|)|-|,||*|j||+d"	}2t*||2 tj+| |&& ||2ddd#d$\}3}1|'|-|1d!  ||k r|&,| |3|| |d7 }|&(|3|*| |%j- r|'j- s t d%|*  nq5W d   n	1 s-w   Y  t d&|*d  d' |.  |.  qq|   |   |   t d( |dkrm|%j/ |(| d}4|&j/ |(| d}5n|%j |(| d}4|&j |(| d}5tj0|4|%j1 |(| d||d) t d* tj0|5|&j1 |(| d||d) dS )+z
        Test Generation using beam search to compare PyTorch and ONNX model.
        It will print top 1 and top k errors on the given test inputs.
        zstart test generation: (top_k=z top_k_no_order=z max_steps=z test_inputs=z max_inputs=)r             r<   )rF   context_lenpast_sequence_lengthsequence_lengthr$   rU   configmodel_class)
is_float16TorchzQuantized OnnxOnnxz with IO Binding
   r1   r2   Nr3   r&   r+   r,   rZ   r.   Fr   zProcessing step:    )
total_runsg     @@)re   num_seqT)rk   return_numpyinclude_copy_output_latencyzbreak at step: zTotally z
 steps runz	ONNX)pad_token_ideos_token_idz	ONNX with IO binding)2printlenrd   r#   n_headn_embdr$   rp   r   FLOAT16evalr%   r   get_output_shapesget_output_buffersINT8r   	enumerater'   zerosr!   r   rF   start_batchrQ   r1   rJ   no_gradrR   r8   sumr,   rE   longrD   timeitdefault_timerpytorch_inferencer9   add_latencyrY   onnxruntime_inferenceauto_increase_buffer_size$onnxruntime_inference_with_binded_iosave_test_dataany	end_batchr-   pprint_resultsr.   )6sessionmodelr   test_inputs	precisionre   r   top_k_no_order	max_steps
max_inputsverboser   save_test_data_dirr#   rs   rt   r$   rp   test_data_savedrf   init_output_shapesoutput_buffersbaseline_nametreatment_nametorch_metriconnx_metriconnx_io_metricrW   inputsr1   r2   r3   r&   r+   r,   r-   r.   onnx_runneronnx_io_runnertorch_runnerrF   ra   rU   rl   seq_lenpast_seq_len
start_timepytorch_outputonnx_outputavg_latency_msoutput_shapesonnx_io_outputresults_onnxresults_onnx_ior   r   r   test_generation   sx  &













C


z$Gpt2BeamSearchTester.test_generationNc                 C   sD  |du rd}|du rd}t | r|   } t| D ]\}}t|D ]w\}}t|tjs2t|trzt|D ]\}}	|	|krF||d } nq6t|D ]\}}	|	|kr]|d|d  } nqKtd d	dd |D }
td|d  d|
g  q#d	d	d |D }
td| d|
  td
| d||    td qdS )z0
        Print test generation results.
        Nr<   z(----------------------------------------,c                 S      g | ]}t |qS r   str.0token_idr   r   r   
<listcomp>      z7Gpt2BeamSearchTester.pprint_results.<locals>.<listcomp>z
>> Output z: 	c                 S   r   r   r   r   r   r   r   r     r   z
>> Scores zP================================================================================)
r'   	is_tensorrD   r?   rz   r>   r@   rQ   rq   join)
output_idsoutput_scoresro   rp   rW   samplejseqktresultr   r   r   r     s8   


z#Gpt2BeamSearchTester.pprint_results)Fr   F)NN)r   r   r   r    r   r9   rY   r   r   FLOAT32r   r   __classcell__r   r   r5   r   r   $   s.    .J jr   )loggingossysr   r?   r'   gpt2_beamsearch_helperr   r   gpt2_testerr   r   pathrS   r   dirname__file__benchmark_helperr   	getLoggerr   loggerr   r   r   r   r   r   <module>   s    
