o
    ;ήcmt                     @   sn  d Z ddlZddlZddlZddlZddlmZ ddlmZ ddlZddl	Z	ddl
Z
ddlmZmZmZmZmZmZmZmZmZmZmZmZ ddlmZ ddlmZmZmZmZ ddlm Z  e!d	Z"dd
l#m$Z$m%Z% e
j&ddZ&dej'vr}e(e&ej'd< ddl)Z)ddl*m+Z+m,Z,m-Z-m.Z.m/Z/ dd Z0dd Z1de2de2fddZ3dd Z4dd Z5dd Z6e7dkre6  dS dS )a   Benchmarking the inference of pretrained transformer models.
    PyTorch/TorchScript benchmark is based on https://github.com/huggingface/transformers/blob/master/examples/benchmarks.py.
    One difference is that random input_ids is generated in this benchmark.

    For onnxruntime, this script will convert a pretrained model to ONNX, and optimize it when -o parameter is used.

    Example commands:
        Export all models to ONNX, optimize and validate them:
            python benchmark.py -b 0 -o -v -i 1 2 3
        Run OnnxRuntime on GPU for all models:
            python benchmark.py -g
        Run OnnxRuntime on GPU for all models with fp32 optimization:
            python benchmark.py -g -o
        Run OnnxRuntime on GPU with fp16 optimization:
            python benchmark.py -g -o -p "fp16"
        Run TorchScript on GPU for all models:
            python benchmark.py -e torchscript -g
        Run TorchScript on GPU for all models with fp16:
            python benchmark.py -e torchscript -g -p "fp16"
        Run ONNXRuntime and TorchScript on CPU for all models with quantization:
            python benchmark.py -e torchscript onnxruntime -p "int8" -o
        Run OnnxRuntime with the ROCM provider and graph optimization script:
            python benchmark.py -g -m bert-base-cased --provider rocm --optimizer_info by_script --disable_embed_layer_norm

    It is recommended to use run_benchmark.sh to launch benchmark.
    N)datetime)Enum)ConfigModifierOptimizerInfo	PrecisionallocateOutputBufferscreate_onnxruntime_sessionget_latency_resultinference_ortinference_ort_with_io_bindingoutput_detailsoutput_fusion_statisticsoutput_summarysetup_logger)FusionOptions)create_onnxruntime_inputexport_onnx_model_from_ptexport_onnx_model_from_tfload_pretrained_model)QuantizeHelper )MODEL_CLASSESMODELSF)logicalOMP_NUM_THREADS)
AutoConfig	AutoModelAutoTokenizer	GPT2ModelLxmertConfigc           3      C   s  dd l }g }| rd| vrd| vrtd |S d}|dkr3tj}d}d| vr3td |S |tjkrAtd	| d
 |D ]}t| d }|
D ]z}|t|krZ np|d | }t| d |_	t
|}d|v rt , t|t| d t| d t| d |||||| |||||||\}}} }!W d    n1 sw   Y  d|v rt|t| d t| d t| d |||||| |||||||\}}} }!|sqNt|| |d||d}"|"d u rqNdd |" D }#g }$| rdnd}%tj||d}&tt|t|t| |&jg}'tt||&jg}(|D ]})|)dkrq|D ]}*|!d ur/|*|!kr/q!d|v r7tjntj}+t| |)|*||&|+},d|j||%||| ||||)|*| tt d}-t d!||)|*g |rtt"|"|,|-|	|)|}.nG|"#|#|,}/|'g}0t$t|/D ]}1|1dkrt| d dkr|0%|( q|0%|' qd|v rtj&ntj'}2t(|"|,|-|	|#|/|$|0|)|%|2|}.t |. |%|. q!qqNqC|S )Nr   CUDAExecutionProviderROCMExecutionProviderzvPlease install onnxruntime-gpu package instead of onnxruntime, and use a machine with GPU for testing gpu performance.tensorrt   TensorrtExecutionProviderzhPlease install onnxruntime-gpu-tensorrt package, and use a machine with GPU for testing gpu performance.zOptimizerInfo is set to zA, graph optimizations specified in FusionOptions are not applied.   pt      tfT)enable_all_optimizationnum_threadsverbosec                 S   s   g | ]}|j qS  )name).0node_argr-   r-   I/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/benchmark.py
<listcomp>   s    z#run_onnxruntime.<locals>.<listcomp>cudacpu	cache_dironnxruntimeengineversion	providersdevice	optimizer	precision
io_binding
model_nameinputsthreads
batch_sizesequence_lengthcustom_layer_numr   z)Run onnxruntime on {} with input shape {}gpt))r7   get_available_providersloggererrorr   NOOPTwarningr   len
model_typer   parsetorchno_gradr   r   r   get_outputsr   from_pretrainednumpyprodmaxhidden_sizeint64int32r   __version__get_layer_numstrr   nowinfoformatr
   runrangeappendlonglongintcr   )3use_gpuprovidermodel_namesmodel_classconfig_modifierr>   r+   batch_sizessequence_lengthsrepeat_timesinput_countsoptimizer_infovalidate_onnxr6   onnx_dirr,   	overwritedisable_ort_io_bindinguse_raw_attention_maskmodel_fusion_statisticsmodel_sourceargsr7   resultswarm_up_repeatr@   all_input_names
num_inputsinput_namesfusion_optionsonnx_model_fileis_valid_onnx_model
vocab_sizemax_sequence_lengthort_sessionort_output_namesoutput_buffersr<   configmax_last_state_sizemax_pooler_sizerC   rD   input_value_type
ort_inputsresult_templateresultort_outputsoutput_buffer_max_sizesi	data_typer-   r-   r1   run_onnxruntimeZ   sX  















	


Kr   c                    s.  g }| rt j std |S t d |D ]}tj||	|
d}|| t	|||
|d}t
j||
d}||jv r?|j| nd}td|  td|   |tjkr\|  t | rbd	nd
}|| |tjkrtt|}|D ]}|dkr}qv|D ]}|d ur||krqtd|||g t jd|jd ||ft j|dzP|	rt j|n|   tj fdd|dd}|	rdndt jd| rdnd
d|d|d||||  t!t"# d}|$t%|| t| |&| W q t'y } zt(| t j)  W Y d }~qd }~ww qvq|S )NzYPlease install PyTorch with Cuda, and use a machine with GPU for testing gpu performance.F)torchscriptr6   )r   r6   custom_model_classr5      zModel zNumber of parameters zcuda:0r4   r   z%Run PyTorch on {} with input shape {}r'   )lowhighsizedtyper<   c                      s    S Nr-   r-   	inference	input_idsr-   r1   <lambda>s  s    zrun_pytorch.<locals>.<lambda>repeatnumberr   rO   NAr3   r   r8   )*rO   r3   is_availablerH   rI   set_grad_enabledr   rR   modifyr   r   max_model_input_sizesdebugnum_parametersr   FLOAT16halfr<   toINT8r   quantize_torch_modelr]   r^   randintr~   longjittracetimeitr   rY   rZ   r[   r   r\   updater	   ra   RuntimeError	exceptionempty_cache)rd   rf   rg   rh   r>   r+   ri   rj   rk   r   r6   r,   rv   r@   r   model	tokenizermax_input_sizer<   rC   rD   runtimesr   er-   r   r1   run_pytorch/  s   











-r   do_eager_modeuse_xlac                    s*   ddl m dd l fdd}|S )Nr   )wrapsc                    sT     fdd} j d fdd}du r(du s&J d|S |S )	Nc                         | i |S r   r-   ru   kwargsfuncr-   r1   run_in_eager_mode     zFrun_with_tf_optimizations.<locals>.run_func.<locals>.run_in_eager_mode)experimental_compilec                     r   r   r-   r   r   r-   r1   run_in_graph_mode  s   zFrun_with_tf_optimizations.<locals>.run_func.<locals>.run_in_graph_modeTFzcCannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.)function)r   r   r   r   r)   r   r   r   r1   run_func  s   
z+run_with_tf_optimizations.<locals>.run_func)	functoolsr   
tensorflow)r   r   r   r-   r   r1   run_with_tf_optimizations  s   r   c                    s  g }dd l jj| | sjg d | r$j s$td |S | r`j	d}zj|d d jj
|d d jjdd W n ty_ } zt| W Y d }~nd }~ww |tjksj|tjkrntd|D ]}tj||	d |  t| |	|dd	tj||	d}||jv r|j| nd
}|D ]}|dkrq|D ]}|d ur||krqtd|||g dd l}|  fddt|| D }j|||fj dzzt!dddfdd}t!dddfdd}t!ddd fdd}| j"r|nt# t$r|  t%j&fdd|dd}dj'd| r0dndd|d|d||||( t)t*+ d }|,t-|| t| |.| W q ty} } zt| dd!l/m0} |1 }|2  W Y d }~qd }~ww qqp|S )"Nr   GPUzVPlease install Tensorflow-gpu, and use a machine with GPU for testing gpu performance.Tz/gpu:0)r<   z+Mixed precision is currently not supported.r5   )r   r6   r   is_tf_modelr   z(Run Tensorflow on {} with input shape {}c                    s   g | ]} d  jd qS )r   r'   )r   r~   )r/   r   )r   rngr-   r1   r2     s    z"run_tensorflow.<locals>.<listcomp>)shaper   F)r   r   c                      s    ddS )NF)trainingr-   r-   r   r   r-   r1   encoder_forward  s   z'run_tensorflow.<locals>.encoder_forwardc                      s     ddS )NF)decoder_input_idsr   r-   r-   r   r-   r1   encoder_decoder_forward  r   z/run_tensorflow.<locals>.encoder_decoder_forwardc                     s8   j dd jg} j dd jg}| |ddS )Nr'   F)visual_feats
visual_posr   )randomnormalvisual_feat_dimvisual_pos_dim)featspos)r   r   r   r)   r-   r1   lxmert_forward  s   z&run_tensorflow.<locals>.lxmert_forwardc                      s     S r   r-   r-   )r   r-   r1   r     s    z run_tensorflow.<locals>.<lambda>r'   r   r   r   r3   r4   r   r8   )r3   )3r   r   	threading set_intra_op_parallelism_threadsset_visible_devicestestis_built_with_cudarH   rI   list_physical_devicesexperimentalset_memory_growth
distributeOneDeviceStrategyr   r   r   r   r   NotImplementedErrorr   rR   r   r   r   r   r]   r^   r   Randomr`   constantrX   r   is_encoder_decoder
isinstancer   r   r   rY   rZ   r[   r   r\   r   r	   ra   numbar3   get_current_devicereset)rd   rf   rg   rh   r>   r+   ri   rj   rk   r6   r,   rv   physical_devicesr   r@   r   r   rC   rD   r   valuesr   r   r   r   r   r3   r<   r-   )r   r   r   r   r   r)   r1   run_tensorflow  s   









Kr   c                  C   st  t  } | jddddtg dtt ddt  d | jd	dd
tdddgdd | jddtd ttddt d | jddddtdgg ddd | jdddtt	j
dddd | jddtt	j
dddd | jdd dd!d"d# | jd$dtd d%d | jd&d'ttjttd(d) | jd*dd!d+d# | jd,dd!d-d# | jd.d/ttjttd0d) | jd1d2dd!d3d# | jd4d5dd d6d7 | jd8d9dd d:d7 | jd;d<dd d=d7 | jd>d?ddd
gtg d@dAdB | jdCdDddEtdFdG | jdHdIdtd
gdJ | jdKdLdtg dMdJ | jdNdd!dOd# | jddP | jdQdRddtdSgdTdU | jdVdtd dWd t|  |  }|S )XNz-mz--modelsF+)zbert-base-casedzroberta-basegpt2z Pre-trained models in the list: z, )requirednargstypedefaultchoiceshelpz--model_sourcer'   r&   r)   zExport onnx from pt or tfz--model_classz!Model type selected in the list: )r   r   r   r   r   z-ez	--enginesr7   )r7   rO   r   r   zEngines to benchmarkz-cz--cache_dir.cache_modelsz%Directory to cache pre-trained models)r   r   r   r   z
--onnx_dironnx_modelszDirectory to store onnx modelsz-gz	--use_gpu
store_truezRun on gpu device)r   actionr   z
--providerzExecution provider to usez-pz--precisionzfPrecision of model to run. fp32 for full precision, fp16 for half precision, and int8 for quantization)r   r   r   r   z	--verbosezPrint more informationz--overwritezOverwrite existing modelsz-oz--optimizer_infozjOptimizer info: Use optimizer.py to optimize onnx model as default. Can also choose from by_ort and no_optz-vz--validate_onnxzValidate ONNX modelz-fz--fusion_csvz:CSV file for saving summary results of graph optimization.)r   r   r   z-dz--detail_csvz#CSV file for saving detail results.z-rz--result_csvz$CSV file for saving summary results.z-iz--input_counts)r'   r(   r%   zXNumber of ONNX model inputs. Please use 1 for fair comparison with Torch or TorchScript.)r   r   r   r   r   r   z-tz--test_timesd   z8Number of repeat times to get average inference latency.)r   r   r   r   z-bz--batch_sizes)r   r   r   z-sz--sequence_lengths)             @         z--disable_ort_io_bindingz=Disable running ONNX Runtime with binded inputs and outputs. )rq   z-nz--num_threadsr   zThreads to use)r   r   r   r   r   z--force_num_layersz%Manually set the model's layer number)argparseArgumentParseradd_argumentr[   listr   keysjoinr   ospathr   FLOAT32r   BYSCRIPTintset_defaultsr   add_arguments
parse_args)parserru   r-   r-   r1   parse_arguments/  sH  

					

r  c                  C   s  t  } t| j | jtjkr| jstd d S | jtj	kr(| jr(td d S t
tdd | jD | _td|   tj| js^zt| j W n ty]   td| j  Y nw d| jv }d| jv }d	| jv }d
| jv }t| j}g }| jD ]}t| ttj  |s|r| jdgkrtd |r|t| j| j| j || j|| j!| j"| j#d| j| j7 }|r|t| j| j| j || j|| j!| j"| j#d| j| j7 }|r|t$| j| j| j || j|| j!| j"| j#| j| j7 }i }|r1z2| j% }	|t&| j| j'| j| j || j|| j!| j"| j#| j| j(| j)| j| j*| j| j+| j,|	|| j-| 7 }W q|   tjddd Y q|q|t./ 0d}
|rK| j1pEd|
 d}t2|| t3|dkr`| j!dgkr^td d S | j4pid|
 d}t5|| | j6pxd|
 d}t7|||  d S )Nzfp16 is for GPU onlyzint8 is for CPU onlyc                 s   s     | ]}|d kr
t n|V  qdS )r   N)	cpu_count)r/   xr-   r-   r1   	<genexpr>  s    zmain.<locals>.<genexpr>zArguments: z#Creation of the directory %s failedrO   r   r7   r   r'   zB--input_counts is not implemented for torch or torchscript engine.TF	Exception)exc_infoz%Y%m%d-%H%M%Sbenchmark_fusion_z.csvr   zNo any result avaiable.benchmark_detail_benchmark_summary_)8r  r   r,   r>   r   r   rd   rH   rI   r   sortedsetr+   r]   r  r  existsr6   mkdirOSErrorenginesr   force_num_layersrO   set_num_threadsr   
__config__parallel_inforl   rK   r   modelsrg   ri   rj   
test_timesr   use_mask_indexr   re   rm   rn   ro   rp   rq   rt   r   r\   strftime
fusion_csvr   rL   
detail_csvr   
result_csvr   )ru   enable_torchenable_torchscriptenable_onnxruntimeenable_tensorflowrh   rv   r+   rs   rr   
time_stampcsv_filenamer-   r-   r1   main  s   














r6  __main__)8__doc__r  loggingr  r   r   enumr   rS   onnxpsutilbenchmark_helperr   r   r   r   r   r	   r
   r   r   r   r   r   r{   r   onnx_exporterr   r   r   r   quantize_helperr   	getLoggerrH   huggingface_modelsr   r   r  environr[   rO   transformersr   r   r   r   r   r   r   boolr   r   r  r6  __name__r-   r-   r-   r1   <module>   sD   8

 V`  A 
