o
    ;ήcW                     @   s  d Z ddlZddlZddlZddlZddlZddlmZ ddlZddl	Z	ddl
mZmZ ddlmZ ddlmZmZ ddlmZ ddlmZ ejejejed	d	 dd
lmZmZmZmZ ddl m!Z! e"dZ#dddZ$de%de&fddZ'dd Z(dddZ)e*dkre)  dS dS )a  
This converts GPT2 model to onnx. Examples:
(1) Convert pretrained model 'gpt2' to ONNX
   python convert_to_onnx.py -m gpt2 --output gpt2.onnx
(2) Convert pretrained model 'distilgpt2' to ONNX, and use optimizer to get float16 model.
   python convert_to_onnx.py -m distilgpt2 --output distilgpt2_fp16.onnx -o -p fp16
(3) Convert a model check point to ONNX, and run optimization and int8 quantization
   python convert_to_onnx.py -m ./my_model_checkpoint/ --output my_model_int8.onnx -o -p int8

    N)Path)MODEL_CLASSESGpt2HelperFactory)Gpt2TesterFactory)DEFAULT_TOLERANCEPRETRAINED_GPT2_MODELS)version)
AutoConfigz..)	Precisioncreate_onnxruntime_sessionprepare_environmentsetup_logger)QuantizeHelper c                 C   s  t  }|jdddtddt d |jddtd	tt d
dt  d |jddtt	j
dddd |jddtt	j
dddd |jdddddd |jdd |jddddd |jdd |jddtddd |jd d!dtd"d#d |jd$d%dttjttd&d |jd'd(dtd)d*d |jd+d,dtd-d.d |jd/ddd0 |jdd1 |jd2d3ddd0 |jdd4 |jd5ddd0 |jdd6 |jd7ddd8d |jdd9 |jd:td;d<d= |d>}|jd?tdd@d= |jdAtdBdCd= |jdDtdBdEd= |jdFddGtdHdI |jdJtdBdKd= |dL}|jdMddNdO |jdPtdQdRd= |jdStddTd= |dU}|jdVdWdddXd |jddY |jdZddd[d |jdd\ |jd]dGg d^d_ |jd`dGg dad_ |jdbdGg dcd_ |jddddded |jddf || }|S )gNz-mz--model_name_or_pathTz2Model path, or pretrained model name in the list: z, )requiredtypehelpz--model_classFGPT2LMHeadModelz!Model type selected in the list: )r   r   defaultchoicesr   z--cache_dir.cache_modelsz%Directory to cache pre-trained models)r   r   r   r   z--outputonnx_modelsz/Output directory, or model path ends with .onnxz-oz--optimize_onnx
store_truez'Use optimizer.py to optimize onnx model)r   actionr   )optimize_onnxz	--use_gpuzuse GPU for inference)use_gpuz--tolerancer   z;the aboslute and relative tolerance for parity verificationz--input_test_filez-ir   z)Path to the file with inputs to test withz-pz--precisionzoPrecision of model to run. fp32 for full precision, fp16 for half or mixed precision, and int8 for quantizationz-tz--test_casesi  z'Number of test cases per run for parityz-rz--test_runs
   z<Number of runs for parity. It is used for significance test.z	--verbose)r   r   )verbosez-ez--use_external_data_format)use_external_data_formatz--overwrite)	overwritez--use_int32_inputszJUse int32 instead of int64 for input_ids, position_ids and attention_mask.)use_int32_inputsz--beam_size   z2Beam size if greedy/top-p/top-k sampling is needed)r   r   r   z$configurable one step search optionsz--ignore_eosz3If ignore end of sentence token in model inference.z--repetition_penalty   z-Positive. >1 to penalize and <1 to encourage.z--temperaturez&Softmax temperature for output logits.z--excluded_token_ids+z0A list of token ids to be excluded in inference.)r   nargsr   r   z--length_penaltyz<Positive. >1 to penalize and <1 to encourage short sentence.zone step sampling optionsz--do_samplez3If to do sampling instead of beam search or greedy.)r   r   z--do_sample_top_pgffffff?z0Nuclear/top-p sampling accumulation probability.z--do_sample_top_kzUse top-k if non-zero.zVfloat to float16 conversion parameters that works when "--precision fp16" is specifiedz-az--auto_mixed_precisionz^Convert to mixed precision automatically. Other float16 conversion parameters will be ignored.)auto_mixed_precisionz--keep_io_typesz8Use float32 for past inputs, present and logits outputs.)keep_io_typesz--io_block_listz7List of inputs or outputs in float32 instead of float16)r%   r   r   z--op_block_listzzList of operators (like Attention Gather Add LayerNormalization FastGelu MatMul) to compute in float32 instead of float16.z--node_block_listz<List of node names to compute in float32 instead of float16.z--force_fp16_initializersz*Convert all float initializers to float16.)force_fp16_initializers)argparseArgumentParseradd_argumentstrjoinr   listr   keysospathset_defaultsfloatr
   FLOAT32intadd_argument_groupbool
parse_args)argvparsersearch_option_groupsampling_option_groupfp16_option_groupargs r?   [/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/models/gpt2/convert_to_onnx.pyparse_arguments(   sz  
		
	


rA   	onnx_pathr   c                 C   s.   |st j| S tdd t| jdD S )Nc                 S   s   g | ]}|  jqS r?   )statst_size).0fr?   r?   r@   
<listcomp>  s    z'get_onnx_model_size.<locals>.<listcomp>*)r0   r1   getsizesumr   parentrglob)rB   r   r?   r?   r@   get_onnx_model_size  s   rM   c                   C   s   dS )NzGaverage_latency(batch_size=8,sequence_length=1,past_sequence_length=32)r?   r?   r?   r?   r@   get_latency_name
  s   rN   gpt2_parity_results.csvc           2      C   s	  i }ddl m} t|tdk rtdt| }t|j |s3dd l}d	| r+| n|j
dd  }|jdkr>t|j |_td|  |j}|jdsR|jntj|j}	t||	|j |jtjkrm|jsmJ d	|jtjkrz|jszJ d
|jtjkr|jrJ dt|j d }
t|j d }|jdkrd}n
|jdkrd}nd}t |}t!"|}t#j$|j%|d}|dkr|
j$|j%|d|j&|d}n*|dkr|
j$|j%|d|j&|j'|j(|j)|j*|j+|j,|j-|j.|d}n	|
j$|j%||d}t/0|jrdnd}|1 2| |j3s|j4dkrtd |j5|	|j%|j|jtjkg dd}|d }tj6|r>|j7s>t8d|  n1td|  |j9||||j|j3|||j:rXt/j;nt/j<|j:rat/j;nt/j<|j:rjt/j;nt/j<d
 d |j=i}|j>r}|j>|d < |j?r|j?|d!< |j@r|j@|d"< |jAr|jA|d#< |jtjko|j= }|js|jtjkr||jtjkrtB|jnd$ }td%|  |j|||jtjk|jCjD|jCjE|j3fd&|jFi| n|}|jtjkrtd' tGH||d( |j3 tGI|}td) |d( }|jdr&||jkr&|j3s&dd lJ}|K||j |j}td*|  tLtM||j3d+ d+ }tN||jd,|jd-}|jd.kr|d ur|jO|||||j|j|j|||j:rdt/j;nt/j<|j:rmt/j;nt/j<|j:rvt/j;nt/j<|jP|jQ|jd/}|jR||||d0d,|j|||j:rt/j;nt/j<|j:rt/j;nt/j<|j:rt/j;nt/j<d1dd2d3}|jtjkrtd4|  dd lS}ddlTm} tU }tj6|}tV|d5d6d7} d8d9d:d;d<d=d>d?d@d dAd"d!d#d&dBdCdD|dEdFdGdHdIdJdKdLdMg}!|jW| |!dN}"|s|"X  i d8|d9|d:|j%d;|jd<|jd=|jd>|jd?|jPd@|jQd |j=dA|j>d"|j@d!|j?d#|jAd&|jFdBtYdBdCtYdCdD|||dOdG|dP dH|dQ dI|dR dJ|dS dK|dK dL|dL dE|dE dM|dM dFdTZ|i}#tdU|#  |[|# |"\|# W d    n	1 sw   Y  |j]rg }$tV|j]}%t^|%D ]\}&}'|'_ }'t`a|'}(t/btcjd|(dV tcj<dW2|})|rMdX|(v r|rtcjentcjf}*t/btcjd|(dX |*dW2|}+ndY},|)|,kg|rt/jent/jf}+|)h|)|,kd dZ|(v rt/btcjd|(dZ tcj<dW2|}-n|+i jdYd }-|-h|-dk d |j:r2|)2t/j;n|)|j:r=|-2t/j;n|-|j:rH|+2t/j;n|+d[}.ndV|j:rX|)2t/j;n|)i}.|dkse|dkrt/kd|)jld gi }/t/k|)jld dg}0t/jm|)jld dgt/jndW}1|.[|/|0|1d\ |$o|. qW d    n	1 sw   Y  |jp||||$|j|jd]d,dd|jd^tq|jrd_ td`|  |S )aNr   )__version__z3.1.0z/This tool requires transformers 3.1.0 or later. r#   z
Arguments:z.onnxz"fp16/int8 requires --optimize_onnxzfp16 requires --use_gpuzquantization only supports CPU   GPT2LMHeadModel_BeamSearchStepbeam_search_step)GPT2LMHeadModel_ConfigurableOneStepSearchconfigurable_one_step_searchr   )	cache_dir)config
batch_size	beam_sizerW   )rX   rY   rZ   
ignore_eostemperaturerepetition_penaltyexcluded_token_idslength_penalty	do_sampledo_sample_top_pdo_sample_top_krW   )rX   rW   zcuda:0cpu   z4Try --use_external_data_format when model size > 2GB)fp32fp16int8)
new_folderremove_existingrawz,Skip exporting ONNX model since it existed: zExporting ONNX model to )has_position_idshas_attention_maskinput_ids_dtypeposition_ids_dtypeattention_mask_dtyper'   node_block_listop_block_listr(   re   zOptimizing model to r&   zquantizing model...rg   zfinished quantizing modelzOutput path: i   T)enable_all_optimizationr   r   )rtolatolmodel_classrk   rl   rm   rn   ro   test_cases_per_run
total_runsr   d          )rw   use_io_bindingru   rk   rl   rm   rn   ro   rY   sequence_lengthpast_sequence_lengthzfp16 conversion parameters:ar   )modenewline
experimentrun_id
model_nameru   gpu	precision	optimizer
test_casesrunsio_block_listORT_TRANSFORMER_OPTIONSORT_CUDA_GEMM_OPTIONSonnxruntimetop1_match_rateonnx_size_in_MBdiff_50_percentilediff_90_percentilediff_95_percentilediff_99_percentilediff_pass_ratenan_ratetop1_match_rate_per_run)
fieldnamesz.2fmax_diff_percentile_50max_diff_percentile_90max_diff_percentile_95max_diff_percentile_99z{}zresult: 	input_ids)dtypeattention_maskposition_ids)r   r   r   )beam_select_idxinput_log_probsinput_unfinished_sents      )	r   ru   top_ktop_k_no_order	max_steps
max_inputsr   save_test_datasave_test_data_dirzDone. Output model: )stransformersrP   r   parseRuntimeErrorrA   r   r   sysr-   r9   	tolerancer   r   loggerinforW   outputendswithr0   r1   dirnamer   r   r
   r4   r   FLOAT16INT8r   ru   r   create_helperr   create_testerr	   from_pretrainedmodel_name_or_pathrZ   r[   r\   r]   r^   r_   r`   ra   rb   torchdeviceevaltor   n_layerget_onnx_pathsexistsr    warningexport_onnxr!   int32int64r'   r   rp   rq   r(   r,   rX   num_attention_headshidden_sizer&   r   quantize_onnx_modelquantize_torch_modelshutilmover5   rM   r   test_parityr   	test_runstest_performancecsvr   rN   open
DictWriterwriteheadergetenvformatupdatewriterowinput_test_file	enumeraterstripjsonloads
from_numpynumpyasarrayfloat16float32r   masked_fill_longcumsumzerosshapeonesr7   appendtest_generationr   rK   )2r9   experiment_namer   csv_filenameresulttransformers_versionr>   r   rW   
output_dirru   use_padding
model_type
gpt2helper
gpt2testerrX   modelr   onnx_model_pathsraw_onnx_modelfp16_paramsis_io_float16output_pathr   model_size_in_MBsessionparity_resultlatencyr   ort_versionlatency_namecsv_file_existedcsv_filecolumn_names
csv_writerrowtest_inputsread_f_linedatar   numpy_floatr   paddingr   inputsr   r   r   r?   r?   r@   main  sd  

 










 



"	




D
 

0r  __main__)N)Nr   r   rO   )+__doc__r)   r   loggingr0   r   pathlibr   r   r   gpt2_beamsearch_helperr   r   gpt2_beamsearch_testerr   gpt2_helperr   r   	packagingr   r   r	   r1   r   r-   r   __file__benchmark_helperr
   r   r   r   quantize_helperr   	getLoggerr   rA   r,   r7   rM   rN   r  __name__r?   r?   r?   r@   <module>   s8    

 \
  W
