o
    ;ήcC                     @   sP  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZmZ d dlmZmZ d dlmZ ejejejedd d dlmZ edZd9dd	ZG d
d dZdd Zdd Zdd Zdd Z de!fddZ"dd Z#ddgfdefddZ$dd Z%d d! Z&d"d# Z'd$d% Z(d&d' Z)defd(d)Z*defd*d+Z+e,d,kr&e Z-ee-j. e-j/d-k se-j0d.k se-j/e-j0 d/k re1d0 ee-j/e-j0e-jZ2e-j3se4d1d2kre-j5re	j67 d  d3ksJ d4e*e2e- ne+e2e- zee2j8Z9W n   e:d5e2j8  e2j;Z9Y e<d6 e2j8=d7d8Z>e e9e> dS dS ):    N)get_latency_namemain)PRETRAINED_GPT2_MODELS
Gpt2Helper)	OnnxModelz..)setup_logger c              	   C   s  t  }|jdddtddt d |jddtd	d
d |jddtddd |jddtddd |jddddd |jdd |jddddd |jdd |jddddd |jdd |jdddd |jdd |jd ddd!d |jdd" || }|S )#Nz-mz--model_name_or_pathTz2Model path, or pretrained model name in the list: z, )requiredtypehelpz--csvFzgpt2_parity_results.csvz#path of csv file to save the result)r	   r
   defaultr   z--test_casesi  znumber of test cases per runz--runs(   znumber of repeated runs	--use_gpu
store_truezuse GPU for inference)r	   actionr   )use_gpuz--allz'run all combinations of mixed precision)allz-e--use_external_data_format)r	   r   )use_external_data_formatz	--verbose)verbosez--skip_testzEdo not run test, and only rank experiments based on existing csv file)	skip_test)	argparseArgumentParseradd_argumentstrjoinr   intset_defaults
parse_args)argvparserargs r"   W/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/models/gpt2/gpt2_parity.pyparse_arguments"   sZ   
r$   c                   @   s   e Zd Zdd Zdd ZdS )
ParityTaskc                 C   s"   || _ || _|| _g | _d| _d S )Nr   )
total_runs
test_casescsv_pathresultsrun_id)selfr'   r&   r(   r"   r"   r#   __init__^   s
   
zParityTask.__init__c                 C   s   t j  d}| d| j }|  jd7  _zt|d| j d| j g ||| jd}W n   t	d|  Y |rC| j
| d S d S )Nz%Y%m%d%H%M%S_   z-tz-r)experiment_namer*   csv_filenamezFailed to run experiment )datetimenowstrftimer*   r   r'   r&   r(   logger	exceptionr)   append)r+   r   r/   
start_timer*   resultr"   r"   r#   rune   s   
zParityTask.runN)__name__
__module____qualname__r,   r9   r"   r"   r"   r#   r%   ]   s    r%   c                 C   s\   g }dd l }t| dd}||}|D ]}|| qW d    |S 1 s'w   Y  |S )Nr   r   newline)csvopen
DictReaderr6   )r(   rowsr?   csvfilereaderrowr"   r"   r#   load_results_from_csvx   s   

rF   c                 C   s>   t | t  }t | d }t | d }|d |d  |d  S )z@Scoring function based on 3 metrics. The larger score is better.top1_match_rateonnx_size_in_MBi  
   d   )floatr   )rE   latency_in_msrG   rH   r"   r"   r#   score   s   rM   c           
         s  t   t d i  |D ]}| |d < qtt|   fdddd}t| d|  td| d	 d
}d}d
}| D ]G\}}	|	|krJ|}|	}|d7 }|D ]4}|d |krtd||	|t|t	  t|d |d |d |d dkrd|d vrdnd  nqRq@d S )Nz
**********r*   c                    s   | d t  | d  fS )Nr.   r   )rM   )itemrow_mapr"   r#   <lambda>       zprint_wins.<locals>.<lambda>T)keyreversez Wins:z	Based on z* wins and a scoring function, the ranking:r   r.   z]{:02d}: WINs={:02d}, run_id={}, latency={:5.2f} top1_match={:.4f} size={}_MB experiment={} {}rG   rH   
experimentORT_CUDA_GEMM_OPTIONS4Half2 (Half2 Disabled)r   )
printdictsorteditemsr4   debuginfoformatrK   r   )
winsrB   	test_namerE   sorted_winsrankprevious_valuecountrS   valuer"   rO   r#   
print_wins   sX   



ri   c                 C   s  i }i }| D ]}|d }d||< d||< qt |ddd
}g d}tj||d}|  g d}	t| }
t|
d	 D ]}| | }t|d	 |
d	D ]}| | }d
}|	D ]}|| || krad} nqS|seqIt|d trddl}|	|d }|	|d }n|d }|d }zt
jj||d
dd\}}W n ty   d}d}Y nw t
jj||dd
d\}}|dk rt|d t|d kr||d   d	7  < n
||d   d	7  < |dk rt|d t|d kr||d   d	7  < n
||d   d	7  < |d |d |d t|d |d |d t|d ||||d}|| qIq;W d   n	1 s+w   Y  td|  t|| d t|| d dS )zRun U test and T test.r*   r   wr   r=   )
model_namerun_id_1experiment_1top1_match_rate_1run_id_2experiment_2top1_match_rate_2U_statisticU_pvalueT_statisticT_pvalue)
fieldnames)rk   r'   runsr.   TFtop1_match_rate_per_runNz	two-sided)use_continuityalternative)axis	equal_varg?rG   rk   rV   z(U-Test and T-Test results are output to zU-TestzT-Test)r@   r?   
DictWriterwriteheaderlenrange
isinstancer   jsonloadsscipystatsmannwhitneyu
ValueError	ttest_indrK   writerowr4   r`   ri   )rB   output_csv_path
utest_wins
ttest_winsrE   r*   rC   column_nameswriterrequired_match_columnsnum_resultsiresult1jresult2all_matchedcolumnr   abutest_statisticutest_pvaluettest_statisticttest_pvaluer"   r"   r#   run_significance_test   s   


Or   raw_onnx_modelc                 C   s   t | }t|}| }|jjd j|v sJ ||jjd j }|jdkr2t	d|j  |jS t
d|j d|j  d S )Nr   MatMulz#Found last MatMul node for logits: z-Failed to find MatMul node for logits. Found z	 of node )onnxloadr   output_name_to_nodegraphoutputnameop_typer4   r`   warning)r   model
onnx_modelr   noder"   r"   r#   get_last_matmul_node_name  s   

r   c                 C   sN   | j }d| d }| jr|d |ddd|g7 }|r%|dg|  |S )N-m  -o --use_gpu -p fp16r   --io_block_listlogitsz--node_block_list--op_block_list)model_name_or_pathsplitr   r6   extend)r!   last_matmul_node_nameop_block_listr   
parametersr"   r"   r#   get_mixed_precision_parameters&  s   
r   FastGeluLayerNormalizationtaskc                 C   sf   t |||}dt|}tddkrdnd}|r#d| d| }nd| d	| }| || d S )
N,rW   rX   rZ   r   Mixed precision baseline +  in FP32z=Mixed precision baseline (logits output and last MatMul node z	 in FP32))r   r   r]   osgetenvr9   )r   r!   r   r   r   op_block_list_strname_suffixr   r"   r"   r#   run_candidate8  s   r   c                 C   s^   | j }d| d }| jr|d | jr|d d| d }| jr+|d ||fS )Nr   z -o -p fp32r   r   r   )r   r   r   r6   r   )r!   r   fp32_baselinefp16_baseliner"   r"   r#   get_baselinesH  s   


r   c                   C   s   d  S )z$All operators in the optimized modelz7Attention Gather Add LayerNormalization FastGelu MatMul)r   r"   r"   r"   r#   get_all_operatorsW  s   r   c                    s   ddg}|  || d dg}|  || d t }|  || dg dd |D  d	g d
 |D ] dg fdd|D  }|  || | d  d q1dS )z:Step 0 is to check which operator in FP16 causes most lossr   r   zFP16 except logitsz--keep_io_typeszGraph I/O FP32, Other FP16r   c                 S   s   g | ]}|qS r"   r"   .0or"   r"   r#   
<listcomp>i  s    z$run_tuning_step0.<locals>.<listcomp>z--force_fp16_initializerszFP32 except weights in FP16c                    s   g | ]}| kr|qS r"   r"   r   opr"   r#   r   n  rR   zFP32 except z in FP16N)r9   r   )r   r   fp32_logitsfp32_ioop_listr   r"   r   r#   run_tuning_step0\  s    r   c                 C   s0   t  D ]}d|g}| || d| d qdS )zAStep 1 is to figure out which operator in FP32 could benefit mostr   r   r   Nr   r9   r   mixed_precision_baseliner   r   r"   r"   r#   run_tuning_step1r  s   

r   c                 C   s:   t  D ]}|dvrdd|g}| || d| d qdS )zAssumed that you have run step 1 to figure out that Logits FP32 and Add FP32 is important,
    Step 2 is to figure out a combination of two operators (one is Add from step one) to get better result
    )Addr   r   zMixed precision baseline + Add,r   Nr   r   r"   r"   r#   run_tuning_step2|  s   


r   c                 C   s\   t jd|j|jg d}t|d }t| ||g d t| ||dgd t| ||ddgd d S )Nonnx_models
new_folderremove_existingrawr   r   r   )r   get_onnx_pathsr   r   r   r   )r   r!   onnx_model_pathsr   r"   r"   r#   run_parity_disable_half2  s   r   c                 C   s  t jd|j|jg d}t|\}}| |d |js!td d S | |d t	|d }t
| ||g d t
| ||dgd |jrXt| | t||g d}t| | t| | nt
| ||d	dgd t
| ||d
dgd t
| ||g dd t
| ||g dd t
| ||g dd d S )Nr   r   zFP32 baselinez5skip mixed precision since --use_gpu is not specifiedzFP16 baseliner   r   r   r   r   )r   r   r   )r   r   r   Gather)r   r   r   r   r   )r   r   r   r   r   r9   r   r4   r`   r   r   r   r   r   r   r   )r   r!   r   r   r   r   r   r"   r"   r#   
run_parity  sZ   



r   __main__rJ      i'  zNot enough test cases or runs to get stable results or test significance. Recommend test_cases >= 100, runs >= 20, test_cases * runs >= 10000.rW   rX      zhalf2 kernel is not avaiable in current GPU device. Please set environment variable ORT_CUDA_GEMM_OPTIONS=0 or use supported GPU like V100 or T4zFailed to load csv z#Start running significance tests...z.csvz
.stats.csv)N)?r   r?   r1   loggingr   sysr   scipy.statsr   torchconvert_to_onnxr   r   gpt2_helperr   r   r   r   pathr6   r   dirname__file__benchmark_helperr   	getLoggerr4   r$   r%   rF   rM   ri   r   r   r   r   r   r   r   r   r   r   r   r   r:   r!   r   r'   rw   r   r   r   r   r   cudaget_device_capabilityr(   rB   r5   r)   r`   replacesummary_csvr"   r"   r"   r#   <module>   sv    

;	.]


>
$

