o
    ;ήcgK                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZ d dlZd dlZd dlZd dlZd dlmZ d dlZeeZG dd	 d	e	ZG d
d de	ZG dd dZdejiZdddddi fddZ d.ddZ!d/ddZ"dd Z#dd Z$dd Z%dd Z&d0d d!Z'ej(d fd"d#Z)d$d% Z*d1d'd(Z+d)eeee,ef   fd*d+Z-d,d- Z.dS )2    N)ThreadPoolExecutor)datetime)Enum)sleep)AnyDictListOptional)versionc                   @       e Zd ZdZdZdZdd ZdS )	Precisionfp32fp16int8c                 C      | j S Nvalueself r   P/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/benchmark_helper.py__str__"      zPrecision.__str__N)__name__
__module____qualname__FLOAT32FLOAT16INT8r   r   r   r   r   r      s
    r   c                   @   r   )OptimizerInfono_optby_ort	by_scriptc                 C   r   r   r   r   r   r   r   r   -   r   zOptimizerInfo.__str__N)r   r   r   NOOPTBYORTBYSCRIPTr   r   r   r   r   r    &   s
    r    c                   @   s$   e Zd Zdd Zdd Zdd ZdS )ConfigModifierc                 C   
   || _ d S r   
num_layers)r   r*   r   r   r   __init__2      
zConfigModifier.__init__c                 C   s   | j d u rd S t|dr| j |_td| j   t|dr+| j |_td| j   t|dr?| j |_td| j   d S d S )Nnum_hidden_layersz6Modifying pytorch model's number of hidden layers to: encoder_layersz7Modifying pytorch model's number of encoder layers to: zdecoder_layers z7Modifying pytorch model's number of decoder layers to: )r*   hasattrr-   loggerinfor.   decoder_layers)r   configr   r   r   modify5   s   



zConfigModifier.modifyc                 C   r   r   r)   r   r   r   r   get_layer_numB   r   zConfigModifier.get_layer_numN)r   r   r   r+   r4   r5   r   r   r   r   r'   1   s    r'   float32TFc                    s(  d }zt  }	|rt jj|	_nt jj|	_|rd|	_|dkr)||	_t	d|	j  |r/d|	_
nd|	_
t	d|   |rn|dkrEddg}
n,|d	krNd
dg}
n#|dkrWg d}
n|dkr`ddg}
n|dkrig d}
nddg}
ndg}
 r| fdd|
D }
t j| |	|
d}W |S    tjddd Y |S )NTr   z%Session option: intra_op_num_threads=   zCreate session for onnx model: dmlDmlExecutionProviderCPUExecutionProviderrocmROCMExecutionProvidermigraphx)MIGraphXExecutionProviderr=   r;   cudaCUDAExecutionProvidertensorrt)TensorrtExecutionProviderrA   r;   c                    s$   g | ]}| v r| | fn|qS r   r   ).0nameprovider_optionsr   r   
<listcomp>   s   $ z.create_onnxruntime_session.<locals>.<listcomp>)	providers	Exception)exc_info)onnxruntimeSessionOptionsGraphOptimizationLevelORT_ENABLE_ALLgraph_optimization_levelORT_ENABLE_BASICenable_profilingintra_op_num_threadsr0   debuglog_severity_levelInferenceSessionerror)onnx_model_pathuse_gpuproviderenable_all_optimizationnum_threadsrR   verboserG   sessionsess_optionsrI   r   rF   r   create_onnxruntime_sessionL   sF   







r`   c                 C   s8   | rt jddd d S t jdd tdtj d S )NDEBUGz8[%(filename)s:%(lineno)s - %(funcName)20s()] %(message)s)levelfmtz%(message)s)rc   transformers)coloredlogsinstalllogging	getLoggersetLevelWARNING)r]   r   r   r   setup_logger   s   
rk   c                 C   s   | rt j| st |  |rt j|st | |r5|dkr+dt v s*J dn
dt v s5J dtdtj	  tdt
j	  tdtj	  ttj	td	ks]J tt
j	td
ksjJ ttj	td	kswJ d S )Nr9   r:   zBPlease install onnxruntime-directml package to test GPU inference.rA   z=Please install onnxruntime-gpu package to test GPU inference.zPyTorch Version:zTransformers Version:zOnnxruntime Version:z1.10.0z4.12.0)ospathexistsmakedirsrL   get_available_providersr0   r1   torch__version__rd   r
   parse)	cache_dir
output_dirrY   rZ   r   r   r   prepare_environment   s(   



rv   c              
   C   s   t | tt|  d }tj| tjdd }|d|  }t| d|dt| dd dt| dd dt| dd d|d|dS )Ng     @@)dtypez{:.2f}Z   _   c   )
test_timeslatency_variancelatency_90_percentilelatency_95_percentilelatency_99_percentileaverage_latency_msQPS)sumfloatlennumpyvarfloat64format
percentile)latency_list
batch_size
latency_msr|   
throughputr   r   r   get_latency_result   s   r   c                 C   sv   t |dddd!}g d}tj||d}|  | D ]}|| qW d    n1 s,w   Y  td|  d S )Na asciimodenewlineencoding)enginer
   rI   device	precision	optimizer
io_binding
model_nameinputsthreadsr   sequence_lengthcustom_layer_numr   r{   r   r   r|   r}   r~   r   
fieldnamesz&Detail results are saved to csv file: )opencsv
DictWriterwriteheaderwriterowr0   r1   )resultscsv_filenamecsv_filecolumn_names
csv_writerresultr   r   r   output_details   s   r   c                    s  t |dddd}g d g }|jD ]}|jD ]}|d| d|  qqtj| | d}|  |jD ]}d	D ]}	|jD ]y}
d
D ]t}|j	D ]n}i }| D ]`}|d |kr|d |	kr|d |
kr|d |kr|d |kr fdd|
 D }|s|| |dd |D  n D ]}|| || ksJ q|d }|d }|d |d| d| < qN|r|| qHqCq?q:q6W d    n1 sw   Y  td|  d S )Nr   r   r   r   )r   r   r   r   r
   rI   r   r   r   r   r   b_sr   )         )TFr   r   r   r   r   r   c                    s   i | ]\}}| v r||qS r   r   )rD   kvheader_namesr   r   
<dictcomp>  s    z"output_summary.<locals>.<dictcomp>c                 S   s   i | ]}|d qS )r   r   )rD   r   r   r   r   r     s    r   r   r   z'Summary results are saved to csv file: )r   batch_sizessequence_lengthsappendr   r   r   modelsenginesr\   itemsupdater   r0   r1   )r   r   argsr   
data_namesr   r   r   r   input_countengine_namer   r   rowr   headersr   r   sr   r   r   output_summary   s\   











0r   c                 C   s   t |ddddO}g dttt|    }tj||d}|  |  D ]'}t	t
 | | d< tj| | d< tj| | d	< || | d
< || |  q(W d    n1 sZw   Y  td|  d S )Nr   r   r   r   )model_filenamer   rd   rq   r   r   rd   rq   r   z(Fusion statistics is saved to csv file: )r   listnextitervalueskeysr   r   r   strr   nowrd   rr   rq   r   r0   r1   )model_fusion_statisticsr   r   r   r   keyr   r   r   output_fusion_statistics  s   r   c                    sd   i }t j fddd|d t j fddd|d}|| |ddi |t|| |S )Nc                          d  S r   runr   
ort_inputsort_sessionr   r   <lambda>-      zinference_ort.<locals>.<lambda>r   numberrepeatc                      r   r   r   r   r   r   r   r   .  r   r   F)timeitr   r   r   )r   r   result_templaterepeat_timesr   warm_up_repeatr   r   r   r   r   inference_ort+  s   
r   c              
      s*  i }   | D ]/}t|| |	}t|| jtv r(tt|| j n|
} ||j	j
d||j|  q
t|dkrFt|||	 t|D ]\}} ||| j	j
dtj|| j||   qJtj fddd|d tj fddd|d}|| |ddi |t|| |S )	Nr   c                      
     S r   run_with_iobindingr   r   r   r   r   r   f     
 z/inference_ort_with_io_binding.<locals>.<lambda>r   r   c                      r   r   r   r   r   r   r   r   l  r   r   T)r   r   rq   
from_numpytor   rw   IO_BINDING_DATA_TYPE_MAP
bind_inputr   typeshapedata_ptrr   allocateOutputBuffers	enumeratebind_outputr   r6   r   r   r   r   )r   r   r   r   ort_output_namesort_outputsoutput_buffersoutput_buffer_max_sizesr   r   	data_typer   r   rE   np_input
input_typeiort_output_namer   r   r   r   inference_ort_with_io_binding5  sR   	

	
r   c                 C   s&   |D ]}|  tj|tj|d qd S )N)rw   r   )r   rq   emptyr6   )r   r   r   r   r   r   r   r   v  s   r   {   c                 C   s<   t |  tj |  t|  tj|  tj|  dS )z5Set random seed manually to get deterministic resultsN)randomseedr   rq   manual_seedr@   manual_seed_all)r   r   r   r   set_random_seed~  s
   

r  returnc               
   C   s   ddl m} m}m}m}m}m}m} z>|  g }| }t|t	s#W d S t
|D ]#}	|||	}
t|
tr8 W d S ||	|||	|
j|
j|
jd q'|  |W S  | yh } ztd| W Y d }~d S d }~ww )Nr   	NVMLErrornvmlDeviceGetCountnvmlDeviceGetHandleByIndexnvmlDeviceGetMemoryInfonvmlDeviceGetNamenvmlInitnvmlShutdown)idrE   totalfreeused-Error fetching GPU information using nvml: %s)py3nvml.py3nvmlr  r  r  r  r  r	  r
  
isinstanceintranger   r   r  r  r  print)r  r  r  r  r  r	  r
  r   device_countr   r1   rW   r   r   r   get_gpu_info  s4   $



	
r  c              	   C   s  G dd d}|d}| r|  }|d u rd S t }| }||j }z||}| }W d|_| }	nd|_| }	w |	d u rL	 W d    d S td| d|	  t|dkrt|	dkrt|t|	krd}
t|D ]\}}|d }|	| d }|| }t|
|}
qp|
W  d    S W d    d S W d    d S W d    d S 1 sw   Y  d S |	 }t =}| }||j	}z||}| }W d|_| }	nd|_| }	w td	|d
d|	d
d |	| W  d    S 1 sw   Y  d S )Nc                   @   s<   e Zd Zd
ddZdd Zdeeeee	f   fddZ
d	S )z%measure_memory.<locals>.MemoryMonitorTc                 S   r(   r   )keep_measuring)r   r  r   r   r   r+     r,   z.measure_memory.<locals>.MemoryMonitor.__init__c                 S   sB   dd l }d}	 t||t  jd }td | js 	 |S q)Nr   T   {Gzt?)	psutilmaxProcessrl   getpidmemory_inforssr   r  )r   r  	max_usager   r   r   measure_cpu_usage  s   z7measure_memory.<locals>.MemoryMonitor.measure_cpu_usager  c           
   
      s>  ddl m}m}mm}mm}m} g g  zo|  | }t|t	s-t
d|  W d S dd t|D fddt|D  	 t|D ]%}||}t|tr`t
d|   W d S t| |jd	 |< qGtd
 | jsunqC|   fddt|D W S  |y }	 zt
d|	 W Y d }	~	d S d }	~	ww )Nr   r  z*nvmlDeviceGetCount result is not integer: c                 S   s   g | ]}d qS r   r   rD   r   r   r   r   rH     s    zKmeasure_memory.<locals>.MemoryMonitor.measure_gpu_usage.<locals>.<listcomp>c                    s   g | ]} |qS r   r   r#  )r  r  r   r   rH     s    Tz%nvmlDeviceGetMemoryInfo returns str: r  r  c                    s    g | ]}| | | d qS ))	device_idrE   max_used_MBr   r#  )gpu_namemax_gpu_usager   r   rH     s    r  )r  r  r  r  r  r  r	  r
  r  r  r0   rW   r  r   r  r  r   r  )
r   r  r  r  r	  r
  r  r   r1   rW   r   )r&  r'  r  r  r   measure_gpu_usage  s>   $



z7measure_memory.<locals>.MemoryMonitor.measure_gpu_usageNT)r   r   r   r+   r!  r	   r   r   r   r   r(  r   r   r   r   MemoryMonitor  s    
"r*  FzGPU memory usage: before=z  peak=r   r   r%  zCPU memory usage: before=z.1fz
 MB, peak=z MB)
r(  r   submitr   r  r  r   r   r  r!  )is_gpufuncr*  monitormemory_before_testexecutor
mem_thread	fn_thread_r   max_usedr   memory_beforebeforeafterr  r   r   r   measure_memory  sl   <



(






$r8  r)  r   r"  )r   )/r   rg   rl   r   r   concurrent.futuresr   r   enumr   timer   typingr   r   r   r	   re   r   rq   rd   	packagingr
   rL   rh   r   r0   r   r    r'   r6   r   r`   rk   rv   r   r   r   r   r   longlongr   r   r  r   r  r8  r   r   r   r   <module>   sV   
		

B
"4

A
&