o ;ήcpE@s,ddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z ddl Z ddlZddlZddlmZmZeGdddZeGdd d Zd&d d Zd dZddZddZddZddZddZddZddZddZdd Zd!d"Z d#d$Z!e"d%krdZ#e!dSdS)'N) dataclass)datetime)Path)generate_test_dataget_bert_inputsc@sfeZdZUeed<eed<eed<eed<eed<eed<eed<eed<eed <eed <eed <d S) TestSetting batch_sizesequence_length test_cases test_timesuse_gpuuse_io_bindingproviderintra_op_num_threadsseedverbose log_severityN)__name__ __module__ __qualname__int__annotations__boolstrrrN/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/bert_perf_test.pyr s  rc@s6eZdZUeed<eed<eed<eed<eed<dS) ModelSetting model_pathinput_ids_namesegment_ids_nameinput_mask_name opt_levelN)rrrrrrrrrrr/s  rc Csddl}|||rd|vrtd|rI|dkr ddg}n,|dkr)ddg}n#|d kr2gd }n|d kr;ddg}n|d krDgd }nddg}ndg}|}||_|jj|_|durb|j j |_ n+|dkrl|j j |_ n!|dkrv|j j |_ n|dkr|j j|_ n |dkr|j j |_ n||_ |dur||_|j|||d} |r|dkrd| vsJ| S|dkrd| vsJ| S|d krd| vsJd| vsJ| S|d krd| vsJ| S|d krd| vsJd| vsJ| Sd| vsJ| Sd| vsJ| S)NrCUDAExecutionProviderzWarning: Please install onnxruntime-gpu package instead of onnxruntime, and use a machine with GPU for testing gpu performance.dmlDmlExecutionProviderCPUExecutionProviderrocmROCMExecutionProvidermigraphx)MIGraphXExecutionProviderr(r&cudatensorrt)TensorrtExecutionProviderr#r&r"c) providersr*r-) onnxruntimeset_default_logger_severityget_available_providersprintSessionOptionslog_severity_level ExecutionModeORT_SEQUENTIALexecution_modeGraphOptimizationLevelORT_ENABLE_ALLgraph_optimization_levelORT_DISABLE_ALLORT_ENABLE_BASICORT_ENABLE_EXTENDEDrInferenceSession get_providers) rr rrr<rr1execution_providers sess_optionssessionrrrcreate_session8st               rEcCs,tjtjtjtjtjtjtjtji}||S)N)torchfloat32npfloat16int32int64longlong) torch_typetype_maprrr numpy_types rOcs4fdd|D}fdd|D}||fS)Nc"i|] \}}|t|qSrrF from_numpyto.0namearraydevicerr "z/create_input_output_tensors..crPrrQrTrXrrrZr[)items)inputsoutputsrY input_tensorsoutput_tensorsrrXrcreate_input_output_tensorssrac Csx|}|D]\}}|||jjdt|j|j|q|D]\}}| ||jjdt|j|j|q#|SNr) io_bindingr\ bind_inputrYtyperOdtypeshapedata_ptr bind_output)sessr_r`rcrVtensorrrrcreate_io_bindings(rlcCsg}g}|jr dnd}t|D]I\}}|||} || i} tt|D] } | | | || <q&t|| |\} } t|| | }||t }||t |}||q||fS)Nr+cpu) r enumeraterunappendrangelenrarlrun_with_iobindingtimeit default_timer)rD all_inputs output_names test_settingresults latency_listrY test_case_idr]resultr^ir_r`rc start_timelatencyrrr%onnxruntime_inference_with_io_bindings"       rc Cstt|dkr||t|g}g}t|D]\}}t}|||}t|} |||| q||fSrb)rrrorandomchoicernrtrurp) rDrvrwryrzr{r]r~r|rrrronnxruntime_inferences     rc Csj|}dtj|}|d|j|jdd7}|d|jd|j d|j d|j d |j 7}|S) Nz model={},z4graph_optimization_level={},intra_op_num_threads={},zGraphOptimizationLevel.ORT_z batch_size=z,sequence_length=z ,test_cases=z ,test_times=z ,use_gpu=) get_session_optionsformatospathbasenamer<rreplacerr r r r )rrDrxrCoptionrrr to_strings0rcCsXt|j|j|j||j|jd}dd|D}t|j||}||vr*td|dStd|g}|j rKt |j D]} t ||||\} } | | q9nt |j D]} t|||\} } | | qPt|d} t| } t| d}t| d}t| d }t| d }t| d }|jd | }| ||||||f||<td t| dt|ddS)N)rcSsg|]}|jqSr)rV)rUoutputrrr sz run_one_test..zskip duplicated test:z Running test:2KZ_r/g@@z,Average latency = {} ms, Throughput = {} QPS.2f)rErr rr!r get_outputsrr4r rqr rextendrrHrW statisticsmean percentilerr) model_settingrx perf_resultsrvrrDrwkeyall_latency_listr}ryrz latency_msaverage_latency latency_50 latency_75 latency_90 latency_95 latency_99 throughputrrr run_one_testsX           rcCs,tjt|||||fd}||dS)N)targetargs)multiprocessingProcessrstartjoin)rrxrrvrprocessrrr launch_tests rc Cs|jdurt|||||jdStjdd}tjdd}tt||g}tdtd|D] }||vr7||q,|j dd|D] }t|||||q@dS)NF)logicalTr.)reverse) rrpsutil cpu_countlistsetrqminrpsort) rrxrrvr logical_corescandidate_threadsr}rrrrrun_perf_tests&s(     rc Csrt|j|j|j|j\}}}td|jd|jd|jt |j|j|j|j |j |||dd }t ||||dS)Nz Generating z samples for batch_size=z sequence_length=F)random_mask_length) rrrrr r4r rr rrrr)rrxr input_ids segment_ids input_maskrvrrrrun_performance>s*  rcCst}|jddtdd|jdddtddd |jd d dtd d|jd dtddd|jdddtddd|jddtgdddd|jddtddd|jdddd d!|jdd"|jd#dtd$gd%d&d'|jd(ddd)d!|jdd*|jd+ddd,d!|jdd-|jd.dtdd/d|jd0d1dtdd2d|jd3dtdd4d|jd5dtdd6d|jd7dtdd8d|}|S)9Nz--modelTzbert onnx model path)requiredrehelpz-bz --batch_size+zKbatch size of input. Allow one or multiple values in the range of [1, 128].)rrenargsrz-sz--sequence_lengthz maximum sequence length of inputz --samplesF z!number of samples to be generated)rredefaultrz-tz --test_timesrzJnumber of times to run per sample. By default, the value is 1000 / samplesz --opt_level)rr.r"r/r/zZonnxruntime optimization level: 0 - disable all, 1 - basic, 2 - extended, 99 - enable all.)rrechoicesrrz--seedzPrandom seed. Use the same seed to make sure test data is same in multiple tests.z --verbose store_truezprint verbose information)ractionr)rz--log_severityr")rr.r"rz.0:Verbose, 1:Info, 2:Warning, 3:Error, 4:Fatal)rrerrrz --use_gpuzuse GPU)r z--use_io_bindingzuse io_binding)r z --providerzExecution provider to usez-nz--intra_op_num_threadsz>=0, set intra_op_num_threadsz--input_ids_namezinput name for input idsz--segment_ids_namezinput name for segment idsz--input_mask_namezinput name for attention mask)argparseArgumentParser add_argumentrr set_defaults parse_args)parserrrrrparse_argumentsYs       rc Cst}|jdkrtdtd|j|_t}|}t|j }t |dks0t|dkr0t dt |j |j|j|j|j}|D]%}t||j|j|j|j|j|j|j|j|j|j }td|t|||q?t|ddd d }tj !t"|j j#d $|jrd nd d!ddtt%|D|jt&'(d}t)|dddK} t*j+| ddd} d} |D]6\} } | ,d}| durgd} | -dd|D| .| dd| D}|-dd|D| .|qWdn1swYtd|dS)Nrr.rz batch_size not in range [1, 128]z test settingFcSs|dS)Nr.r)xrrrszmain..)rrzperf_results_{}_B{}_S{}_{}.txtGPUCPU-cSsg|]}t|qSr)rrUrrrrrszmain..z %Y%m%d-%H%M%Szw+r)newline  ) delimiterlineterminator,)z Latency(ms) Latency_P50 Latency_P75 Latency_P90 Latency_P95 Latency_P99zThroughput(QPS)cSg|] }|ddqS)=rsplitrrrrrcSsg|]}t|dqS)r)rrrrrrscSr)rr.rrrrrrrzTest summary is saved to)/rr maxrsamplesrManagerdictrrr Exceptionrmodelrrr r!rr r r rrrrrr4rsortedr\rrrrparentrrrnowstrftimeopencsvwriterrrwriterow)rmanagerrbatch_size_setrrrxsorted_results summary_filetsv_file tsv_writerheadersr perf_resultparamsvaluesrrrmainsp           r__main__)Nr")$rrrrrrrt dataclassesrrpathlibrnumpyrHrrFbert_test_datarrrrrErOrarlrrrrrrrrrr__spec__rrrrsF    M  8yL