o ;ήcRs@sLddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl m Z mZmZddlZddlZddlmZmZmZddlmZddlZejejejeddddlZe dZ!d eee"e ffd d Z#d5d dZ$      d6d eee"e ffddZ%d ee"e ffddZ&ddZ'd7ddZ(d ee"e ffddZ)d eee"e ffddZ*d eee"e ffddZ+d eee"e ffd d!Z,d8d"d#Z-d$d%Z.d eee"e ffd&d'Z/d eee"e ffd(d)Z0  *d9d+d,Z1d:d.d/Z2d;d0d1Z3d2d3Z4e5d4kr$e4dSdS)<N)ProcessPoolExecutor)datetime)AnyDictList)PRETRAINED_LONGFORMER_MODELSLongformerHelperLongformerInputs)LongformerModelz..returnc s:|dkr t|g}|D]} |D]} |D]} td| d| d| t| | | |} | } tjfdd|dd}id d d tj d d ddddddd|d|dddd|d| d| d| dt t ddddd ddddd!}| t|| td"|||qqq |S)#Nr batch_size= sequence_length= global_length=csSNr input_listmodelrf/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/models/longformer/benchmark_longformer.pySsz$test_torch_latency..)repeatnumberenginetorchversiondevicecuda optimizerr precisionfp32 io_binding model_name descriptionz [torch]inputsthreads batch_sizesequence_length global_lengthrmemoryNAdiff_maxdiff_90_percentile)diff_95_percentilediff_99_percentileuse_compact_memory%s)rset_num_threadsloggerinforget_dummy_inputsto_listtimeitr __version__strrnowupdatebenchmark_helperget_latency_resultappend)rrr# batch_sizessequence_lengthsglobal_lengths test_times num_threadsresultsr(r)r*r%_runtimesresultrrrtest_torch_latency;st         #rITcCsd|d|d|}td|dt||||}|} |d| } |} || } t| d | d} td| |r^t | sP| dkr^t d | dt d | dt| S) Nr rrz$Comparing Torch and ORT outputs for ...rzlast_state max diff = gMbP?ztorch last_state:zort last_state:)r4r5rr6get_ort_inputsrunr7npamaxcpunumpymathisnanprintfloat)rr ort_sessionr(r)r*verbose parameters dummy_inputs ort_inputs ort_outputsr torch_outputsmax_diffrrr test_parityps r]Fr!cs g}|D]|D]|D]jjdksJdtdddd| d| d| d t}|}rGt|d|}id |d |d d dddt t j dddt | dt | dt | dt dt dt dt |dt t dddddddd||d}| st|t|jj}t|t|}tj|||d d!g|g||gtjd" }n tj|||d#}|sfd$d%t|D}t||d<t|d&|d<t|d'|d(<t|d)|d*<||qq q|S)+NrzPLimitation of current implementation: number of global token <= attention_windowzTesting batch_size=rrz optimizer=z , precision=z io_binding=rJr#r$r%r&r OnnxRuntimerrrr rr'r(r)r*rCrr+r r-r.)r/r0r1 use_half4 last_statepooler) result_template repeat_timesort_output_namesrZoutput_buffersoutput_buffer_max_sizesr(r data_type)rbrcr(c s g|] }tqSr)r]).0rFr(rr*rrUr)rVrr s z$test_ort_latency..Z_r/cr0)configattention_windowr4r5rr6rKrSrLr: onnxruntimer9intrr;max hidden_sizer=inference_ort_with_io_bindingrMlonglong inference_ortrange percentiler?)rrr#r$rUr@rArBrCrDrr disable_io_bindingrVr1r_disable_parityrErXrYrZrbmax_last_state_sizemax_pooler_sizerH diff_resultsrrirtest_ort_latencys              ]r~c shtdddddd fdd}tjd |d }|d S) NzTesting memory for model=z , batch_size=z, sequence_length=z, global_length=z , test_times=z, num_threads=csZddi}d|i}tjdd|d}t}|}tD]}|d|}q"dS)Narena_extend_strategykSameAsRequestedCUDAExecutionProviderTuse_gpuenable_all_optimizationrDprovider_options)r=create_onnxruntime_sessionrr6rKrwrL)cuda_provider_optionsrsessionrXrYrFr(rr*rDonnx_model_pathr)rCrr inferences  z"test_ort_memory..inferenceT)is_gpufunc) onnx_modelr(r)r*rCrDr+)r4r5r=measure_memory) rrr(r)r*rCrDr memory_usedrrrtest_ort_memorys& rcCs,|tvrt|n|}t|}|||Sr)rr from_pretrainedto)r#rtorch_model_name_or_dirrrrrload_torch_model%s   r.cCs^tj||d}tj||d}tj||d}tj|r%|}|Stj|r-|}|S)Nz.onnx _fp32.onnx _fp16.onnx)ospathjoinisfile)r#onnx_dirroptimized_fp32_modeloptimized_fp16_modelrrrfind_onnx_model.s  rcCst|jdkr tdt|jdkrtdt|jdkr!td|j}|js+t|n|j}tj t |||jd|jd|jd|j |j S)Nrz5For memory test, only one batch_size (-b) is allowed.z:For memory test, only one sequence_length (-s) is allowed.z8For memory test, only one global_length (-g) is allowed.r)lenr@ RuntimeErrorrArBronnxrrr empty_cacherrCrD)argsrr#rrrr test_memory:s$ rc Cs|j}|js t|n|j}|dp|d}|dsdnd}t||}|j}ddi}d|i} tj|dd|| d } | durEtd |t j d d d k} |} | sV| d 7} |j rd| |dkr`dnd7} n | |dkrkdnd7} t |||| | |j|j|j|j||||j|j| |j |jS)Nrrr!fp16rrrTrz,Failed to create ORT session from ONNX file ORT_LONGFORMER_COMPACT_MEMORY1z[non_compact_memory]z[half4]z[float4]z[half2])rrrendswithrrDr=rrrenvirongetr_r~r@rArBrCryrVrz) rrr#r optimizedr rrDrrrr1r$rrrtest_ortQsV rc Cs.t|j|}t|||j|j|j|j|j|jSr)rrrIr@rArBrCrD)rrrrrr test_torchs rcCsd|jkr t||St||S)Nrp)rrrrrrrr test_latencys   rc Cs\t}|jdddtdddtd|jdd dtd d d gd d |jddddtdd|jdddtdgd|jdddtgddd|jddtddd|jdd dtd!gd"d|jd#d$dtd!d%d|jd&dd'd(d)|jd*dd'd+d)|jd,dd'd-d)|jdd.|jd/dd'd0d)|jdd1|jd2dd'd3d)|jdd4| |}|S)5Nz-mz--modelFlongformer-base-4096z=Checkpoint directory or pre-trained model names in the list: z, )requiredtypedefaulthelpz-ez--enginerprzEngine to benchmark.)rrrchoicesrz-tz --test_timesz8Number of repeat times to get average inference latency.)rrrrz-bz --batch_sizes+r)nargsrrz-sz--sequence_lengths)zSequence lengths. It could have multiple values in latency test.If --export_padding is not used, sequence length shall be multiple of window size.)rrrrz--onnxzOnnx model pathz-gz--global_lengthsrzGNumber of global tokens. It could have multiple values in latency test.z-nz --num_threadszThreads to use.z--disable_io_binding store_truezDo not use IO Binding.)ractionrz--memoryz%Test memory usage instead of latency.z --verbosezPrint more information.)rVz --use_half4zUse half4 kernel.)r_z--disable_parityzDo not run parity test.)rz) argparseArgumentParser add_argumentr:rrkeysrq set_defaults parse_args)argvparserrrrrparse_argumentss           rcCsdd|D}t|dkrtddSt|dddd)}gd }tj||d }||D] }t|||q-|Wdn1sGwYtd |dS) NcSsg|]}d|vr|qSaverage_latency_msrrhrHrrrrjsz"output_details..rzNo latency results for output.ar asciimodenewlineencoding)rrrr rr"r#r%r'rrCr$r(r)r*r1r_r-r.r/r0r+QPSrlatency_variancelatency_90_percentilelatency_95_percentilelatency_99_percentile fieldnamesz&Detail results are saved to csv file: )rrSopencsv DictWriter writeheaderwriterowflush)rE csv_filenamelatency_resultscsv_file column_names csv_writerrHrrroutput_detailss   (rcCs:tdtdtd}|jrt||gSt||S)NF{zcuda:0)rset_grad_enabledr=set_random_seedrr+rrrrrrrL-s     rLcCsftjs tdt}t|t|g}t|dksJ|dWdS1s,wYdS)NzYPlease install PyTorch with Cuda, and use a machine with GPU for testing gpu performance.rr) rr is_availablerrlistmaprLr) argumentsexecutorrErrr launch_test<s $rrcCsB|rdnd}|tjd<td||rdndtjd<td|r$dndg} d} gd} |g} d D]} | D]}| D]}d D]}|rid }td |d | d|d|d|d | d| d}| t|7} d}|rodnd}|r{| d|dn| d|d}tj |st d|d |d|d|d|d|d| }|s|d7}|r|d7}|dkr|d7}d}z"|rt|dd}t |}t|d | d}t |}Wnt y}zt d |d}~wt YqAt|dkr |r|dd!nd"|dd!<nt d#td$|| |7} qAq=q9q5| S)%Nr0rzORT_LONGFORMER_COMPACT_MEMORY=ORT_LONGFORMER_USE_HALF4zORT_LONGFORMER_USE_HALF4={}r)rrrr)r)rz-e z -t z -b z -s z -g z -m  rprr_frrzonnx file not exists:z --onnx z --disable_io_bindingz --use_half4z --disable_parityz -t 10 --memoryzKeyboard Interruptedr+zN/Az%length of latency_results should be 1r2)rrr4r5formatrsplitrLrexistsrrKeyboardInterrupt traceback print_excr)r1 run_torch run_memoryuse_io_bindinguse_fp16use_merged_qkv_weightsr_r(compact_memoryrErCrAr@r#r)r* engine_namer file_format onnx_pathrmemory_resultsrexcrrr run_testsFs          :rrc s@t|dddd }gdttdd|D}|ttdd|D}|ttd d|D}|g}|D]}|D] } |d | d |qCq?tj||d } | |D]} i} i} | d d|Di}|dd|D|D]b}|d| kr||rfdd| D}| s| |nD]}| |||krt dq|d} |d}d | d |}zt ||}Wn t yYqw| ||7<||d7<q| r|D]}||vr||dkr| |||| |<qd| |<q| | qa|WddS1swYdS)Nrr rr)r#r rrr*r1r_r$cSg|]}|dqS)r$rrrrrrjz"output_summary..cSr)r(rrrrrrjrcSr)r)rrrrrrjrb_srcSi|]}|dqSrrrhkrrr z"output_summary..cSr r rr rrrrrr$csi|] \}}|vr||qSrr)rhrv header_namesrrrszDescription shall be uniquer(r)rr)rrsetsortr?rrrr<itemsrrT ValueErrorrr)rEr data_fieldrdescription_listr@rA data_namesr)r(rr$row sum_latency count_latencyrHheadersrkeylatencyrrroutput_summarysd      $r!cCs\t|dd|d}|r |S|r"|t|dd|d7}|t|dd|d7}|t|dd|d7}|S)zARun experiments to compare different algorithms on one batch sizeTF)rrr_r()r)rr( is_baseline test_resultsrrrrun_experimentss8r$cCstjdt}t|jttj dkr.t |}t d}d|d}t||dSt}td|gd}gd}|rR|d d d krRgd }gd}|r_td d|d dnd}tjdddk}d||rrdnd} td| d|d|d} g} t| D]} |D]} td| |d}t|d| |7} qqdD]}t| | d|d|qg} t| D]} |D]} td| |d}t|d| |7} qqdD]}t| | d|d|qdS) Nspawnrz %Y%m%d-%H%M%Sbenchmark_detail_z.csvz GPU info: %s)rrr)rr(rrtotall )@ rr'rr(rz (?u)[^-\w.]rFnamegpuORT_LONGFORMER_BASELINErrlongformer_base_ _baseliner zexperiment_name=z, fp16_batch_sizes=z, fp32_batch_sizes=T)rr(r"zlongformer_base_fp16.csv)rrr+r.Fzlongformer_base_fp32.csv)rmultiprocessingset_start_methodrr= setup_loggerrVrsysrrrr;strftimer get_gpu_infor4r5resubrrrrwr$r!)rr# time_stamprgpu_listfp16_batch_sizesfp32_batch_sizesgpu_namer"experiment_name total_runs all_resultsrFr( fp16_results metric_name fp32_resultsrrrmainsT           rD__main__)T)Fr!FTFFF)rr)TFTTTTTrr)F)6rrloggingrQrr7r4r8rconcurrent.futuresrrtypingrrrrPrMrlongformer_helperrrr transformersr rprr?rdirname__file__r= getLoggerr4r:rIr]r~rrrrrrrrrrLrrr!r$rD__name__rrrrs|       5 u  .  7 _1  S I% 1