o ;ήcZ[@sddlZddlZddlZddlmZddlZddlZddlmZddl m Z m Z m Z ddl mZddlmZddlmZddlmZmZmZmZejejejed d dd lmZmZm Z d ej!d <e"e#Z$dej%iZ&dEddZ'ddZ(ddZ)ej*fddZ+ddZ,ddZ-ddZ.ddZ/ dFdd Z0d!e1d"e1d#e2d$e3d%e3d&e d'e3d(e3fd)d*Z4d+e1d,e1d-e1fd.d/Z5d0d1Z6 dFd2d3Z7d4d5Z8dGd7d8Z9d9d:Z:d;d<Z;d=d>ZdCdDZ?dS)HN)Path)AffinitySetting) OptimizerInfo Precisioncreate_onnxruntime_session) MODEL_CLASSES)QuantizeHelper)torch_onnx_export) AutoConfig AutoTokenizer LxmertConfigTransfoXLConfigmodelsgpt2)PRETRAINED_GPT2_MODELSGPT2ModelNoPastStateTFGPT2ModelNoPastState2TF_CPP_MIN_LOG_LEVELtriucCs|dusJt|jdkr|d|dksJtd}|tjdtjd|}|d|dd|df}t||t |S)Nrr)rdtype) lenshapesize torch_functorchonesuint8wherebool zeros_like)xdiagonalout torch_triutemplatemaskr+M/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/onnx_exporter.py triu_onnx!s & r-cCs tt_dSN)r-rrr+r+r+r,replace_torch_functions+s r/cCstdt_dS)Nr)rrrr+r+r+r,restore_torch_functions/sr0c Cstjjd|d||f|d}d|i}d|vr#tj||g|d}||d<d|vr4tj||g|d} | |d<|jr;||d<t|tr^tjdd|j  tj |d <tjdd|j  tj |d <t|t rotj|jgtj d|d <|S) Nrr)lowhighrr input_idsattention_maskrtoken_type_idsdecoder_input_ids visual_feats visual_posz@tf_transfo_xl_model/transformer/pos_emb/einsum/Einsum/inputs_1:0)numpyrandomrandintr zerosis_encoder_decoder isinstancer randnvisual_feat_dimastypefloat32visual_pos_dimr hidden_size) vocab_size batch_sizesequence_length input_namesconfig data_typer3inputsr4 segment_idsr+r+r,create_onnxruntime_input3s$    rMcCs&i}|D] }||vr||||<q|Sr.r+)rKrHremaining_model_inputs input_namer+r+r, filter_inputsMs  rPcCs$t|ttfrdd|DgS|gS)NcSsg|]}t|qSr+)flatten.0ir+r+r, Vszflatten..)r>listtuple)rKr+r+r,rQUs$rQcCs0|D]}t|ttfs||nt||q|Sr.)r>rVrWappendupdate_flatten_list)rKres_listrTr+r+r,rYYs rYc Cs|djd}dd|D}ddtt|D}t|D]%\}}ddi||<||j}t|D]\}} | |krC|||d iq2q||fS) Nr3cSsi|]}|dddqS)rFseq_len)rrr+)rSkeyr+r+r, bsz&build_dynamic_axes..cSsg|] }dt|dqS)output_r)strrRr+r+r,rUdsz&build_dynamic_axes..rrFr\)rkeysranger enumerateupdate) example_inputsoutputs_flattenrG dynamic_axes output_namesrT output_namedimsjdimr+r+r,build_dynamic_axes_s  rmc CsNt||dd}|durt|ddSt|ddd|D}|||}t|t|krEtdt|dt|dStt|D]Q} t t || ||  } | d krntd | d | |rrd nd } |rxd nd } tj || ||  | | dstd| d| d| dSqKtd|dS)NF)enable_all_optimizationz is an invalid ONNX modelz is a valid ONNX modelcSsi|] \}}||qSr+)r9)rSktr+r+r,r^~sz'validate_onnx_model..z"Number of output tensors expected z, got g-C6?zMax absolute diff=z for output tensor g?g?)rtolatolzOutput tensor z is not close: rtol=z, atol=z0inference result of onnxruntime is validated on T) rloggererrorinfoitemsrunrrbr9amaxabscpuallclose) onnx_model_pathreexample_outputs_flattenuse_gpufp16rh test_sessionexample_ort_inputsexample_ort_outputsrTabs_diffrqrrr+r+r,validate_onnx_modelns: $   ronnx_dir model_name input_countoptimized_by_scriptr~ precisionoptimized_by_onnxruntimeuse_external_datac Csddlm}|dd|} |s| d|} n|rdnd} | d|d|d| } |r/| d7} |} |rG|sGtj|| } tj| sGt| tj| | dS) Nr)subz [^a-zA-Z0-9_]_gpurz_ortz.onnx)rerospathjoinexistsmakedirs) rrrrr~rrrrnormalized_model_namefilenamedevice directoryr+r+r,get_onnx_file_paths    r file_pathsuffixreturncCs&t|}t|j|j||jS)a Append a suffix at the filename (before the extension). Args: path: pathlib.Path The actual path object we would like to add a suffix suffix: The suffix to add Returns: path with suffix appended at the end of the filename and before extension )rr`parentjoinpathstem with_suffixr)rrrr+r+r,add_filename_suffixsrcCsh|stj|s*t|jjdddddlm}m}||||dd}||||<dSt d|dS)NTparentsexist_okr)get_fusion_statisticsoptimize_by_onnxruntimec)r~optimized_model_path opt_level'Skip optimization since model existed: ) rrrrrmkdir optimizerrrrsru)r|ort_model_pathr~ overwritemodel_fusion_statisticsrrrr+r+r,optimize_onnx_model_by_ortsrc  Cs|stj|slt|jjdddddlm} ddlm } | dur&| |} | |t j |kr3d| _ t j|kr;d| _| ||||d| |dd}|dksO|d krS||| |<t j |krd|jdd ||| dStd |dS) NTrr) FusionOptions)optimize_modelF) num_headsrDroptimization_optionsr~only_onnxruntime bert_kerasbert_tf) keep_io_typesr)rrrrrrfusion_optionsrrruse_raw_attention_maskrFLOAT16enable_gelu_approximationINT8enable_embed_layer_normuse_dynamic_axesget_fused_operator_statisticsconvert_float_to_float16save_model_to_filersru)r|r model_typenum_attention_headsrDr~rrrruse_external_data_formatrrr opt_modelr+r+r,optimize_onnx_models8         rcCsz|dkr|tvr |Stddt|tvrdSddl}|d|dkr'dS|d|dkr1dS|d |dkr;d Sd S) NzValid model class:  rrz-squad$AutoModelForQuestionAnsweringz-mprc$"AutoModelForSequenceClassificationrAutoModelWithLMHead AutoModel)r Exceptionrrrsearch)rcustom_model_classrr+r+r,modelclass_dispatchersrFcCszt||}|dkr|rtj|||dStj|||dS|r!d|}td|gd}td|t||}|j|||dS)Nr)rI cache_dirTF transformers)fromlistzModel class name: )rrfrom_pretrainedr __import__rsrugetattr)rrIrr is_tf_modelmodel_class_nametransformers_module model_classr+r+r,load_pretrained_model$s  rcCs@tj||d}t|drd|_||t||||d}||fS)Nr return_dictF)rIrr)r rhasattrrmodifyr)rrrconfig_modifierrImodelr+r+r, load_pt_model7s   rcCsHtj||d}||t}|t||||dd}|||fS)NrT)rIrrr)r rrr get_affinityr set_affinity)rrrrrIaffinity_settingrr+r+r, load_tf_modelCs rcCs ddlm}||\}}||fS)Nr)tf2pt_pipeline)convert_tf_models_to_pytorchr)rrrIrr+r+r,load_pt_model_from_tfXs  rc Csd}|r t| |||d|}|tjkr| || jfS|tjks'|tjks'|tjkrqt||t |d||d|}t | ||| j | j ||| | | || |} |rUt| ||||tjk|}|tjkrqt d| t| | |t d| |tjkr|rt| d}t| ||| | | || jfS)NTFzQuantizing model: zFinished quantizing model: r)rrNOOPTrEBYSCRIPTrrrrrrrrDrsrurquantize_onnx_modelBYORTrr)rrrrrHr~r optimize_info validate_onnxrrrIrr|rer}rhris_valid_onnx_modelrrr+r+r,validate_and_optimize_onnxbs       rcCsxt||||\}}|tj||d}||jvr|j|nd}|jddd}t||}|d i|}t|tt fsCJdt |t |}t |g}t ||t|d| | d|}|satj|std|t|jjd d d t||\}}tt|t ||t|||d ||d tntd |t|||||| | | | | ||||||d|\}}}||||fS)NrrThis is a sample inputpt)return_tensorsz%type of output is not list or tuple: FExporting ONNX model to {}Tr) rargsfrHrhrgdo_constant_folding opset_versionr!Skip export since model existed: r+)rrzr rmax_model_input_sizes encode_plusrPr>rVrWtyperQrYrrrrrrsruformatrrrrmr/r valuesrar0r)rrrrrrrrrHr~roptimizer_inforrrrrrIr tokenizermax_input_sizereexample_outputsr}r|rgrhonnx_model_filerrEr+r+r,export_onnx_model_from_ptsx        rc(Csddl}|jgdtj||d}|jdur|ddi||jvr(|j|nd}t||||\}}| t ||j dd|d d d }t ||}|j rY|j dd|d d d j|d <|d kru|jdd|jg|d<|jdd|jg|d<z|jr|d|_WnY||dd}d}|dks|dkrdg}|d}ddlm}||}t||t |d| | d|}|r|ddn|}|stj|sYtd||st|jj d d dddl!}ddl"}|j#$|j#j%g}|&D]\} }!dgt |!j'}"|(|j)t*|"|!j+| dq|j,j-|t*||||d\}#}#|rX|.|d}$|$/tj0|Wdn 1s6wYtj1tj0|d}tj|rRt2|t3||ntd||d}t4|||||| | | | | ||||||||\}%}&}'|%|&|'|fS) NrGPUr pad_tokenz[PAD]rrtf max_lengthT)rr padding truncationr6zunc-nlp/lxmert-base-uncasedrr7r8F)trainingzxlnet-base-casedzxlnet-large-casedlast_hidden_state)nestrr)name)input_signatureopset large_model output_pathrz__MODEL_PROTO.onnxr_tf)5 tensorflowrIset_visible_devicesr rr add_special_tokensrrresize_token_embeddingsrrrPr=r3r:normalr@rC use_cachetensorflow.python.utilrrQrrrrrsrurrrrzipfiletf2onnxlogging set_levelERRORrvrrX TensorSpecrWrconvert from_kerasZipFile extractalldirnamerremoverenamer)(rrrrrrrrrHr~rrrrrrrr rrrIrrerrhrr}r|tf_internal_model_pathr r!specsrvaluerjrzoptimized_onnx_pathrrEr+r+r,export_onnx_model_from_tfs          r2)rNr.)F)@r"rsyspathlibrr9raffinity_helperrbenchmark_helperrrrhuggingface_modelsrquantize_helperrtorch_onnx_export_helperr rr r r r rrXrr*__file__ gpt2_helperrrrenviron getLogger__name__rsrrr-r/r0int64rMrPrQrYrmrr`intr#rrrrrrrrrrrr2r+r+r+r,sp            , !  6   [ b