o ;ήc@sddlZddlZddlZddlmZddlmZddlmZm Z m Z m Z ddl m Z ddlmZmZmZmZmZmZmZmZddlmZeeZGdd d ZdS) N)deque)Path)DictListOptionalTuple)convert_float_to_float16)AttributeProto GraphProto ModelProto NodeProto TensorProtohelper numpy_helper save_model)SymbolicShapeInferenceHelperc@seZdZddZddZddZidfdd Zd d Zd d ZddZ ddZ ddZ ddZ ddZ ddZddZddZddZd d!Zdd#d$Zdd%d&Zdd'd(Zdd)d*Zed+d,Zd-d.Zed/d0Zd1d2Zd3d4Zd5d6Zdd7d8Zdd9d:Zdd;d<Z gfd=d>Z!d"d"gd"fd?d@Z"dAdBZ# " "ddCdDZ$ddFdGZ%ddHdIZ&dJdKZ'dLdMZ(ddOdPZ)dQdRZ*ddSdTZ+ddUdVZ,dWdXZ-dYe.fdZd[Z/ed\e0d]e.fd^d_Z1d`daZ2dbdcZ3ddddeZ4ddfdgZ5ddhdiZ6djdkZ7dldmZ8ddndoZ9ddpdqZ:edrdsZ;dtduZdzd{Z?ed|d}Z@d~dZAe  E dddZBdddZCddZDddZEedeFdeFdeGfddZHddZIde.fddZJd"S) OnnxModelcCs||dSN) initializeselfmodelrJ/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/onnx_model.py__init__szOnnxModel.__init__cCs"||_i|_d|_d|_d|_dSNT)r_node_name_suffixshape_infer_helperenable_shape_infer all_graphsrrrrrs  zOnnxModel.initializecCs d|_dS)NF)rrrrrdisable_shape_inferences z!OnnxModel.disable_shape_inferenceFcCsd|jr0|jdus |rt|j|_z |j|r|jWSWdSd|_tdtdYdSdS)NFzfailed in shape inferencer)rrrrinferprintsysexc_info)rdynamic_axis_mappingupdaterrrinfer_runtime_shape"s  zOnnxModel.infer_runtime_shapecCsDi}|D]}|jD]}||vr|g||<q |||q q|Sr)nodesinputappend)rinput_name_to_nodesnode input_namerrrr,0s   zOnnxModel.input_name_to_nodescCs*i}|D] }|jD]}|||<q q|Sr)r)output)routput_name_to_noder- output_namerrrr0:s    zOnnxModel.output_name_to_nodecCs,g}|D] }|jD]}||q q|Sr)graphsr-r+)r all_nodesgraphr-rrrr)As    zOnnxModel.nodescCs|jjSr)rr4r rrrr4HszOnnxModel.graphcCs|jdur|jSg|_|jjg}|rY|d}|j||jD]6}|jD]0}|jtj j krt|jD]\}}|jD] }||vr|Sq qt|jSr) enumerater-r*len)rr4outputsidxr-r*rrrget_topological_insert_idxs   z#OnnxModel.get_topological_insert_idcCs(|D] }||jvr|j|qdSr)r2r-removerGrrr remove_nodes   zOnnxModel.remove_nodecCs|D]}||qdSr)rR)rnodes_to_remover-rrr remove_nodess zOnnxModel.remove_nodesNcCsV|dus ||jjjkr|jjj|gdS||}|||j}|j||dSr) rr4rAr-extendrKrPr/insert)rr-rJr4 insert_idxrrradd_nodes  zOnnxModel.add_nodecCs@|dur|jjj|dS|D] }||j}|||qdSr)rr4r-rUrArX)r nodes_to_addnode_name_to_graph_namer-rJrrr add_nodess  zOnnxModel.add_nodescCH|dus ||jjjkr|jjj|gdS||}|j|gdSr)rr4rA initializerrUrK)rtensorrJr4rrradd_initializer zOnnxModel.add_initializercCr\r)rr4rAr*rUrK)rr*rJr4rrr add_inputr`zOnnxModel.add_inputcCHt|tr t|ts Jtt|jD]}|j||kr!||j|<qdSr)r;strrangerMr*)r-old_input_namenew_input_namejrrrreplace_node_input  zOnnxModel.replace_node_inputcC"|jjjD] }t|||qdSr)rr4r-rrh)rrerfr-rrrreplace_input_of_all_nodesz$OnnxModel.replace_input_of_all_nodescCrbr)r;rcrdrMr/)r-old_output_namenew_output_namergrrrreplace_node_outputrizOnnxModel.replace_node_outputcCrjr)rr4r-rro)rrmrnr-rrrreplace_output_of_all_nodesrlz%OnnxModel.replace_output_of_all_nodescCs4|D]}|jD] }|j|kr|Sq qdSr)r2r]rA)rrAr4r^rrrget_initializers    zOnnxModel.get_initializercCs*g}|D] }|j|kr||q|Sr)r)op_typer+)rrrr)r-rrrget_nodes_by_op_types    zOnnxModel.get_nodes_by_op_typecCsD|dur|}g}|jD]}||vr||D]}||qq |Sr)r,r/r+)rr-r,childrenr/rrr get_childrens   zOnnxModel.get_childrencCs:|dur|}g}|jD] }||vr|||q |Sr)r0r*r+)rr-r0parentsr*rrr get_parentss zOnnxModel.get_parentscCs@|dur|}t|j|krdS|j|}||vrdS||Sr)r0rMr*)rr-ir0r*rrr get_parents zOnnxModel.get_parentcCs^t|jD]'\}}||vr,||}|j|kr ||vr ||fStd|d|jqdS)a Find parent node based on constraints on op_type. Args: node (str): current node name. parent_op_type (str): constraint of parent node op_type. output_name_to_node (dict): dictionary with output name as key, and node as value. exclude (list): list of nodes that are excluded (not allowed to match as parent). Returns: parent: The matched parent node. None if not found. index: The input index of matched parent node. None if not found. zTo find first z , current NN)rLr*rrloggerdebug)rr-parent_op_typer0excluderxr*parentrrrmatch_first_parents zOnnxModel.match_first_parentc Cs|dusJ|dus|dksJ|dur|}|dur1|||||\}}|dur/|||S|t|jkrHtd|dt|jdS||||}|dur^|j|kr^||vr^|S|durntd|d|jdS)a" Find parent node based on constraints on op_type and index. When input_index is None, we will find the first parent node based on constraints, and return_indice will be appended the corresponding input index. Args: node (str): current node name. parent_op_type (str): constraint of parent node op_type. input_index (int or None): only check the parent given input index of current node. output_name_to_node (dict): dictionary with output name as key, and node as value. exclude (list): list of nodes that are excluded (not allowed to match as parent). return_indice (list): a list to append the input index when input_index is None. Returns: parent: The matched parent node. Nrz input_index z >= node inputs zExpect z, Got ) r0rr+rMr*r{r|ryrr) rr-r} input_indexr0r~ return_indicerindexrrr match_parents$  zOnnxModel.match_parentcCs^t|D](\}}t|tst|tsJg}|||d|d||}|r,|||fSqdS)Nr)NN)rLr;rrmatch_parent_path)rr-pathsr0rxpathrmatchedrrrmatch_parent_paths4szOnnxModel.match_parent_pathsc Cst|t|ks J|dur|}|}g}t|D]1\}} |j|| |||g|d} | durDtjd|d||d| dddS|| | }q|S)a  Find a sequence of input edges based on constraints on parent op_type and index. When input_index is None, we will find the first parent node based on constraints, and return_indice will be appended the corresponding input index. Args: node (str): current node name. parent_op_types (str): constraint of parent node op_type of each input edge. parent_input_index (list): constraint of input index of each input edge. None means no constraint. output_name_to_node (dict): dictionary with output name as key, and node as value. return_indice (list): a list to append the input index when there is no constraint on input index of an edge. Returns: parents: a list of matched parent node. N)r~rzFailed to match index=z parent_input_index=z op_type=T) stack_info)rMr0rLrr{r|r+) rr-parent_op_typesparent_input_indexr0r current_nodematched_parentsrxrrmatched_parentrrrr=s. zOnnxModel.match_parent_pathTc Csj|||}t|}t|dkr3|}|j|kr|S|r-|||}|D]}||q%t|dksdSr5)rurrMr6rr appendleft) rr- child_typer, recursivertdqrchildrrrfind_first_child_by_typeos       z"OnnxModel.find_first_child_by_typec Csz|dur|}|||}t|}t|dkr;|}|j|kr#|S|r5|||}|D]}||q-t|dksdSr5)r0rwrrMr6rrr) rr- parent_typer0rrvrrrrrrfind_first_parent_by_type~s       z#OnnxModel.find_first_parent_by_typecCsh|dD]}|jd|kr#|jD]}|jdkr"t|jSqq||}|dur2t|SdS)NConstantrvalue)rsr/r7rArto_arraytrq)rr1r-attr]rrrget_constant_values    zOnnxModel.get_constant_valuecCs6t|jD]\}}||}|dur||fSqdS)Nrz)rLr*r)rr-rxr*rrrrget_constant_inputs   zOnnxModel.get_constant_inputư>cCs8||\}}|dur|jdkrt|||kr|SdS)Nrr)rsizeabs)rr-expected_valuedeltarxrrrrfind_constant_inputs"zOnnxModel.find_constant_inputc Csd||}|durt|d|ddSt|j|kr0t|d|d|d|jdSdS)N z is not initializer.Fz shall have z dimensions. Got shape T)rr{r|rMshape)rr1 dimensions descriptionrrrr$is_constant_with_specified_dimensions "z.OnnxModel.is_constant_with_specified_dimensioncCs||||dkSr5)r)rr-rrrrrhas_constant_inputszOnnxModel.has_constant_inputc Cs|dur|}||jd}g}t|}t|dkrK|}||vr$q||vrE|||jD]}||vrD||}|D]} || qsz'OnnxModel.get_dtype..N)rr4 value_infor elem_typefind_graph_inputr8find_graph_output)rrtensor_type_map graph_input graph_outputrrr get_dtypes     zOnnxModel.get_dtyper-attribute_namecCs,|jD]}|j|krt|}|SqdSr)r7rArget_attribute_value)r-rr?rrrrget_node_attributes   zOnnxModel.get_node_attributecCs~|}d}|D]"}|jdkr,|j|d|d}|r,|jdkr,|jd|jd<|d7}q |dkr=td||dSdS)avRemove Cast node that are followed by another Cast node like --> Cast --> Cast --> Note that this shall be used carefully since it might introduce semantic change. For example, float -> int -> float could get different value than the original float value. So, it is recommended to used only in post-processing of mixed precision conversion. rCast)r0rzRemoved %d cascaded Cast nodesN)r0r)rrryr*r{info prune_graph)rr0 removed_countr-rrrrremove_cascaded_cast_nodess    z$OnnxModel.remove_cascaded_cast_nodescsjdddurtddSfdd}g}D] }|jdkr>||jd}||jd}|r>||kr>||q|rt }t }|D]3}t t|j|@rqt t|j|@sp |jd|jdn qO |jd|jd|qOtd t|dSdS) zKRemove cast nodes that are not needed: input and output has same data type.T)r'Nz>Skip removing useless cast nodes since shape inference failed.cs:|}|r |Sj|jjdrj|jjjSdS)Nr)r known_vi_r8rrr)input_or_output_namedtyper shape_inferrr get_data_types z:OnnxModel.remove_useless_cast_nodes..get_data_typerrz4Removed %d Cast nodes with output type same as input)r(r{rr)rrr*r/r+setrCrFboolrprkrRrM)rrrSr- input_dtype output_dtypegraph_input_namesgraph_output_namesrrrremove_useless_cast_nodes s2         z#OnnxModel.remove_useless_cast_nodescCstd|jd|ddS)NzbThe function convert_model_float32_to_float16 is deprecated. Use convert_float_to_float16 instead!T)use_symbolic_shape_infer keep_io_types)r{warningr)rcast_input_outputrrr convert_model_float32_to_float163sz*OnnxModel.convert_model_float32_to_float16c sdvrdd<|j}|rt|}|j|ddd}d|i}|fdddDt|fi|}||||d S) aAConvert a model to half (default) or mixed precision. To use mixed precision, user need specify which graph inputs, outputs, operator type or list of nodes shall keep in float32. By default, we use symbolic shape inference to get shape and type information. If not, ONNX shape inference will be used. Note that symbolic/ONNX shape inference might fail, and the conversion might not proceed without shape and type information. Args: use_symbolic_shape_infer (bool, optional): use symbolic shape inference instead of onnx shape inference. Defaults to True. keep_io_types (Union[bool, List[str]], optional): It could be boolean or a list of float32 input/output names. If True, model inputs/outputs should be left as float32. Defaults to False. op_block_list (List[str], optional): List of operator types to leave as float32. Defaults to None, which will use `float16.DEFAULT_OP_BLOCK_LIST` as default. node_block_list (List[str], optional): List of node names to leave as float32. Defaults to None. force_fp16_initializers(bool): force converting all float initializers to float16. Default to false, which will convert only the one needed to avoid precision loss. min_positive_val (float, optional): minimal positive value. Defaults to 1e-7. max_finite_val (float, optional): maximal finite value. Defaults to 1e4. rTF) auto_mergeguess_output_rankdisable_shape_infercsi|] }|vr||qSrr)rkeykwargsrrrVs z6OnnxModel.convert_float_to_float16..)rmin_positive_valmax_finite_val op_block_listnode_block_listforce_fp16_initializersN)rr infer_shapesr'rrrr)rrrrr parameters fp16_modelrrrr9s    z"OnnxModel.convert_float_to_float16c Cs|r|dr |n|d}n|d}d}||jvr!|j|d}n.|D])}|jrN|j|rNzt|jt|d}t|d|}Wq%tyMYq%wq%||j|<|t |S)arCreate a unique node name that starts with a prefix (default is operator type). The name will not be duplicated with any name that generated or existed in current graphs. Args: op_type (str): operator type name_prefix (str, optional): prefix of node name. Defaults to None. Returns: str: node name _rrN) endswithrr)rA startswithintrMmax ValueErrorrc)rrr name_prefixprefixsuffixr-rrrrcreate_node_nameks"      zOnnxModel.create_node_namecC&|jjjD] }|j|kr|SqdSr)rr4r*rA)rr.r*rrrr  zOnnxModel.find_graph_inputcCrr)rr4r/rA)rr1r/rrrrrzOnnxModel.find_graph_outputc Cs|dur|}g}|||}t|}t|dkrC|}||vr#q||vr=|||jD] }||vr<|||q/t|dks|Sr5)r0rwrrMr6r+r*r) rr-rr0rrvrrr*rrrget_parent_subgraph_nodess"      z#OnnxModel.get_parent_subgraph_nodescCsrg}|jD]}||r||vr||q|r7||g}|D]}|jD]}||r5||vr5||q%q |S)z@ Find graph inputs that linked to current node. )r*rr+r)rrr graph_inputsr* parent_nodesr-rrrget_graph_inputss     zOnnxModel.get_graph_inputscCs,d}|jD]}||kr|S|d7}qdS)Nrrr)r*) node_output child_noderr*rrrrs   zOnnxModel.input_indexcCsr|}g}|}|D]}|jdkr|jd|vr||q ||t|dkr7tdt|dSdS)NrrzRemoved unused constant nodes: ) r,r)rrr/r+rTrMr{r|)rr, unused_nodesr)r-rrrremove_unused_constants   z OnnxModel.remove_unused_constantc Cs~t|dkrtddS|durdd|jjjD}|}g}|D]}||vrD||}||vr4q%||g}| || |q%g}|jjj D] }||vrW| |qL| |g} |jjjD] }|j |vrp| |qd| D] }|jjj|qs|} g} |jjjD] } | j | vr| | q| D] } |jjj| q| s| s|rtdt| t| t||dS)a@ Prune graph to keep only required outputs. It removes unnecessary inputs and nodes. Nodes are not linked (directly or indirectly) to any required output will be removed. Args: outputs (list): a list of graph outputs to retain. If it is None, all graph outputs will be kept. rz)Skip prune_graph since graph has subgraphNcSg|]}|jqSrrA)rr/rrr z)OnnxModel.prune_graph..z.zremove z unused inputs: cSrrr)rr]rrrr9rz unused initializers: zremaining initializers:)rr4r-rrr{r|r*r+rArQrMr]rr) rverboser4remaining_input_namesr-r.inputs_to_remover*names_to_removeweights_to_removeweights_to_keepr]rrrrsF           zOnnxModel.update_graphc Csb|D],}|jD]&}||vrq||vr-||D]}||vr,td|d|dSqqqdS)Nz,it is not safe to remove nodes since output z is used by FT)r/r{r|)rrS keep_outputsr,r0node_to_remover  impacted_noderrris_safe_to_fuse_nodes@s    zOnnxModel.is_safe_to_fuse_nodescCsdgt|j}i}g}t|jD]6\}}tdd|jD||<||dkr0||j|q|jD]}||vr?|g||<q3|||q3qdd|jD}dd|jD}||} | d} | D]*}| |krkqd|} ||vr||D]}||d||<||dkr||j|quqdd} t|} | | kr|| jD]'} | |vr|| D]}||d||<||dkr||j|| d} qq| d} | | ks| t|jkrt d| d t|jd |j| | d |j |dS) Nrcss|]}|rdVqdS)rNr)rrrrr Vsz3OnnxModel.graph_topological_sort..cSrrr)rinitrrrrbrz4OnnxModel.graph_topological_sort..cSrrrrrrrrcrrzGraph is not a DAG: end=z, len(graph.node)=z, graph.node[end]=r-) rMr-rLsumr*r+r]sortr/ RuntimeError ClearFieldrU)r4 deps_count deps_to_nodes sorted_nodesnode_idxr-r.initializer_namesrrBprev_input_namestartendr/rrrgraph_topological_sortOs^          z OnnxModel.graph_topological_sortcCst|jjdSr)rr'rr4r rrrtopological_sortszOnnxModel.topological_sortc Cst|jjddd|rht|j}|jddd|d}|r#t|jnd}tj|r8td|t ||rNtj|rMtd|t |n t |r[t d|dt ||d||||ddSt ||dS) NT)rvexist_okz.datazDelete the existed onnx file: z'Delete the existed external data file: zOutput directory (z!) for external data is not empty.)save_as_external_dataall_tensors_to_one_filelocationsize_thresholdconvert_attribute) rrmkdirrAosrexistsr{rrQlistdirrr) r output_pathr+r,r.r/ output_direxternal_data_pathr-rrrsaves4         zOnnxModel.savecCs|td||dr+t|d}|ttWdn1s%wYn t |j|||td|dS)Nz Sort graphs in topological orderz.jsonwzModel saved to ) r{rr(ropenwritercrrr7)rr4use_external_data_formatr,outrrrsave_model_to_files   zOnnxModel.save_model_to_filecCs2g}|jjjD]}||jdur||q|S)z[ Returns real graph inputs (excluding initializers from older onnx model). N)rr4r*rqrAr+)rrr*rrr'get_graph_inputs_excluding_initializerss  z1OnnxModel.get_graph_inputs_excluding_initializerscCs*|jjD] }|jdvr|jSqtd)zGet opset version of onnx domain Raises: RuntimeError: ONNX model has no opset for default domain. Returns: int: opset version of onnx domain. )zai.onnxz*ONNX model has no opset for default domain)r opset_importdomainversionr)ropsetrrrget_opset_versions  zOnnxModel.get_opset_versiontensor1tensor2returncCsP|j|jks |j|jkrdS|dr|dr|j|jkSt|t|kS)a"Returns True when two tensors have same value. Note that name can be different. Args: tensor1 (TensorProto): initializer 1 tensor2 (TensorProto): initializer 2 Returns: bool: True when two intializers has same value. Fraw_data) data_typedimsrrHrr)rErFrrrhas_same_values   zOnnxModel.has_same_valuecCst|dkr tdt|jjj}dg|}t|dD]'}||dkr(qt|d|D]}t |jjj||jjj|rE|||<q/qd}t|D] }||dkrm|d7}| |jjj|j |jjj||j qM|dkr| t d|ddSdS)zRemove initializers with duplicated values, and only keep the first one. It could help reduce size of models (like ALBert) with shared weights. Note: this function does not process subgraph. rz9remove_duplicated_initializer does not process subgraphs.rrzRemoved z# initializers with duplicated valueN)rMr2r{rrr4r]rdrrKrkrArr#)rinitializer_countsamerxrgcountrrrremove_duplicated_initializers0       z'OnnxModel.remove_duplicated_initializerrcCs@t|dkr tddd|jjjDdd|jjjD}|jjjD]}|j |vr:||j |vr:||j |_ q&|jjj D]J}t t|jD]}|j||vre||j||vre||j||j|<qIt t|jD]}|j||vr||j||vr||j||j|<qmq@|jjj D] }|j |vr||j |_ qdS)zAdd prefix to initializer or intermediate outputs in graph. Main graph inputs and outputs are excluded. It could help avoid conflicting in name of node_args when merging two graphs. Note: this function does not process subgraph. rz/add_prefix_to_names does not process subgraphs.cSrrr)rrxrrrrrz1OnnxModel.add_prefix_to_names..cSrrr)rorrrrrN) rMr2r{rrr4r*r/r]rAr-rdr)rrexcludedr]r-rgrrrradd_prefix_to_names s2 (    zOnnxModel.add_prefix_to_namesrrzr)r)T)F)FTr)F)FT)K__name__ __module__ __qualname__rrr!r(r,r0r)r4r2rCrFrHrKrPrRrTrXr[r_ra staticmethodrhrkrorprqrsrurwryrrrrrrrrrrrrrrcrr rrrrrrrrrrrrrrrr'r(r7r=r>rDr rrKrOrRrrrrrs          1 2     &  2"     ;- 7 *  r)loggingr1r$ collectionsrpathlibrtypingrrrrfloat16ronnxr r r r r rrrrr getLoggerrSr{rrrrrs   (