o ;ήc-@sddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z m Z mZmZmZmZmZmZmZmZmZmZmZmZmZddlmZGdddZdS) N)onnx_pb) ONNXModel)TENSOR_NAME_QUANT_SUFFIXQuantizationModeQuantizedValueQuantizedValueType QuantType __producer__ __version__add_infer_metadataattribute_to_kwargcompute_scale_zp find_by_nameget_qmin_qmax_for_qTypeget_qrange_for_qTypemodel_has_infer_metadata quantize_datasave_and_reload_modeltensor_proto_to_array)CreateOpQuantizerc@s.eZdZ dDddZddZddZdd Zd d Zd d ZddZ ddZ ddZ ddZ ddZ ddZddZddZddZd d!ZdEd"d#ZdEd$d%Zd&d'Zd(d)Zd*d+ZdFd-d.Zd/d0ZdGd2d3Z 1 1 4 1dHd5d6Z 7 1 1 4 1dId8d9ZdJd:d;Z 7 1dKdd?Zd@dAZ dBdCZ!dS)L ONNXQuantizerNc CsHt|st|}dd|jjD|_|jdd|jjD|jdd|jjDt||_ |s8|j ||_ ||_ ||_ ||_d|_| rK| ni|_d|jvoW|jd|_d|jvob|jd|_d|jvom|jd|_|tjk} d |jvr{| n|jd |_d |jvrdn|jd |_|tjkrtjjntjj|_|tjkrtjjntjj|_ ||_| |_| |_ | |_!g|_"d|_#d |_$i|_%|j%d d|jjD|j%d d|jjD|j j jj&D]}|j%dd|jDq|'|_(|j t)vrt*d+|j |,|_-d|_.d|_/d|_0d|_1i|_2|j 3|_4i|_5dS)NcSi|]}|j|qSname).0virrN/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/quantization/onnx_quantizer.py 7z*ONNXQuantizer.__init__..cSrrrrotrrrr8r cSrrrritrrrr9r FEnableSubgraphForceQuantizeNoInputCheckMatMulConstBOnlyWeightSymmetricActivationSymmetric/cSi|]}|jdqSrrr!rrrrnr cSr+r,rr#rrrror cSsi|]}|dqSr,r)r output_namerrrrqsz unsupported quantization mode {}fixed_quantization_range_uint8fixed_quantization_range_int8 fixed_zero fixed_zero_zp)6rrgraph value_info value_infosupdateoutputinputrmodelreplace_gemm_with_matmul per_channel reduce_rangemodestaticfuse_dynamic_quant extra_optionsenable_subgraph_quantizationforce_quantize_no_input_checkq_matmul_const_b_onlyr QInt8is_weight_symmetricis_activation_symmetric onnx_proto TensorProtoINT8UINT8activation_qType weight_qType tensors_rangenodes_to_quantizenodes_to_excludeop_types_to_quantize new_nodesparent graph_scope tensor_namesnodecheck_opset_version opset_versionr ValueErrorformatcalculate_quantization_paramsquantization_paramsfixed_qrange_uint8_namefixed_qrange_int8_namefixed_zero_namefixed_zero_zp_namequantized_value_mapget_non_initializer_inputsgenerated_value_namesused_scale_zp_map)selfr8r:r;r<r=rKrJrLrMrNrOr?is_weight_int8rTrrr__init__%sh         zONNXQuantizer.__init__c Cs|tjj|d|jjjd}t|t||j|j|j |j |j |j |j |j|j|j|j }||_d|j||_||jjjS)z generate submodel for the subgraph, so that we re-utilize current quantization implementation. quantize the submodel update subgraph and set it back to node zonnx-quantizer) producer_name opset_importsz{}{}/)onnxhelper make_modelr8 opset_importr rr:r;r<r=rKrJrLrMrNrOr?rQrXrRquantize_modelr2)rcsubgraph graph_key warped_model sub_quanitzerrrrquantize_subgraphs0 zONNXQuantizer.quantize_subgraphc Csdd|jD}t|dkr|S|jdkr|jn d|jt|j}i}|jD]I}|jtjj kr@|j| |j d||ji}n+|jtjj krgg}|j D]}|| |d||jt|gqL|j|i}nt|}||q'tjj|j|j|jfd|ji|S) z| Check subgraph, if any, quantize it and replace it. return new_nodes added for quantizing subgraph cSs,g|]}|jtjjks|jtjjkr|qSr)typerhAttributeProtoGRAPHGRAPHS)rattrrrr s  z>ONNXQuantizer.quantize_node_with_sub_graph..rz{}_node_count_{}z{}:{}z{}:{}:{}r) attributelenrrXop_typerPrrrhrsrtrqgrugraphsextendr r5ri make_noder7r6) rcrT graph_attrs node_namekwargsrvkvvaluermrrrquantize_node_with_sub_graphs0 $     $z*ONNXQuantizer.quantize_node_with_sub_graphcCsdd|jjjD}dt|krtd|dj}|dkr'td|dS|dkrMtd||jjj|d|jjj t j d d gd }d |_ |S) NcSs g|] }|jr |jdkr|qS)zai.onnx)domain)ropsetrrrrwsz5ONNXQuantizer.check_opset_version..rz$Failed to find proper ai.onnx domainr zThe original model opset version is {}, which does not support node fusions. Please update the model to opset >= 11 for better performance.zThe original model opset version is {}, which does not support quantization. Please update the model to opset >= 11. Updating the model automatically to opset 11. Please verify the quantized model.rx T)r8rkrzrWversionloggingwarningrXremover~rhri make_opsetidr>)rcai_onnx_domainrVrrrrUs0  z!ONNXQuantizer.check_opset_versioncCstdd|jDS)zQ Detect if model already has QuantizeLinear or DequantizeLinear. css$|] }|jdkp |jdkVqdS)QuantizeLinearDequantizeLinearN)r{)rrTrrr s z.ONNXQuantizer.has_QDQ_nodes..)anyr8nodesrcrrr has_QDQ_nodesszONNXQuantizer.has_QDQ_nodescCs2t||jdur dS|jdur|j|SdS)NTF)rr8 initializerrQfind_initializer_in_path)rcinitializer_namerrrrs   z&ONNXQuantizer.find_initializer_in_pathcCs2|j||D]}|jD]}|j|q qdSN)rPr~r6raadd)rcrrTr-rrr add_new_nodess  zONNXQuantizer.add_new_nodescCs|r td|jD]2}|jr||}t|j}t ||}| t |t|jD]}|j|j D]}|j |q6q.q||jd|jj|j|jdurq|j\}}t|dkrqtdt|t|jj_t|jj_|jjS)NzPlease check if the model is already quantized.Note you don't need to quantize a QAT model. OnnxRuntime support to run QAT model directly.rTrz0Invalid model with unknown initializers/tensors.)rrrr8rr@rrzrPrquantizeranger6rar_dequantize_outputsr2 ClearFieldrTr~rQclean_initializers RuntimeErrorstrr rfr producer_version)rcrTnumber_of_existing_new_nodes op_quantizerir-_initializers_not_foundrrrrls2       zONNXQuantizer.quantize_modelcCst||j}|duSr)rr8r)rc input_namerrrris_input_a_initializer$sz$ONNXQuantizer.is_input_a_initializercCs|jSr)r:rrrris_per_channel(szONNXQuantizer.is_per_channelcCsFt||j}|dur|jtjjkS|jr|jdurdS|j |S)NF) rr8r data_typerFrGFLOATr@rQis_valid_quantize_weight)rc weight_nameweightrrrr+s  z&ONNXQuantizer.is_valid_quantize_weightcCs~||r ||S||jvr)|j|}|jdr'|jjjtj j kr'dSdS|j r5|j r5|j |Std|dS)N tensor_typeTzzFailed to infer data type of tensor: {}. Please add data type info for this tensor if your model has customized operators.F)rrr4keysrrHasFieldr elem_typerFrGrr@rQis_float_tensorrrrX)rc tensor_namerrrrr3s    zONNXQuantizer.is_float_tensorcCsV|jdurt|jdkr|j|jvrdS|j|jvrdS|jdur)|j|jvr)dSdS)NrFT)rMrzrr{rOrN)rcrTrrrshould_quantize_nodeEs   z"ONNXQuantizer.should_quantize_nodecCs$|tjjkr |||S|||S)aZ Create nodes for dynamic quantization of input and add them to nodes_list. parameter input_name: Name of the input. parameter nodes_list: new nodes are appended to this list. parameter qType: type to quantize to. return: scale_name, zero_point_name, scale_shape, zero_point_shape. )rFrGrH+_get_dynamic_input_quantization_params_int8,_get_dynamic_input_quantization_params_uint8)rcr nodes_listqTyperrr&_get_dynamic_input_quantization_paramsUs   z4ONNXQuantizer._get_dynamic_input_quantization_paramscCstjj}|d}|d}tjjd|g|dg|dd}|||d}tjjd|g|dg|dd}|||d } tjd |jdg| dg| } || |d } tjd |jdg| dg| } || |d } tjd | jd| jdg| dg| }||tj|j tjj gt |d g}|j ||d}tjd|jd|j g|g|}||tj|j|gdg}|j |||jggfS)a/ Create nodes for dynamic quantization of input to int8 and add them to nodes_list parameter input_name: Name of the input. parameter nodes_list: new nodes are appended to this list. return: scale_name, zero_point_name, scale_shape, zero_point_shape. _scale _ReduceMin ReduceMin:0rkeepdims _ReduceMax ReduceMax_AbsAbs_Abs_MaxMaxg@ scale_DivDiv)rFrGrHrhrirappendr6 make_tensorr\rrr8add_initializerr^)rcrrrinput_scale_namereduce_min_namereduce_min_nodereduce_max_namereduce_max_nodereduce_min_abs_namereduce_min_abs_nodereduce_max_abs_namereduce_max_abs_node abs_max_name abs_max_nodeinitializer_divscale_div_namescale_div_nodeinitializer_zprrrrbs|           z9ONNXQuantizer._get_dynamic_input_quantization_params_int8cCstjj}|d}|d}|d}tjjd|g|dg|dd}|||d}tjjd |g|dg|dd} || tj|jtjj gt |g} |j | tj|j tjj gd g} |j | |d } tjd | jd|jdg| dg| } || |d }tjd| jd|jg|g|}|||d}tjd |j |jdg|dg|}|||d}tjd|jd|g|dg|}|||d}tjd|j|dg|}|||d}tjjd|j|g||d}||||ggfS)a0 Create nodes for dynamic quantization of input to uint8 and add them to nodes_list parameter input_name: Name of the input. parameter nodes_list: new nodes are appended to this list. return: scale_name, zero_point_name, scale_shape, zero_point_shape. r _zero_pointrrrrrrrg _scale_SubSub _scale_Divr_zero_point_Sub_zero_point_Div_zero_point_FloorFloor_zero_point_CastCast)to)rFrGrIrhrirrrr[rrr8rr]r6)rcrrrr input_zp_namerrrrinitializer_qrangeinitializer_qvaluescale_sub_namescale_sub_noderr zp_sub_name zp_sub_node zp_div_name zp_div_node zp_floor_name zp_floor_node zp_cast_name zp_cast_noderrrrs            z:ONNXQuantizer._get_dynamic_input_quantization_params_uint8cCs|dus|dur>|jdus||jvrtd|dS|j|}|dus+t|dkr3td|||dg}|dg}n|g}|g}g}|d}|j} g} |d } tj || ||} |j | tj | t j j| |} |j | d | || |fS) a\ Create initializers and inputs in the graph for zero point and scale of output. Zero point and scale values are obtained from self.quantization_params if specified. parameter param_name: Name of the quantization parameter. return: result, scale_name, zero_point_name, scale_shape, zero_point_shape. Nz5Quantization parameters for tensor:"{}" not specified)Frxrxrxrxz_Quantization parameters should contain zero point and scale. Specified values for output {}: {}rrrrT)rZrinforXrzrWrJrhrirr8rrFrGr)rc param_name use_scale use_zeropointparamszero_point_values scale_valueszero_point_shapezero_point_namezero_point_type scale_shape scale_nameinit_zp init_scalerrr_get_quantization_paramss0     z&ONNXQuantizer._get_quantization_paramscCs |j|}|t}|d}|dur|durd||} } } n ||\} } } } } g} | r:tjd|| | g|g|}n<|jr?dS|jr^|tj j kr^|d} |d} tjd|g|| | g|}n| || |\} } }}tjd|| | g|g|}t ||| | ||j |<| |gS)a Given an input for a node (which is not a initializer), this function - add nodes to compute zero point and scale for this input if they don't exist. - add new QuantizeLinear node to quantize the input. :param node: node being quantized in NodeProto format. :param input_index: index of input in node.input. :param qType: type to quantize to. :param given_scale_name: if those inputs need to be quanitzed using this scale tensor. :param given_zp_name: if those inputs to be quantized using this zeropoint tensor. :return: List of newly created nodes in NodeProto format. _QuantizeLinearNTrrrDynamicQuantizeLinear)r7rrrhrirr=r>rFrGrIrrr_)rcrT input_indexrgiven_scale_name given_zp_namerr- ql_node_name data_foundrzp_namerr qlinear_noderzp_shaperrr_get_quantize_input_nodes9sN   z'ONNXQuantizer._get_quantize_input_nodescCsDt|tr t|dksJd||jvsJ|d||j|<dS)Nrz(value must be scale(float) and zeropointz has been setted before) isinstancetuplerzrb)rcrrrrrset_quant_scale_zpwsz ONNXQuantizer.set_quant_scale_zpcCs.||jvr |j|S|jdur|j|SdS)NNN)rbrQfind_quantized_valuercrrrrfind_quant_scale_zp|    z!ONNXQuantizer.find_quant_scale_zpcCs.||jvr |j|S|jdur|j|SdSr)r_rQrrrrrrrz"ONNXQuantizer.find_quantized_value?cCs||jvr |j|jS|j|j}t||j}t|}t||j}t|} |t} ||jvr9|j|j} n||jvrI| |\} } } } } nt d |t| |j} t| }|||}t | |t j}t j |t jd|j}tj|| }|j|g| d}t j |t jdd}tj||}|j|g| d}t j|jt jdd}tj||}|j|g||jvsJt|| ||tj|jdkrdnd}||j|<| S) z] Quantized the bias. Zero Point == 0 and Scale == Input_Scale * Weight_Scale z@Expected {} to be in quantized value map for static quantizationdtyperrrrN)r_q_namerrr8rrrrZrrWrXnpasarrayroundastypeint32reshapedimsrh numpy_helper from_arrayr~float32zerosshaperr Initializersize)rc bias_namerrbetaweight_scale_nameweight_initializer weight_scalebias_initializer bias_dataquantized_bias_namerrinputscale_initializer input_scale bias_scalequantized_data bias_np_datapacked_bias_initializerquantized_bias_scale_namebias_scale_datapacked_bias_scale_initializerquantized_bias_zp_name bias_zp_datapacked_bias_zp_initializerquantized_valuerrrquantize_bias_staticsN       z"ONNXQuantizer.quantize_bias_staticcCs||jvp||jvp||jvS)zq only check for value info and newly generated tensor names, initializers are checked separately )r4rSra)rcrrrrcontains_tensors zONNXQuantizer.contains_tensorFc Cs|j||dddd|dS)NFrrTindicesinitializer_use_weight_qTyper;op_level_per_channelaxis from_subgraph_ONNXQuantizer__quantize_inputs)rcrTrDrHrrrquantize_activationsz!ONNXQuantizer.quantize_activationrc Cs|j||d||||dS)NTrCrI)rcrTrDr;rFrGrHrrrquantize_weights zONNXQuantizer.quantize_weightTc Cs8g}g} g} g} |D] } |j| } | |jvr/|j| }||j| |j| |jq t| |j}|durs|j rS|rS| |j |rI|j n|j ||\}}}n|||r[|j n|j |\}}}| || |||q || r|j| d|j|j}|dur||| |j }|durdS|r||n| ||d}|jdkr| |j||jd| |jdq | |jd||jd| |jdq |jdur |jj|| g||||d d \}}}}| |d||d| |dq td | |j| | || fS) a Given a node, this function quantizes the inputs as follows: - If input is an initializer, quantize the initializer data, replace old initializer with new initializer - Else, add QuantizeLinear nodes to perform quantization parameter node: node being quantized in NodeProto format. parameter indices: input indices to quantize. return: (List of quantized input names, List of zero point names used for input quantization, List of scale names used for input quantization, List of new QuantizeLinear nodes created) Nr)NNNNrrrrrT)rEr;rFrGrHz2Invalid tensor name to quantize: {} @graph scope{})r7r_rrr rrr8rr:quantize_weight_per_channelrrKrJquantize_initializerrBfind_node_by_namerPr2rrr~r{r6rQrJrWrXrR)rcrTrDrEr;rFrGrH scale_nameszero_point_namesquantized_input_namesrr node_inputr@r q_weight_namer rrquantize_input_nodesparent_quantized_input_namesparent_zero_point_namesparent_scale_namesrrrr__quantize_inputss                     zONNXQuantizer.__quantize_inputscCs$|j|jvr|j|j}|j|j|jfS|jt}|jd}|jd}t|} t|  ||j |j o4|\} } } } } t j |tjjg| g}t j ||g| g}|j||g|s|tj| t jj|d|j} t j| |}|j|gt|j|||tjd}||j|j<|||fS)a :param weight: TensorProto initializer :param qType: type to quantize to :param keep_float_weight: Whether to quantize the weight. In some cases, we only want to qunatize scale and zero point. If keep_float_weight is False, quantize the weight, or don't quantize the weight. :return: quantized weight name, zero point name, scale name rrrN) rr_rr rrrrflattentolistrDr;rhrirrFrGrr8rr~rrmappingTENSOR_TYPE_TO_NP_TYPEr#r$r%r&rrr*)rcrrr;keep_float_weightr@rTr r weight_datar zero_pointscale q_weight_datascale_initializerzero_initializerq_weight_initializerrrrrNbsF       z"ONNXQuantizer.quantize_initializerc Cs||jvr|j|}|j|j|jfSt||j}|dur#td|t|}|j |} g} g} g} g} g}t | D];}| ||}t | ||jpQ|tjjk|joU|\}}}}}| || || || |||q:t|j }d||<t|d|}t dt|D]}t|||}t||f|}q|t}|d}|d}t||||tjd}||j|<|j|g}t j!"|tjj#|| }t j!"|||| }|j$||g|stj|t j%j&|d|j}t j'(||}|j$|g|||fS)Nz{} is not an initializerrrrrr))r_rr rrr8rrWrr)rtakerrZr[rDrFrGrHr;rlistrrr#rz concatenaterrrr*r$rhrirrr~r\r]r%r&) rcrrK channel_axisr;r^r@rweights channel_count rmin_list rmax_listzero_point_list scale_listquantized_per_channel_data_listrper_channel_datarminrmaxr`raquantized_per_channel_data reshape_dimsquantized_weightschannel_weightsrTr rzero_scale_shapercrdrerrrrMs~                 z)ONNXQuantizer.quantize_weight_per_channelcCs||jvr@||jvr@|j|}|d}|j||j|j}|dur7|j|j|jg}t j d||g|}|S||j dks@JdS)a Given a value (input/output) which is quantized, add a DequantizeLinear node to dequantize it back to float32 parameter value_name: value to dequantize parameter new_nodes_list: List of new nodes created before processing current node return: None if there is already a DequantizeLinear node that dequantizes it A DequantizeLinear node otherwise _DequantizeLinearNrr) r_rar8rOrPr2rrr rhrirr6)rc value_namer@ dqlinear_name dqlinear_nodedqlinear_inputsdequantize_noderrr_dequantize_values  zONNXQuantizer._dequantize_valuecCs6|jjD]}||j}|dur|j|qdS)z Dequantize output if it is quantized parameter new_nodes_list: List of new nodes created before processing current node return: List of new nodes created N)r8r2r6rrrPr)rcr6r~rrrrs   z!ONNXQuantizer._dequantize_outputscCs|jdurdS|jD]D}|jdvrq |jrq ||sq t|j|jddkr-q |jd|j vsA|j d|j vrBq |j|j d|j|jd<q i}|j D]}|j|\}}t |j |jd\}}t |||||j||<qX|S)N)ClipRelurr) symmetric)rLr8rr{rErrzinput_name_to_nodesr7rr6rrJr)rcrTrZrrrrsqminqmaxrrrrYs(   (z+ONNXQuantizer.calculate_quantization_paramsrr)r)F)FFrF)TFFrF)FF)TF)"__name__ __module__ __qualname__rerqrrUrrrrlrrrrrrrrrrrrrrArBrKrLrJrNrMrrrYrrrrr$sX g"% S ] '> B   l: S r)rnumpyrrhonnx.numpy_helperrrF onnx_modelr quant_utilsrrrrr r r r r rrrrrrrrregistryrrrrrrs  L