o ;ήcT@sFddlZddlZddlZddlmZddlmZddlmZm Z ddl Z ddl Z ddl m Z mZmZmZddlZddlmZmZmZmZGdd d eZGd d d ejd ZGd ddZGdddeZGdddeZGdddeZGdddeZGdddejd ZGdddeZ ddej!difdee e"fddZ#dS) N)Enum)Path)OptionalSequence) ModelProto TensorProtohelper numpy_helper) apply_plotclone_model_with_shape_infer load_modelsmooth_distributionc@seZdZdZdZdZdS)CalibrationMethodrr N)__name__ __module__ __qualname__MinMaxEntropy PercentilerrI/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/quantization/calibrate.pyrsrc@s<eZdZeddZejdefddZddZ dd Z d S) CalibrationDataReadercCst|dr t|jp tS)Nget_next)hasattrcallablerNotImplemented)clssubclassrrr__subclasshook__sz&CalibrationDataReader.__subclasshook__returncCt)z9generate the input data dict for ONNXinferenceSession runNotImplementedErrorselfrrrr#szCalibrationDataReader.get_nextcCs|SNrr%rrr__iter__(szCalibrationDataReader.__iter__cCs|}|dur t|Sr')r StopIteration)r&resultrrr__next__+szCalibrationDataReader.__next__N) rrr classmethodr abcabstractmethoddictrr(r+rrrrrs  r) metaclassc@sveZdZ    ddeeefddZdgfdd Zd d Zd d Z ddZ ddZ de fddZ de fddZdS)CalibraterBaseNaugmented_model.onnxFop_types_to_calibratecCs~t|trtt|d|_nt|trt|d|_n t|tr#||_ntd||_||_||_ ||_ d|_ d|_ dg|_ dS)a :param model: ONNX model to calibrate. It can be a ModelProto or a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param symmetric: make range of tensor symmetric (central point is 0). :param use_external_data_format: use external data format to store model which size is >= 2Gb Fz5model should be either model path or onnx.ModelProto.NCPUExecutionProvider) isinstancestrr rmodelr ValueErrorr3augmented_model_path symmetricuse_external_data_format augment_model infer_sessionexecution_providers)r&r7r3r9r:r;rrr__init__3s    zCalibraterBase.__init__r4cCs||_|dS)zz reset the execution providers to execute the collect_data. It triggers to re-creating inference session. N)r>create_inference_session)r&r>rrrset_execution_providersTs z&CalibraterBase.set_execution_providerscCs,t}tjj|_tj|j||jd|_dS)z9 create an OnnxRuntime InferenceSession. ) sess_options providersN) onnxruntimeSessionOptionsGraphOptimizationLevelORT_DISABLE_ALLgraph_optimization_levelInferenceSessionr9r>r=)r&rBrrrr@[s  z'CalibraterBase.create_inference_sessionc Csdd|jjD}|dd|jjD|dd|jjDtdd|jjD}t}ttjtj g}|jj D]6}|j rF|j |j vrqt |j|jD]"}||vrp||}|jdrp|jjj|vrp||vrp||qNq;||fS)z select all quantization_candidates op type nodes' input/output tensors. returns: tensors (set): set of tensor name. value_infos (dict): tensor name to value info. cSi|]}|j|qSrname).0virrr nz>CalibraterBase.select_tensors_to_calibrate..cSrJrrK)rMotrrrrOorPcSrJrrK)rMitrrrrOprPcs|]}|jVqdSr'rK)rMinitrrr qz=CalibraterBase.select_tensors_to_calibrate.. tensor_type)graph value_infoupdateoutputinputset initializerrFLOATFLOAT16noder3op_type itertoolschainkeystypeHasFieldrW elem_typeadd) r&r7 value_infosr^tensors_to_calibratetensor_type_to_calibratera tensor_namerNrrrselect_tensors_to_calibrategs(     z*CalibraterBase.select_tensors_to_calibratecC|jS)z. return: augmented onnx model )r<r%rrrget_augment_modelsz CalibraterBase.get_augment_modelcCr")z abstract method: augment the input model to prepare for collecting data. It will: 1. save augmented model to augmented_model_path. 2. set the self.augment_model r#r%rrr augment_graphszCalibraterBase.augment_graph data_readercCr")z abstract method: collect the tensors that will be used for range computation. It can be called multiple times. r#r&rrrrr collect_datazCalibraterBase.collect_datacCr")zy abstract method: compute the [min, max] range for the tensors to calibrate based on the collected data. r#rsrrr compute_rangeruzCalibraterBase.compute_range)Nr2FF)rrrrrr6r?rAr@rnrprqrrtrvrrrrr12s  ! r1csfeZdZ      ddeeeffdd Zdd Zd d Zd e fd dZ ddZ ddZ Z S)MinMaxCalibraterNr2F{Gz?r3cs|tt|j|||||dg|_d|_t|jjj|_ t dd|jjjD|_ ||_ |r9|dks5|dkr9t d||_dS)a :param model: ONNX model to calibrate. It can be a ModelProto or a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param symmetric: make range of tensor symmetric (central point is 0). :param use_external_data_format: use external data format to store model which size is >= 2Gb :param moving_average: compute the moving average of the minimum and maximum values instead of the global minimum and maximum. :param averaging_constant: constant smoothing factor to use when computing the moving average. r3r9r:r;NcsrSr'rKrMr[rrrrUrVz,MinMaxCalibrater.__init__..rr z;Invalid averaging constant, which should not be < 0 or > 1.)superrwr?intermediate_outputscalibrate_tensors_rangelenr7rXr[num_model_outputsr]model_original_outputsmoving_averager8averaging_constant)r&r7r3r9r:r;rr __class__rrr?s  zMinMaxCalibrater.__init__cst|j|\}}ttttj dgtj d}j j |fdd}|D] }||d||dq/tj|j|jd|_dS) z Adds ReduceMin and ReduceMax nodes to all quantization_candidates op type nodes in model and ensures their outputs are stored as part of the graph output :return: augmented ONNX model r dtypecs~d}|d|}|d}tjj||g|g||d}tjjd|g|g|d}jj||gjjt|t j dgdS)Nr __Reshape)keepdimsrLReshape)inputsoutputsrL) onnxr make_noderXraextendr[appendmake_tensor_value_inforr_)rmreduce_op_namer reduce_outputintermediate_output reduce_node reshape_noder7reshape_shape_namerradd_reduce_min_maxs  z:MinMaxCalibrater.augment_graph..add_reduce_min_max ReduceMin ReduceMaxsave_as_external_dataN)r r7rnr6uuiduuid4r from_arraynparrayint64rXr^rrsaver9r;r<)r&tensorsr reshape_shapertensorrrrrqs     zMinMaxCalibrater.augment_graphcC g|_dSr'r|r%rrrclear_collected_data z%MinMaxCalibrater.clear_collected_datarrcCsR |}|sn |j|jd|qt|jdkrtd||dS)NTrNo data is collected.) rr|rr=runr~r8rvr)r&rrrrrrrts zMinMaxCalibrater.collect_datacCs|s|S|D]F\}}|jr2|d|j||d|d}|d|j||d|d}nt|d||d}t|d||d}||f||<q|S)Nrr )itemsrrminmax)r& old_range new_rangekeyvalue min_value max_valuerrr merge_ranges"$zMinMaxCalibrater.merge_rangecstjdkr jSfddttjdDfddjD}i|D]}|D] \}}|g|q.q(jdfddtdtdD}tfdd D}g}tdtdD]r}d} d} j rt j ||dd } t j ||d dd } nt ||} t ||d } t| tks| jdkrt| } t| tks| jdkrt| } jrt t| t| } |t| | gqi|t| | gqitt||}jrj|_jS|_jS) z Compute the min-max range of tensor :return: dictionary mapping: {added node names: (ReduceMin, ReduceMax) pairs } rcg|] }j|jqSrr= get_outputsrLrMir%rr z2MinMaxCalibrater.compute_range..cg|] }tt|qSrr/ziprMr output_namesrrrNcsg|] }|ddqS)rr) rpartitionr)added_output_namesrrr#src3s&|]}|jvr||fVqdSr')rr)merged_output_dictr&rrrU's    z1MinMaxCalibrater.compute_range..)axisr )r~r|r}ranger setdefaultrrr/rrmeanrrrfintsizefloatr:abstuplerr)r&output_dicts_listdkvcalibrate_tensor_namesmerged_added_output_dictpairsrrrmin_value_arraymax_value_arraymax_absolute_valuenew_calibrate_tensors_ranger)rrrr&rrvsR   zMinMaxCalibrater.compute_range)Nr2FFFrx)rrrrrr6r?rqrrrtrrv __classcell__rrrrrws #. rwcsbeZdZ        ddeeeffd d Zd d Zd dZde fddZ ddZ Z S)HistogramCalibraterNr2F percentile-X@r3c sxtt|j|||||dg|_d|_t|jjj|_ t dd|jjjD|_ d|_ ||_ ||_||_| |_d|_dS)a :param model: ONNX model to calibrate. It can be a ModelProto or a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_bins: number of bins to create a new histogram for collecting tensor values. :param num_quantized_bins: number of quantized bins. Default 128. :param percentile: A float number between [0, 100]. Default 99.99. ryNcsrSr'rKrzrrrrUkrVz/HistogramCalibrater.__init__..)r{rr?r|r}r~r7rXr[rr]r collectormethodnum_binsnum_quantized_binsrrk) r&r7r3r9r;rr:rrrrrrr?Js"  zHistogramCalibrater.__init__cCs`t|j}||\|_}|jD]}||jvr |jj||qtj ||j |j d||_ dS)z make all quantization_candidates op type nodes as part of the graph output. :return: augmented ONNX model rN) r r7rnrkrrXr[rrrr9r;r<)r&r7rjrrrrrqss    z!HistogramCalibrater.augment_graphcCrr'rr%rrrrrz(HistogramCalibrater.clear_collected_datarrcs |}|sn jjd|qtjdkrtdfddttjdDfddjD}i|D]}|D] \}} |g|qCq=t fdd D}j spt j jjjjd _ j |dS) zy Entropy Calibrator collects operators' tensors as well as generates tensor histogram for each operator. TNrrcrrrrr%rrrrz4HistogramCalibrater.collect_data..crrrrrrrrrc3s&|]}|jvr||fVqdSr')rkr) merged_dictr&rrrUs$z3HistogramCalibrater.collect_data..)rr:rrr)rr|rr=rr~r8rrrr/rHistogramCollectorrr:rrrcollectr)r&rrrrrrrclean_merged_dictr)rrr&rrts8    z HistogramCalibrater.collect_datacCs|jstd|jS)z Compute the min-max range of tensor :return: dictionary mapping: {tensor name: (min value, max value)} z9No collector created and can't generate calibration data.)rr8compute_collection_resultr%rrrrvs z!HistogramCalibrater.compute_range)Nr2FrFrrr) rrrrrr6r?rqrrrtrvrrrrrrIs  )%rcs:eZdZ       d deeeffdd ZZS) EntropyCalibraterNr2Fentropyrr3c $tt|j||||||||ddS)a :param model: ONNX model to calibrate. It can be a ModelProto or a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_bins: number of bins to create a new histogram for collecting tensor values. :param num_quantized_bins: number of quantized bins. Default 128. )rr:rrN)r{rr?) r&r7r3r9r;rr:rrrrrr?  zEntropyCalibrater.__init__)Nr2FrFrrrrrrrr6r?rrrrrr rcs:eZdZ       d deeeffdd ZZS) PercentileCalibraterNr2Frrrr3c r)a :param model: ONNX model to calibrate. It can be a ModelProto or a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_quantized_bins: number of quantized bins. Default 128. :param percentile: A float number between [0, 100]. Default 99.99. )rr:rrN)r{rr?) r&r7r3r9r;rr:rrrrrr?rzPercentileCalibrater.__init__)Nr2FrFrrrrrrrrrrc@s,eZdZdZejddZejddZdS)CalibrationDataCollectorzL Base class for collecting data for calibration-based quantization. cCr")z Generate informative data based on given data. name_to_arr : dict tensor name to NDArray data r#r& name_to_arrrrrrsz CalibrationDataCollector.collectcCr")z? Get the optimal result among collection data. r#r%rrrr sz2CalibrationDataCollector.compute_collection_resultN)rrr__doc__r-r.rrrrrrrs  rc@s`eZdZdZddZddZddZdd Zd d Zd d Z ddZ ddZ ddZ ddZ dS)ra` Collecting histogram for each tensor. Percentile and Entropy method are supported. ref: https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py ref: https://docs.nvidia.com/deeplearning/tensorrt/pytorch-quantization-toolkit/docs/_modules/ pytorch_quantization/calib/histogram.html cCs(i|_||_||_||_||_||_dSr')histogram_dictrr:rrr)r&rr:rrrrrrr?s  zHistogramCollector.__init__cCror')rr%rrrget_histogram_dict$sz%HistogramCollector.get_histogram_dictcCsHtd|jdkr||S|jdkr |jr||S||Std)Nz/Collecting tensor data and making histogram ...rr3Only 'entropy' or 'percentile' method are supported)printr collect_valuer:collect_absolute_valuer8rrrrr's     zHistogramCollector.collectcCsL|D]\}}t|}|}|jdkr!t|}t|}nd}d}t|}||jvrCtj ||j d\}}||||f|j|<q|j|}|d} |d} |d} |d} t|} | | dkr| d| d}t | d|| ||}t | |f} tj || d\}}|dt | | 7<||t| |t| |f|j|<qdS)z5 Collect histogram on absolute value r)binsrr N)rrasarrayflattenrrrabsoluter histogramrarangehstackr~)r&rrdata_arrrrhist hist_edges old_histogramold_minold_maxold_histold_hist_edges temp_amaxwidth new_bin_edgesrrrr6s4          z)HistogramCollector.collect_absolute_valuec Cs|D]Y\}}t|}|}|jdkr!t|}t|}nd}d}tt|t|}||jvrE|j|}| ||||||j|<qtj ||j | |fd\}} || |||f|j|<qdS)z1 Collect histogram on real value rrN) rrrrrrrrrmerge_histogramrr) r&rrrrr thresholdr rrrrrr[s,         z HistogramCollector.collect_valuecCs|\}}}} } || kr)tj|t|| | fd\} } | ||t||t| || fS| dkrAtj|t|| |fd\} }| |7} n9t|}d| |}t|| |d}|d|}||| }tj||| |fd\} }| ||||7<| |t||t| ||fS)Nrrrr )rrr~rrr)r&r rnew_minnew_max new_thresholdr r r r  old_thresholdnew_histrrr old_num_bins old_stridehalf_increased_bins new_num_binsrrrr{s2    z"HistogramCollector.merge_histogramcCsX|jr t|jdkrtdtd|j|jdkr|S|jdkr(|Std)Nrz=Histogram has not been collected. Please run collect() first.z@Finding optimal threshold for each tensor using {} algorithm ...rrr)rr~r8rformatrcompute_entropycompute_percentiler%rrrrs  z,HistogramCollector.compute_collection_resultcCsr|jdks |jdkrtd|j}|j}i}tdt|td|jtdd|||D]\}}|d}|d}|}t ||} |j rgt | |d} t ||  t || f||<n"d|d } t | d | } t | | } t || t || f||<|d } |d }||d| kr| ||df||<||d|kr||d|f||< q5|S) Nrdz.rrinfr)copy scipy.statsrrrzerosrrdeepcopyr"astyperrr5ndarrayargmin)r&rrr.rrrrzero_bin_indexnum_half_quantized_bin kl_divergence thresholdsr start_index end_indexsliced_distributionpleft_outliers_countright_outliers_countnonzerosquantized_binsnum_merged_binsindexstartendqnormmin_kl_divergence_idxr,rrrrrr+sf            z(HistogramCollector.get_entropy_thresholdN)rrrrr?rrrrrrrrr+rrrrrs% ! - rr2Fr3c CsBd}|tjkr1d|vr dn|d}d|vrdn|d}d|vr!dn|d} t||||||| d}n]|tjkr`d|vrs>  m+p""=