o
    ;ήce                     @   s~   d dl Z d dlZd dlZdddddddd	Zd
d Zdd ZdddZdd Zdd ZdddZ	dd Z
edkr=e
  dS dS )    N)bert
BertConfig zQhttps://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip)r   r   r   zThttps://storage.googleapis.com/bert_models/2019_05_30/wwm_cased_L-24_H-1024_A-16.zip)r   r   r   zRhttps://storage.googleapis.com/bert_models/2018_10_18/uncased_L-24_H-1024_A-16.zip)albertAlbertConfigr   zBhttps://storage.googleapis.com/albert_models/albert_base_v1.tar.gz)r   r   r   zChttps://storage.googleapis.com/albert_models/albert_large_v1.tar.gz)gpt2
GPT2Config	GPT2Modelz0https://storage.googleapis.com/gpt-2/models/117M)r   r   r	   z0https://storage.googleapis.com/gpt-2/models/124M)zbert-base-uncasedzbert-base-casedzbert-large-uncasedzalbert-basezalbert-largez
gpt-2-117Mz
gpt-2-124Mc                 C   sb   t | }| dd }tj||}t|d}||j W d    |S 1 s*w   Y  |S )N/wb)	requestsgetsplitospathjoinopenwritecontent)tf_ckpt_urlckpt_dirrcompressed_file_namecompressed_file_dirf r   \/tmp/pip-target-vg8gfxp4/lib/python/onnxruntime/transformers/convert_tf_models_to_pytorch.pydownload_compressed_file9   s   

r   c                 C   sj   d }t | D ]	}t j| |} t j|r| }tt|d }|dd dd }t j||S )Nz/*data-00000-of-00001.r   r
   r   )	r   listdirr   r   isfilestrglob
rpartitionr   )r   sub_folder_dirounique_file_nameprefixr   r   r   get_ckpt_prefix_pathB   s   r)   	tf_modelsc              	   C   s  dd l }tj|tj |}tj|| }tj|s#t	| t
|  d }dd l}|d|d kret||}dd l}||d}	|	| t| W d    t|S 1 s\w   Y  t|S |d|d krt||}
dd l}||
d}|| t|
 W d    t|S 1 sw   Y  t|S dD ]+}t|d | }ttj||d}||j W d    n1 sw   Y  qt|S )	Nr      z.zip$r   z.tar.gz$)
checkpointzmodel.ckpt.data-00000-of-00001zmodel.ckpt.indexzmodel.ckpt.metar
   r   )pathlibr   r   r   Path__file__parentabsoluteexistsmakedirsTFMODELSresearchr   zipfileZipFile
extractallremover)   tarfiler   r   r   r   r   )
model_nametf_models_dirr-   base_dirr   r   r5   zip_dirr7   zip_reftar_dirr;   tar_reffilenamer   r   r   r   r   download_tf_checkpointP   sD   






rD   c                 C   s   t |  d }td|gd}t||}|dd }t|d }t|dkr)| n|t|d }t |  d dkrIdd	lm	} |
|}	||	fS t |  d }
td|
gd}t||
}||}	||	fS )
N   transformersfromlistr
   r   z/*config.json   r   )AutoModelForPreTraining)r4   
__import__getattrr$   r#   lenfrom_json_filer"   rF   rJ   from_config)r<   tf_checkpoint_pathconfig_nameconfig_modulemodel_configparent_pathconfig_pathconfigrJ   
init_modelmodel_categroy_namemodulemodel_categroyr   r   r   init_pytorch_model   s   
$

r[   c           
      C   s   dt |  d  }td|gd}|du rt||}nt |  d dkr%tdddlm} |j}|du r8||||n||||}	|	  |	S )	Nload_tf_weights_in_r   rF   rG   Fr   z*Only support tf2 ckeckpoint for Bert model)/convert_bert_original_tf2_checkpoint_to_pytorch)r4   rK   rL   NotImplementedErrorrF   r]   load_tf2_weights_in_berteval)
r<   rV   rW   rP   is_tf2load_tf_weight_func_namerY   load_tf_weight_funcr]   modelr   r   r    convert_tf_checkpoint_to_pytorch   s   
re   Fc                 C   sB   | t vr
t| d t| }t| |\}}t| ||||}||fS )Nz not implemented)r4   r^   rD   r[   re   )r<   ra   rP   rV   rW   rd   r   r   r   tf2pt_pipeline   s   rf   c                  C   s   dd l } dd l}| d}t D ]=}t|\}}|jt| d u s$J |jd|jd d|j	d}z|| W q t
yN } z|| W Y d }~qd }~ww d S )Nr   r   rE   )      )lowhighsizedtype)loggingtorch	getLoggerr4   keysrf   
model_typerandint
vocab_sizelongRuntimeError	exception)rm   rn   loggerr<   rV   rd   inputer   r   r   tf2pt_pipeline_test   s   
rz   __main__)r*   )F)r#   r   r   r4   r   r)   rD   r[   re   rf   rz   __name__r   r   r   r   <module>   s*   .	
3


