
    VpfY                        d Z ddlZddlZddlZddlmZ ddlZddlZddlZ	ddl
mZ ddl
mZ ddlZddlmZ ddlmZ ddlmZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZ ddlm Z m!Z" ddlZ#ddl$Z%ej&        '                                  ej(                    Z)d Z*d Z+ G d dej,                  Z- ej.        d           G d dej,                              Z/e0dk    r  ej1         ej2                               dS dS )z3Tests for serialization and deserialization of GDA.    N)partial)absltest)parameterized)	test_util)array)
xla_bridge)NamedShardingGSPMDShardingSingleDeviceSharding)PartitionSpec)serialization)LayoutDeviceLocalLayoutc                  ^    t                               t          j        d                     d S )N   )_exit_stackenter_contextjtuset_host_platform_device_count     w/var/www/html/nettyfy-visnx/env/lib/python3.11/site-packages/jax/experimental/array_serialization/serialization_test.pysetUpModuler   )   s'    C>qAABBBBBr   c                  8    t                                            d S N)r   closer   r   r   tearDownModuler   ,   s    r   c                   `   e Zd Zd Z ej        d          d             Zd Zd Zd Z	d Z
 ej        ej        ej        g          d	             Z ej        ej        ej        g          d
             Zd Zd Zd Zd Zd Z ej        dd          d             Zd Zd Zd Zd ZdS )CheckpointTestc                 0    t          j        ||           d S r   )osrename)selftemp_ckpt_dirfinal_ckpt_dirs      r   _on_commit_callbackz"CheckpointTest._on_commit_callback2   s    Im^,,,,,r   cpuc           	      `   t          j        dd          }dt          dd          }t          j                  }t          ||          t          j        |t          j	                  
                              t          j        fd          }t          j        |                     d          j                  }t#          j        t'          |                    t#          j                    }|                    |ggt-          | j        ||          	           |                                 fd
}t3          j                     t7          j         |                       t3          j                    \  }}	|                     |	d           t#          j                  }
t3          j                      t7          j        |
          !                                 t3          j                    \  }}	| "                    |	d           t3          j#                     d S )N)      xy)i   i   r,   r-   dtypec                     |          S r   r   idxsrcs    r   <lambda>z8CheckpointTest.test_memory_consumption.<locals>.<lambda>?   s    CH r   memprofon_commit_callbackc                     K   t          j        t          j        d                     d {V } |                                  d S )Ni@@ )byte_limiter)r   async_deserialize_LimitInFlightBytesblock_until_ready)r	inp_shapeshardingtspecs    r   deserialize_with_byte_limitzKCheckpointTest.test_memory_consumption.<locals>.deserialize_with_byte_limitI   st      /%"6yAAC C C C C C C C Ca r   i i)$r   create_global_meshPmathprodr	   jnparangenpint32reshaper   make_array_from_callbackpathlibPathcreate_tempdir	full_pathr   get_tensorstore_specstrGlobalAsyncCheckpointManager	serializer   r&   wait_until_finishedtmstartasynciorunget_traced_memory
assertLessr:   clear_tracesr<   assertGreaterstop)r#   global_meshpspecnuminpckpt_dirmanagerrA   unused_currentpeakdeserialize_wo_limitr>   r?   r3   r@   s              @@@@r   test_memory_consumptionz&CheckpointTest.test_memory_consumption5   s   (<<KIc3KKE
)I

C[%00H
*S
)
)
)
1
1)
<
<C

(8 C |D//	::DEEH.s8}}==E8::G	w"4#;XxPP  R R R !!!       HJJJK++--.../11ND 	OOD*%%%(:%$ $O
 K$%%77999/11NDtZ(((GIIIIIr   c           	         t          j        dd          }d}t          dd          }t          j        |          }t          ||          }t          j        |t          j	                  
                    |          t          j        ||fd          }t          j        |                     d          j                  }t#          j        t'          |                    }|j        d |j        d	|d
<   t          j        dg          }	t-          j                     	 t#          j                    }
|
                    |g|gt5          | j        ||                     |
                                 t-          j                    \  }}|                     |j        d|	 z  dz   z             t-          j                      d S # t-          j                      w xY w)N)   ri   r+   ) @  rj   r,   r-   r.   c                     |          S r   r   r1   s    r   r4   zACheckpointTest.test_memory_consumption_for_save.<locals>.<lambda>k   s    S r   memprofsave)shape
compressorchunksmetadatar'   r6   ri   g      ?)!r   rB   rC   rD   rE   r	   rF   rG   rH   rI   rJ   r   rK   rL   rM   rN   rO   r   rP   rQ   rm   test_device_matchesrU   rV   rR   rS   r   r&   rT   rY   rZ   nbytesr]   )r#   r^   r>   r_   r`   r?   ra   rb   r@   is_cpurc   rd   re   r3   s                @r    test_memory_consumption_for_savez/CheckpointTest.test_memory_consumption_for_savec   s   (<<K&Ic3KKE
)I

C[%00H
*S
)
)
)
1
1)
<
<C

(81111 C |D//>>HIIH.s8}}==E) E* $eW--FHJJJ:<<g%'$&(      !!###133nd
oodCJ!6z*:S*@ABBBgiiiiibgiiiis   "BG Gc           	      F   t          j        dd          }d}t          dd          }t          j        |          }t          j        |t
          j                                      |          t          j
        |t          ||          fd          }t          j        |                     d          j                  }t          j        |                     | d	          j                  }t!          |          g}t#          j                    }	|	                    |g|t)          | j        ||          
           |	                                 |	                    t          ||          g|          \  }
|                     |
t          j                   |                     t          j        |
j        d         j                  t          j	        dgdggt
          j                             |                     t          j        |
j        d         j                  t          j	        dgdggt
          j                             |                     |
j        d         j        j        d           |                     |
j         t
          j                   d S )Nr*   r)   r+   r   r)   r,   r-   r.   c                     |          S r   r   r2   global_input_data1s    r   r4   zECheckpointTest.test_checkpointing_with_path_variant.<locals>.<lambda>       &s+ r   ckpt_variant/firstr6   r   r)   ri      r)   ri   )!r   rB   rC   rD   rE   rH   rG   rI   rJ   r   rK   r	   rL   rM   rN   rO   rQ   r   rR   serialize_with_pathsr   r&   rT   deserialize_with_pathsassertIsInstance	ArrayImplassertArraysEqualasarrayaddressable_shardsdataassertEqualrm   r/   )r#   r^   r>   r_   r`   a1rb   
ckpt_path1
ckpt_pathsrc   m1rz   s              @r   $test_checkpointing_with_path_variantz3CheckpointTest.test_checkpointing_with_path_variant   sf   (<<KIc3KKE
)I

C 3bh777??	JJ		'=e44++++
- 
-B |D//??IJJHx///00:< <J j//"J8::G  	j"4#;XxPP ! R R R !!!

(
(	{E	*	*+Z9 9CB"eo...2:b&;A&>&CDD8aS1#Jbh???A A A2:b&;A&>&CDD8aS1#Jbh???A A AR*1-28&AAARXrx(((((r   c           
         t          j        dd          }d}t          dd          }t          j        |          }t          j        |t
          j                                      |          t          j
        |t          ||          fd          }t          j        |                     d          j                  }t          j        |                     | d	          j                  }t          j        |||z   t
          j                                      |          t          j
        |t          ||          fd
          }t          j        |                     | d          j                  }	d }
t          j        dd          }t          j
        dt          |t          d                     |
          }t          j        |                     | d          j                  }t!          |          t!          |	          t!          |          g}t"          j                            t(          j        |          }t)          j                    }|                    |||g|t1          | j        ||                     |                                 t)          j        t          ||          t          |t          d                    t          |t          d                     g|          \  }}}|                     |t          j                   |                     t          j        |j         d         j!                  t          j	        dgdggt
          j                             |                     t          j        |j         d         j!                  t          j	        dgdggt
          j                             | "                    |j         d         j!        j#        d           | "                    |j$        t
          j                   |                     |t          j                   |                     t          j        |j         d         j!                  t          j	        ddgddggt
          j                             |                     t          j        |j         d         j!                  t          j	        ddgddggt
          j                             | "                    |j         d         j!        j#        d           | "                    |j$        t
          j                   |                     |t          j                   tK          |j                   D ]\  }}| "                    |j&        tO          d           f           | "                    |j(        |           |                     t          j        |j!                  t          j	        g t
          j)                             | "                    |j$        t
          j)                   d S )Nrv   r+   rw   r,   r-   r.   c                     |          S r   r   ry   s    r   r4   z=CheckpointTest.test_checkpointing_jax_array.<locals>.<lambda>   r{   r   ckptr}   c                     |          S r   r   r2   global_input_data2s    r   r4   z=CheckpointTest.test_checkpointing_jax_array.<locals>.<lambda>   r{   r   z/secondc                 B    t          j        g t           j                  S Nr.   rH   r   float32_s    r   cb3z8CheckpointTest.test_checkpointing_jax_array.<locals>.cb3       Xb
++++r   r   r,   r   z/thirdr6   r   r)   ri   r~   r               r)   r)   )*r   rB   rC   rD   rE   rH   rG   rI   rJ   r   rK   r	   rL   rM   rN   rO   rQ   jax	tree_utiltree_mapr   rP   rR   rS   r   r&   rT   run_deserializationr   r   r   r   r   r   r   rm   r/   	enumerateindexslice
replica_idr   )r#   r^   r>   r_   r`   r   rb   r   a2
ckpt_path2r   global_mesh1da3
ckpt_path3r   tspecsrc   r   m2m3isrz   r   s                         @@r   test_checkpointing_jax_arrayz+CheckpointTest.test_checkpointing_jax_array   sx   (<<KIc3KKE
)I

C 3bh777??	JJ		'=e44++++
- 
-B |D//77ABBHd11X2E2E2EFFPQQJ S3Ybh( ( ((/	(:(: 		'=e44++++
- 
-B d11X2F2F2FGGQRRJ, , ,*488M		'mM1T7733S
: 
:Bd11X2E2E2EFFPQQJj//3z??C
OODJ]##M$F
SSF8::G	Rf"4#;XxPP  R R R !!!2	{E	*	*	{AcFF	+	+	}agg	.	.	0 		 JBB 	"eo...2:b&;A&>&CDD8aS1#Jbh???A A A2:b&;A&>&CDD8aS1#Jbh???A A AR*1-28&AAARXrx((("eo...2:b&;A&>&CDD8b"XBx$8IIIK K K2:b&;A&>&CDD8b"XBx$8IIIK K KR*1-28&AAARXrx((("eo..."/00 Q Q1
qwt///
q|Q'''
RZ//"BJ1O1O1OPPPPRXrz*****r   c           
         t          j        dd          }d}t          dd          }t          j        |          }t          j        |t
          j                                      |          t          j
        |t          ||          fd          }t          j        |                     d          j                  }|d	z  }t          j        |||z   t
          j                                      |          t          j
        |t          ||          fd
          }|dz  }	d }
t          j        dd          }t          j
        dt          |t          d                     |
          }|dz  }t!          |          t!          |	          t!          |          g}t"          j                            d |          }t)          j                    }t-          j        d          5 }|                    |||g|t3          | j        ||          |           d d d            n# 1 swxY w Y   |                                 t)          j        t          ||          t          |t          d                    t          |t          d                     g|          \  }}}|                     |t          j                   |                     t          j         |j!        d         j"                  t          j	        dgdggt
          j                             |                     t          j         |j!        d         j"                  t          j	        dgdggt
          j                             | #                    |j!        d         j"        j$        d           | #                    |j%        t
          j                   |                     |t          j                   |                     t          j         |j!        d         j"                  t          j	        ddgddggt
          j                             |                     t          j         |j!        d         j"                  t          j	        ddgddggt
          j                             | #                    |j!        d         j"        j$        d           | #                    |j%        t
          j                   |                     |t          j                   tM          |j!                  D ]\  }}| #                    |j'        tQ          d           f           | #                    |j)        |           |                     t          j         |j"                  t          j	        g t
          j*                             | #                    |j%        t
          j*                   d S )Nrv   r+   rw   r,   r-   r.   c                     |          S r   r   ry   s    r   r4   zECheckpointTest.test_checkpointing_ocdbt_transaction.<locals>.<lambda>   r{   r   r   firstc                     |          S r   r   r   s    r   r4   zECheckpointTest.test_checkpointing_ocdbt_transaction.<locals>.<lambda>  r{   r   secondc                 B    t          j        g t           j                  S r   r   r   s    r   r   z@CheckpointTest.test_checkpointing_ocdbt_transaction.<locals>.cb3  r   r   r   r   r   thirdc                 .    t          j        | d          S )NTocdbt)r   rP   )ps    r   r4   zECheckpointTest.test_checkpointing_ocdbt_transaction.<locals>.<lambda>  s    -4QdCCC r   T)atomic)r7   transactionr   r)   ri   r~   r   r   r   r   r   r   )+r   rB   rC   rD   rE   rH   rG   rI   rJ   r   rK   r	   rL   rM   rN   rO   rQ   r   r   r   r   rR   tsTransactionrS   r   r&   rT   r   r   r   r   r   r   r   r   rm   r/   r   r   r   r   r   )r#   r^   r>   r_   r`   r   rb   r   r   r   r   r   r   r   r   r   rc   r   r   r   r   r   r   rz   r   s                          @@r   $test_checkpointing_ocdbt_transactionz3CheckpointTest.test_checkpointing_ocdbt_transaction   s   (<<KIc3KKE
)I

C 3bh777??	JJ		'k5))++++
 
B
 |D//77ABBHG#J 3c	BBBJJ  
	'k5))++++
 
B
 H$J, , , *488M		'mM1T7733S
 
B G#Jj//3z??C
OODJ]##CCZ F 8::G	t	$	$	$ r2,
$&(  "                   !!!2+u--+qvv..-411	

 	 JBB 	"eo...

2(+011
1#s28,,,   	

2(+011
1#s28,,,   	R*1-28&AAARXrx((("eo...

2(+011
2r(RH%RX666   	

2(+011
2r(RH%RX666   	R*1-28&AAARXrx((("eo..."/00 Q Q1
qwt///
q|Q'''
RZ//"BJ1O1O1OPPPPRXrz*****s   /1H,,H03H0)input_dtypec                 6   t          j        dd          }d}t          j        |          }t	          j        ||                              |          fd}t          j        |t          |t          dd                    |          }t          j        |                     d          j                  }t          |          g}t           j                            t&          j        |          }	t'          j                    }
|
                    |g|	t/          | j        ||          	           |
                                 t          t          j        d
d          t          dd                    }t'          j        |g|	dgt          j        g          \  }t	          j        dgdgdggt          j                  t	          j        dgdgdggt          j                  t	          j        dgdgdggt          j                  t	          j        dgdgdggt          j                  t	          j        dgdgdggt          j                  t	          j        dgdgdggt          j                  t	          j        dgdgdggt          j                  t	          j        dgdgdggt          j                  d}|j        D ]?}|                     t	          j        |j                  ||j         j!                            @tE          j#        tI          |j%        j&                            }t'          j        |g|	dgt          j        g          \  }|j        D ]0}|                     |j        '                    d                     1d S )Nr   r+   rw   r.   c                     |          S r   r   )r   rz   s    r   cb1zJCheckpointTest.test_checkpointing_with_bigger_shape_jax_array.<locals>.cb1Q  s    &&r   r,   r-   r   r6   rv      r)   r   r)   r*   ri   r~         r   
      	      r            r   ri   r)   r~   r*   r   r   r   r   )(r   rB   rD   rE   rH   rG   rJ   r   rK   r	   rC   rL   rM   rN   rO   rQ   r   r   r   r   rP   rR   rS   r   r&   rT   r   r   r   r   r   r   deviceidr
   get_replicatedlistdevicesflatastype)r#   r   r^   global_input_shaper`   r   arrrb   r   r   rc   dsr   expected_datalnew_dsr   rz   s                    @r   .test_checkpointing_with_bigger_shape_jax_arrayz=CheckpointTest.test_checkpointing_with_bigger_shape_jax_arrayH  sb   (<<K
)&
'
'C3k:::BB ' ' ' ' '

(M+qc{{CCSJ JC|D//88BCCHh--J]##M$F
SSF8::G	v"4#;XxPP  R R R !!!	s-fjAA1S#;;	O	OB

+RD&7)-/ZL: :CB 8aS1#sO2:6668aS1#sO2:6668aS1#t$BJ7778aS1#t$BJ7778bTB4!%RZ8888bTB4!%RZ8888aS1#sO2:6668aS1#sO2:666	 	M " M M
RZ//qx{1KLLLL)${/B/G*H*HIIF

+VHfvh
U
UCB" K K
QV%7%>%>y%I%IJJJJK Kr   c                    t          j        dd          }d}t          j        |          }t	          j        ||                              |          fd}t          j        |t          |t          dd                    |          }t          j        |                     d          j                  }t          |          g}t           j                            t&          j        |          }	t'          j                    }
|
                    |g|	t/          | j        ||          	           |
                                 t          t          j        d
d          t          dd                    }t5          j        d          }t'          j        |g|	dg|g          \  }t5          j        dgdgdgg|          t5          j        dgdgdgg|          t5          j        dgdgdgg|          t5          j        dgdgdgg|          t5          j        dgdgdgg|          t5          j        dgdgdgg|          t5          j        dgdgdgg|          t5          j        dgdgdgg|          d}|j        D ]?}|                     t	          j        |j                   ||j!        j"                            @tG          j$        tK          |j&        j'                            }t'          j        |g|	dg|g          \  }|j        D ]0}|                     |j         (                    |                     1d S )Nr   r+   rw   r.   c                     |          S r   r   )r   global_input_datas    r   cbz7CheckpointTest.test_checkpointing_with_int4.<locals>.cb  s    u%%r   r,   r-   r   r6   rv   int4r   r   r)   r*   ri   r~   r   r   r   r   r   r   r   r   r   r   r   r   ))r   rB   rD   rE   rH   rG   rJ   r   rK   r	   rC   rL   rM   rN   rO   rQ   r   r   r   r   rP   rR   rS   r   r&   rT   rF   r/   r   r   r   r   r   r   r   r
   r   r   r   r   r   )r#   r   r^   r   r`   r   r   rb   r   r   rc   r   target_dtyper   r   r   r   r   r   s                     @r   test_checkpointing_with_int4z+CheckpointTest.test_checkpointing_with_int4x  s]   (<<K
)&
'
'C	#[999AA & & & & &

(M+qc{{CCRI IC|D//88BCCHh--J]##M$F
SSF8::G	v"4#;XxPP  R R R !!!	s-fjAA1S#;;	O	OB9V$$L

+RD&7)-9N< <CB
 9qcA3_L9999qcA3_L9999qcA3%\:::9qcA3%\:::9rdRD1#&l;;;9rdRD1#&l;;;9qcA3_L9999qcA3_L999	 	M " M M
RZ//qx{1KLLLL)${/B/G*H*HIIF

+VHfvh
W
WCB" M M
QV%6%=%=l%K%KLLLLM Mr   c           	         t          j        dd          }d}t          j        d          t	          |t          d                     }t          j        ||fd          }t          j        | 	                    d          j
                  }t          |          g}t          j                            t          j        |          }t          j                    }|                    |g|t'          | j        ||                     |                                 t	          t          j        dd          t          d                     }	t          j        |	g|dgt          j        g          \  }
|
j        D ]L}|                     t          j        |j                                      t          j                             Md S )Nr)   r,   r   r*   c                     |          S r   r   )r2   r   s    r   r4   zDCheckpointTest.test_checkpointing_scalar_jax_array.<locals>.<lambda>  s    49 r   r   r6   )r   rB   rH   r   r	   rC   rK   rL   rM   rN   rO   rQ   r   r   r   r   rP   rR   rS   r   r&   rT   r   r   r   r   r   r   r   )r#   r^   r   r   array1rb   r   r   rc   r   r   r   r   s               @r   #test_checkpointing_scalar_jax_arrayz2CheckpointTest.test_checkpointing_scalar_jax_array  s   (55K8A;;Dk1T77++A+A44446 6F|D//88BCCHh--J]##M$F
SSF8::G	&"4#;XxPP  R R R !!!	s-dS::AdGG	D	DB

+				 CB " J J
RZ//RZ1H1HIIIIJ Jr   c                 z   t          j        dd          }t          j        d          }t	          j        |                                          }t          j        t          |t          d                     g|g          \  }|j        D ]/}|                     t          j        |j                  |           0d S )Nr   r,   i   )r   rB   rH   rG   r   r   specr   r   r	   rC   r   r   r   r   )r#   r^   r   r@   r   r   s         r   ,test_deserialize_tensorstore_array_jax_arrayz;CheckpointTest.test_deserialize_tensorstore_array_jax_array  s    (55K9T??DHTNN!!E

+	{AdGG	,	,-	 CB " 7 7
RZ//66667 7r   c           	          ddddddddd}|                      t          j        |                     |                      t          j        dd	dd
ddd                     d S )Nri   r)   bcr~   )arp   r*   r   defzarrgfile)    @   )ro   rm   thing)driverkvstorerp   one_more)
assertTruer   _spec_has_metadatar#   r   s     r   test_spec_has_metadataz%CheckpointTest.test_spec_has_metadata  s     
 
 
 
  D 	OOM4T::;;;OO(!   *
 *
 	 		 	 	 	 	r   c                 l    ddddddidd}|                      t          j        |                     d S )Nri   r)   r   r~   r   r*   r   assertFalser   r  r  s     r   test_spec_has_no_metadataz(CheckpointTest.test_spec_has_no_metadata  s[     
 
 
 
 
D 	]5d;;<<<<<r   c                 X    i }|                      t          j        |                     d S r   r
  r  s     r   test_empty_spec_has_no_metadataz.CheckpointTest.test_empty_spec_has_no_metadata  s,    D]5d;;<<<<<r   )gcszgs://my/ckpt/dir/path)filez/my/ckpt/dir/pathc                    t          j        |d          }|                    d          }|r@|                     |d         d         t          j                            |                     nN|                     |d         d         t           j         dt          j                            |                      |                     |d         d         d           d S )NTr   zgs://r  basez://path)r   rP   
startswithr   r!   r  dirname_DEFAULT_DRIVER)r#   r  r   is_gcs_paths       r   test_get_tensorstore_spec_ocdbtz.CheckpointTest.test_get_tensorstore_spec_ocdbt  s    
 -d$???D//'**K U
tIv.0E0EFFFF
tIv.'7SSBGOOD<Q<QSSU U UT)_V,f55555r   c                     d}|                      t          d          5  t          j        |d           d d d            d S # 1 swxY w Y   d S )Nzmy/ckpt/pathz"Checkpoint path should be absoluteTr   )assertRaisesRegex
ValueErrorr   rP   )r#   r  s     r   +test_get_tensorstore_spec_not_absolute_pathz:CheckpointTest.test_get_tensorstore_spec_not_absolute_path  s    D			
 D
F 
F ; ;(T::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;s   AA	Ac                 h   d}t          j        |d          }|                     t          j        |                     d}t          j        |d          }|                     t          j        |                     ddddd	d
dd}|                     t          j        |                     d S )Nzgs://some-buck/pathTr   z/tmp/checkpointcastrI   r   r   zs3://some-bucket/path)r  r  )r  r  )r  r/   r  )r   rP   r  is_remote_storager  )r#   gs_pathgs_spec
local_path
local_specnested_tspecs         r   test_maybe_cloud_storagez'CheckpointTest.test_maybe_cloud_storage  s    #G0EEEGOOM3G<<==="J3JdKKKJ]4Z@@AAA ")3JKK
 
 L 	OOM3LAABBBBBr   c           	      Z   t          j        dg          s|                     d           t          j        dd          }t	          j        d                              dd          }t          |t          dd	                    }t          j
        ||          }t          j        d
 t          t          j                                                |                                                                          }|                     |j        j        j        |j        j        d d d                    t-          j        |                     d          j                  }t-          j        |                     | d          j                  }t          j                            t8          j        |g          }t9          j                    }	|	                    |g|tA          | j!        ||                     |	"                                 t9          j#        |g|          \  }
|                     |
j        |           | $                    |
tJ          j&                   | '                    |
|           |
j(        D ](}| '                    |j)        ||j*                            )d S )Ntpuz"Layouts are only supported on TPUsrv   r+   r   r   r*   r,   r-   c                     | j         S r   )Tr   s    r   r4   z6CheckpointTest.test_load_with_layout.<locals>.<lambda>.  s    13 r   )out_shardingsr   r}   r6   )+r   rq   skipTestrB   rH   rG   rJ   r	   rC   r   
device_putjitr   DLLAUTOlowercompileoutput_layoutsr   layoutdevice_local_layoutmajor_to_minorrL   rM   rN   rO   r   r   r   rP   rR   rS   r   r&   rT   r   r   r   r   r   r   r   r   )r#   meshnp_inpr   r   
out_layoutrb   	ckpt_pathr   rc   outs              r   test_load_with_layoutz$CheckpointTest.test_load_with_layout%  s^   "E7++ :
mm8999!&*55DYr]]""1a((FdAc3KK((A
.
#
#CfSX6F6FGGGMM WYY~~'' SZ3B3B44R4HJ J J |D//77ABBHT00H1D1D1DEEOPPI]##M$FTTF8::G	v"4#;XxPP  R R R !!!,j\6BBDCSZ,,,#u///3'''# 6 6
QVVAG_55556 6r   c                 N   t           j        }d}t          j        t          j        |                                        |                              |          }t          j        | 	                    d          j
                  }t          j        j                            t          j                              }t          j                            t$          j        |g          }t%          j                    }|                    |g|d            |                                 t%          j        |g||g|g          \  }|                    t           j                  }	|                     |	j        t           j                   |                     |	|	z   |	dz             d S )Nrw   	test_ckptc                      d S r   r   r   r   r   r4   z?CheckpointTest.test_deserialization_with_int4.<locals>.<lambda>U  s    4 r   r6   )	shardingstensorstore_specsglobal_shapesdtypesr)   )rF   r   rG   rH   rE   rJ   r   rL   rM   rN   rO   r   r?   r
   r   r   r   r   r   rP   rR   rS   rT   r   int8r   r/   r   )
r#   r/   rm   r   rb   r?   r   rc   deserialized_arrr;  s
             r   test_deserialization_with_int4z-CheckpointTest.test_deserialization_with_int4E  st   HEE
*RWU^^
$
$
,
,U
3
3
:
:5
A
AC|D//<<FGGH |)88GGH]##*XJ F 8::G	'<    
 !!! &9* gw	   
!
!#(
+
+CSY)))39cAg.....r   N) __name__
__module____qualname__r&   r   skip_on_devicesrg   rt   r   r   r   r   productrH   rI   rF   bfloat16r   r   rD  r   r   r   r  r  r  named_parametersr  r  r%  r<  rF  r   r   r   r   r   0   s       - - - 3u+ + +Z! ! !F) ) )@B+ B+ B+H\+ \+ \+| =bh%=>>>-K -K ?>-K^ =ch%9:::/M /M ;:/MbJ J J<	7 	7 	7  2= = == = = "=!&# 6 6	 6; ; ;C C C&6 6 6@/ / / / /r   r   T)jax_enable_memoriesc                   @    e Zd Z ej        d          d             ZdS )TransferShardTestr'   c                 `   t          j        d                              d          }t          t	          j                    d         d          }t	          j        ||          }|j        d         }t          j	        t          j        |                    }|                     ||           d S )Nr   )r*   r*   r   r   )memory_kind)rH   rG   rJ   r   r   r   r-  r   rW   rX   r   transfer_shard_to_hostr   )r#   r8  r?   r   shardnp_outs         r   test_transfer_shard_to_hostz-TransferShardTest.test_transfer_shard_to_hosti  s    Yr]]""6**F#CKMM!$4(KKKH
.
*
*C"1%E[=eDDEEF66*****r   N)rG  rH  rI  r   rJ  rV  r   r   r   rP  rP  f  s=         3u+ + + + +r   rP  __main__)
testLoader)3__doc__rW   
contextlibrD   	functoolsr   r!   rL   tracemallocrU   absl.testingr   r   r   	jax.numpynumpyrF   jax._srcr   r   r   r   xbjax.shardingr	   r
   r   r   rC   $jax.experimental.array_serializationr   jax.experimental.layoutr   r   r/  rH   tensorstorer   configparse_flags_with_absl	ExitStackr   r   r   JaxTestCaser   with_configrP  rG  mainJaxTestLoaderr   r   r   <module>rm     s@   : 9             				      ! ! ! ! ! ! & & & & & & 



       % % % % % %       % % % % % % K K K K K K K K K K + + + + + + > > > > > > D D D D D D D D         
     " " ""j"$$C C C  s/ s/ s/ s/ s/S_ s/ s/ s/l T***+ + + + + + + +*+ z(-,3,..////// r   