
    Vpfc                        d Z ddlmZ ddlZddlmZ ddlZddlZddlZddl	Z	ddl
mZ ddlZddlmZ ddlZddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ  ddl!m"Z# ddl"Z$ddl%Z& ej'                     ddl(m)Z)  ej*                    Z+da,d Z-d Z. G d de)j/                  Z0e1dk    r  ej2         ej3                               dS dS )a(  Tests for the jax2tf conversion of pjit.

 To verify that the tests do run indeed on multiple devices you can run

  perftools/gputools/profiler/jfprof.sh jax/experimental/jax2tf/tests:sharding_test_tpu -- -c opt --test_filter=ShardingTest.test_shmap_all_to_all --test_arg=--vmodule=jax2tf=3 --

    )SequenceN)partial)Any)absltest)compiler)config)	test_util)
xla_bridge)lax)jax2tf)pjit)	shard_map)NamedSharding)Mesh)PartitionSpec)tf_test_utilc                  `   t                               t          j        d                     t          j        dg          rjt
          j        j                            d          } t
          j	        
                    |            t
          j        j                            |           ad S d ad S )N   tpu )r   )_exit_stackenter_contextjtuset_host_platform_device_counttest_device_matchestf
distributecluster_resolverTPUClusterResolverr   experimental_connect_to_clusterr   experimentalinitialize_tpu_systemtopology)resolvers    k/var/www/html/nettyfy-visnx/env/lib/python3.11/site-packages/jax/experimental/jax2tf/tests/sharding_test.pysetUpModuler&   ;   s    C>qAABBB 	eW%% }-@@R@HHHI--h777v"88BBHHHHHH    c                  8    t                                            d S N)r   close r'   r%   tearDownModuler,   G   s    r'   c                   2    e Zd ZdZ fdZddddee         fdZ	 	 d.d	Zd
ee         de	fdZ
d Zddd
ee         fdZ ej        d dD                        ej        dg          d/d                        Z ej        dg          d             Z ej        dg          d             Z ej        d dD                        ej        dg          d0d                        Z ej        d dD                       d1d             Zd! Z ej        d" d#D                       d2d%            Z ej        ed&'          d(             Z ej        d)          d*             Z ej        d+ d,D                       d3d-            Z xZS )4ShardingTestz;Tests that inspect the HLO for the sharding annotations.
  c                 n   t                                                       t          j        dg          rt	          j        d          t          t          j                              dk     rt	          j        d          t          j
        t          j                    d d                   | _        d S )Ngpuz%Sharding HLO tests not useful for GPU   z&Test requires at least 2 local devices)supersetUpr   r   unittestSkipTestlenjaxdevicesnparray)self	__class__s    r%   r3   zShardingTest.setUpN   s    	GGMMOOO
w'' GEFFF
3;==AFGGG8CKMM"1"-..DLLLr'      r1   )num_replicasnum_partitionsargsc                D    |j         |                     d          }t          |          }t          j        d| j        |           t          j        dg          rt          j	                    }t          j        ||z            }t          j        |d|f          }|dk    }	t          j        ||||	          }
|                    ||
                                          d                                         }t          j        d	| j        |j        |           d
S d
S )z=Log the HLO generated from JAX before and after optimizations	stablehlo)dialectz[%s] got JAX HLO %sr   r=   )r>   r?   device_assignmentuse_spmd_partitioningr   z-[%s] got JAX optimized HLO for platform %s %sN)lowercompiler_irstrlogginginfo_testMethodNamer   r   r
   get_backendr9   arangereshaper   get_compile_optionscompilehlo_modules	to_stringplatform)r;   f_jaxr@   r>   r?   jax_compjax_hlobackendrE   rF   compile_optionsjax_optimized_hlos               r%   log_jax_hlozShardingTest.log_jax_hloW   s=    u{D!--k-BBH(mmGL&(<gFFF w'' N&((g)N\$ABB*%6^8LMM,q0 4#'- 5	  o "//
?$ $$/KMM!55>Y[[ lB')9;LN N N N NN Nr'   r=   r=   r=   r1   c                     |                      t          j                    d           t          j        j        j                            t          ||          S )Nr   )computation_shaper>   )	assertEqualr   device_under_testr   r   r!   DeviceAssignmentbuildr#   )r;   r^   r>   s      r%   rE   zShardingTest.device_assignmento   sQ     	S*,,e4446/55$5! 6 # # #r'   args_tfreturnc                    t          j        |dd          }t          j        d| j         |j        | j                                                   dt          j	                    
                                 d} |j        | } |dt          j	                    
                                          }t          j        d	| j        |           t          j        d
d|          S )zGet the unoptimized HLO from TFFT	autographjit_compilez[%s] Got TF graph %sz/device:z:0hlo)stageplatform_namez[%s] got TF HLO %szmetadata=.*r   )r   functionrJ   rK   rL   get_concrete_functiongraphas_graph_defr   r`   upperexperimental_get_compiler_irresub)r;   f_tfrc   f_tf_fundevice_nametf_hlo_generatortf_hlos          r%   rx   zShardingTest.tf_hlow   s    {45dCCCHL'%//9?LLNNP P P AS244::<<@@@K<x<gF3#8#:#:#@#@#B#B  F L%t';VDDD 6."f---r'   c                 
    d|fS )N>=r+   )r;   values     r%   GEQzShardingTest.GEQ   s    %=r'   r+   checksc                   |                      ||          }|D ]\  }}t          t          j        ||                    }t	          |t
                    r$|                     ||d| d| d| d|            `t	          |t                    r<|d         dk    r0|                     ||d         d| d|d          d| d|            J d
S )a  Check the sharding in TF.

    Args:
      f_tf: the TF callable
      args_tf: the TF args
      checks: a list of tuples. The first element is a regular expression, the
        second element is an integer representing the expected number of
        occurrences of the regular expression in the TF HLO. As a special case,
        the second element can be the result of `self.GEQ(v)` to check that
        the number of occurrences is greater or equal to a value.
    zregular expression `z` expected to occur z times but occurs z* times in the TF HLO.
This is the TF HLO:
r   rz   r=   z` expected to occur at least FN)	rx   r6   rr   findall
isinstanceintr_   tupleassertGreaterEqual)r;   rt   rc   r~   rx   check_reexpected_countcounts           r%   check_shardingzShardingTest.check_sharding   sH    [[w''F$*   ."*Xv..//e	NC	(	( >:H : :: :16: :17: :	< 	< 	< 	<
 ne,, 1Bd1J1J>!$:H : :&q): :=B: :17: :	< 	< 	< 	< 	 r'   c           	      H    g | ]}d D ]}t          d| d| ||           S missingNPz_in_shardings=z_out_shardings=)testcase_namein_shardingsout_shardingsdict.0r   r   s      r%   
<listcomp>zShardingTest.<listcomp>   k        1	   V,VV}VV$MC C C   r'   r   )kwargs)xr1   r   c           	          d i }|dk    r|dk    rt          d d          nd |d<   |dk    r|dk    rt          dd           nd |d<   t          j        fi |d}t          j        t          j        |          t          j                                      |          }                     |gd	
           t          j	        dd           fd            }|dk    rdnd}t          j        j        r	|dv rdnd}n|dnd}|dk    rdnd}	|dnd}
                     t          j                  |gd|fd|	fg           |dvr?|=                     t          j                  |gd|fd|
fd||z   |	z   |
z   fg            |          } ||          }                     |                                |           d S )Nc                 4    t          j        | j                  S r)   jnpsinTr   s    r%   rU   z+ShardingTest.test_pjit_basic.<locals>.f_jax       WQS\\r'   r   r   r   r   r   
      dtyper1   r?   FTrf   c                 "   t          j                  }t          j        dg          r[t          j        j        j                            |t	          j	        |           g
                    g d                    d         S  ||           S Nr   r\   r^   rE   r   r   convertr   r   r   compatv1r   rewriteconvert_to_tensorrE   r   f_convertedrU   r;   s     r%   rt   z*ShardingTest.test_pjit_basic.<locals>.f_tf   s    N5))k		 %	)	) y|''".q112"44".,, 5   (   	 	 {1~~r'   r=   r   Nr   Ef32\[10,20\].*custom_call_target.*Sharding.*sharding.*devices=\[1,2\]Ef32\[20,10\].*custom_call_target.*Sharding.*sharding.*devices=\[2,1\]r}   @f32\[10,20\].*custom_call_target.*Sharding.*sharding.*replicated@f32\[20,10\].*custom_call_target.*Sharding.*sharding.*replicatedcustom_call_target.*Sharding)r   r   r9   rN   prodfloat32rO   r[   r   rl   r   #jax2tf_default_native_serializationr{   r   r   r   assertAllClosenumpy)r;   r   r   pjit_kwargsx_shaper   rt   
count_in_Pcount_in_replicatedcount_out_Pcount_out_replicatedres_jaxres_tfrU   s   `            @r%   test_pjit_basiczShardingTest.test_pjit_basic   s      Ky  5AS5H5HQtS\\\dk.!	!!6Cs6J6JaTlllPTk/"Ie++{++EG
	"''"""*555==gFFAUQC222[5d333	 	 	 	 	 43	 #c))qJ17 =!-1B!B!BAA!-!5AA1$++!!K - 5111us V V
  	 	 	 ,,,1J
usP "P!#,--;>RRT
  	 	 	 eAhhGT!WWF00000r'   c                 J   t          t          j        t          d d          t          dd           fd           d             d}t          j        t          j        |          t          j                                      |          }d}t          j        t          j        |          t          j                                      |          }|                     ||gd           t          j
        |          fd	}|                     ||gg d
           d S )Nr   r   r   c                     | |z  S r)   r+   )r   ys     r%   rU   z2ShardingTest.test_pjit_variable_arg.<locals>.f_jax        Ulr'   r   r   )r      r1   r   c                 @     t          j                  |           S r)   )r   r   )r   rU   x_vs    r%   <lambda>z5ShardingTest.test_pjit_variable_arg.<locals>.<lambda>  s    *V^E**322 r'   ))r   r=   )zEf32\[20,30\].*custom_call_target.*Sharding.*sharding.*devices=\[2,1\]r=   )z@f32\[10,30\].*custom_call_target.*Sharding.*sharding.*replicatedr=   )r      r}   )r   r   r   r9   rN   r   r   rO   r[   r   Variabler   )r;   shape_xr   shape_yr   rt   rU   r   s         @@r%   test_pjit_variable_argz#ShardingTest.test_pjit_variable_arg   s1    TYacllAc4LL%A! ! ! ! ! G
	"''"""*555==gFFAG
	"''"""*555==gFFAUQF1555
+a..C22222Dqc	
 	
 	
      r'   c           	          t          j        dt           j                  }t          j        ddt           j                  t          t          j        t          d          fd           fd            t          j	        dd	           fd
            } 
                    t          j                  |gdd                     d          fg            |          } ||          }                     ||           d S )Nr   r      r   r   c                     | z  j         S r)   )r   )r   consts    r%   rU   z7ShardingTest.test_pjit_closed_over_const.<locals>.f_jax  s    %i]r'   FTrf   c                 "   t          j                  }t          j        dg          r[t          j        j        j                            |t	          j	        |           g
                    g d                    d         S  ||           S r   r   r   s     r%   rt   z6ShardingTest.test_pjit_closed_over_const.<locals>.f_tf#  s    N5))k		 %	)	) y|''".q112"44".,, 5 0 0 ( 
 
 	 	 {1~~r'   )zEf32\[10,20\].*custom_call_target.*Sharding.*sharding.*devices=\[2,1\]r=   r   r=   r}   )r9   onesr   r   fullr   r   r   r   rl   r   r   r   r|   r   )r;   r   rt   r   r   r   rU   s   `    @@r%   test_pjit_closed_over_constz(ShardingTest.test_pjit_closed_over_const  s5   

+++AHXq
333ETYaffYdCCC    DC [5d333	 	 	 	 	 43	 	us QXXa[[
  	 	 	 eAhhGT!WWF(((((r'   c                 Z    g | ](}d D ]#}dD ]}t          d| d| d| |||          $)S ))Nr   Nz2*b1,_z_,b22*b1,b2z_nested_pjit=z_constraint=_poly=)r   nested_pjit
constraintpolyr   )r   r   r   r   s       r%   r   zShardingTest.<listcomp>?  s        # 
 5   ZZZ*ZZTXZZ"zF F F    r'   )TFTr   c                   
 |dk    rt          dd           nd 
t          t          j        d d           
fd            }d}t          j        t          j        |          t          j                                      |          }|                     ||gd           t          j
        ||	          }|dk    rrdnd
nd}|dk    rrdnd
nd}	|                     ||gdd|fd|	fddd|z   |	z   fg           d S )Nr   r   r   c                     t          j        | | gd          }r" t          j        d           |          }n t          j                            |          }t          j        ||gd          S )Nr=   axisc                     | S r)   r+   )r   s    r%   r   zKShardingTest.test_pjit_sharding_constraint.<locals>.f_jax.<locals>.<lambda>O  s     r'   r   )r   concatenater   r7   r   with_sharding_constraint)r   r   constraint_shardingr   s     r%   rU   z9ShardingTest.test_pjit_sharding_constraint.<locals>.f_jaxJ  s     /1a&q
)
)
)a	 E9DIkk0C$79 9 99:< < G,,Q0CDD_aV!,,,,r'   r   r   r1   r   )polymorphic_shapesr=   r   )r   r=   zEf32\[10,40\].*custom_call_target.*Sharding.*sharding.*devices=\[2,1\]z@f32\[10,40\].*custom_call_target.*Sharding.*sharding.*replicated)z@f32\[10,80\].*custom_call_target.*Sharding.*sharding.*replicatedr=   r   r}   )r   r   r   r9   rN   r   r   rO   r[   r   r   r   )r;   r   r   r   rU   shaper   rt   count_inner_shardingcount_inner_replicatedr   s    `        @r%   test_pjit_sharding_constraintz*ShardingTest.test_pjit_sharding_constraint>  sp    +5*;*;!C,,,TYT! ! !- - - - -! !- E
	"'%..
333;;EBBAUQC222>%D999D 9Cc8I8I3AA!!q:D:K:K;5aaAAQRqc UU!#P#% U,a2F.FI_._`
      r'   c           	      H    g | ]}d D ]}t          d| d| ||           S r   r   r   s      r%   r   zShardingTest.<listcomp>o  r   r'   Nc                 F   t           j        j        s|                     d           t	          t          j                              }d}t          |          |k     rt          j	        d| d          t          j        |d |                                       d          }t
          j                            |d          }d i }|dk    r)|d	k    rt          |t!          d d
                    nd |d<   |dk    r)|d	k    rt          |t!          d
d                     nd |d<   t#          j        fi |d}t          j        t          j        |          t          j                                      |          }	fd}
|d	k    r|                     d          nd}t           j        j        r|dv r|                     d          nd}n||                     d          nd}|d	k    r|                     d          nd}t           j        j        r|dv r|                     d          nd}n||                     d          nd}|                     |
|	|	j        gd|fd|fg           |dvr+|dvr)|                     |
|	|	j        gd|fd|fg           d S d S d S )Nz)TODO: failure in non-native serializationr1   zTest requires z local devices)r1   r   c                 4    t          j        | j                  S r)   r   r   s    r%   rU   z*ShardingTest.test_grad_pjit.<locals>.f_jax~  r   r'   r   r   r   r   r   r   r   c                    t          j        d          5 }|                    |            t          j                    5 }|                    |             t          j                  |           }d d d            n# 1 swxY w Y   |                    || |          }d d d            n# 1 swxY w Y   |                    ||           }|S )NT
persistent)output_gradients)r   GradientTapewatchr   r   gradient)r   res_cttapetape2r   dy_dxd2y_dx2rU   s          r%   	f_grad_tfz.ShardingTest.test_grad_pjit.<locals>.f_grad_tf  sE   ?d+++ Dt

3_ 	.%
++c


(6>%((--&	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. fcFCCD D D D D D D D D D D D D D D eS))gns5   )B. 3A?3B.?B	B.B	B..B25B2r   r   r=   r   r   r}   r   )r   r   r{   skipTestlistr7   local_devicesr6   r4   r5   r9   r:   rO   shardingr   r   r   r   rN   r   r   r|   r   r   )r;   r   r   r  sizemesh_devicesmeshr   r   r   r   r   r   r   r   rU   s                  @r%   test_grad_pjitzShardingTest.test_grad_pjitn  s    5; A
mm?@@@*,,--MD
=D  CtCCCDDD8M%4%01199$??L<\622D   Ky  -9S-@-@dAdCLL)))d .!	!!-:c-A-AdAc4LL)))t /"Ie++{++EG
	"''"""*555==gFFA     !- 3 3!J17 G+7;L+L+LDHHQKKKRS+7+?DHHQKKKQ!.#!5!5$((1+++1K17 I,9=N,N,NTXXa[[[TU,9,ATXXa[[[q	Aqs8 VWabUWbc	
     ---,FW2W2W
)aXPRefUWbc
       .-2W2Wr'   c                    t          j                    d d         }t          |          dk     rt          j        d          d }t          |d          }t          t          t          |                    d          }t          |t           j	        
                    d                    }t          |t           j	        
                    d                    }t          j        t          j        t          j        ||                    d          }t          j        t          j        t          j        ||                    d          }t!          j        d	t           j        
          }	t          j        |	          }
t          j        d          5 }|                    |
            ||
          }|                    ||
          }d d d            n# 1 swxY w Y   t          j        d          5 }|                    |
            ||
          }|                    ||
          }d d d            n# 1 swxY w Y   |                     ||           d S )Nr1   zTest requires 2 local devicesc                 0    t          j        | dz            S )Ng       @)r   sumr   s    r%   rU   z=ShardingTest.test_grad_sharding_different_mesh.<locals>.f_jax  s    WQV__r'   i)r  )r   F)rg   )r1      r   Tr   )r7   r  r6   r4   r5   r   r  reversedr   r  r   r   rl   r   r   r   r9   r   r   r   r   r   r   r   )r;   r8   rU   r  mesh_rev	shardingsshardings_revrt   f_tf_revinpinput_vr   r   g
res_tf_revg_revs                   r%   !test_grad_sharding_different_meshz.ShardingTest.test_grad_sharding_different_mesh  s    !!"1"%G
7||a=>>>   DD'**++S11HdCL$>$>v$F$FGGI!(CL,F,Fv,N,NOOM;v~diI&N&N&NOO!&( ( (D{6>$)E*V*V*VWW%*, , ,H
'&

+
+
+Ck#G	D	)	)	) )T
jjtG}}f
--
(
(a) ) ) ) ) ) ) ) ) ) ) ) ) ) )
 
D	)	)	) 1T
jj8G$$jmmJ00e1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 	5!!!!!s$   7GG"%G">7IIIc                 6    g | ]}t          d | |          S )z_func=)r   funcr   )r   r  s     r%   r   zShardingTest.<listcomp>  s;       
 $t444  r'   )pjit_shardedpjit_replicatednested_pjit_shardednested_pjit_replicatedr  c                    t           j        j        rt          j        d          t          t          j        t          d          fd           d             }t          t          j        d d           d             }d }d }d}t          j	        t          j
        |          t          j        	                              |          }|d
k    r|}n|dk    r|}n|dk    r|}n|dk    r|}nJ t          | j        d          5   ||          }	|                     t           d          5   t#          j        |          |           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nz8There is no error in eager mode for native serializationr   r   c                     | | z   S r)   r+   as    r%   f_pjit_shardedz:ShardingTest.test_pjit_eager_error.<locals>.f_pjit_sharded  r   r'   c                     | | z   S r)   r+   r!  s    r%   f_pjit_replicatedz=ShardingTest.test_pjit_eager_error.<locals>.f_pjit_replicated  r   r'   c                 x    |  t          j         t          j        t          d          fd           |           z   S )Nr   r   )r   r   r   r   r!  s    r%   f_nested_pjit_shardedzAShardingTest.test_pjit_eager_error.<locals>.f_nested_pjit_sharded  s4    O37!C&&$OOOPQRRRRr'   c                 \    |  t          j         t          j        d d           |           z   S )Nr   )r   r   r   r!  s    r%   f_nested_pjit_replicatedzDShardingTest.test_pjit_eager_error.<locals>.f_nested_pjit_replicated  s+    J37TJJJ1MMMMr'   )r   r   r   r  r  r  r  Fr   
axis_nameszUfunction with sharded arguments or results must be used under a `tf.function` context)r   r   r{   r4   r5   r   r   r   r9   rN   r   r   rO   r   r8   assertRaisesRegex
ValueErrorr   r   )
r;   r  r#  r%  r'  r)  r   r"  rU   _s
             r%   test_pjit_eager_errorz"ShardingTest.test_pjit_eager_error  sU    17 ZXYYY TYaffY! ! ! ! ! TYT! ! ! ! !S S SN N N E
	"'%..
333;;EBBA~ee	"	"	"ee	&	&	&#ee	)	)	)&eel	dlv	.	.	. ! !
%((a!!

ac c ! ! 	ua   ! ! ! ! ! ! ! ! ! ! ! ! ! ! !! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s6   'E0.EE0E	E0E	 E00E47E4z@all_to_all .* are only implemented properly for TPUs and GPUs .*)categorymessagec           
          t          j        dg          rt          j        d          t	           j        d          }t          j        dt          j                  	                    d          }t          t          j        t          dd           ft          d d                    t          t          |t          dd           ft          d d          	          d
                         t          j        dd           fd            }|5   |          }t          j        |dd          \  }}t          j        |dd          \  }}t          j        |dd          \  }	}
t          j        ||	gd          }t          j        ||
gd          }t          j        ||gd          }                     ||            ||          }                     ||           d d d            d S # 1 swxY w Y   d S )Ncpu(TODO(b/268295912): ShardingRemover crashr   r*     r   r  r  r   r  in_specs	out_specsc                 4    t          j        | dddd          S )Nr   r=   T)
split_axisconcat_axistiled)r   
all_to_all)bs    r%   rU   z1ShardingTest.test_shmap_all_to_all.<locals>.f_jax  s    
 ^AsqatLLLLr'   FTrf   c                 &   t          j        d          }t          j        dg          r[t          j        j        j                            |t	          j	        |           g
                    g d                    d         S  ||           S )NTnative_serializationr   r\   r   r   r   r   )r"  r   rU   r;   s     r%   rt   z0ShardingTest.test_shmap_all_to_all.<locals>.f_tf  s    N5tDDDk		 %	)	) y|''".q112"44".,, 5 0 0 ( 
 
 	 	 {1~~r'   r1   r   r   r=   r   r   r4   r5   r   r8   r9   rN   r   rO   r   r   r   r   r   rl   splitr   r   )r;   r  r"  rt   r   b0b1b00b01b10b11resr   rU   s   `            @r%   test_shmap_all_to_allz"ShardingTest.test_shmap_all_to_all  sm    w'' JHIII#///D
	%rz***226::ATYS$<</4G G GYTd||o4? ? ?M M? ?G GM [5d333	 	 	 	 	 43	 
 
+ 
+agx11%%%fb""aa(((hc3"aa(((hc3>3*1---b>3*1---bNB8!,,,c
'3'''tAwwf
&'***
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+s   CG<<H H z<TODO(b/268295912): ShardingRemover crash,on all platforms!!!c           
         t          | j        d          }t          t          j        t	          dd           ft	          dd                     t          t
          |t	          dd           ft	          dd                     d                         }|5  t          j        d                              d          } ||          }t          j	        |dd	
          \  }}||}}t          j
        ||gd	
          }|                     ||           t          j        t          j        |d          dd          } ||          }	|                     |	|           d d d            d S # 1 swxY w Y   d S )Nr   r*  r   r7  c                     t          j        dd          fdt                    D             }t          j        | d|          S )Nr=   r   c                 $    g | ]}||d z   z  fS r=   r+   r   j	axis_sizes     r%   r   z[ShardingTest.test_repro_xla_bug_shmap_collective_permute.<locals>.f_jax.<locals>.<listcomp>>  '    AAA1q1q5I%&AAAr'   permr   psumrangeppermuter?  rV  rS  s     @r%   rU   zGShardingTest.test_repro_xla_bug_shmap_collective_permute.<locals>.f_jax8  N    
 (1c""iAAAAi0@0@AAAd\!St,,,,r'   r5  r6  r1   r   r   TrA  Frf   )r   r8   r   r   r   r   r9   rN   rO   rD  r   r   r   rl   r   r   )
r;   r  rU   r"  r   rE  rF  expectedrt   r   s
             r%   +test_repro_xla_bug_shmap_collective_permutez8ShardingTest.test_repro_xla_bug_shmap_collective_permute4  s   #///DTYS$<</3G G GYTd||o3? ? ?- -? ?G G-
 
 , ,
)E


"
"6
*
*aagx11%%%fb"2"bRq111h
'8,,, [
.T
:
:
:t- - -d tAwwf
&(+++, , , , , , , , , , , , , , , , , ,s   CE..E25E2c                 6    g | ]}t          d | |          S )r   )r   r   r   )r   r   s     r%   r   zShardingTest.<listcomp>R  s;       
 $t444  r'   r   c           
      V   
 t          j        dg          rt          j        d          t	           j        d          }t          j        dt          j                  	                    d          }t          t          j        t          dd           ft          dd                     t          t          |t          dd           ft          dd           	          d
                         
t          j        dd          
 fd            }|5   
|          }t          j        |dd          \  }}||}}t          j        ||gd          }                     ||            ||          }	                     |	|           d d d            d S # 1 swxY w Y   d S )Nr3  r4  r   r*  r5  r   r6  r   r7  c                     t          j        dd          fdt                    D             }t          j        | d|          S )Nr=   r   c                 $    g | ]}||d z   z  fS rP  r+   rQ  s     r%   r   zMShardingTest.test_shmap_collective_permute.<locals>.f_jax.<locals>.<listcomp>b  rT  r'   rU  rW  r[  s     @r%   rU   z9ShardingTest.test_shmap_collective_permute.<locals>.f_jax\  r\  r'   FTrf   c                 .   t          j        d          }t          j        dg          r\t          j        j        j                            |t	          j	        |           g
                    g d                    d         }n ||           }|S )NT)rB  r   r   r\   r   r   r   r   )r"  r   rK  rU   r   r;   s      r%   rt   z8ShardingTest.test_shmap_collective_permute.<locals>.f_tfe  s    N5t6:< < <k		 %	)	) il&&".q112"44".,, 5 0 0 ' 
 
 	 k!nnjr'   r1   r   r   rC  )r;   r   r  r"  rt   r   rE  rF  r]  r   rU   s   ``        @r%   test_shmap_collective_permutez*ShardingTest.test_shmap_collective_permuteQ  s    w'' JHIII#///D
	%rz***226::ATYS$<</3G G GYTd||o3? ? ?- -? ?G G-
 [5d333      43 
 , ,agx11%%%fb"2"bRq111h
'8,,,tAwwf
&(+++, , , , , , , , , , , , , , , , , ,s   A9FF"%F")r\   r=   )r   r   )Tr   r   )r   N)r  r)   )__name__
__module____qualname____doc__r3   r   r   r[   rE   rI   rx   r|   r   r   parameterized_filterable	with_meshr   r   r   r   r  r  r/  ignore_warningUserWarningrL  r4   skipr^  rd  __classcell__)r<   s   @r%   r.   r.   K   s        / / / / /  !N N NXc] N N N N2 +7%&# # # #.(3- .C . . . .,  
   (3-    >  3  1     3=(A1 A1 A1  A1F 3=(  < 3=(!) !) !)F  3  '	     3=($ $ $  $L  3  1    > > > >@ "  "  "D  3 D    (! (! (! (!T 3{`b b b$+ $+b b$+V 8=OPP, , QP,8  3 5    
$, $, $, 
$, $, $, $, $,r'   r.   __main__)
testLoader)4rh  collections.abcr   
contextlib	functoolsr   rJ   mathosrr   typingr   r4   absl.testingr   r7   jax._srcr   r   r	   r   r
   r   jax.experimentalr   r   jax.experimental.shard_mapr   jax.shardingr   r   r   r   	jax.numpyr   r   r9   
tensorflowr   parse_flags_with_absljax.experimental.jax2tf.testsr   	ExitStackr   r#   r&   r,   JaxToTfTestCaser.   re  mainJaxTestLoaderr+   r'   r%   <module>r     sU    % $ $ $ $ $             				 				        ! ! ! ! ! ! 



             % % % % % %             # # # # # # ! ! ! ! ! ! 0 0 0 0 0 0 & & & & & &       + + + + + +                    7 6 6 6 6 6"j"$$
 
 
  o, o, o, o, o,</ o, o, o,j z(-,3,..////// r'   