
    Vpf                      U d Z ddlmZ ddlZddlmZ ddlZddlmZ ddlm	Z	m
Z
mZmZ ddlZddlmZmZmZ ddlZddlZddlZddlmZmZmZmZmZ ddlZddlZddlZdd	l m!Z! dd
l m"Z" ddl m#Z# ddl m$Z$ ddl m%Z% ddl m&Z& ddl m'Z' ddl m(Z) ddl m*Z+ ddl m,Z, ddl m-Z- ddl m.Z. ddl m/Z/ ddl m0Z0 ddl m1Z1 ddl m2Z2 ddl m3Z3 ddl m4Z5 ddl6m7Z7 ddl8m9Z9 ddl8m:Z: ddl;m<Z< ddl;m=Z= dd l;m>Z? dd!l;m@Z@ dd"l;mAZA dd#lBmCZCmDZDmEZE dd$lFmGZH dd%lFmIZI dd&lJmKZK dd'lLmMZM dd(lNmOZO dd)lPmQZR dd*lSmTZTmUZUmVZVmWZWmXZXmYZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z` dd+lambZbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZjmkZk dd,llmmZmmnZn  G d- d.eo          ZpeHjq        ZresebcZtZs eju        ev          Zweexeyezeexeyf         d/f         f         Z{e-j|        Z|e-j}        Z}e-j~        Z~e-j        Ze-j        Zee~e}e|f         Ze+j        Ze/j        Zeeef         Ze-j        Zd0 Ze.j        ddd6            Zi Zd7ed8<    ed9:          dd=            Zd> Zeeej        j        <   d? Ze7D ]Zeee<   d@ Zeee$j        <   dA Zeee$j        <   	 dddLZddNZi ZdOedP<   ddQZeee:<   ddVZe	d/e	egef         f         Zi ZdWedX<   ddZZi ZdWed[<   ddoZddpZ G dq dre          ZddsZddvZedd~            Z edddg          Z G d de$j                  Ze)j        d             ZddZddZddZ G d de$j                  Ze)j        dd            Z ej        d1           G d d                      Z G d de          Z G d de          ZddZe)j        d             ZddZddZe.j        dd            ZddZezee	f         Ze:eefiZded<   ddZ G d de1j                  ZddZej         G d d¦                      Z G dÄ de1j                  Zdń Z G dƄ dǦ          Z G dȄ dɦ          ZƐdd̈́ZǐddτZ G dЄ dѦ          Z e$j        dҦ          Zej        Zeˠ                    e           dӄ ZdԄ ZdՄ ZѐddڄZdۄ Zd܄ Zee?j        e<    ee?j        deeЬަ          e?j        e<   ee?j        e<   ee<j        e<   ee<j        e<    ee<j        e˦          e<j        e<   d߄ Zee$j        e<   d Zd Zd ZddZd ZddZddZdddZ e@j        ee           ddZddZddZddZ e$j        d          Zej        d             ZddZ ee@j        e          dd            Z e$j        d          Zej        d              Z ee@j        e          dd            ZddZ G d dej                  Zej         G d d                      Z G d d	e          ZezeeReXeVf         eeedf         f         ZddZddZeeReXf         ZddZekd             Z G d de          Z ekdd            Zekdd            Zekdd            Z G d d          ZddZ ed:          dd             Zekdd)            Z ed:          dd-            Zdd/Z	d0 Z
i Zd1ed2<   ek	 ddd4            ZekdÐd6            Ze
eeCeDdf                  Z G d7 d8e          Z ed:          dĐd;            Zd< Ze.j        dŐdJ            ZdƐdMZ G dN dIe1j                  ZdǐdTZdȐdWZdɐdYZi Z edZeR[          Zdʐd`ZdːdeZ ee/j        e           d̐dgZ ee/j        e           d͐diZ ee_e           dj Z 	 ddkZ!dΐdpZ"dϐdrZ#dАdtZ$du Z%dv Z&ekdw             Z'dx Z(dy Z)dz Z*ej         G d{ d|                      Z+ G d} d~e          Z,d Z- G d de1j                  Z.	 ddѐdZ/d Z0e/j1        Z1dҐdZ2dӐdZ3dԐdZYed             Z4dS (  z1Implementation of pmap and related functionality.    )annotationsN)contextmanager)
namedtuple)CallableSequenceIterableIterator)partial	lru_cachecached_property)Any
NamedTupleTypeVarUnioncast)api_util)compiler)config)core)dispatch)dtypes)effects)linear_util)mesh)op_shardings)sharding_specs)profiler)sharding_impls)source_info_util)stages)	tree_util)util)
xla_bridge)array_types)DShapedArray)ShapedArray)ad)batching)partial_eval)mlir)xla)DeviceLocalLayout
AutoLayoutLayout)
xla_client)xla_extension_version)ir)hlo)PartitionSpec)Sharding)ArrayMappingArrayMappingOrAutoOrUnspecifiedAUTOUNSPECIFIEDUnspecifiedValueget_array_mappingis_autois_unspecifiedis_unspecified_or_autoarray_mapping_to_axis_resourcesSingleDeviceShardingGSPMDSharding)
safe_mapsafe_zippartition_list	wrap_nametuple_updatetuple_deletedistributed_debug_logunzip2HashableFunctionweakref_lru_cache)AbstractRef	RefEffectc                      e Zd ZdS )WeakRefListN__name__
__module____qualname__     Z/var/www/html/nettyfy-visnx/env/lib/python3.11/site-packages/jax/_src/interpreters/pxla.pyrN   rN   P           $rT   rN   .c                    | S NrS   )xs    rU   identityrZ   k   s    rT   T	shardingsSequence[JSharding]returnSequence[xc.ArrayImpl]c                   t          |          dk    rB|d         }|rt          j        |          }t          t	          |                   |g|           S t          j        d           }t          t          ||                     D ]\  }\  }}|rt          j        |          }|t	          |                   }|d         	                    |           |d         	                    |           |d         	                    |           d gt          |          z  }|
                                D ]:\  }	\  }
}}t          |	         ||          }t          |
|          D ]
\  }}|||<   ;t          d |D                       sJ |S )N   r   c                     g g g fS rX   rS   rS   rT   rU   <lambda>zshard_args.<locals>.<lambda>w   s    RRL rT      c              3     K   | ]}|d uV  	d S rX   rS   ).0results     rU   	<genexpr>zshard_args.<locals>.<genexpr>   s'      66FV4666666rT   )lenr+   canonicalize_dtypeshard_arg_handlerstypecollectionsdefaultdict	enumeraterB   appenditemsall)r[   argscanonicalizeargbatchesishardingbatchresultstindicesasoutsouts                  rU   
shard_argsr   m   s    	YY!^^
q'C ("3''cd3ii(#	::: #$8$899'%htY&?&?@@  a#x ("3''cDIIE	!HOOA	!HOOC	!HOOH &*FSYY$6'#MMOO  a'1aa A&&D7D))  3gajj 
66g666	6	6666	.rT   zBdict[Any, Callable[[Sequence[Any], Sequence[Any]], Sequence[Any]]]rj   i   )maxsizenum_addressable_devicesintc                *    t          d           ff| z  S rX   )slice)r   s    rU   _get_replicated_slicesr      s    ;;.	4	44rT   c                     t          d          )Nznumpy masked arrays are not supported as direct inputs to JAX functions. Use arr.filled() to convert the value to a standard numpy array.)
ValueErrorxsr[   s     rU   _masked_array_errorr      s     V 	W 	W WrT   c           	     
   g }t          | |          D ]\  }|j        }j        t          j        k    r2t          j        j        t          j        t                              t          j
                  }|j        rgt          |          z  }nGt          |                    j                                                            }fd|D             }|                    t#          ||||                     |S )Ndtypec                     g | ]
}|         S rS   rS   )re   rv   rY   s     rU   
<listcomp>z _shard_array.<locals>.<listcomp>   s    &&&!&&&rT   )rB   _addressable_device_assignmentr   r   float0npzerosshapeboolr   shaped_abstractifyis_fully_replicatedrh   tupleaddressable_devices_indices_mapvaluesro   batched_device_put)	r   r[   ry   rw   devicesavalshardsr{   rY   s	           @rU   _shard_arrayr      s    'b),, 
H 
Hka5Gw&-
(17"(4..
1
1
1a&q))D# 'sS\\!ffh>>qwGGNNPPQQg&&&&g&&&fNN%dHfgFFGGGG	.rT   c                6    t          |d | D                       S )Nc                    g | ]	}|j         
S rS   )_datare   rY   s     rU   r   z!_shard_darray.<locals>.<listcomp>   s    444A444rT   r   r   s     rU   _shard_darrayr      s!    	I44444	5	55rT   c                6    t          |d | D                       S )Nc                    g | ]	}|j         
S rS   )_bufr   s     rU   r   z(_shard_mutable_array.<locals>.<listcomp>   s    3331333rT   r   r   s     rU   _shard_mutable_arrayr      s!    	I33333	4	44rT   r   core.ShapedArrayrw   	JShardingr   Sequence[Any]r   Sequence[jax.Device]	committedr   c                   ddl m fdt          ||          D             }t          |          t          |          k    r                    | |||d          S t          j        | ||t          |          |          S )Nr   arrayc                    g | ]N\  }}t          |j                  r4t          j        |j                  r|                                |hk    L|OS rS   )
isinstance	ArrayImplr   is_single_device_shardingrw   r   )re   rY   dr   s      rU   r   z&batched_device_put.<locals>.<listcomp>   sh     
# 
# 
#1EO,,
#0<<
# iikkaS    !  rT   T)r   _skip_checks)jax._srcr   rB   rh   r   xcr   list)r   rw   r   r   r   bufsr   s         @rU   r   r      s     
# 
# 
# 
#W-- 
# 
# 
#$ 	YY#b''??h	  F F F		tXr4==)	L	LLrT   axisc                    	 t          t          |                   | ||          S # t          $ r%}t          dt          |                     |d }~ww xY w)Nz!No _shard_aval handler for type: )_shard_aval_handlersrk   KeyError	TypeError)sizer   r   errs       rU   _shard_avalr      sg    OT

+D$===	 O O O
DT

DD
E
E3NO   #& 
A AAz>dict[type[core.AbstractValue], Callable[[int, int, Any], Any]]r   c                f   	 |j         |         | k    rt          d|  d| d|j                    n# t          $ r t          d          d w xY wt          j        j        r*|                    t          |j         |d                    S |                    t          |j         |                    S )Nz
Axis size z does not match dimension z
 of shape z/Cannot split a {x.dim}D value along axis {axis}r`   r   )	r   r   
IndexErrorr   pmap_no_rank_reductionvalueupdaterE   rF   )r   r   rY   s      rU   _shard_abstract_arrayr      s    Rwt} *D * *D * * !* * + + +  
 R R R
F
G
GTQR"( 788,qwa88899988,qw558666s	   .1 Acore.AbstractValuer{   tuple[Index, ...] | None#Callable[[list[xc.ArrayImpl]], Any]c                    	 t          t          |                    | ||          S # t          $ r%}t          dt          |                      |d}~ww xY w)a  Returns a function for handling the raw buffers of a single output aval.

  Args:
    aval: The local output AbstractValue.
    sharding_spec: Indicates how the output is sharded across devices, or None
      for non-array avals.
    indices: The pre-computed result of spec_to_indices, or None for non-array
      avals.

  Returns:
    A function for handling the Buffers that will eventually be produced
    for this output. The function will return an object suitable for returning
    to the user, e.g. an Array.
  !No pxla_result_handler for type: N)local_result_handlersrk   r   r   )r   rw   r{   r   s       rU   local_aval_to_result_handlerr      so    &C $t**.tXwGGG	 C C C
8DJJ88: :?BCCr   z1dict[type[core.AbstractValue], PxlaResultHandler]r   'Callable[[Sequence[xc.ArrayImpl]], Any]c                    	 t          t          |                    | ||          S # t          $ r%}t          dt          |                      |d}~ww xY w)a  Returns a function for handling the raw buffers of a single output aval.

  Args:
    aval: The global output AbstractValue.
    out_axis_resources: A PartitionSpec specifying the sharding of outputs.
      Used for creating GSDAs.
    global_mesh: The global device mesh that generated this output. Used
      for creating GSDAs.

  Returns:
    A function for handling the Buffers that will eventually be produced
    for this output. The function will return an object suitable for returning
    to the user, e.g. an Array.
  r   N)global_result_handlersrk   r   r   )r   out_shardingr   r   s       rU   global_aval_to_result_handlerr      so    "C!$t**-dL)LLL	 C C C
8DJJ88: :?BCCr   r   funlu.WrappedFunbackend
str | None	axis_namecore.AxisName	axis_sizeglobal_axis_sizeSequence[Any] | Nonenamestrin_axesSequence[int | None]out_axes_thunk"Callable[[], Sequence[int | None]]donated_invarsSequence[bool]is_explicit_global_axis_sizer   c       
           	
 t           j        j        r=t           j        j        r,
s*t	          d 	D                       s	 
fd}|S t          t          j        |          }t           	
g|R  \  }}t           j	        j        r:t          dfd j        fdfdt          t          j        |          fd|f           |S )Nc              3     K   | ]}|V  d S rX   rS   re   r   s     rU   rg   z%xla_pmap_impl_lazy.<locals>.<genexpr>!  s"      2M2M12M2M2M2M2M2MrT   c                 6    t          g| R 
	d
S )N)
r   r   r   r   r   r   r   r   r   r   )
_emap_impl)rr   r   r   r   r   r   r   r   r   r   r   r   s    rU   _emap_apply_fnz*xla_pmap_impl_lazy.<locals>._emap_apply_fn"  sN     Sd S SGy"+>N 'dG'5'55QS S S SrT   zRunning pmapped functionzpython functionr   zabstract argsfingerprint)r   disable_jitr   
eager_pmapany
unsafe_mapr+   abstractifyparallel_callabledistributed_debugrG   fmap)r   r   r   r   r   r   r   r   r   r   r   rr   r   abstract_argscompiled_funr   s   ```````````     rU   xla_pmap_impl_lazyr     sa     	6#4#: 	
&	/22M2Mn2M2M2M/M/M	S S S S S S S S S S S S S S S S_d33-/	7Iy*:GT~~"4 &34 4 4, # 85t<,ce4$g.*C,F,FG(+6	8 8 8
 
rT   c                ,    t          | g|R i |} || S rX   )r   )r   rr   paramsr   s       rU   xla_pmap_implr   9  s,    #C9$999&99,	t	rT   c                  $    e Zd ZU ded<   ded<   dS )EmapInfor   r   r   r   NrP   rQ   rR   __annotations__rS   rT   rU   r  r  =  s*         rT   r  c       
           ddl m} t          d |	D                       rt          d          |
rt          d          t	          ||          }fd|D             }t          j        t          |          5 }t          j                    5  t          j	        ||          5  |
                                fdt          ||          D             } | j        | }t          j        |          }t          d	 |D                       \  }}d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   ~d d d            n# 1 swxY w Y    |            }t!          j        |          j        }|d
v rdnd}g }t          |||          D ]\  }}}t'          j        d          5  |}t+          ||j                  rd} t'          j        d d|                              f||d nt3          |          ||          t5          j        |          |          }|                    |           d d d            n# 1 swxY w Y   |S )Nr   r   c              3     K   | ]}|V  d S rX   rS   r   s     rU   rg   z_emap_impl.<locals>.<genexpr>O  s"      ##q######rT   z,Buffer donation not supported in eager pmap.z9Non-default global_axis_size not supported in eager pmap.c                     g | ]
}|i n|iS rX   rS   )re   in_axisr   s     rU   r   z_emap_impl.<locals>.<listcomp>V  s&    WWWGgoIw+?WWWrT   )	emap_infoc                8    g | ]\  }}t          ||          S rS   	MapTracer)re   rt   r}   rz   s      rU   r   z_emap_impl.<locals>.<listcomp>Z  s)    JJJ&#q1c1%%JJJrT   c              3  2   K   | ]}|j         |j        fV  d S rX   val
shard_axesre   rz   s     rU   rg   z_emap_impl.<locals>.<genexpr>]  s+      $P$PqaeQ\%:$P$P$P$P$P$PrT   >   tpucudarocm)r`   rS   Fc                    |S rX   rS   )_rY   s     rU   rb   z_emap_impl.<locals>.<lambda>k  s    q rT   )r   out_axesr   r   donate_argnums)r   r   r   NotImplementedErrorr  r   new_base_mainMapTracenew_sublevelextend_axis_envwith_cur_sublevelzipcall_wrappedr   
full_raiserH   xbget_backendplatformjaxr   r   r   pmapgetr   r   arangero   )r   r   r   r   r   r   r   r   r   r   r   rr   r   r	  r  maintracersansout_tracersoutvalsout_axes_srcr  r$  r  new_outvalsout_axis_srcout_axisoutvaldonate_argnums_r   rz   s     `                           @rU   r   r   A  s    ##N##### N
L
M
MM! -
 , - - - w(()WWWWwWWW*(i888 D				 Q Qd29iNN Q Q

 
 
"
"aJJJJCj4I4IJJJgCg&cc**k$$P$PK$P$P$PPPg|Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q 	               ^(^G$$-(#'>>>44B.+(+L(G(L(L  $lHf			  &o	FEO	,	, *CH
.l&&y112"?44W(* * * +-)I*>*>H Hc                
sn   ;ED5&A,DD5D""D5%D"&D5)E5D9	9E<D9	=EEE0BII	I	idxtuple[int | None, ...]c                T     t           fdt                     D                       S )Nc              3  t   K   | ]1\  }d n%t          fdd |         D                       z
  V  2d S )Nc              3  ,   K   | ]}|d uo|k     V  d S rX   rS   )re   jrv   s     rU   rg   z*_map_schedule.<locals>.<genexpr>.<genexpr>z  s0      @@1q}.Q@@@@@@rT   )sum)re   lrv   r4  s     @rU   rg   z _map_schedule.<locals>.<genexpr>y  so       + +1a ytt3@@@@BQB@@@@@@+ + + + + +rT   )r   rn   )r4  s   `rU   _map_scheduler<  t  sC    
 
 + + + +$S>>+ + + 
+ 
+ +rT   r   infonameslist[core.AxisName]all_axeslist[tuple[int | None, ...]])tuple[Callable, dict[core.AxisName, int]]c                   g }t          t          t          |                              D ]\  }t          fd|D                       }t	          d |D                       rOt          j        | ||d|j        |j        d nt          |j                            } |	                    |           d t          t          |                    D             }| |fS )Nc              3  (   K   | ]}|         V  d S rX   rS   )re   arg_axisrv   s     rU   rg   z_multi_pmap.<locals>.<genexpr>  s'      99HHQK999999rT   c              3     K   | ]}|d uV  	d S rX   rS   )re   r  s     rU   rg   z_multi_pmap.<locals>.<genexpr>  s'      
6
677$
6
6
6
6
6
6rT   r   )r   r   r  r   r   c                    i | ]\  }}||	S rS   rS   re   rv   r   s      rU   
<dictcomp>z_multi_pmap.<locals>.<dictcomp>  s    KKK4D!KKKrT   )
reversedr   rn   r   r   r%  r&  r   r   ro   )	r   r=  r>  r@  
used_namesr   r   out_shard_axesrv   s	           @rU   _multi_pmaprM    s     *$y//0011 
 
ga999999999G

6
6g
6
6
666 
(
,</44T$,5G5GJ J Ja KK9Xj5I5I+J+JKKK.	
N	rT   FakePrimitivemultiple_resultsbindc                  N     e Zd Z fdZd Zd Zd Zd Zd Zd Z	d Z
d	 Z xZS )
r  c               B     t                      j        |  || _        d S rX   )super__init__r	  )selfr	  rr   	__class__s      rU   rT  zMapTrace.__init__  s"    EGGdDNNNrT   c                $    t          | |i           S rX   r  )rU  r  s     rU   purezMapTrace.pure  s    T3###rT   c                8    t          | |j        |j                  S rX   )r  r  r  )rU  tracers     rU   subliftzMapTrace.sublift  s    T6:v'8999rT   c                     j         j        d         }t          d |D                       \  }}t           fdt          j        j        j        D                       t          fd|D                       }t          fdt          	                                          f          }t          |||          \  }	t	          j                    5  t          j        d          5   |	| }
d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   j        r fd|
D             S t           |
          S )Nr	  c                *    g | ]}|j         |j        fS rS   r  r  s     rU   r   z.MapTrace.process_primitive.<locals>.<listcomp>  s!    FFFq|4FFFrT   c              3  B   K   | ]}|j         j        u |j        V  d S rX   )
main_tracer)  r   )re   r   rU  s     rU   rg   z-MapTrace.process_primitive.<locals>.<genexpr>  s?       0 0Q\TY.. &....0 0rT   c              3  \   K   | ]&}t          t          |j                            V  'd S rX   )r<  r   r'  )re   r}   r>  s     rU   rg   z-MapTrace.process_primitive.<locals>.<genexpr>  s7      JJ!]3que#4#455JJJJJJrT   c                      j         | i S rX   rP  )rr   r   	primitives    rU   rb   z,MapTrace.process_primitive.<locals>.<lambda>  s    ~y~t'Fv'F'F rT   Fc                2    g | ]}t          |          S rS   r  )re   r  rL  rU  s     rU   r   z.MapTrace.process_primitive.<locals>.<listcomp>  s%    FFFsic>22FFFrT   )r)  payloadrH   r   r   thread_local_statetrace_stateaxis_envrI   rp   rM  eval_contextr%  r   rO  r  )rU  rc  r*  r   r=  valsr  r@  r   f_mappedr-  r>  rL  s   `` `       @@rU   process_primitivezMapTrace.process_primitive  s   9[)DFFgFFFGGD* 0 0 0 0$"9"E"N 0 0 0 0 0EJJJJzJJJJJHFFFFF#U6<<>>%:%:;	= 	=A*1dE8DDHn				    coe44    $g                                                           ! GFFFFFgFFFFT7N333s6   D0D6DD	D	D	
DD Dc                    t           rX   )r  )rU  call_primitiver   r*  r   s        rU   process_callzMapTrace.process_call  s    
rT   c           	        |d         t          d          t          j        j        s>t	          fdf          }t          d|          }|                     |||          S |d         |d         |d         |d	         f\  }}t          d
 |D                       \  }	}
fdt          |	||
          D             }
t          j
                    5  t          j        | j                  5  | j                                        }t          t          t           |          |	|
          } j        | }t          |j        |          }t          d |D                       \  }}~~~~d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          fdt          || |                      D                       \  }}t          t          t           |           ||          S )Nr   +Nested pmap with explicit devices argument.c                 &     j         g| R i |S rX   rb  )rr   kwargsr   map_primitives     rU   rb   z&MapTrace.process_map.<locals>.<lambda>  s%    "4-"4S"J4"J"J"J6"J"J rT   TrO  rP  r   r   r   r   c              3  2   K   | ]}|j         |j        fV  d S rX   r  r  s     rU   rg   z'MapTrace.process_map.<locals>.<genexpr>  +      EEqual3EEEEEErT   c                    g | ]C\  }}}|9t          t          j        |          |                                |          i|n|DS rX   )_annot_to_flatr   ndimr   )re   vaxr}   r   s       rU   r   z(MapTrace.process_map.<locals>.<listcomp>  sh     B B BaQ ^ nRWQZZRHHNANN)*B B BrT   c              3  2   K   | ]}|j         |j        fV  d S rX   r  r  s     rU   rg   z'MapTrace.process_map.<locals>.<genexpr>  s+      GGaQUAL1GGGGGGrT   c              3  F   K   | ]\  }}}t          |||          V  d S rX   )_match_annot)re   r{  r}   dstr   r   s       rU   rg   z'MapTrace.process_map.<locals>.<genexpr>  sU       Q Q(q!S 'y)Q3GG Q Q Q Q Q QrT   )r   r   r   r   rI   rN  rl  rH   r  r   r  r  r)  r  r   r
   r  r   r!  )rU  rt  r   r*  r   rP  fake_primitiver   r   rj  r  rz   
in_tracersr+  r,  r   outaxesr   r   s    ``              @@rU   process_mapzMapTrace.process_map  s   i$DEEE# E
J
J
J
J
J#
   d %dFFFn##NGVDDD5;K5Hy6"23VK5H5J1Iw	EEWEEEEED*B B B B"%dGZ"@"@B B BJ 
			 * *d29iSS * *
)
%
%
'
'awy!,,dJ??jCj)cc**kGG;GGGGGlc7
Zk* * * * * * * * * * * * * * * * * * * * * * * * * * * * * *  Q Q Q Q Q,/Wnn>N>N,O,OQ Q Q Q QLCwy$''g666s7   F6A=E?3F?F	FF	FFFc               Z   |rd}t          |          ~~~t          d |D                       \  }}t          || j        |          \  }}	t	          j                    5   |j        | }
d d d            n# 1 swxY w Y   t          t          t          |           |
 |	                      S )Nzcustom_jvp with symbolic_zeros=True not supported with eager pmap. Please open an issue at https://github.com/google/jax/issues !c              3  2   K   | ]}|j         |j        fV  d S rX   r  r  s     rU   rg   z3MapTrace.process_custom_jvp_call.<locals>.<genexpr>  rw  rT   
r  rH   _emap_subtracer)  r   r  r   r   r
   r  )rU  primr   jvpr*  symbolic_zerosmsgin_valsr   r  out_valss              rU   process_custom_jvp_callz MapTrace.process_custom_jvp_call  s     %Nc$$$c>EEWEEEEEGW"3	7;;MC				 , ,!!7+h, , , , , , , , , , , , , , ,wy$''88::>>>s   A66A:=A:c                ^   |rd}t          |          ~~~~~t          d |D                       \  }	}
t          || j        |
          \  }}t	          j                    5   |j        |	 }d d d            n# 1 swxY w Y   t          t          t          |           | |                      S )Nzcustom_vjp with symbolic_zeros=True not supported with eager pmap. Please open an issue at https://github.com/google/jax/issues !c              3  2   K   | ]}|j         |j        fV  d S rX   r  r  s     rU   rg   z3MapTrace.process_custom_vjp_call.<locals>.<genexpr>  rw  rT   r  )rU  rc  r   fwdbwdr*  	out_treesr  r  r  r   r  r  s                rU   process_custom_vjp_callz MapTrace.process_custom_vjp_call  s     %Nc$$$3YEEWEEEEEGW"3	7;;MC				 , ,!!7+h, , , , , , , , , , , , , , ,wy$''88::>>>s   !A88A<?A<c                   t          fdt          j        j        j        f          }t          d|          }t          j                    5  t          j                            t          j
        j                  }d d d            n# 1 swxY w Y   t          | |j        di          }|                     ||fi           S )Nc                L    t           j                            j                  S rX   )r%  lax
axis_indexr   )r  frames    rU   rb   z-MapTrace.process_axis_index.<locals>.<lambda>  s    #'$$UZ00 rT   Fru  r   )rI   r%  r  r  r   rN  r   ri  iotar   int32r   r  rl  )rU  r  rP  r  rangedummy_tracers    `    rU   process_axis_indexzMapTrace.process_axis_index  s    0000		UZ(* *D #EEEEN				 1 1gll28UZ00e1 1 1 1 1 1 1 1 1 1 1 1 1 1 1T55:q/::L!!.</2FFFs   0BBB)rP   rQ   rR   rT  rX  r[  rl  ro  r  r  r  r  __classcell__)rV  s   @rU   r  r    s            $ $ $: : :4 4 4  7 7 76
? 
? 
?? ? ?G G G G G G GrT   r  c              '     K   |                                  }t          t          t          |          ||          }|i fV }t          |j        |          }t          d |D                       \  }}~~~~||fV  d S )Nc              3  2   K   | ]}|j         |j        fV  d S rX   r  r  s     rU   rg   z!_emap_subtrace.<locals>.<genexpr>  s+      IIqual3IIIIIIrT   )r  r   r
   r  r!  rH   )	r)  r   r  rz   r  r+  r,  r  r  s	            rU   r  r    s      
!79a(('7;;*"n#AL#&&+II[IIIII(HS+(rT   rz  mapped_axesIterable[int]
annotation
int | Nonec                l    |d S t          |          fdt          |           D             |         S )Nc                    g | ]}|v|	S rS   rS   )re   rv   mapped_axes_s     rU   r   z"_annot_to_flat.<locals>.<listcomp>  s#    	:	:	:A\$9$9!$9$9$9rT   )setr  )rz  r  r  r  s      @rU   ry  ry    s?    [!!,	:	:	:	:U4[[	:	:	::	FFrT   r  r   shard_axis_srcdict[core.AxisName, int]dst_annotation$tuple[Any, dict[core.AxisName, int]]c                   t          |          }|                    | d           }t          t          j        |          |d u z   |                                |          t          j                    5  |k    r|}nt          |          t                    cxk    rt          k    r>n n;t          j        ||          }t          t          j        |          ||          }nB|97t          j        ||          }fd|                                D             }nt          d d d            n# 1 swxY w Y   ||fS )Nc                *    i | ]\  }}|||k    z   S rS   rS   )re   nr   r  s      rU   rI  z _match_annot.<locals>.<dictcomp>  s'    MMMda1q>MMMrT   )dictpopry  r   rz  r   r   ri  rk   r   r(   moveaxis	_moveaxis	broadcastrp   r  )	r   r   r  r  r  shard_axis_outsrcr2  r  s	           @rU   r  r    s    ''.9d++#rws||sd{3^5J5J5L5L%	' 	'# 
  
 
czzff	cd3ii	&	&	&	&3	&	&	&	&	& c3//f ~sCHHnn	!#y#66fMMMMn6J6J6L6LMMMnn
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
	s   4B6D66D:=D:r  r  r  c                    d g| z  }|                                 D ]
\  }}|||<   |                    |          }|                    |||k     z
  |           d t          |          D             S )Nc                    i | ]
\  }}|||S rX   rS   rH  s      rU   rI  z_moveaxis.<locals>.<dictcomp>   s#    	D	D	Dga43C$3C3C3CrT   )rp   r  insertrn   )rz  r  r  r  lstkr{  r   s           rU   r  r    s}    %)FTM#    daCFF	$**SC#I%%%	D	D3	D	D	DDrT   c                  @    e Zd ZddgZddZed             Zd Zd	 Zd
S )r  r  r  tracer  r  c                     | _         | _        | _        t           fd j                                        D                       sJ d S )Nc              3  8   K   | ]}|j         j        k     V  d S rX   )r  rz  )re   r  rU  s     rU   rg   z%MapTracer.__init__.<locals>.<genexpr>)  s,      GGssTX]"GGGGGGrT   )_tracer  r  rq   r   )rU  r  r  r  s   `   rU   rT  zMapTracer.__init__%  sU    DKDH DOGGGGdo.D.D.F.FGGGGGGGGGGrT   c                    t          j        | j                  }t          | j                  }t          |                                          d d d         D ]#}t          j        |j	        |         ||          }$|S N)
r+   r   r  r  r  sortedr   r   mapped_avalr   )rU  r   r  axis_idxs       rU   r   zMapTracer.aval+  st    ?48$$Ddo&&J:,,..//"5 D Ddj2HdCCddKrT   c                    | S rX   rS   rU  s    rU   
full_lowerzMapTracer.full_lower3  s    KrT   c                    d | j                                         D             }| j         dd                    |           dS )Nc                "    g | ]\  }}| d | S )=rS   )re   r  r{  s      rU   r   z%MapTracer.__str__.<locals>.<listcomp>7  s&    AAAAQ****AAArT   {,})r  rp   r  join)rU  
named_axess     rU   __str__zMapTracer.__str__6  sH    AA)>)>)@)@AAAJh22#((:..2222rT   N)r  r  r  r  )	rP   rQ   rR   	__slots__rT  propertyr   r  r  rS   rT   rU   r  r  "  so        l#)H H H H   8  3 3 3 3 3rT   r  backend_namec                
   t          | |||||| j        |||
  
        \  }}}}}t          | |||||||	|
|d t          j                    |||||          }|                                }t          |j        |j        g          S )N)r   r   r   r   r   r   avals)lowering_platformslowering_parametersclosed_jaxprr   replicasr   pci)	get_pmap_jaxprrP   lower_parallel_callabler*   LoweringParameterscompilerN   unsafe_callr   )r   r  r   r   r   r   r   r   r   r   r   r  r  
xc_backendr  r   r  pmap_computationpmap_executables                      rU   r   r   :  s     5C	<,<CL'#5	52 52 521,
Hfc
 -	9i!17D~"E43J3L3Lh   %,,../	o1?3NO	P	PPrT   )frozenc                      e Zd ZU ded<   ded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   ded<   ed             Zed             ZdS )ParallelCallableInfor   r   	xc.Clientr   r   r   r   r   r   Sequence[xc.Device] | Noner   Iterable[int | None]r   r   r   Sequence[core.AbstractValue]r  c                l      j         r) fd j         D             }t          |          dk    sJ nd }|S )Nc                X    g | ]&}|j         t          j         j                  k    $|'S rS   )process_indexr"  r   )re   r   rU  s     rU   r   z6ParallelCallableInfo.local_devices.<locals>.<listcomp>f  s?     C C C12#3DL#A#AAA AAArT   r   )r   rh   )rU  r   s   ` rU   local_devicesz"ParallelCallableInfo.local_devicesc  s\    | C C C C C C CcXX\\\\\cJrT   c                *    |                                  S rX   )r   r  s    rU   r  zParallelCallableInfo.out_axesm  s       rT   N)rP   rQ   rR   r  r   r  r  rS   rT   rU   r  r  W  s         )))...%%%%4444%%%%  ? ! ! ?! ! !rT   r  c                  B    e Zd ZU ded<   ded<   ded<   ded<   ded<   d	S )
	ShardInfor  sharded_avalsSequence[core.ShapedArray]out_sharded_avalsglobal_sharded_avalsr   num_local_shardsnum_global_shardsNr  rS   rT   rU   r  r  r  sN         ----////4444rT   r  c                  .    e Zd ZU ded<   ded<   ded<   dS )ReplicaInfor   jaxpr_replicasnum_local_replicasnum_global_replicasNr  rS   rT   rU   r  r  z  s6         rT   r  jaxpr
core.Jaxprc                `    t          j        |           }||z  }||z  }t          |||          S rX   )r   r  r  )r   r   r   r  r  r  s         rU   find_replicasr    s>     *511. >1(>9	^%79L	M	MMrT   c              '     K   t          d t          | |          D                       }|i fV } |            }t          d t          ||          D                       V  d S )Nc              3  h   K   | ]-\  }}||n!t           j                            ||f          V  .d S N)
dimensions)r%  r  squeeze)re   r  rt   s      rU   rg   z)_change_argument_ranks.<locals>.<genexpr>  sV        
'3 _cc#'//#7*/"M"M     rT   c              3  h   K   | ]-\  }}||n!t           j                            ||f          V  .d S r  )r%  r  expand_dims)re   rY   r   s      rU   rg   z)_change_argument_ranks.<locals>.<genexpr>  sY        
!T <aaSW00w0GG     rT   )r   r  )r   r   rr   ry   r  s        rU   _change_argument_ranksr    s      	  gt,,   
 
$ *'^(  (++   	 	     rT   r  4tuple[core.Jaxpr, list[Any], ReplicaInfo, ShardInfo]c           
         t           fdt           j         j                  D                       }|}t          j        j        rt          | j         j                  }n|}t          j
         j         j        d           5  t          j        d|j        t          j                  5  t#          j        ||t#          j        |d                    \  }}}d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t)          j        ||j                  }t          j        |          }t1          |          t1           j                  k    s+J t1          |          t1           j                  f            t5          | j         j                  }|j        }|j        }	t=          |||||	          }
||||
fS )Nc              3  R   K   | ]!\  }}|t          j        ||          n|V  "d S rX   )r   r   )re   r   r   r  s      rU   rg   z*stage_parallel_callable.<locals>.<genexpr>  sS       : :
$ 150@k#-t,,,d: : : : : :rT   zMFinished tracing + transforming {fun_name} for pmap in {elapsed_time:.9f} secfun_nameeventr&  )r   rB   r   r  r   r   r   r  r   r   r  r   r   r   log_elapsed_timerP   JAXPR_TRACE_EVENTpetrace_to_jaxpr_finaldebug_info_finalr   jaxpr_debug_info
debug_infoapply_outfeed_rewriterrh   r  r  r   r  r  r  )r  r   r  orig_funr   r  constsr  r  r  r   s   `          rU   stage_parallel_callabler    s     : : : : ci88: : : : :- ("( 
 ck33E
F
FCC
CCM3+?FF @ @		"WX%?
A 
A 
A @ @ *,)@
}b1#v>>*@ *@&e@ @ @ @ @ @ @ @ @ @ @ @ @ @ @@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ 
#E8+>
?
?%

)%
0
0%			3s|#4#4	4	4	4	
c#,//71	4	4	4 5#-1EFF(02&)+ +& 
&	((s6   'C>-.C'C>'C+	+C>.C+	/C>>DDr  r  r  r  c
                $   ||t          j        |d                   }
nt          j        |          }
t          ||
|||||||		  	        }t	          ||           \  }}}}t          j        ||h          }t          j        ||          }||
|||fS Nr   )r"  get_device_backendr#  r  r  r   remove_named_axis_effectsClosedJaxpr)r   r  r   r   r   r   r   r   r   r  r   r  r   r  r  r   r  s                    rU   r  r    s     \1#GAJ//GGn\**G
GY	+;W~u	& 	&# %<C$E$E!%6

(
<
<%!%00,	w&#	55rT   r  tuple[str, ...] | Noner  mlir.LoweringParametersr  core.ClosedJaxprr  r  r   PmapComputationc
                  t          j                    dk    r|r||k    rt          d| d| d          |j        }d}d}|st          j        |          dk    r|rd}nd}t                              t          j                  rt                              d|j	                   t                              d|j
                   t                              d	|j        |j                   t                              d
|           t                              d|j                   t          j        |          dk    r}|r{|j        t          j        |          k    r^|j        |k    r't          d|dt          j        |                     t          d|j         dt          j        |                     |r"|j        dk    rt          d|j                   t"          j        j        rt          j        nt          j        }t                              |          r<t                              |d| j        t/          |           |j        |	|j                   t3          j        |j        |f|f          }t7          j        t;          |d                    }d |D             }t=          j        tA          |j
                  |j!                  }d| j         }|
p|j!        f}tE          ||d           5  tG          tH          j%        &                    |j$                            }|rt          d          tG          tH          j%        '                    |j$                            }t=          j(        dtS          |          t<          j*                  5  tW          j,        |||||t3          j-        |          |||d d |j.        o|j.        j/        |j.        o|j.        j0        |j        |          }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   tc          |j2        ||||||||j3        |j4        |j        j.        |j5                  S )Nr`   zSpecified axis_size z" doesn't match received axis_size .FTzsharded_avals: %szglobal_sharded_avals: %sz(num_replicas: %d  num_local_replicas: %dzdevices: %szlocal_devices: %szOn multi-host platforms, the input to pmapped functions must have leading axis size equal to the number of local devices if no `devices` argument is specified. Got axis_size=, num_local_devices=zOn multi-host platforms, pmapped functions must run across all devices, i.e. num_replicas * num_partitions should equal the number of local devices. Got num_replicas=z, and num_local_devices=zOn multi-host platforms, pmapped functions that both have `devices` specified and contain an inner_pmap must specify an `axis_size` (or remove the `devices` argument). Got nested_replicas=z@Compiling %s (%d) for %d devices with args %s. (num_replicas=%d)r&  c                    g | ]}|d u S rX   rS   )re   r   s     rU   r   z+lower_parallel_callable.<locals>.<listcomp>+  s    666dTT\666rT   pmap_z(Ordered effects not supported in `pmap`.MFinished jaxpr to MLIR module conversion {fun_name} in {elapsed_time:.9f} secr  )ordered_effectsbackend_or_name	platformsaxis_context
name_stackdonated_argsreplicated_argsarg_shardingsresult_shardings	arg_namesresult_namesnum_replicasr  )r.  r  r  r   
tuple_argsunordered_effectsr,  	keepalivehost_callbacksr  shape_poly_state)6r"  process_countr   r   loggerisEnabledForloggingDEBUGdebugr  r  r  r  r  r  local_device_countr  r   log_compilesr   WARNINGlogrP   idr  r   AxisEnvr   new_name_stackrD   r   should_tuple_argsrh   r$  maybe_extend_axis_envr   r   r,  	filter_infilter_not_inr  r   JAXPR_TO_MLIR_MODULE_EVENTr*   lower_jaxpr_to_moduleReplicaAxisContextr  r5  result_pathsr%  moduler:  r;  r<  )r   r   r   r   r   r   r   r   r   r  r  r  r  r   r  r   r  r   no_nested_shardingmust_run_on_all_deviceslog_priorityrh  r0  r2  r8  module_namer.  r,  r9  lowering_results                                 rU   r  r    sd   0 	A">
i
'
'
	"/ 	" 	"	" 	" 	"# # # 
%!	% '	  1$$	 	' " #''' 9
LL$f&:;;;
LL+V-HIII
LL;-x/JL L L
LL(((
LL$c&7888w!##(?#!6w!?!???)++
?1:
? 
? !3G<<
? 
?@ @ @ 	> !3	> 	>  27;;		> 	>? ? ?  $H3a77
# 
	 # #$ $ $ %+$7$=P7=,&& N
JJ|Q|RWW'0LN N N
 #"YL3C2EG G(.yv/F/FGG*66g666/)#f.I*J*J*1*:< <*&&&+ 7W%5$7)Y(8$?? 3 3)),*>??A AO CABBB--l.BCCE E		"WZ(K
M 
M 
M 3 3 2

)!%8BB%)$C)9)C'IE,<,I313 3 3o3 3 3 3 3 3 3 3 3 3 3 3 3 3 33 3 3 3 3 3 3 3 3 3 3 3 3 3 34 
/#, 8 &:+<)8#2#<(7(F*6*<*G*9*J	
L 	
L 	
L 	Ls8   ;B"PAP:PP
	
PP
	PP!$P!r   r&   c                    ||S t          |          t          u r0t          t          |j        ||           |j        |j                  S t          |          )N)	weak_type)rk   r   r&   rE   r   r   rY  r   )r   r   r   r   s       rU   _pmap_unmap_shaped_arrayrZ  V  sZ     
\$;DzzS|DJd;;TZ!%1 1 1 1oorT   zdict[type, AvalMapHandlerPair]_pmap_aval_mapping_handlerscore.AxisSizec                   t           j        j        st          j        | |||          S t
                              t          |          d          \  }}| || |||          S t          d| dt          |                     )NNNzno unmapping handler for z	 of type )	r   r   r   r   unmapped_avalr[  r'  rk   r   )r   r   r   r   r  handlers         rU   _pmap_unmapped_avalra  e  s    		&	, ;dItT:::*..tDzz<HH*!W74D$///
KKKtDzzKK
L
LLrT   c                  X    e Zd ZU ded<   ded<   ddZddZej        ddd            Zd	S )r%  	ir.Module_hlozPmapExecutable | None_executabler2   c                0    d | _         || _        || _        d S rX   )re  rd  compile_args)rU  r2   rg  s      rU   rT  zPmapComputation.__init__u  s    DDI$DrT   r]   c                    | j         S rX   rd  r  s    rU   	stablehlozPmapComputation.stablehlo|  
    9rT   NPmapExecutablec                x    | j         |+t          j        | j        fi | j        d|i}||| _         |S | j         S Ncompiler_options)re  UnloadedPmapExecutablefrom_hlord  rg  rU  ro  
executables      rU   r  zPmapComputation.compile  si    #3#?)2
)- -(- -+- - -j 
	!%rT   )r2   rc  r]   rc  rX   r]   rl  )	rP   rQ   rR   r  rT  rj  r   annotate_functionr  rS   rT   rU   r%  r%  q  s|         ///$$$$% % % %          rT   c                D    t          | t                    s
J |             | S rX   )r   r&   r   s    rU   _cast_to_shaped_arrayry    s%    	D+	&	&,,,,,	+rT   c                      e Zd ZU ded<   ded<   ded<   ded<   d	ed
<   ded<   ded<   ded<   ded<   ded<   ded<   d Zd)dZe	 	 d*d+d(            ZdS ),rp  r   compiledxb.XlaBackendr   r  local_input_avalsr\   input_shardingsSequence[ShapedArray]local_output_avalsoutput_shardingslist[core.Effect]r9  r,  r   r:  r;  core.JaxprDebugInfor  c                   g }t          | j        | j                  D ]\  }}t          |t          j                  s
J |            t          |t          j                  s
J |            |                    |j	        t          j        |j        |j	                  nd            t          | j        | j                  }t!          | j        | j                                        |          }t'          | j        d| j        ||| j        | j        | j        t1          | j                  t5          t7          t9          |                              d           }|S )Nzparallel computation)rB   r}  r~  r   r   PmapShardingr   r&   ro   sharding_specr   spec_to_indicesr   local_avals_to_results_handlerr  r  InputsHandlerr{  r  ExecuteReplicatedr   r9  r,  r:  r   r;  r  r  rh   )rU  input_indicesr   spechandle_outshandle_argsexecute_funs          rU   build_execute_funz(UnloadedPmapExecutable.build_execute_fun  sZ   Mt5t7KLL 7 7
dn9::@@D@@@d.//55555+ 
(T5G
H
H
H157 7 7 7 11H151FH HK 4 $ ; ; = =}N NK#DM3I$(L+{$($:$($8$.$()<$=$=$'c-.@.@(A(A$B$BDJ JK rT   r]   rl  c                ~    t          | j        dd           }t          | j        | j        || j        | j        |           S )Nr   )getattrr{  rl  r  r}  r  )rU  r   s     rU   loadzUnloadedPmapExecutable.load  s@    $-==Kt-{ 5t= = =rT   Nr2   rc  r  r  r  r  r   r  r8  r   	list[Any]r.  Sequence[str]r<  "mlir.ShapePolyLoweringState | Nonec                   ~
||j         rt          j        |           } j        }||j        t          j        j                  k    rGd}t          |	                    |j        t          j        j                  j
                            |j        |j        k    r4fdt          t          j        j                            D             }n+t          j        j                  d |j                 }n|j        t          j                  k    rd                    t#          t$          j                            }|j        j        k    r0t          dj         dt          j                   d| d          t          d	|j         d
t          j                   dj         d| d	          |j        t          |          k    r(t          d|j        dt          |          d          d}t)          j        |                              j
        |f          }t/          j        j
        ||d|t/          j        |           j                  }||_        t          j        j                  t)          j        fd|j        D                       }fdt;          |j        j                  D             }tA          ||          }fdt;          |j!        j"                  D             }fdt;          |j!        j"                  D             }tA          ||          }tG          j$        dj%        tF          j&                  5  t/          j'        j        | |||          }d d d            n# 1 swxY w Y   tQ          |j        j)        ||||||||	          *                                S )Nzocompiling computation that requires {} logical devices, but only {} XLA devices are available (num_replicas={})c                N    g | ]!}t          j        |j                  D ]}|"S rS   )r"  r  r   )re   r  r   r  s      rU   r   z3UnloadedPmapExecutable.from_hlo.<locals>.<listcomp>  sU     I I I+M3;GGI I  I I I IrT   r   z, ztLeading axis size of input to pmapped function must equal the number of local devices passed to pmap. Got axis_size=r(  z$.
(Local devices available to pmap: )zpmapped function requires zR local devices to run due to nested pmapped or other parallel functions, but only z" are available.
(outer axis size: z#, local devices available to pmap: z#compiling computation that creates z shards, but z devices were specifiedr`   F)r7  num_partitionsdevice_assignmentuse_spmd_partitioningenv_options_overridesdetailed_loggingr   c                *    g | ]}|j         k    |S rS   )r  )re   r   r  s     rU   r   z3UnloadedPmapExecutable.from_hlo.<locals>.<listcomp>  s,     ( ( (Q_-M-M-M-M-MrT   c           
         g | ]>\  }}t          j        j        j        t	          t
          |          j        |          ?S rS   )r   pmap_sharding_specr  r   r   r&   r   )re   r   r  r  r  s      rU   r   z3UnloadedPmapExecutable.from_hlo.<locals>.<listcomp>	  s^     J J J D' 	)'d##)7	4 	4J J JrT   c           
     p    g | ]2\  }}|)t          t          j        j        ||                    n|3S rX   )ry  ra  r   r   )re   r   r1  r  s      rU   r   z3UnloadedPmapExecutable.from_hlo.<locals>.<listcomp>  sg     P P P D(  	s}hMM	O 	O 	O%)P P PrT   c                b    g | ]+\  }}t          j        j        j        |j        |          ,S rS   )r   r  r  r   r   )re   r   r1  r  r  s      rU   r   z3UnloadedPmapExecutable.from_hlo.<locals>.<listcomp>  sN     5 5 5 D( 	)'
H	N 	N5 5 5rT   @Finished XLA compilation of {fun_name} in {elapsed_time:.9f} secr  )r{  r   r}  r~  r  r  r9  r,  r:  r;  r  )+uses_dim_varsr*   refine_polymorphic_shapesr   r  r"  device_countr   r   formatr  r  r  r=  r  rh   r  r   r   r   r   r   reshaper   get_compile_optionsuse_detailed_loggingparameter_is_tupled_argumentsr  flatrB   r  r   _get_pmap_shardingr  r  r   r  r   BACKEND_COMPILE_EVENTcompile_or_get_cachedrp  r  r  )r2   r  r  r   r8  r9  r,  r;  r:  r  r.  r<  ro  r   r  local_devices_strr  r  compile_optionslocal_device_assignmentinput_sharding_specsin_shardingslocal_unmapped_avals	out_specsout_shardingsr{  r  s    ``                       @rU   rq  zUnloadedPmapExecutable.from_hlo  s    	#(8(F#*3//ckG		!BOCK$@$@	@	@9F$<$&OCK$@$@$,$@B B C C 	C 
	!F$;	;	;I I I I%0@0M0M*N*N I I I "3;7778P9P8PQ		 C(9$:$:	:	: IIc#s/@&A&ABB"cm33HH H478I4J4JH H 4EH H HI I I ,6+B , ,%():%;%;, , $'=, , )	, , ,- - - 
	!S\\	1	1j1113w<<<<A B B 	B N$&HW$5$5$=$=		%~6%8 %821%+#.!6s;;  O 5?O1$S[11M h ( ( ( ($)( ( (  J J J J J &f&:CKHH	J J J
 &&=&:< <LP P P P 'v'?NN	P P P
5 5 5 5 5 '$cl4 45 5 5I
 ''>	JJM		"J!?
A 
A 
A   /
+s-
 h               ")$/&+'%)+ + + ,04662s   )NNNru  r^  )r2   rc  r  r  r  r  r   r  r8  r   r9  r  r,  r  r;  r  r:  r   r  r  r.  r  r<  r  )rP   rQ   rR   r  r  r  staticmethodrq  rS   rT   rU   rp  rp    s         ---1111&&&&++++''''&&&&$$$$''''  (= = = =  GK $w2 w2 w2 w2 <w2 w2 w2rT   rp  c                  Z    e Zd Zg dZd Zed	d            Zd Zej	        d             Z
dS )
rl  xla_executable_unsafe_callbuild_unsafe_callr   in_avals_jaxpr_debug_info_unloaded_executablec                h    || _         d | _        || _        || _        || _        || _        || _        d S rX   r  )rU  r  r  r   r  r  unloaded_executables          rU   rT  zPmapExecutable.__init__7  s?    (DD.D"DDM-D 3DrT   r]   Callable[..., Any]c                P    | j         |                                 | _         | j         S rX   r  r  r  s    rU   r  zPmapExecutable.unsafe_callA  (     0022drT   c                    | j         S rX   r  r  s    rU   xla_extension_executablez'PmapExecutable.xla_extension_executableI      rT   c                    t          t          j        |          }t          | j        || j                    | j        | S rX   )r   r+   r   check_arg_avals_for_callr  r  r  )rU  rr   	arg_avalss      rU   callzPmapExecutable.callL  s<     COT**IT]It7MNNN4T""rT   Nr]   r  )rP   rQ   rR   r  rT  r  r  r  r   rv  r  rS   rT   rU   rl  rl  2  s        ' ' ')4 4 4    8   # # # # #rT   rl  c                       fd|D             S )Nc                :    g | ]}t          j        |          S rS   )r   r  )re   r  r   s     rU   r   z&_get_pmap_sharding.<locals>.<listcomp>U  s&    	G	G	G.
%gt
4
4	G	G	GrT   rS   )r   specss   ` rU   r  r  T  s    	G	G	G	G	G	G	GGrT   c                  &    e Zd ZdZddZd Zd ZdS )r  )r`  r  r  r  Nc                d    t          t          |          | _        || _        || _        || _        d S rX   )r
   r   r`  r  r  r  )rU  r  r  r  s       rU   rT  zInputsHandler.__init__[  s1    :|44DL&D$D&DrT   c                ,    |                      |          S rX   )r`  )rU  input_bufferss     rU   __call__zInputsHandler.__call__a  s    <<&&&rT   c                8    d| j          d| j         d| j         dS )NzInputsHandler(
local_devices=z,
in_shardings=z,
input_indices=r  )r  r  r  r  s    rU   r  zInputsHandler.__str__d  sC    3!/3 3 -3 3 "/3 3 3 4rT   r^  )rP   rQ   rR   r  rT  r  r  rS   rT   rU   r  r  X  sK        K)' ' ' '' ' '4 4 4 4 4rT   r  c                      e Zd ZdZd Zd ZdS )ResultsHandlerhandlersr  	out_avalsc                0    || _         || _        || _        d S rX   r  )rU  r  r  r  s       rU   rT  zResultsHandler.__init__p  s    DM&DDNNNrT   c                @    d t          | j        |          D             S )Nc                *    g | ]\  }} ||          S rS   rS   )re   hr   s      rU   r   z+ResultsHandler.__call__.<locals>.<listcomp>v  s$    EEE4AAdGGEEErT   )rB   r  )rU  out_bufss     rU   r  zResultsHandler.__call__u  s"    EE8DM8#D#DEEEErT   N)rP   rQ   rR   r  rT  r  rS   rT   rU   r  r  k  s>         9)  
F F F F FrT   r  unmapped_local_out_avalsr  local_shardingsc                    d t          ||           D             }d t          | ||          D             }t          |||           S )Nc                    g | ]>\  }}t          |                    |j                                                            ?S rS   )r   devices_indices_mapr   r   )re   r}   r   s      rU   r   z2local_avals_to_results_handler.<locals>.<listcomp>|  sV     U U UQ q,,TZ88??AABB U U UrT   c                8    g | ]\  }}}t          |||          S rS   )r   )re   r   r}   idcss       rU   r   z2local_avals_to_results_handler.<locals>.<listcomp>~  s:       
$4 #4D11  rT   rB   r  )r  r  out_indicesr  s       rU   r  r  y  sp    U U (:R S SU U U+ #$<o{[[  ( 
/3K	L	LLrT   global_out_avalsc                ^    fdt          | |          D             }t          |||           S )Nc                8    g | ]\  }}t          ||          S rS   )r   )re   global_avalr}   r   s      rU   r   z3global_avals_to_results_handler.<locals>.<listcomp>  s9       
+q $KI>>  rT   r  )r  r[   r   r  s     ` rU   global_avals_to_results_handlerr    sK       $%5yAA  ( 
)-=	>	>>rT   c                  R    e Zd ZdZg dZ	 dddZd Zd Zej	        d             Z
dS )r  zIThe logic to shard inputs, execute a replicated model, returning outputs.)r  r   r   
in_handlerout_handlerhas_unordered_effectsr,  r:  has_host_callbacks_local_deviceskept_var_idxmutpgle_profiler__weakref__Nr  r  r  r  r9  r  r,  r:  r   r  r   r  set[int]r  MutationData | Noner  profiler.PGLEProfiler | Nonec                   || _         || _        || _        || _        || _        t          |          | _        || _        | j                                         | _	        || _
        |	| _        |
| _        || _        || _        d S rX   )r  r   r   r  r  r   r  r,  r  r  r:  r  r  r  r  )rU  r  r   r   r  r  r9  r,  r:  r  r  r  r  s                rU   rT  zExecuteReplicated.__init__  s     )DDIDL DO"D!%&7!8!8D*D-;;==DDN0D$DDH&DrT   c                H      j         r fd j         D             }g ||}|S )Nc                d    g | ],}t           j                            |j                  j        -S rS   )r   runtime_tokensget_token_inputr  r   )re   effrU  s     rU   r   z;ExecuteReplicated._add_tokens_to_inputs.<locals>.<listcomp>  sB        
!
1
1#t7J
K
K
P  rT   )r,  )rU  
input_bufstokenss   `  rU   _add_tokens_to_inputsz'ExecuteReplicated._add_tokens_to_inputs  sP     *   )  f *V)j)jrT   c                   t          | j                  D ]8\  }}t          j                            ||                    |                     9t          | j        |          D ]\  }}t          |          dk    sJ t          |          dk    r9t          j        	                    |t          j        |d                              gg }|D ]H}t          |j        t          j                  sJ |                    |j        j        d                    It          j        |          }	t'          j        d|	|          }
t          j        	                    |t          j        |
                     d S )Nr   r`   )r   )rn   r  r   r  set_output_runtime_token	get_tokenr  r,  rh   set_token_resultr   Tokenr   rw   r   r?   ro   _device_assignmentPositionalShardingr%  $make_array_from_single_device_arrays)rU  
token_bufssharded_tokenrv   devicer  	token_buftoken_devicestokenr}   global_token_arrays              rU   _handle_token_bufsz$ExecuteReplicated._handle_token_bufs  s    t233 . .	666
-))!,,. . . .d2J?? 
 
Y^^a	Y1		00dj16N6NOOOO 	E 	EEENN,OPP
P
P
P


u~@C
D
D
D
D-m<< E!Y
 
 	00.//	
 	
 	
 	

 
rT   c                     fdt          |          D             } j        rg | j        j        }                     |          }t          j                             j                  5   j        s j	        s j
        r                     |          } j                            |d          }|                    t           j                            }|                                }                     ||           n j                            |          }t%          j                    rI|                                }|D ]}t%          j         j        |                                |          }n|                     j        j                  } j        F j                                        r-t          |          dk    r|d                                          d d d            n# 1 swxY w Y    j        |S g }	t9           j        j        |          D ]*\  }
}|
|||
         _        |	                    |           +|	S )Nc                ,    g | ]\  }}|j         v |S rS   )r  )re   rv   rY   rU  s      rU   r   z.ExecuteReplicated.__call__.<locals>.<listcomp>  s*    DDD$!QQ$2C-C-CA-C-C-CrT   T)with_tokensr   ) rn   r  in_mutr  r   PGLEProfilerr  r  r,  r  r  r  r  execute_sharded,disassemble_prefix_into_single_device_arraysrh   consume_tokenr  r   needs_check_special%disassemble_into_single_device_arrayscheck_specialr   r  consume_with_handlersr  
is_runningblock_until_readyr  out_mutr   ro   )rU  rr   r  ry   result_token_bufssharded_runtime_token
out_arraysarraysr   out_rv   os   `           rU   r  zExecuteReplicated.__call__  s   DDDD)D//DDDDx '&t&dho&d&&J			$	$T%7	8	8 # #

 B$"< B$B//
;;
%55D 6 
 
 $PP$%%' ' ' 5 5 7 7 13HIIII%55jAA		%	'	' GBBDD
  	4 	4F

 F
3
3
3
3z**++D,<,EFF


(T-?-J-J-L-L
(#hhllA  """3# # # # # # # # # # # # # # #6 xjddh&,,  $!Q=$q',,
++a....ks   -E;G44G8;G8rX   )r  r  r  r  r9  r  r,  r  r:  r   r  r   r  r  r  r  r  r  )rP   rQ   rR   __doc__r  rT  r  r  r   rv  r  rS   rT   rU   r  r    s        QQ6 6 6) >B' ' ' ' '*  
 
 
0 ) ) ) ) )rT   r  xla_pmapc                T   t          | |d                   \  }}t          | |d                   \  }	}t          ||d                   \  }}
|
dg|z  z   }
t          |t          |	          t          |
          t          |                    }t          ||d                   \  }}dg|z  |z   }t          ||d                   \  }}dg|z  |z   }t          ||d                   \  }}t          |t          |          t          |          t          |                    }||fS )Nr   r   r  r   )r   r  r   F)rC   r  r   )unks_ininst_inkept_outs_knownkept_outs_stagednum_resparams_knownparams_stageddonated_invars_knownr  in_axes_knownout_axes_knownnew_params_knowndonated_invars_stagedin_axes_stagedout_axes_stagednew_params_stageds                   rU   (_pmap_partial_eval_custom_params_updaterr@    sX    +7LAQ4RSS#G\)-DEE-$_l:6NOO!^!QC'M1.,m0D0D#(#8#8)./C)D)DF F F
 ,G]CS5TUU! 'G+.CC$WmI.FGG!^3=>1.%&6j8QRR!_=%2G2G$)/$:$:*/0E*F*FH H H 
,	,,rT   c                P    t          j        | d         t           j        d|          S )Nr   r   )r   r_  no_axis_name)r6  r   s     rU   #_pmap_partial_eval_custom_res_makerrC    s!    		L5t7H!T	R	RRrT   c           	     H   |j         d         }t          ||j         d         d           5  t          j        |j         d         |           \  }}d d d            n# 1 swxY w Y   t	          ||j         d                   \  }}t	          ||j         d                   \  }}t	          | |j         d                   \  }}t          |j         |t          |          t          |          t          |                    }	t          |          st          |           s|j        s|d fS t          j
        |j        |h          }
t          j        d t          |j        |          D             d	 t          |j        |           D             |j        |	|
|j                  }||fS )
Nr   r   
call_jaxprr   r   r  )rE  r   r   r  c                    g | ]	\  }}||
S rS   rS   re   r{  useds      rU   r   z"_pmap_dce_rule.<locals>.<listcomp>(  s!    ???wq$$????rT   c                    g | ]	\  }}||
S rS   rS   rG  s      rU   r   z"_pmap_dce_rule.<locals>.<listcomp>)  s!    AAAwq$DAAAArT   )r   rK  r  	dce_jaxprrC   r  r   r   r   r   filter_named_axis_effectsnew_jaxpr_eqnr  invarsoutvarsrc  source_info)used_outputseqnr   	new_jaxprused_inputsr  r   r   r  
new_paramseffsnew_eqns               rU   _pmap_dce_rulerW    s   j%)Y
3E(FMM R R\#*\*BLQQI{R R R R R R R R R R R R R R R$[#*=M2NOO!^k3:i+@AA*!W|SZ
-CDD+!XCJ9#(#8#8!'NNU8__F F F* 
[		  #l"3"3  I<M  ))*;i[IID??#cj+66???AA#ck<88AAAz4: :G s   $AA!Ar   core.ParamDictkept_inputsnum_new_inputsc                    | d         }|s|rd|z  }nEt          |          t          |          k    sJ d t          ||          D             }dg|z  |z   }t          | t          |                    S )Nr   Fc                    g | ]	\  }}||
S rS   rS   )re   r   kepts      rU   r   z8_xla_call_partial_eval_update_params.<locals>.<listcomp>8  s!    PPPGAt4PaPPPrT   Fr   )rh   r  r  r   )r   rY  rZ  r   s       rU   $_xla_call_partial_eval_update_paramsr`  .  s     *+.	 ? ?.NN{s>222222PPs>;'G'GPPPNW~->N	fU>%:%:	;	;	;;rT   c                v    | d         }d t          ||          D             }g ||R }t          | |          S )Nr   c                    g | ]	\  }}||
S rS   rS   )re   r   nzs      rU   r   z.xla_call_jvp_update_params.<locals>.<listcomp>?  s!    LLLEArLaLLLrT   r_  r  r  )r   nz_tangentsr   donated_tangentsnew_donated_invarss        rU   xla_call_jvp_update_paramsrh  =  sS    *+.LLS%E%ELLL;;*:;;	f%7	8	8	88rT   c                    | d         }d t          ||          D             }d |D             }t          | g ||R           S )Nr   c                    g | ]	\  }}||
S rS   rS   )re   r   us      rU   r   z5_xla_call_transpose_update_params.<locals>.<listcomp>E  s!    OOO41aQOQOOOrT   c                    g | ]}|d S r\  rS   )re   rc  s     rU   r   z5_xla_call_transpose_update_params.<locals>.<listcomp>F  s    :::"r::::rT   r_  rd  )r   undef_primalsnonzero_ctsr   donated_primalsdonated_cotangentss         rU   !_xla_call_transpose_update_paramsrq  C  s`    *+.OO3~}#E#EOOO/:::::	f%L%L9K%L%L	M	M	MMrT   rE  )res_avalc                     d vr S |s S  fd}t           d          d         d           5  t          j         d         |          }d d d            n# 1 swxY w Y   t           |          S )NrE  c                4    | d         v r| fn
 |           S )Nr   rS   )r   r   substs    rU   shadowed_substz(_pmap_axis_subst.<locals>.shadowed_subst[  s&    f[111D77uuT{{BrT   r   r   )rE  )rK  r   subst_axis_names_jaxprr  )r   ru  traverserv  rR  s   ``   rU   _pmap_axis_substry  V  s    M	 MC C C C C CVK0#$67? ? < <+F<,@,:< <I< < < < < < < < < < < < < < < 
f	+	+	++s   AAAc                   t          j        t          j        | j        t          j        | j                  z  t          j                            }t          j        t          j        | j        d         t          j                            }t          j
        t          j        t          j                    |          |          S r  )r*   ir_constantr   r   nrepsmathprodsizesuint32r2   	remainderdivide
replica_id)rh  divmods      rU   _unravel_index_hlor  e  s    hx~8>!:!::BIFF	H 	H#(."4bi@@AA#	sz#."2"2C88#	>	>>rT   c                   | t           j        u r|S t          | t           j                  r=t	          j        | j        t          j                  r$| j        j        	                    | j                  } t          | j                  }t          j        t          j        dt          j                            }|gt#          |          z  }|                    |t'          |                     |                                }|                    |d           t+          j        ||t          j        |                    }t+          j        t          j        |           |          S t5          |           )NrS   r   r`   )r   abstract_tokenr   r&   r   
issubdtyper   extended_rulesphysical_element_avalr   r   r*   r{  r   r   r  rh   r  r  copyr2   dynamic_slicedense_int_arrayr  aval_to_ir_typer   )	r   rh  rY   r  dimszeroidxsdims_unsqueezeddynamic_slice_results	            rU   
_hlo_shardr  k  s/   	T   H$()) V_55 AZ44TZ@@d
DBHRry999::D6CIIDKK+H55666iikkO7A&&&,	4%o668 8;t+D113GHHH
D//rT   c                    	 t          fdt          | j                  D                       S # t          $ r t	          d           d w xY w)Nc              3  .   K   | ]\  }}|k    |V  d S rX   rS   )re   rv   r   r   s      rU   rg   z_axis_read.<locals>.<genexpr>  s1      OOWQTY=N=Nq=N=N=N=NOOrT   zunbound axis name: )maxrn   r>  r   	NameError)rh  r   s    `rU   
_axis_readr  ~  sn    AOOOO	(. 9 9OOOOOO	 A A A
5)55
6
6D@As	   ,0 Arh  sharding_impls.AxisEnvtuple[tuple[int, ...]]c                <   t          |t          t          f          s|f}t          t          t	          t
          |           |                    }t          | j        t          j	        | j
                            \  }}|rJ | j
        |fz   }t          ||          S rX   )r   r   r   r   r
   r  divmodr|  r}  r~  r  _axis_groups)rh  r   	mesh_axestrailing_sizeragged	mesh_specs         rU   axis_groupsr    s    	D4-	(	( 7DJwz8<<dCCDD) 8>1J1JKK-n//)	i	+	++rT   c                   t          j        t          j        |                                         |           }t          j        t          j        ||t          j        t          |                              t          j        t          j        | |                    df          }t          t          t          |j
                            S )a}  Computes replica group ids for a collective performed over a subset of the mesh.

  Args:
    mesh_spec: A sequence of integers representing the mesh shape.
    mesh_axes: A sequence of integers between 0 and `len(mesh_spec)` (exclusive)
      indicating over which axes the collective is performed.
  Returns:
    A tuple of replica groups (i.e. tuples containing replica ids).
  r  )r   r(  r}  r~  r  r  rh   taker   r   T)r  r  r  groupss       rU   r  r    s     
49Y''	(	(	0	0	;	;$:k$	29S^^#<#<==yI..//46 6& 
z%**	+	++rT   ctxmlir.LoweringRuleContextc                   |t           j        u r|S t          |t           j                  rt	          |j                  }|                    |j        d         g|z             }t          j	        | d|          }t          j
        t          j        dt          j                            }t          |          g|gt          |          z  z   }	t!          j        |t          j        dg                    }
t!          j        ||
|	          }t          j        t+          ||j        d                             }t!          j        ||          }|dk    rt	          t1          dt          |                              }|                    |d           t	          |          }|                    ||j        d                    t!          j        |t          j        |                    }|S t7          |          )Nr  r   r   rS   r   r`   )r   r  r   r&   r   r   r   r  r*   full_like_avalr{  r   r   r  r  rh   r2   r  r  dynamic_update_slicedense_int_elementsr  r>  cross_replica_sumr  r  	transposer   )r  r   rh  r1  rY   r  padded_avalpaddedr  r  broadcast_resultreplica_groupsr   permtransposed_dimss                  rU   _hlo_unshardr    s   	T   H$()) 
D++X^B%7$84$?+@@K a55FBHRry999::Dx(()TFSYY,>>D}Q(<aS(A(ABB%f.>EEF,(HN2.//1 1N


7
7C1}}%3t99%%&&d
kk(AT

oXx~b'9:::M#t3D99::cJ
D//rT   envc                \    t          j        | j        | j        |fz   | j        |fz             S rX   )r   rH  r|  r>  r  )r  r   r   s      rU   _extend_axis_envr    s2    			39w+> #	TG 3
5 
5 5rT   r  c                   ~
t          j        | j        j                    j        j        j        r|t          d          t           j        j        ||          d |j        D             }fdt          |||          D             }t          ||d           5   j                            t          j                            }t          j        || j                            t#          j        |d                    t          j                    dg|R d j        i\  }}d d d            n# 1 swxY w Y   d |j        D             } fd	t          ||	|          D             }|S )
Nrq  c                    g | ]	}|j         
S rS   rx  re   r{  s     rU   r   z"_pmap_lowering.<locals>.<listcomp>  s    000af000rT   c              3  L   K   | ]\  }}}|t          |||          n|V  d S rX   )r  )re   r   in_noder  new_envs       rU   rg   z!_pmap_lowering.<locals>.<genexpr>  sc       D D 	gw  tWgw/// 'D D D D D DrT   )r/  r&  rS   dim_var_valuesc                    g | ]	}|j         
S rS   rx  r  s     rU   r   z"_pmap_lowering.<locals>.<listcomp>  s    222!qv222rT   c           	     >    g | ]\  }}}t          |||          S rS   )r  )re   r   r1  shardr  r  s       rU   r   z"_pmap_lowering.<locals>.<listcomp>  sE     
O 
O 
O#dHe sD'8U
;
; 
O 
O 
OrT   )r*   check_backend_matchesmodule_contextr.  rh  r>  r   r  rM  r  rK  replacer   rP  jaxpr_subcompr0  extendr"   rD   TokenSetr  rN  )r  r   r   r   r   r   rE  r   r   r  r   r   in_nodesr  in_nodes_shardedsub_ctxsharded_outsr  r  r~   r  s   `                   @rU   _pmap_loweringr    s    Wc&8&BCCC 	 & D7+>
B
C
CCS/8)-/ /' 10j/000(D D D D #&h'"B"BD D D
 Y(8$?? + + ((#6w?? ) A AG(dnT6::;;+ /+ + + )	+ +OL!+ + + + + + + + + + + + + + + 32z1222)
O 
O 
O 
O 
O'*9h'M'M
O 
O 
O$	+s   BD66D:=D:r5   c                ,   t          |t                    sJ t          |j                  }|                                D ]2\  }}||         | |         z  dk    sJ ||xx         | |         z  cc<   3|                    t          |                    S )Nr   r   r   r&   r   r   rp   r   r   )
axis_sizesr   r   r   r   r   s         rU   tile_aval_ndr    s    	D+	&	&&&&
tz

%MMOO % %jdD;D))Q....	$KKKJt$$KKKK	5<<	(	((rT   r  c                    t          |t                    sJ t          |j                  }|                                D ]\  }}||xx         | |         z  cc<   |                    t          |                    S )Nr   r  )r  r  r   r   r   r   s         rU   untile_aval_ndr    s{    	D+	&	&&&&
tz

%NN$$ $ $jdD	$KKK:d##KKKK	5<<	(	((rT   axesc           	     `    t          | j        |t          | j        j        ||                    S rX   )r  r   r  
local_meshr   r  r   s      rU   mesh_local_to_globalr    s1    	
D$T_%:D$GG
I 
I IrT   c           	     `    t          | j        j        |t          | j        ||                    S rX   )r  r  r   r  r  s      rU   mesh_global_to_localr    s.    	-t$TZt<<
> 
> >rT   full_to_shardc                .    t          |j        ||           S rX   )r  r   rY   r  r   r  s       rU   _full_to_shard_abstract_evalr    s     
dj$	*	**rT   manual_axes_set&frozenset[sharding_impls.MeshAxisName]r   Meshc                  	
 |j         
t          
                                          }d t          |j                  D             	t          t                    }fd|j        D             }	fd|D             	fd|D             z   }dg| j        z  }|                    t          j
        
fd|D                                  |                    t          j
        
fd|D                                  t          j                    }t          j        j        j        |_        ||_        ||_        ||_        t          j        j        j        t          j        j        j        g|_        |S )	zoCreate an OpSharding proto that declares all mesh axes from `axes` as manual
  and all others as replicated.
  c                    i | ]\  }}||	S rS   rS   )re   rv   r   s      rU   rI  z manual_proto.<locals>.<dictcomp>	  s    BBBGAtaBBBrT   )keyc                    g | ]}|v|	S rS   rS   )re   r   r  s     rU   r   z manual_proto.<locals>.<listcomp>  s-     5 5 5dO33 333rT   c                     g | ]
}|         S rS   rS   re   r|   
axis_orders     rU   r   z manual_proto.<locals>.<listcomp>  s    666z!}666rT   c                     g | ]
}|         S rS   rS   r  s     rU   r   z manual_proto.<locals>.<listcomp>  s    222z!}222rT   r`   c                     g | ]
}|         S rS   rS   re   r|   named_mesh_shapes     rU   r   z manual_proto.<locals>.<listcomp>  s    KKKa.q1KKKrT   c                     g | ]
}|         S rS   rS   r  s     rU   r   z manual_proto.<locals>.<listcomp>  s    GGGa.q1GGGrT   )r   r   r   rn   
axis_namesr  r   rz  ro   r}  r~  r   
OpShardingTypeOTHERrk   tile_assignment_dimensionsiota_reshape_dimsiota_transpose_perm
REPLICATEDMANUALlast_tile_dims)r   r  r   
mesh_shapemanual_axesreplicated_axestad_perm	tad_shapeprotor  r  s    `       @@rU   manual_protor    s    Z$++--..*BBy'A'ABBB*C000+5 5 5 5do 5 5 5/ 7666o6662222k2223(cDIo)49KKKK?KKKLLMMM49GGGG;GGGHHIII
-//%}!'%*%.%"&%&%-,79K9RS%	,rT   r  c                  | j         \  }| j        \  }t          j        |t	          |                                        |j                                                  }t          t          |j                            t          |
                                          z
  }t          j        | ||||          }	t          |||          }
t          j        | |	||
|          fS N)unspecified_dims)avals_in	avals_outr   NamedShardingr>   _to_xla_hlo_shardingrz  to_protor  r  r   r*   wrap_with_sharding_opr  wrap_with_full_to_shard_op)r  rY   r  r   r  aval_inaval_outsharding_protor  sxr   s              rU   _full_to_shard_loweringr    s     \('m)("4)H)N)NOOGL))((**  w|,,--DKKMM0B0BB!#q'>3CE E E"
wT
2
2%

)#r8U;KM M M 
O OrT   shard_to_fullc                .    t          |j        ||           S rX   )r  r   r  s       rU   _shard_to_full_abstract_evalr  /  s     

D!	,	,,rT   c                  | j         \  }| j        \  }t          |||          }t          t	          |j                            t          |                                          z
  }t          j        | ||||          }	t          j
        |t          |                                        |j                                                  }
t          j        | |	||
|          fS r  )r  r  r  r  r  rz  r   r*   r
  r   r  r>   r  r	  wrap_with_shard_to_full_op)r  rY   r  r   r  r  r  r   r  r  r  s              rU   _shard_to_full_loweringr  4  s     \('m)(
wT
2
2%w|,,--DKKMM0B0BB!#q'53CE E E" "4)H)N)NOOHM**88::  
)#r8^*:< < 
> >rT   -Iterable[JSharding | AUTO | UnspecifiedValue]c                4    | D ]}t          |          r dS dS NTFr;   )r[   r}   s     rU   check_if_any_autor  D  s0      aqzz TT	rT   c                  (    e Zd ZdZdZdZdZdZd ZdS )MismatchTyper   r`   rc         c                `    | j         dk    rdS | j         dk    rdS | j         dk    rdS | j          S )NIN_SHARDINGzexplicit input shardingOUT_SHARDINGzexplicit output shardingCONTEXT_DEVICESr   )r   r  s    rU   r  zMismatchType.__str__R  sE    yM!!&&	n	$	$''	'	'	'Yi>rT   N)	rP   rQ   rR   ARG_SHARDINGr"  SHARDING_INSIDE_COMPUTATIONr#  r!  r  rS   rT   rU   r  r  K  s=        ,, !/+    rT   r  c                      e Zd ZU ded<   ded<   ded<   edd	            Zedd            ZddZed             Zed             Z	d Z
d ZdS )DeviceAssignmentMismatchSequence[xc.Device]dar  m_typezdispatch.SourceInfo | NonerO  r]   Sequence[int]c                $    d | j         D             S )Nc                    g | ]	}|j         
S rS   )rG  r   s     rU   r   z7DeviceAssignmentMismatch.device_ids.<locals>.<listcomp>d  s    """QAD"""rT   )r)  r  s    rU   
device_idsz#DeviceAssignmentMismatch.device_idsb  s    ""$'""""rT   r   c                J    | j         d         j                                        S r  )r)  r$  upperr  s    rU   r$  z!DeviceAssignmentMismatch.platformf  s    71:$$&&&rT   c                <    | j         t          j        k    rd| dndS )N z's )r*  r  r#  rU  api_names     rU   _maybe_api_namez(DeviceAssignmentMismatch._maybe_api_namej  s)    #{l.JJJxPRRrT   c                V    | j         dn dt          j        | j         j                    S )Nr3  z at )rO  r   	summarizer  s    rU   source_info_strz(DeviceAssignmentMismatch.source_info_strm  s3     &N$.t/?/KLLNNrT   c                &    d| j          d| j         S )Nzdevice ids z on platform )r.  r$  r  s    rU   _dev_ids_plat_strz*DeviceAssignmentMismatch._dev_ids_plat_strt  s    FFFt}FFFrT   c                j    | j         t          j        k    r| j        o| j        j         d| n| j         S )Nz inside )r*  r  r%  rO  eqn_namer4  s     rU   
m_type_strz#DeviceAssignmentMismatch.m_type_strx  sA    {lFFF =D$4$=QQxQQQLPKYrT   c                z    |                      |           d|                     |           d| j         | j         S )Nr2  z with )r6  r>  r;  r9  r4  s     rU   _strzDeviceAssignmentMismatch._str|  sU    ##H-- > >0I0I > >%>'+';> > ?rT   N)r]   r+  )r]   r   )rP   rQ   rR   r  r  r.  r$  r6  r9  r;  r>  r@  rS   rT   rU   r'  r'  \  s         ))))# # # 8# ' ' ' 8'S S S S   8 G G 8GY Y Y? ? ? ? ?rT   r'  c                      e Zd ZdS )DeviceAssignmentMismatchErrorNrO   rS   rT   rU   rB  rB    rV   rT   rB  	xc.Devicec                 V    t           j        j        pt          j                    d         S r  )r   default_devicer   r"  r  rS   rT   rU   _get_default_devicerF    s!    			$	=(8(:(:1(==rT   Iterable[ShardingInfo]'tuple[xc.Client, tuple[xc.Device, ...]]c           	     \   d }|d}nt          |          }| D ]\  }}}t          |          r|(t          |          r|j        j        ||fn	|j        ||f}t          |          r|j        j        n|j        }|s4|d         |k    r't          t          | t          |||          g          ||k    r9t          t          |t          j	        d           t          |||          g          ||r|}n|t                      f}n|d         }t          j        |d                   |fS )NrS   r   )r   r<   r;   r   _flat_devices_tupler  rB  r'  r  r#  rF  r"  r  )r[   r   first_sharding_inforv   s_typerO  arr_device_assignmentfinal_device_assignments           rU    _get_and_check_device_assignmentrO    s    _GGGnnG ) S Saa "?Fqzz ;16%v{
;
;$fk:  ;B!**^AF66!J^ 	S	Q	#8	8	8+$&9:$%:FKPP-R S S 	S 
9
 
)	)	)+$Wl.JDQQ$%:FKPP-R S S 	S 
*  W %"24461!4		6q9	:	:<S	SSrT   %tuple[core.Jaxpr, set[int], set[int]]c                    dgt          | j                  z  }t          j        | |          \  }}}d t	          |          D             }d t	          |          D             }|||fS )NTc                    h | ]	\  }}||
S rS   rS   re   rv   bs      rU   	<setcomp>z&prune_unused_inputs.<locals>.<setcomp>  s!    >>>$!QA>A>>>rT   c                    h | ]	\  }}||
S rS   rS   rS  s      rU   rU  z&prune_unused_inputs.<locals>.<setcomp>  s!    <<<1!<!<<<rT   )rh   rN  r  dce_jaxpr_constsrn   )r   rP  rR  used_constsrS  kept_const_idxr  s          rU   prune_unused_inputsrZ    sv     #em,,,,(*(;E<(P(P%)[+>>)K"8"8>>>.<<	+ 6 6<<<,	NL	00rT   c                F  
 t          j        t          ||                    }t          | t          j                  sJ | j        }| j        }| j        }	|s|st          d |	D                       r*t          t          t          |	                              nWt          |          \  }

fdt          |          D             }t          fdt          |          D                       }
t!          j        |          }t	          j        ||          } | ||fS )Nc              3  j   K   | ].}t          |d           ot          j        |j                   V  /dS )r   N)hasattrr   is_constant_shaper   )re   r|   s     rU   rg   z_dce_jaxpr.<locals>.<genexpr>  sU       
 
 !W


Ed&<QW&E&E"E 
 
 
 
 
 
rT   c                "    g | ]\  }}|v 	|S rS   rS   )re   rv   crY  s      rU   r   z_dce_jaxpr.<locals>.<listcomp>  s'    EEEDAqn1D1Da1D1D1DrT   c              3  *   K   | ]\  }}|v 	|V  d S rX   rS   )re   rv   rY   r  s      rU   rg   z_dce_jaxpr.<locals>.<genexpr>  s1      XXAa<FWFW1FWFWFWFWXXrT   )r   rI  rD   r   r   r!  r   r  r  r   r  r  rh   rZ  rn   r   r   r  )r  r5  r  keep_unusedr   auto_spmd_loweringr0  r   r  r  rY  r  s             @@rU   
_dce_jaxprrd    sA     .y8/L/LMM*	L$"2	3	3333

%&"( ' 	 
 

 
 
   uS]]++,,LL*=e*D*D'E><EEEEIf--EEEFXXXX>)B)BXXXXXN

)%
0
0%!%00,	~|Z	??rT   c                  $    e Zd ZU ded<   ded<   dS )MutationDatazlist[core.MutableArray]r  zlist[int | None]r'  Nr  rS   rT   rU   rf  rf    s*         !!!!rT   rf  ;tuple[core.ClosedJaxpr, Sequence[int | None], MutationData]c           
     j   ddl m} t          |           \  } }t          j         || j        | j                   }t          j        t          | j
                            fdt          | j                  D             }d |                                D             }t          t          |j        t#          t          |j                                                }t%          t          |j        t#          t          |j
                                                }||t'          ||          fS )Nr   discharge_statec                `    i | ]*\  }}t          |t                    |t                    +S rS   )r   rK   next)re   rv   r|   counts      rU   rI  z#_discharge_refs.<locals>.<dictcomp>  sC     . . .$!QQ,,.q$u++ . . .rT   c                    i | ]\  }}||	S rS   rS   )re   rv   r9  s      rU   rI  z#_discharge_refs.<locals>.<dictcomp>  s    2221q!222rT   )jax._src.state.dischargerj  _move_mutable_constsr   r!  r   r  itrm  rh   r  rn   r  rp   r   r   r'  r  r   rf  )	r   rj  r  rR  	inout_map	outin_mapinout_aliasesr'  rm  s	           @rU   _discharge_refsru    s    766666&u---%U\ J JK)
(3u''
(
(%. . . .)EN*C*C . . .)22	 1 1222)IM5Y5G1H1H+I+IJJKK-Y]E#i.A*B*B$C$CDDEE'	M<#@#@	@@rT   0tuple[core.ClosedJaxpr, list[core.MutableArray]]c                `   | j         }d | j        D             }t          || j                  \  }}t          ||j                  \  }}g |j        |R }t          j        |||j        |j                  }t          j
        |||j        |j        |d           }t          j        ||          |fS )Nc                B    g | ]}t          |t          j                  S rS   )r   r   MutableArray)re   r`  s     rU   r   z(_move_mutable_consts.<locals>.<listcomp>  s%    
I
I
I:a*++
I
I
IrT   )r   r  rC   	constvarsrM  r  make_jaxpr_effectsrN  eqnsr   Jaxprr!  )	r  r   hoistr  r  rz  mutvarsrM  r   s	            rU   rp  rp    s     
%
I
I\5H
I
I
I%!%)<==.&&%eU_==)W$U\$G$$&!)VU]EJOO'
*Yuzd$ $%		%	(	(&	00rT   c                    ddl m}  || j        | j                  \  }}| j        j        |_        t          j        ||          S )Nr   ri  )ro  rj  r   r  r  _debug_infor   r!  )r   rj  jaxpr_r  s       rU   _discharge_internal_refsr    sM    666666"?5;==.&&{-&		&&	)	))rT   c                  "    e Zd Zd	dZd Zd ZdS )
SemanticallyEqualShardingsr[   ,tuple[GSPMDSharding | UnspecifiedValue, ...]r  tuple[core.AbstractValue]c                d    d t          ||          D             }|| _        || _        || _        d S )Nc                `    g | ]+\  }}t          |          r|nt          ||j                  ,S rS   )r=   to_gspmd_shardingrz  re   r}   r|   s      rU   r   z7SemanticallyEqualShardings.__init__.<locals>.<listcomp>  sJ     + + +Aq $A&&H,=a,H,H+ + +rT   )r  _gspmd_shardingsr[   r  )rU  r[   r  gspmd_shardingss       rU   rT  z#SemanticallyEqualShardings.__init__  sF    + +	5))+ + +O ,DDNDJJJrT   c                X    t          t          d | j        D                                 S )Nc              3  `   K   | ])}t          |t                    r|j        |j        fn|V  *d S rX   )r   r@   _hlo_sharding_hashmemory_kindre   r}   s     rU   rg   z6SemanticallyEqualShardings.__hash__.<locals>.<genexpr>  s\       O O34:a''	/	q}---.O O O O O OrT   )hashr   r  r  s    rU   __hash__z#SemanticallyEqualShardings.__hash__  sI     O O8<8MO O O O O P P PrT   c                    t          |t                    sdS t          d t          | j        |j                  D                       S )NFc              3     K   | ]f\  }}t          |t                    rDt          |t                    r/t          j        |j        |j                  o|j        |j        k    n||k    V  gd S rX   )r   r@   r   are_op_shardings_equal_hlo_shardingr  )re   r}   r-  s      rU   rg   z4SemanticallyEqualShardings.__eq__.<locals>.<genexpr>  s        
 Aq q-((	-7=-I-I		,Q_ao	N	N 
,]am+!V	     rT   )r   r  rq   r  r  )rU  others     rU   __eq__z!SemanticallyEqualShardings.__eq__  sZ    e788 U  
 -u/EFF     rT   N)r[   r  r  r  )rP   rQ   rR   rT  r  r  rS   rT   rU   r  r    sI           P P P
	 	 	 	 	rT   r  r|  Nonec           
     N   | dk    rt          j        d| d           | t          j        |          k    r+t	          d| d|  dt          j        |           d          t          j                    dk    r*| dk    st          j        |d          rt          d	          d S d S )
Nr`   zThe jitted function a?   includes a pmap. Using jit-of-pmap can lead to inefficient data movement, as the outer jit does not preserve sharded data representations and instead collects input and output arrays onto a single device. Consider removing the outer jit unless you know what you're doing. See https://github.com/google/jax/issues/2926.zcompiling computation `z` that requires z replicas, but only z XLA devices are available.r/  zjit of multi-host pmap not implemented (and jit-of-pmap can cause extra data movement anyway, so maybe you don't want it after all).)	warningswarnr"  r  r   r=  r   jaxpr_has_primitiver  )r|  r   r   r   s       rU   )_raise_warnings_or_errors_for_jit_of_pmapr    s    
QYYM	:t 	: 	: 	:; ; ; R_W%%%%
	F$ 	F 	F 	F 	F((	F 	F 	FG G G !aii8/zBBi
	MN N N irT   i   DeviceLocalLayout | Nonec                   t          |          rd S t          j        |j        t          j                  rd S t          j        |j                  sd S |                    |j                  }|j	        d         }	 t          j        |j                            |j        ||                    }n#  Y d S xY w| ||n| S | j        S r  )r=   r   r  r   r  r   r^  r   shard_shaper  r,   from_pjrt_layoutclientget_default_layoutdevice_local_layout)
arg_layoutjit_in_layoutrw   r   r  r   als          rU   _maybe_get_default_layoutr  7  s     H%% 4tz6?33 4		
	+	+ 4$$TZ00+!!$!
		+	##DJQ??
A 
ABB44 &22J6		''s   ;3B/ /B4rt  None | tuple[None | int, ...]propagated_out_mem_kindstuple[None | str, ...]r.  tuple[str, ...]mesh_shape_tupletuple[tuple[str, int], ...]c                   | j         }|j        }|j        }| j        }| j        }t          j        j        rt          j        nt          j	        }t                              |          rt                              |d|||           t          j        |          }t          ||||           |dk    rWt!          t"          ||          }t!          t"          ||          }dgt%          |          z  }t'          j        ||	          }|}n2d }d }d }t'          j        |dd          }t'          j        |          }d}| d| }|dk    rht.          j                            | j                  } t.          j                            |           } t%          |           dk    rt9          d|            t;          t.          j                            | j                            }!t          j        dt?          |          t          j         	          5  tC          j"        || fi d
|!d|d|d|d|d|
d|d|d|d|d|d|j#        o|j#        j$        d|j#        o|j#        j%        d|d|d|d|d|d|d|}"d d d            n# 1 swxY w Y   t          j&        t%          |          |j'                  }#t;          t.          j                            | j                            }$|"j(        |"j)        |"j*        |$|!||#|"j+        fS )NzCCompiling %s with global shapes and types %s. Argument mapping: %s.r`   FrS   r  r   zHThe following ordered effects are not supported for more than 1 device: r+  r  r,  r-  r.  r/  r0  r1  r2  r3  r4  
in_layoutsout_layoutsr5  r6  r7  r  all_default_mem_kindinput_output_aliasesr  r  r  ),r   r[   r  r  r   rD  r   r@  rE  rA  r>  r?  rF  r   r  r  r   _to_logical_shardingrh   r   ShardingContextrH  rP  r   r,  rL  shardable_ordered_effectsrM  r   r   r  r   rN  r*   rO  r  r5  rQ  rJ  r$  rR  r:  r;  r<  )%r  r5  r  r   semantic_in_shardingssemantic_out_shardingsr  r  num_devicesr  r   r0  r  rt  r  r.  r  r  r   r  r  global_in_avalsr  rU  r|  in_mlir_shardingsout_mlir_shardingsr2  axis_ctxr  rh  rV  unsupported_effectsr,  rW  r8  r9  s%                                        rU   _cached_lowering_to_hlor  T  s    
%&0,(2- )/!+$*$7$=P7=,&& 8
JJ|',8 8 8 
!%
(
(%+E7HeLLL aZZ0/<PP13C]SSgO 4 44O-k;LMMH NN O%eR44H0::HN((h((+1__!1;;L<PQQ!;II 
!##	52	5 	56 6 6 0::<;OPPQQ/ WZ(KM M M + + 0+ + + (+  	+
 )+ X+ :+ $^+ (+ ('+ ,++ :+  K+ "Au'7'A+ %G%*:*G+  U!+" &~#+$ 21%+& +]'+( ":!9)+* 0/++, *)-+O+ + + + + + + + + + + + + + +4 )#o*>*>@PQQ*++L,@AAC C

 /";

(*;_
_=
? ?s   A0J

JJr  tuple[xc.Device, ...]xc.DeviceListc                *    t          j        |           S rX   )r   
DeviceList)r  s    rU   _create_da_objectr    s     
(	)	))rT   -Iterator[sharding_impls.TransferToMemoryKind]c              #     K   | j         D ]2}|j        t          j        u rd |j        d         D             E d {V  3t          j        |           D ]}t          |          E d {V  d S )Nc              3  N   K   | ] }t          |t          j                  |V  !d S rX   )r   r   TransferToMemoryKindr   s     rU   rg   z+jaxpr_transfer_mem_kinds.<locals>.<genexpr>  sP       I I>#FGGI! I I I I I IrT   r   )r|  rc  r   device_put_pr   r   	subjaxprsjaxpr_transfer_mem_kinds)r   rQ  subjaxprs      rU   r  r    s      Z I Ic
}---I ISZ	2 I I I I I I I I I I.'' 2 2h'11111111112 2rT   c                n    	 | j         }n#  Y dS xY w|D ] }t          |          r|j        |k    r dS !dS r  )default_memory_kindr=   r  )	da_objectr[   default_mem_kindrv   s       rU   "are_all_shardings_default_mem_kindr    sd     444  aa   }(((UU )	s   
 zdict[Any, Any]memory_kind_propagate_ruletuple[None | str]c                t   i | j         }fd}fd}d }|d gt          |j                  z  }nd |D             }t          ||j        |           t          ||j        d gt          |j                  z             |j        D ]}t          ||j                  }t                              |j        t          ||j        t          |j
                                      }	 |	|i |j        }
|j        j        s|
g}
t          ||j
        |
           t          t          ||j
                            S )Nc                N    t          |           t          j        u rd S |          S rX   rk   r   Literalvarr  s    rU   readz2get_out_memory_kinds_via_propagation.<locals>.read  $    CyyDL  Ts8OrT   c                    || <   d S rX   rS   r  r  r  s     rU   writez3get_out_memory_kinds_via_propagation.<locals>.write      CHHHrT   c                     | j         rd g|z  nd S rX   )rO  )r  num_outvarsr  __s       rU   _default_rulez;get_out_memory_kinds_via_propagation.<locals>._default_rule  s    #'#8BD6KdBrT   c                >    g | ]}t          |          rd n|j        S rX   )r=   r  r  s     rU   r   z8get_out_memory_kinds_via_propagation.<locals>.<listcomp>  s;     - - - 5Q77JddQ] - - -rT   )r   rh   rM  rA   rz  r|  r  r'  rc  r
   rN  r   rO  r   )r  r  r   r  r  r  invar_mem_kindrQ  in_mem_kindsruleout_mem_kindsr  s              @rU   $get_out_memory_kinds_via_propagationr    s    	#

%    
    C C C Vc%,///NN- -+- - -N
5%,///
5%/D6C,@,@#@AAAZ 0 0cD#*--L%))w}cmS=M=MNNP PDD,5#*55M=) &$omUCK////	xem,,	-	--rT   tuple[None | DeviceLocalLayout]c                   ddl m} i | j        }fd}fd}t          ||j        d gt          |j                  z             t          ||j        d gt          |j                  z             |j        D ]M}|j        |j	        u r|j
        d         g}nd gt          |j                  z  }t          ||j        |           Nt          t          ||j                            S )Nr   pjitc                N    t          |           t          j        u rd S |          S rX   r  r  s    rU   r  z-get_out_layouts_via_propagation.<locals>.read  r  rT   c                    || <   d S rX   rS   r  s     rU   r  z.get_out_layouts_via_propagation.<locals>.write  r  rT   layout)r   r  r   rA   rM  rh   rz  r|  rc  sharding_constraint_pr   rN  r   )r  r  r   r  r  rQ  out_eqn_layoutsr  s          @rU   get_out_layouts_via_propagationr    s    
#

%    
     5%,U\):): :;;;
5%/D6C,@,@#@AAAZ 2 2c }222H-.ooS[!1!11oUCK1111	xem,,	-	--rT   c                  (    e Zd ZU dZded<   ded<   dS )AllArgsInfoz4Avals and debug_info for all arguments prior to DCE.r  r  core.JaxprDebugInfo | Noner  N)rP   rQ   rR   r.  r  rS   rT   rU   r  r    s0         <<&&&&((((((rT   r  r}   r@   c           
         t          | t                    r| S t          | j        |                     |          | j        t          | dd                     S )N_internal_device_list)r  _device_list)r   r@   r  r  r  r  )r}   rz  s     rU   r  r    s^    =!! H	q+Q-C-CD-I-I$%M%,Q0G%N%N
P 
P 
P PrT   c                D   t          d | j        D                       rt          |           \  } }}g R t          ft	          |j                  z  z   g R dt	          |j                  z  z   g |R dt	          |j                  z  z   }t          t          ||                    t          fd|j	        D                       \  }}t          d           J n1d x}}t          d | j        D                       rt          |           } | |||||fS )Nc              3  @   K   | ]}t          |t                    V  d S rX   )r   rL   re   es     rU   rg   z(_discharge_refs_jaxpr.<locals>.<genexpr>'  s,      @@aAy	!	!@@@@@@rT   rX   r\  c              3  Z   K   | ]%}|t                    n|         |         fV  &d S rX   )rl  )re   rv   r  r  out_layouts_s     rU   rg   z(_discharge_refs_jaxpr.<locals>.<genexpr>-  sU       ( (  i\l1oz!}-M( ( ( ( ( (rT   c              3  J   K   | ]}t          |t          j                  V  d S rX   )r   r   InternalMutableArrayEffectr  s     rU   rg   z(_discharge_refs_jaxpr.<locals>.<genexpr>3  s/      
X
Xa:a899
X
X
X
X
X
XrT   )r   r   ru  r8   rh   r  iterr  rH   r'  rl  r  )	r  r  r  r   r  r  rt  r  r  s	    ``     @rU   _discharge_refs_jaxprr  %  se   @@<+?@@@@@ <'6|'D'D$L-#\##{ns3:&FFL:'C
OO";;J'~''(S__*DDNM;7788L!' ( ( ( ( ( (( ( ( " "M; d##++++MC

X
X<CW
X
X
XXX <-l;;l
sL*
-
6 6rT   r5  r  r  Sequence[MaybeSharding]r  r  MaybeLayoutr  rb  devices_from_contextr  r  MeshComputationc                 + t          t          j                            ||g                    }t	          | j        | j        j                  }t          | |||||          \  } }+}t          +fdt          |          D                       }t          +fdt          |          D                       }t          | |||||          \  } }}}}}}}| j        }| j        }| j        }t          |           }t          d t          ||          D                       }t          |          t          |          cxk    rt          |          k    s6n J t          |          t          |          t          |          f            t!          t#          j        t'          j        |                              }t+          t          j        d t#          j        |          D             d t#          j        |          D             d |D                       |	          \  }}|
p|j        f}t/          |	p]t          |          dk    pJt1          d |D                       p1t1          d	 |D                       pt1          d
 |D                                 }t3          t          |                    }t5          |t          j        ||d |D                                 }|st7          | |          }ndt          |          z  }t9          ||          }t9          ||          }t'          j        |          } d}!t<          j        j         rKt          j        ||d |D                       D ]*}"tC          |"tD          j#                  r|"j$        j%        }! n+tM          | |||||||t          |          | rt          |          nd||||||||!          \  }#}$}%}&}'}(})}*tO          tQ          |          |#||fi d|d|d|d|ddd|)d|d|&d|'d|%d|$d+d|d|d|d |d!|d"|d#|(d$|*d%|d&|d'|d(d) |D             S )*aL  Lowers a computation to XLA. It can take arbitrary shardings as input.

  The caller of this code can pass in a singleton UNSPECIFIED because the
  number of out_avals might not be known at that time and
  lower_sharding_computation calculates the number of out_avals so it can apply
  the singleton UNSPECIFIED to all out_avals.
  c              3  *   K   | ]\  }}|v 	|V  d S rX   rS   )re   rv   r}   r  s      rU   rg   z-lower_sharding_computation.<locals>.<genexpr>[  s1      RRTQ\@Q@Qq@Q@Q@Q@QRRrT   c              3  *   K   | ]\  }}|v 	|V  d S rX   rS   )re   rv   r;  r  s      rU   rg   z-lower_sharding_computation.<locals>.<genexpr>\  s1      NN41aA<M<MQ<M<M<M<MNNrT   c              3  <   K   | ]\  }}|t           j        n|V  d S rX   )r,   r7   )re   r-  ps      rU   rg   z-lower_sharding_computation.<locals>.<genexpr>k  sO       O O!Q 12',,1 O O O O O OrT   c              3  4   K   | ]}|t           j        d fV  d S rX   )r  r$  re   rv   s     rU   rg   z-lower_sharding_computation.<locals>.<genexpr>w  s,      
Z
ZAA|($/
Z
Z
Z
Z
Z
ZrT   c              3  4   K   | ]}|t           j        d fV  d S rX   )r  r"  re   r-  s     rU   rg   z-lower_sharding_computation.<locals>.<genexpr>x  s,      
[
[AA|($/
[
[
[
[
[
[rT   c              3  :   K   | ]\  }}|t           j        |fV  d S rX   )r  r%  )re   jsrO  s      rU   rg   z-lower_sharding_computation.<locals>.<genexpr>y  sL       A Ar; 8+F A A A A A ArT   r`   c              3  6   K   | ]}t          |           V  d S rX   r<   r  s     rU   rg   z-lower_sharding_computation.<locals>.<genexpr>  s-      	6	6AnQ
	6	6	6	6	6	6rT   c              3  <   K   | ]\  }}t          |           V  d S rX   r  re   r  r  s      rU   rg   z-lower_sharding_computation.<locals>.<genexpr>  s1      	L	LURnR  
 	L	L	L	L	L	LrT   c              3  6   K   | ]}t          |           V  d S rX   r  r  s     rU   rg   z-lower_sharding_computation.<locals>.<genexpr>  s-      	7	7AnQ
	7	7	7	7	7	7rT   c                    g | ]\  }}|S rS   rS   r  s      rU   r   z.lower_sharding_computation.<locals>.<listcomp>  s    >>>ur1>>>rT   rX   Nc                    g | ]\  }}|S rS   rS   r  s      rU   r   z.lower_sharding_computation.<locals>.<listcomp>  s    777A777rT   )r  r  r  r  r  r  spmd_loweringTr8  rc  r9  r,  r;  r:  r  r  r   r  r   r  r  
pmap_nrepsr<  r  all_args_infor  intermediate_shardingsc                    g | ]\  }}|S rS   rS   )re   r}   r  s      rU   r   z.lower_sharding_computation.<locals>.<listcomp>  s    JJJDAqaJJJrT   ))r  rq  chainfrom_iterabler  r  r   r  rd  r   rn   r  r  r  rB   rh   r   r"   stable_uniquer   get_intermediate_shardingsrO  r$  r   r   r  r  r  r   jaxpr_has_prim_requiring_devicesr   use_shardy_partitionerr   r   r   r  r   shape_tupler  r
  r   ),r  r5  r  r  r  r  r  r   rb  r	  r  r  r  rc  r  r0  rt  r  r   r  r  out_layouts_via_propunique_intermediate_shardingsr   r  r.  r   r  r  r  r  r  prim_requires_devicesr  rw   rR  r:  r;  r9  r,  r|  r8  r<  r  s,                                              @rU   lower_sharding_computationr,  :  sH   2 )hlM:;;= = l3\5G5RSS-;EHh^< <8,j RRRRY|%<%<RRRRR,NNNN9Z#8#8NNNNN* 2G\:~}2 2/<\:=+ 
% )/!+
 9FF O O"*;8L"M"MO O O O O+ 
]		s;//	H	H	H	H37G3H3H	H	H	H	H	H	-#k**C0@,A,AKC	H	H	H
 #'t'9)%00(2 (2 #3 #3?h
Z
Z9KL9Y9Y
Z
Z
Z
[
[9KM9Z9Z
[
[
[A A"?A A AB B
    ' !7W%5$7)  8	
q 8		6	6	6	6	6668 
	L	L.K	L	L	LLL8 
	7	7	7	7	7779 9)  &7 8 899);h|]>> =>>>@ @A A 
 ?Cl $  $  '-=)>)>> 5O% %5%' '"CEJJ"( Hm7767779 9   
Hn:	;	; #=4
 +BXx2Gz;I0:uYdN'.(+* +* +*(69n&7*& 
	*oo	
L 
L 
L
 &o
L ('
L  <
L "M
L D
L 
L ,+
L *)
L &o
L $^
L 	
L   <!
L" ##
L$ g%
L& "	'
L( 	)
L* +
L, +-
L. /
L0 ('1
L2 0/3
L4 "M5
L6 "M7
L8 KJ,IJJJJ9
L LrT   MaybeSharding | AUTOJSharding | Nonec                   t          |          st          |          rd S t          | t          t          t
          f          rt          |t                    sJ |S t          | t          j                  rd S t          |           rX   )
r<   r;   r   r&   r%   rK   r   r   AbstractTokenr   )r   rw   s     rU   r  r    s     H !2!2 4$lK@AA h	*****O$*++ 4
D//rT   c                  F    e Zd ZU ded<   ded<   ddZddZdddZddZdS )r
  rc  rd  zMeshExecutable | Nonere  r   r   r2   r   r   r.  r  c                Z    || _         || _        || _        || _        || _        d | _        d S rX   )_namerd  _donated_invars
_platformsrg  re  )rU  r   r2   r   r.  rg  s         rU   rT  zMeshComputation.__init__  s7     DJDI)DDO$DDrT   r]   c                    | j         S rX   ri  r  s    rU   rj  zMeshComputation.stablehlo  rk  rT   NMeshExecutablec                    | j         |1t          j        | j        | j        fi | j        d|i}||| _         |S | j         S rn  )re  UnloadedMeshExecutablerq  r3  rd  rg  rr  s      rU   r  zMeshComputation.compile  sm    #3#?)2
*di- -#'#4- -+- - -j 
	!%rT   dict[str, float]c                    | j         d         }t          j        |          rt          d|j         d          t
                              ||                                                                           S )Nr   z3Lowered.cost_analysis not implemented on platform 'zE'. Use compile().cost_analysis() for post-compilation cost estimates.)	rg  r"  using_pjrt_c_apir  r$  xehlo_module_cost_analysisr2   as_hlo_module)rU  r   s     rU   cost_analysiszMeshComputation.cost_analysis  sz    	*G	7## .-- - -. . . &&w

0H0H0J0JKKKrT   )r   r   r2   rc  r   r   r.  r  rt  rX   r]   r7  )r]   r:  )rP   rQ   rR   r  rT  rj  r  r@  rS   rT   rU   r
  r
    s         ///$$$$          L L L L L LrT   r(  num_out_avalsnum_ordered_effectsr  -Sequence[sharding_impls.GSPMDSharding] | Nonec                   ddl m} t          j        j        rB|rd g|z  }n?	 |                                 d         }|dk    r
||d          }n#  d g|z  }Y n	xY wd g|z  }t          |          |k    sJ t          |          |f            t                    dk    rfd|D             S |                    |           \  }}|sd S |dk    r
||d          }|dk    rt          |          dk    rd S t          |          dk    rt          |          |k     r||z  }t          |          |cxk    rt          |          k    s)n J t          |          |t          |          f            fdt          ||          D             S )Nr   r  r`   c                R    g | ]#}t           j                            |           $S r  )r   r@   get_replicated)re   mkr  s     rU   r   z5get_out_shardings_from_executable.<locals>.<listcomp>$	  sA        (778IWY7ZZ   rT   c                D    g | ]\  }}t          j        ||           S rG  )r   r@   )re   osrJ  r  s      rU   r   z5get_out_shardings_from_executable.<locals>.<listcomp>=	  s@     
9 
9 
9b" 
&'8""
M
M
M 
9 
9 
9rT   )	r   r  r   enable_memoriesr   get_output_memory_kindsrh   get_op_sharding_from_executablerB   )	r  r  rB  rC  r  r  omkr  out_op_shardingss	    `       rU   !get_out_shardings_from_executablerR  		  s,    ! ! %F]"cc%4466q9""'(()#%f}$&=
 C	S]	"	"	"SXX}$=	"	"	" 		q         <<^LL!	 41'(;(<(<= aC 011Q664 		aC(8$9$9M$I$I'-7			-	;	;	;	;3s88	;	;	;	;	;	
]CHH>6	;	;	;
9 
9 
9 
9 !1377
9 
9 
9 9s   *A Anum_in_avalsSequence[GSPMDSharding] | Nonec                2   ddl m} t                    dk    rt          j                  g|z  S |                    |           \  }}|sdS |dk    r
||d         }t          |          |k    sJ t          |          |f            fd|D             S )z!Returns input shardings from XLA.r   r  r`   Nc                0    g | ]}t          |          S rS   )r@   )re   rL  r  s     rU   r   z._get_in_shardings_from_xla.<locals>.<listcomp>W	  s4     
% 
% 
% )2
.
. 
% 
% 
%rT   )r   r  rh   r@   rI  rO  )r  r  rS  rC  r  in_op_shardingsr  s    `     rU   _get_in_shardings_from_xlarX  A	  s    
  		q  ():;;<|KK;;NKK/1	 41%&9&:&:;O	_			-	-	-	/L0*	-	-	-
% 
% 
% 
%#
% 
% 
% %rT   Utuple[Sequence[sharding_impls.NamedSharding], Sequence[sharding_impls.NamedSharding]]c                z    ddl m} |                    |           \  }}fd|D             fd|D             fS )Nr   r  c                :    g | ]}t          j        |          S rS   r   r  )re   rv   r   s     rU   r   z=_get_mesh_pspec_shardings_from_executable.<locals>.<listcomp>d	  s&    
C
C
CQ>'a00
C
C
CrT   c                :    g | ]}t          j        |          S rS   r\  )re   r-  r   s     rU   r   z=_get_mesh_pspec_shardings_from_executable.<locals>.<listcomp>e	  s&    
D
D
DQ>'a00
D
D
DrT   )r   r  get_pspec_from_executable)r  r   r  in_pspec	out_pspecs    `   rU   )_get_mesh_pspec_shardings_from_executablera  ]	  sk     66~tLL(I
C
C
C
C(
C
C
C
D
D
D
D)
D
D
D
F FrT   
_ShardingT)boundsharding_clstype[_ShardingT]r`  @Callable[[sharding_impls.GSPMDSharding, _ShardingT], _ShardingT]c                    |t           | <   d S rX   )_orig_out_sharding_handlers)rd  r`  s     rU   _register_out_sharding_handlerri  m	  s     /6l+++rT   out_ssharding_impls.GSPMDSharding	orig_in_ssharding_impls.NamedShardingc                6    t          j        | |j                  S rX   )r   !_gspmd_to_named_sharding_via_meshr   rj  rl  s     rU   _gspmd_to_named_shardingrq  s	  s     
	9%	P	PPrT   !sharding_impls.PositionalShardingc                L    t          j        | j        |j        | j                  S rX   )r   _op_sharding_to_pos_shardingr  r  r  rp  s     rU   _gspmd_to_positional_shardingru  |	  s,     
	4979J
L 
L LrT   r?   c                r    t          |t                    sJ t          | j        d         | j                  S )Nr   rH  )r   r?   r  r  rp  s     rU    _gspmd_to_single_device_shardingrw  	  sD    	I3	4	4444	q!u/@
B 
B 
B BrT   c                6   g }t           t          |                   }t          | |          D ]\  }}t          |t          j                  r|t          j        ur|j|h|j        |j        k    rXt	          j	        |j
        |                    |j                            r&|j        |j        k    r|                    |           	 |                     |||                     #  |                    |           Y xY w|                    |           |S rX   )rh  rk   rB   r   r   r@   r   r  rz  r  r  r  r  ro   )r  r  rl  	orig_avalr   orig_handlerr-  out_avals           rU   $_get_out_sharding_from_orig_shardingr|  	  s    
#,T)__=,mY77  ka1n233 +++
 
H$8
-9>
)
)3oy==inMMO O * my444

9	
**\\!Y//
0
0
0
0	
**Q-----	jjmmmm	*s   C''D c                &   t          d |D                       r|S t          | |          D ]1\  }}|*t          |          t          v rt	          ||||          c S 2|1|D ].}|*t          |          t          v rt	          |||d           c S /|S )Nc              3  L   K   | ]}t          |t          j                   V   d S rX   )r   r   r@   r  s     rU   rg   z/maybe_recover_user_shardings.<locals>.<genexpr>	  s2      PPQZ>788	8PPPPPPrT   )rq   rB   rk   rh  r|  )old_shardingsnew_shardings	old_avals	new_avalsr   oio_avalrv   s           rU   maybe_recover_user_shardingsr  	  s     	PP-PPPPP ]I66 0 0jb&	~$r((&AAA1
B0 0 0 0 0 '# / /	
477&AAA39a/ / 	/ 	/ 	/ 
rT   ulDeviceLocalLayout | AutoLayoutxlr,   c                    t           dk    r2t          | t                    r| j        | j        |j        k    S | |k    S | |k    S )Ni  )r0   r   r,   _tilingmajor_to_minor)r  r  s     rU   is_user_xla_layout_equalr  	  sJ    c!!"'(( RZ-?""3332Xo8OrT   whatc           	     X    t          | |          st          d| d|  d| d          d S )Nz&Unexpected XLA layout override: (XLA)  != z (User z layout))r  AssertionError)r  r  r  s      rU   _check_user_xla_layoutr  	  s_    	!"b	)	) !
	  	  	  	  	 	  	  	 ! ! !! !rT   Mtuple[Sequence[DeviceLocalLayout | None], Sequence[DeviceLocalLayout | None]]c                   	 |                                  }|                                 }n+#  dt          |          z  dt          |          z  fcY S xY w|dk    r||d          }||d          }g }t          ||          D ]T\  }}t	          j        |          }t          |t                    rt          ||d           |                    |           Ug }	t          ||          D ]T\  }}
t	          j        |          }t          |
t                    rt          |
|d           |	                    |           Ut          d |D                       sJ t          d |	D                       sJ ||	fS )NrX   r   inputoutputc              3  @   K   | ]}t          |t                    V  d S rX   r   r,   r  s     rU   rg   z/_get_layouts_from_executable.<locals>.<genexpr>	  s-      FF!Z,--FFFFFFrT   c              3  @   K   | ]}t          |t                    V  d S rX   r  r  s     rU   rg   z/_get_layouts_from_executable.<locals>.<genexpr>	  s-      GG!Z,--GGGGGGrT   )
get_parameter_layoutsget_output_layoutsrh   rB   r,   r  r   r  ro   rq   )r  r  r  rC  in_layouts_xlaout_layouts_xlanew_in_layoutsrY   rv   new_out_layoutsr-  s              rU   _get_layouts_from_executabler  	  s   A#99;;N$7799OOAS__$gK0@0@&@@@@@1#$7$8$89N%&9&:&:;O.~z22  da*1--A!&'' ,Q7+++ !/44  da*1--A!&'' -Q8,,, 1	FF~FFF	F	FFFF	GGGGG	G	GGGG		((s	   (+ &Ac                t    t          j        t          j        |                                         |           S rX   )r   r(  r}  r~  r  )r  s    rU   get_logical_mesh_idsr  	  s*    	49Z((	)	)	1	1*	=	==rT   c                   |	dk    r|	d}}n|r
d|j         }}n	|j         d}}|	dk    rd }n|                    ||f          }|
d n|
                    dd           }t          j        ||||t
          j        j        ||
|t          j        |           |
  
        }|j	        }|rz|J t          |j                                                  |_        t          t          |j                                                                                d          |_        ||_        t          |          |_        t          |          |_        |S )Nr`   fdo_profile)
r7  r  r  r  r'  use_auto_spmd_partitioningr  r  r  r   r  )r   r  r  r   r  r   r'  r   r  executable_build_optionsr   r   r   !auto_spmd_partitioning_mesh_shaper  auto_spmd_partitioning_mesh_idsr  -allow_spmd_sharding_propagation_to_parameters)allow_spmd_sharding_propagation_to_output)computationr   r  r8  rc  allow_prop_to_inputsallow_prop_to_outputsr   np_devr  ro  r7  r  xla_device_assignmentr  r  optss                    rU   create_compile_optionsr  	  so    !^^#-q.LL 2#$fk.LL#);.L!^^ !"NNL.+IJJ)1!%%mT::  0#-)#:@!3,4[AA  / 
	1$ -1$*2C2C2E2E-F-FD*T$*"3"3"5"56677	 	( 3=//7;<P7Q7Q$4378M3N3N$0	rT   c                  
  t          j        
fdt          g          t          j        t	          
                              }|d }nt          t          ||                    }t          | |||||||	|||          }t          j	        d|t          j
                  5  t          j        |	| ||||          }d d d            n# 1 swxY w Y   |S )Nc                    |          S rX   rS   )rv   r)  s    rU   rb   z%_cached_compilation.<locals>.<lambda>.
  s    r!u rT   )otypesr  r  )r   	vectorizeobjectr(  rh   r  rB   r  r   r  r  r   r  )r  r   r   r  r8  rc  r  r  r;  r   r)  r  compiler_options_keyscompiler_options_valuesr  devro  r  r  s             `        rU   _cached_compilationr  %
  s3    	7____fX666ryR7I7IJJ#"H%:<STTUU*4
4F17	:') )/
  H89; ; ;   3c?N N               
s   %CCCc                   t          | |t          |          |          }||S g }t          |||          D ]\  }}}	t          |          r]|	t          j        ur9t          j        |	j        t          j	                  rt          j        |	|          }|                    |           s|                    |	j                  }
|                    |	j                  }t          j        |	j        t          j	                  s?|
                                s+t!          j        |
|          st%          d| d| d          |                    |           !t'          ||||          }|S )a_  Returns in_shardings extracted from XLA or checks and returns original
  shardings.

  If in_shardings exist on `jit` or on `jax.Array`, then this function will
  check that sharding against what XLA returns as in_shardings. If they don't
  match, an error is raised.

  If in_sharding is unspecified, then the sharding returned by XLA is returned.
  N(Unexpected XLA sharding override: (XLA) r   (User sharding))rX  rh   rB   r<   r   r  r   r  r   r  r   logical_shardingro   r  rz  	is_manualr   r  r  r  )r  r  r  r  rC  in_shardings_xlanew_in_shardingsxla_sorigr   	xla_hlo_s
orig_hlo_ss               rU   !_maybe_get_and_check_in_shardingsr  C
  s    0'_)=)=  #$4l$35 5 $ $eT4d $
d)
)
)

DJ
8
8 */e<<e$$$$,,TY77i,,TY77j
FO<< !!##29jII u  $     	 d####1$oH H 
rT   c                   t          | |t          |          ||          }||S g }t          |||          D ]/\  }}	}
t          |	          r]|
t          j        ur9t          j        |
j        t          j	                  rt          j        |
|          }|                    |           s|                    |
j                  }|	                    |
j                  }t          j        |
j        t          j	                  sO|                                s;t!          j        ||          r|j        |	j        k    rt'          d| d|	 d          |                    |	           1|S )Nr  r  r  )rR  rh   rB   r<   r   r  r   r  r   r  r   r  ro   r  rz  r  r   r  r  r  )r  r  r  r  rC  r  out_shardings_xlanew_out_shardingsr  r  r   r  r  s                rU   "_maybe_get_and_check_out_shardingsr  o
  s    8'-=)>)>/1 1 #$5}$46 6 % %eT4d %
d)
)
)

DJ
8
8 */e<<u%%%%,,TY77i,,TY77j
FO<< !!##29jII  000u  $     	 t$$$$	rT   c                J    t                    dk    rfd| D             S | S )Nr`   c                v    g | ]5}t          |t                    rt          d          |j                  n|6S )r   rH  )r   r@   r?   r  )re   r-  r  s     rU   r   z*finalize_out_shardings.<locals>.<listcomp>
  sY     K K K78z!]++3 !21!51=QQQQ12K K KrT   )rh   )r  r  s    `rU   finalize_out_shardingsr  
  sP    	q  K K K K<IK K K K	rT   c                     e Zd ZU ded<   ded<   ded<   ded<   d	ed
<   ded<   d	ed<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded <   d!ed"<   d# Zd>d&Ze	 	 	 	 	 	 	 	 d?d@d=            Zd(S )Ar9  r   r  #xc.DeviceList | Sequence[xc.Device]r  r|  r   r  input_avalsr\   r~  output_avalsr  r   r   r   r   r  r9  r,  r   r:  r;  r  r  r  r  rc  z"Sequence[DeviceLocalLayout | None]r  r  AllArgsInfo | Noner  r  r  c                    t          | j                  }t          | j        | j        | j                  }t          | j        | j        | j	        ||| j
        | j        | j        t          | j                  | j        | j        | j                  }|S rX   )r  r~  r  r  r  r   r  r  r   r   r9  r,  r:  r   r;  r  r  r  )rU  r  r  r  s       rU   r  z(UnloadedMeshExecutable.build_unsafe_call
  s     455K140$.B BK $TYkT+T-A4>T !!4#4dh	 K
 rT   r]   r7  c                    t          | j        | j        | j        | j        | j        | j        | j        | j        | j	        | j
        | j        |           S rX   )r7  r  r  r  r  r~  r  rc  r  r  r  r  r  s    rU   r  zUnloadedMeshExecutable.load
  sQ    $-t/E*D,=.0E143D/4+;,d4 4 4rT   r`   NTr2   rc  r  r  r  Sequence[JSharding | AUTO]r  -Sequence[JSharding | AUTO | UnspecifiedValue]r  r8  r  r  r  r   r<  r  r  r   Sequence[JSharding] | Nonec                   ||j         rt          j        |          }|!t          |                                          nd }|!t          |                                          nd }t          |t          j                  r|}nt          t          |                    }~t          d |D                       }t          d |D                       }d } |r<t          j                            ||g          D ]}!t          |!          r	|!j        }  nt          || | |||||t          |          ||||||          }"|rL| J t!          |"|           \  }#}$d t#          |#|          D             }d t#          |$|          D             }n|dk    r`| J t%          |"|t          |          |t'          |
                    }t)          |"|t          |          |t'          |
          |          }nBt+          |"                                t'          |          t'          |                    \  }}}}t/          |"||t'          |
                    \  }}t1          |||||          }t3          ||          }t5          di d|"d|d|d	|d
|d|d|d|d| d|	d|
d|d|d|d|d|d|d|d|d|                                S )Nc              3  R   K   | ]"}t          |          pt          |          V  #d S rX   r<   r;   r  s     rU   rg   z2UnloadedMeshExecutable.from_hlo.<locals>.<genexpr>
  sI       !8 !8%& "0!2!2!@gajj !8 !8 !8 !8 !8 !8rT   c              3  R   K   | ]"}t          |          pt          |          V  #d S rX   r  r  s     rU   rg   z2UnloadedMeshExecutable.from_hlo.<locals>.<genexpr>
  sI       ": ":&' #1"3"3"Awqzz ": ": ": ": ": ":rT   c                :    g | ]\  }}t          |          r|n|S rS   r  )re   rY   rv   s      rU   r   z3UnloadedMeshExecutable.from_hlo.<locals>.<listcomp>  sA     L L L!Q #1::,aa1 L L LrT   c                :    g | ]\  }}t          |          r|n|S rS   r  )re   rY   r-  s      rU   r   z3UnloadedMeshExecutable.from_hlo.<locals>.<listcomp>  sA     O O O1a $AJJ-qqA O O OrT   r`   r  r  r   r  r~  r  r  r   r   r9  r,  r:  r;  r  r  rc  r  r  r  r  rS   )r  r*   r  r   keysr   r   r   r  r  rq  r"  r#  r;   r   r  ra  rB   r  rh   r  _get_metadata_jit_pmapr  r  r  r  r9  r  )%r   r2   r  r  r  r  r  r8  rc  r9  r,  r;  r:  r  r   r  r   r  r  r  r  r<  r  r  ro  r  r   r  r  r)  r  r  r   rv   r  r  r  s%                                        rU   rq  zUnloadedMeshExecutable.from_hlo
  sM   : #(8(F#*3//c$4$@ "! ! !FJ  '7&B $!!# # #HL #R]33 7bbU#45566b  !8 !8*6!8 !8 !8 8 8! ": ":+8": ": ": : : D x%%|]&CDD  !1:: 	$
%	 )T4&(<u^44gr:6	G GN  S,U
$-  - ))L L"*+;\"J"JL L LlO O#+,=}#M#MO O Omm 
q|||8L%))_  " " ;M5996F  "68 8 6L((**C,=,=s=?Q?Q6S 6S2mY ;
K_1E1EG GJ 1m_6F   M +="==M! % % %%~%"% % $O	%
 %% &%% '% )% T% ,+% (% )% &~% "\% C%  .-!%" :#%$  K%%& $m'%( $m)%( &*TVV),rT   rA  )r`   NNTNNNN)6r   r   r2   rc  r  r  r  r  r  r  r  r  r  r   r8  r   rc  r   r9  r  r,  r  r;  r  r:  r   r  r  r   r|  r  r  r   r   r  r  r  r  r  r   r  r  r<  r  r  r   r  r  r  r  r   r  r]   r7  )rP   rQ   rR   r  r  r  r  rq  rS   rT   rU   r9  r9  
  si        8888$$$$&&&&%%%%''''///)))&&&&$$$$00001111####----
 
 
4 4 4 4 ( "#*.FJ,037 $=ADH5l, l, l, l, <l, l, l,rT   r9  c                  j    e Zd ZU ded<   ded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   ded<   dS )MeshExecutableFastpathDatazxc.LoadedExecutabler  r   out_pytree_defr\   r  r  r  r  r   out_committedzIterable[bool]kept_var_bitvecr(  arg_handler_devicesz"Sequence[tuple[Index | None, ...]]arg_handler_indicesNr  rS   rT   rU   r  r  2  s~         %%%%####$$$$""""!!!!****999999rT   r  c                l    t          j        | |          }t           j                            |d           S rX   )r!   tree_unflattendispatch_registryflatten)out_treeout_flat
out_unflats      rU   reflatten_outputs_for_dispatchr  ?  s/     '(;;*		$	,	,Z	>	>>rT   c                  j    e Zd Zg dZ	 	 dddZedd            Zd	 Zd
 ZddZ	ddZ
d Zd Zd ZdS )r7  )r  r  r  r  r  _in_shardings_out_shardings_auto_spmd_lowering_kept_var_idx_in_layouts_out_layouts_all_args_infor  Nr  r  c                    || _         || _        || _        || _        d | _        || _        || _        || _        || _        |	| _	        |
| _
        || _        || _        d S rX   )r  r  r  r  r  r  r  r  r  r  r  r  r  )rU  r  r  r  r  r  r  rc  r  r  r  r  r  s                rU   rT  zMeshExecutable.__init__O  sr    
 )D.D DMDND%D'D1D%D!D#D'D 3DrT   r]   r  c                P    | j         |                                 | _         | j         S rX   r  r  s    rU   r  zMeshExecutable.unsafe_calld  r  rT   c                    | j         S rX   r  r  s    rU   r  z'MeshExecutable.xla_extension_executablel  r  rT   c                B     fdt          |          D             } j        |} j        }d }n|} j        j        } j        j        }t	          t
          j        |          }t          |||           t          | j	         j
        | j                     j        | S )Nc                ,    g | ]\  }}|j         v |S rS   r  )re   rv   r|   rU  s      rU   r   z'MeshExecutable.call.<locals>.<listcomp>p  s*    OOODAqqD<N7N7Na7N7N7NrT   )rn   r  r  r  r   r+   r   r  %check_array_xla_sharding_layout_matchr  r  r  r  )rU  rr   args_after_dce	kept_args	ref_avalsr  all_arg_avalss   `      rU   r  zMeshExecutable.callo  s    OOOOIdOOOOON" i-ijji%.i&1j33MYzBBB)*D,<j   4T""rT   r\   c                    | j         S rX   )r  r  s    rU   r~  zMeshExecutable.input_shardings  s    rT   c                    | j         S rX   )r  r  s    rU   r  zMeshExecutable.output_shardings  r  rT   c                J    d t          | j        | j                  D             S )Nc                4    g | ]\  }}t          ||          S rS   r.   re   r;  r}   s      rU   r   z0MeshExecutable.input_layouts.<locals>.<listcomp>  s:     H H H1 1aLL H H HrT   )rB   r  r  r  s    rU   input_layoutszMeshExecutable.input_layouts  s9    H H !143EFFH H H HrT   c                J    d t          | j        | j                  D             S )Nc                4    g | ]\  }}t          ||          S rS   r  r  s      rU   r   z1MeshExecutable.output_layouts.<locals>.<listcomp>  s:     J J J1 1aLL J J JrT   )rB   r  r  r  s    rU   output_layoutszMeshExecutable.output_layouts  s9    J J !2D4GHHJ J J JrT   c           
          t           j        t                    r j        j        s j        j        rd S  fd}t
          j                             j        j        d |g g g t          j
        d           S )Nc                     t          j                  }t          j        j        |g| R i |\  }}}t	          |          \  }}t          d |D                       }|rd |D             }d |D             }	fdt          t          |                    D             }
d t          j	        j
                  D             }t          j        ||j        ||	|
j        j        j        j        j        j        	  	        }nd }||dfS )Nc              3  J   K   | ]}t          |t          j                  V  d S rX   )r   r   r   r   s     rU   rg   zIMeshExecutable.create_cpp_call.<locals>.aot_cache_miss.<locals>.<genexpr>  s.      HH!*Q55HHHHHHrT   c                    g | ]	}|j         
S rS   rx  r  s     rU   r   zJMeshExecutable.create_cpp_call.<locals>.aot_cache_miss.<locals>.<listcomp>  s    ...QV...rT   c                    g | ]	}|j         
S rS   )
_committedr  s     rU   r   zJMeshExecutable.create_cpp_call.<locals>.aot_cache_miss.<locals>.<listcomp>  s    888!888rT   c                "    g | ]}|j         v S rS   r  )re   rv   rU  s     rU   r   zJMeshExecutable.create_cpp_call.<locals>.aot_cache_miss.<locals>.<listcomp>  s1     ; ; ;   22 ; ; ;rT   c                    g | ]N\  }}|t           j        ur9t          j        |j        t          j                  rt          j        ||          n|OS rS   )r   r  r   r  r   r  r   physical_shardingr  s      rU   r   zJMeshExecutable.create_cpp_call.<locals>.aot_cache_miss.<locals>.<listcomp>  sg     
 
 
 1 +++0A!'6?0[0[+ ,Q222
 
 
rT   F)r    CompiledCallParamsCompiledr  r  rq   r  rh   r  r  r  r  r  r  r  r  r  r  )rr   rs  r   r~   r  	args_flatout_tree_dispatchuse_fastpathr  r  r  r  fastpath_datain_tree	no_kwargsr  rU  s                rU   aot_cache_missz6MeshExecutable.create_cpp_call.<locals>.aot_cache_miss  sa   (y'8LLf"(/"6v"O"O"O"O"O"OdHi$B
H% %!h!HHxHHHHHl	 ..X...	88x888; ; ; ;$)#i..$9$9; ; ;
 
 D.>>	
 
 
 3!2LM?'5'5	7 7 =%''rT   c                2    t          |g| g          d         S r  r   )rY   r}   s     rU   rb   z0MeshExecutable.create_cpp_call.<locals>.<lambda>  s    *aS1#2F2Fq2I rT   )r   r  r  r  r  r   _xlar  r   r!   r  )rU  r  r  r  r  s   ```` rU   create_cpp_callzMeshExecutable.create_cpp_call  s    t'):;;  6 3 T( ( ( ( ( ( ( (6 7<<t^RR#%I%IK K KrT   r^  )r  r  r  )r]   r\   )rP   rQ   rR   r  rT  r  r  r  r  r~  r  r  r
  r  rS   rT   rU   r7  r7  G  s          ) 48#'	4 4 4 4 4*    8  # # #&      H H HJ J J#K #K #K #K #KrT   r7  r  r  c           
        t          |           t          |          k    r/t          dt          |            dt          |                     |d |j        D             }n*t          |           fdt                    D             }g }t	          | ||          D ]]\  }}}t          j        ||          sB|                    d| d|                                 d|                                            ^|r_d}d		                    |d |                   }	t          |          |k    rd
| dt          |           }
nd}
t          d|
 d|	           d S )NzComputation compiled for z inputs but called with c                    g | ]}d | d 	S )'rS   )re   r   s     rU   r   z,check_arg_avals_for_call.<locals>.<listcomp>  s     DDDTDDDrT   c                $    g | ]}|d z    d S )r`   /rS   )re   rv   num_argss     rU   r   z,check_arg_avals_for_call.<locals>.<listcomp>  s+    @@@1AE&&H&&@@@rT   z	Argument z compiled with z and called with    
z
The first z of ThezNArgument types differ from the types for which this computation was compiled. z mismatches are:
)
rh   r   r5  r  rB   r   	typematchro   	str_shortr  )r  r  r  r5  errorsref_avalarg_avalr   max_num_errors
str_errorsnum_mismatch_strr%  s              @rU   r  r    s   ^^s9~~%%
	,C	NN 	, 	,y>>	, 	,- - - !DD)9)CDDDII9~~H@@@@h@@@I&"*9i"K"K * *h$>(H-- *mm)d ) )8+=+=+?+? ) )$$&&) )* * *  	GN6/>/233J
6{{n$$GnGG#f++GG
	F%	F 	F9C	F 	FG G G	G 	GrT   c                    t           j                            |           }|g|z  }|g|z  }d}|||t          |           fS )NT)r   r@   rI  r   )r  num_in_shardingsnum_out_shardingsgsr  r  r   s          rU   r  r    sR     #22=AA"((,$**- )	}i}1E1E	EErT   c                v    | D ]5}t          |          st          |          r!t          |dd          r dS 6dS )N_device_backendFT)r<   r;   r  )r[   rv   s     rU   !check_device_backend_on_shardingsr7    sW      aa GAJJ q#U++ TT	rT   in_xla_shardingsin_xla_layoutsSequence[DeviceLocalLayout]r  r  c                B   ddl m} |dgt          |           z  nfdt          |j                  D             }g }d}t          | |||          D ]\  }	}
}}t          |	|          st          |
          r)t          |
g          }|	j	        j
        |
j
        k    r?|                    d|	j	         d|
 d| d	|	j                                         d
f           |s|	j        rt          j        |	j	                            |	j                  |
                    |	j                            s?|                    d|	j	         d|
 d| d	|	j                                         d
f           |si|	j        rb|	j        j        V|T|	j        j        |k    rD|                    d|	j        j         d| d| d	|	j                                         df           |rt+          |d |                   \  }}d                    |          }t/          d |D                       rd}nt/          d |D                       rd}nd}t          |          |k     rdt          |           dn| dt          |           }t1          d| d| d| d|           d S )Nr   )r   r3  c                "    g | ]\  }}|v 	|S rS   rS   )re   rv   r|   r  s      rU   r   z9check_array_xla_sharding_layout_match.<locals>.<listcomp>  s1     " " "$!QL      rT   r&  z<Got input sharding(s) that compiled object was called with: z4 and sharding(s) the computation was compiled with: z	 for arg z with shape: rw   z:Got input layout(s) that compiled object was called with: z2 and layout(s) the computation was compiled with: r  r'  c              3  "   K   | ]
}|d k    V  dS )rw   NrS   re   r  s     rU   rg   z8check_array_xla_sharding_layout_match.<locals>.<genexpr>%  s&      
0
0q1
?
0
0
0
0
0
0rT   zsharding(s)c              3  "   K   | ]
}|d k    V  dS )r  NrS   r>  s     rU   rg   z8check_array_xla_sharding_layout_match.<locals>.<genexpr>'  s&      00qQ(]000000rT   z	layout(s)zsharding(s) and layout(s)zthe z mismatchesz mismatches out of z"Compiled object called with input z does not match the z- the computation was compiled with. Here are z:
)jax._src.arrayr   rh   rn   r5  rB   r   r=   r7  rw   r  ro   r   r*  r  r   r  r  rz  r  r  rH   r  rq   r   )r  r8  r9  r  r  r   r5  r+  
num_errorsrt   r   r  r   db_xsfirst_errorserror_kindsr/  kind_strr0  s       `              rU   r  r    s    '&&&&& %5$<rdS    " " " ")$4$>?? " " " 
 &*#&	C C # #c2r4c9%% b!! -rd33E |2>11mmJ\J JJ J $J J3683E3E3G3GJ J     cn /L--ch77##CH--/ / mmJ\J JJ J $J J3683E3E3G3GJ J     cn 
&2r~
&",,mm+Z++ ++ +)-+ + H&&((+ + 	    9 &vkzk': ; ;L+<((J

0
0K
0
0
000 -hh	00K000	0	0 -hh,h+.v;;+C+C's6{{''''77#f++77  8x 8 8#8 8 '8 8 ,68 89 9 99 9rT   pspecr3   r6   c                J    t          j        | d          }t          |          S )Nzpspec to array_mapping)r   prepare_axis_resources_get_array_mapping)rF  parsed_pspecs     rU   r:   r:   5  s(    6%' ',	L	)	))rT   c               /  f   K   t          j        | i |5  d V  d d d            d S # 1 swxY w Y   d S rX   )r   r  )rr   rs  s     rU   rK  rK  ;  s      T,V,, 
 
	EEE
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   &**)T)r[   r\   r]   r^   )r   r   )
r   r   rw   r   r   r   r   r   r   r   )r   r   )r   r   rw   r   r{   r   r]   r   )r   r   r   r   r]   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r]   r   )r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r4  r5  r]   r5  )
r   r   r=  r  r>  r?  r@  rA  r]   rB  )rz  r   r  r  r  r  r]   r  )r   r   r   r   r  r   r  r  r  r  r]   r  )
rz  r   r  r  r  r   r  r   r]   r  )r   r   r  r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r  r   r   r   r   r]   r  )r  r  r   r   r]   r  )r   r   r  r   r   r   r   r   r   r   r   r  r   r   r   r  r   r   r  r  )$r   r   r   r   r   r   r   r   r   r  r   r   r   r  r   r   r   r   r  r  r  r"  r  r#  r  r$  r   r  r  r  r   r  r  r  r]   r%  )
r   r   r   r   r   r  r   r&   r]   r&   )r   r\  r   r  r   r   r]   r   )r   r   r]   r&   )r  r  r  r\   r]   r  )r  r  r[   r\   r   r   r]   r  )r   rX  rY  r   rZ  r   r]   rX  )rh  r  r]   r  )r  r  )r  r  r   r   )r   r5   )r  r5   )r  r5   )r   r   r  r  r   r  )r  r5   r   r  r  r  )r  r  r  r5   r   r  r  r  )r[   r  r]   r   )r]   rC  )r[   rG  r   r  r]   rH  )r   r  r]   rP  )r   r$  r]   rg  )r  r$  r]   rv  )r   r$  r]   r$  )
r|  r   r   r  r   r   r   r  r]   r  )r]   r  )
rt  r  r  r  r.  r  r  r#  r  r  )r  r  r]   r  )r   r  r]   r  rX   )r  r$  r]   r  )r  r$  r]   r  )r}   r   rz  r   r]   r@   )r  r$  r5  r   r  r   r  r  r  r  r  r  r  r  r   r   rb  r   r	  r  r  r"  r  r#  r  r  r]   r
  )r   r   rw   r-  r]   r.  )
r  r(  rB  r   rC  r   r  r   r]   rD  )r  r(  rS  r   rC  r   r]   rT  )r   r  r]   rY  )rd  re  r`  rf  r]   r  )rj  rk  rl  rm  r]   rm  )rj  rk  rl  rr  r]   rr  )rj  r@   rl  r?   r]   r?   )r  r  r  r,   r]   r   )r  r   )r]   r  )r  r  )r]   r   )
r8  r\   r9  r:  r  r  r  r  r]   r  )rF  r3   r]   r6   (5  r.  
__future__r   enum
contextlibr   rl   r   collections.abcr   r   r   r	   dataclasses	functoolsr
   r   r   	itertoolsrq  r@  r}  typingr   r   r   r   r   r  numpyr   r%  r   r   r   r   r   r   r   r   r   lur   mesh_libr   r   r   r   r   r    r!   r"   r#   r"  jax._src.abstract_arraysr$   jax._src.corer%   r&   jax._src.interpretersr'   r(   r)   r  r*   r+   jax._src.layoutr,   r-   r.   jax._src.libr/   r   r0   jax._src.lib.mlirr1   jax._src.lib.mlir.dialectsr2   jax._src.partition_specr3   jax._src.shardingr4   r   jax._src.sharding_implsr5   r6   r7   r8   r9   r:   rI  r;   r<   r=   r>   r?   r@   jax._src.utilrA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   jax._src.state.typesrK   rL   r   rN   r  r=  r   r   	getLoggerrP   r>  r   r   r   Index
NoShardingChunked	UnstackedShardedAxis
ReplicatedAvalDimShardingr  MeshAxisNameMeshDimAssignmentShardingSpecrZ   rv  r   rj   r  r   r   maMaskedArrayr   _tr   DArrayr   ry  r   r   r   r   r   PxlaResultHandlerr   r   r   r   r   r  r   r<  rM  rN  Tracer  transformation_with_auxr  ry  r  r  Tracerr  cacher   	dataclassr  r  r  r  transformationr  r  r  r  rZ  AvalMapHandlerPairr[  ra  XlaLoweringr%  ry  rp  XlaExecutablerl  r  r  r  r  r  r  MapPrimitive
xla_pmap_prP  r/  def_implr@  rC  rW  r`  rh  rq  call_param_updaterscall_partial_eval_custom_rulepartial_eval_jaxpr_custom_rules	dce_rulescall_transpose_param_updatersmap_transposeprimitive_transposesry  axis_substitution_rulesr  r  r  r  r  r  r  r  register_loweringr  r  r  r  	Primitivefull_to_shard_pdef_abstract_evalr  r  r  shard_to_full_pr  r  r  Enumr  r'  	ExceptionrB  ShardingInforF  rO  MaybeShardingrZ  rd  rf  ru  rp  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r,  r  r
  rR  rX  ra  rh  rb  ri  rq  r  ru  r  rw  r|  r  r  r  r  r  r  r  r  r  r  r9  r  r  r7  r  r  create_mesh_pspec_shardingr7  r  rK  rS   rT   rU   <module>r     sw   8 7 7 " " " " " "  % % % % % %     " " " " " " B B B B B B B B B B B B     9 9 9 9 9 9 9 9 9 9       8 8 8 8 8 8 8 8 8 8 8 8 8 8      



                                           & & & & & & % % % % % % ! ! ! ! ! ! # # # # # #       # # # # # # % % % % % %                   % % % % % % 0 0 0 0 0 0 & & & & & & % % % % % % $ $ $ $ $ $ * * * * * * 4 4 4 4 4 4 & & & & & & % % % % % % A A A A A A A A A A ) ) ) ) ) ) . . . . . .             * * * * * * 1 1 1 1 1 1 3 3 3 3 3 3) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) )
H H H H H H H H H H H H H H H H H H H H H H H H 8 7 7 7 7 7 7 7    $    Wx
C		8	$	$c5%c5j 13 6778&


 $	(&
	7J67}*+z12 *   	    @ Z\  [ [ [ [ 45 5 5 5W W W )< 25$ %    ( (B'R6 6 6"/ 4; 5 5 5(< 4$ % IMM M M M MO O O O
 XZ  Y Y Y Y
7 
7 
7 
7 %: [ !C C C C2 S(C5#:"667 KM  M M M MC C C C. MO  N N N N% % % %N            z      1 1 1 1f+ + + +    $ 
?-?,HIIZG ZG ZG ZG ZGtz ZG ZG ZGx   G G G G       *E E E E3 3 3 3 3 3 3 30 Q Q Q 
Q8 d###! ! ! ! ! ! ! $#!4    
       *   N N N N 
 
 
 )  )  )  )F6 6 6 62 
L L L LD    3=) C12?     	M 	M 	M 	M    f(   2    `2 `2 `2 `2 `2 `2 `2 `2F# # # # #V) # # #DH H H4 4 4 4 4 4 4 4&F F F F F F F F	M 	M 	M 	M? ? ? ?g g g g g g g gT Tz**
? 
  M " " "- - -.S S S     ,< < < <9 9 9N N N &J z "GB,B8: : :  ": . *Z %? z "/P   ,&-gb.>
&K&K 
 #, , , ,< Z (? ? ?  &A A A, , , ,, , ,$   45 5 5 5 (,    @  z> 2 2 2) ) ) )) ) ) )I I I I> > > >
 !$.11"+ + #"+   8 		11O O O 21O !$.11"- - #"- 		11> > > 21>       49   " !? !? !? !? !? !? !? !?H    I    	)%t
+,	#t)> > > >#T #T #T #TJ i!1121 1 1 1 @ @ @.    :    A A A A 1 1 1 1 * * * *       8N N N N0 4( ( ( (8 [? [? [? [?| 4* * * *
2 2 2 2
 
 
 .0  / / / /6: .  .  .  .  .F . . . .: u.
D@AB) ) ) ) )* ) ) ) 4P P P P6 6 6* 
VL VL VL VLr   $L $L $L $L $Lf( $L $L $LN59 59 59 59p% % % %8F F F F ! W\333
6 6 6 6Q Q Q Q
   ":< < <L L L L  %'DF F FB B B B  :< < <  8     &   ! ! ! !!) !) !) !)H> > >, , ,^   :) ) )X  B   W, W, W, W, W, W, W, W,t
: 
: 
: 
: 
: 
: 
: 
:? ? ?lK lK lK lK lKV) lK lK lK` MQG G G G G>
F 
F 
F ,F    D9 D9 D9 D9N* * * * 
 
 
 
 
rT   