
    Vpf\                      U d dl mZ d dlmZmZmZ d dlZd dlmZ d dl	Z	d dl
Zd dlmZ d dlZd dlmZmZmZ d dlZd dlZd dlmZ d dlmZmZmZ d dlmZ d d	lmZ d d
lm Z  d dlm!Z! d dlm"Z" d dlm#Z# d dlm$Z$ d dlm%Z% d dlm&Z' d dlm(Z( d dlm)Z) d dlm*Z* d dlm+Z+ d dlm,Z, d dlm-Z- d dlm.Z. d dlm/Z/ d dl0m1Z1 d dl2m3Z3m4Z4 d dl5m6Z6m7Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@ d dlAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZK d dlLmMZMmNZNmOZO d dlPmQZQ d dlPmRZR d d lPmSZT d d!lPmUZU d d"lPmVZV d d#lWmXZXmYZYmZZZm[Z[m\Z\m]Z] d d$l^m_Z_m`Z`maZambZbmcZc d d%ldmeZemfZf eZge/jh        eicZiZje/jk        elcZlZm e.jn        eo           eZpeZqe.jr        d& es            fdd2            Ztdd4Zuevewexeqd5f         f         Zyd d:Zz ej{        d;d<d=g          Z|d!dBZ} G dC dD          Z~ e~            Zd"dMZd#dQZd$dUZd%dVZd&dXZd'd[Zd\ Z ed]          Zd(d`ZeZeee1f         Z G da dbe!j                  Z edc          Ze'j        d)dg            Zd*duZeeTj        _        d+dyZd,dzZd,d{Zd,d|Zee!j        e!j        <   d,d}Zee!j        e!j        <   eeeq         df         Zd~ Zee!j        e<   d-dZd.dZd/dZd0dZd Z eRj        ee           d Zd1dZd1dZd2dZd Zee!j        _        e'j        d             Zd3dZd4dZd Zd5dZ G d de          Zd Z G d de          Zd Zd6dZd Zd Zd Z G d de!j                  Z G d de!j1                  Zd Zi Zded<   d7dZeee#j        <   d Zeee$j        <    e!j        d          Zd&e_        eĠ                    e8j        j                   eĠ                    e8j        j                    eRj        eeRj        e8j                             ee8j        eĦ          eQj        e<    ee8j        ed           eQj        e<    ee8j        d          e!j        e<   d Z eVj        eeҦ           d Z e!j        d          Zd&e_        eՠ                    d            eՠ                    d             eRj        ed            d ZeeQj        e<   d ZeeQj        e<    ee8j        d          e!j        e<    eVj        ed            i Zded<   dÄ ZdĄ Zdń Zi Zded<   dǄ ZdȄ ZdɄ Zdʄ Zd˄ Zd̄ Zd̈́ Zd΄ Z ej        e6j                                        e9j                                        e:j                                        e>j                                        e;j                                        e<j                                        e=j                                        e(j                                        ej                                        e*j                                        e@j                                        e+j                                                  D ])Z eee!j                  r ee            ee           *e?j        j        e?j        j        e?j        j        e?j        j        e?j        j        fD ]Z ee            ee            ee8j                  dτ             Z ee8j                  dЄ             Z eeĦ          dф             Z eeĦ            eeզ          d҄             Z eeզ            ee8j                    ee8j                    ee8j                    ee8j                    ee8j                  dӄ             Z ee8j                    ee)j                  dԄ             Z ee)j                  dՄ             Z eej                  dք             Z  eej                  dׄ             Z ee!j                  d؄             Z ee#j                  dل             Z ee#j                    ee j                  dڄ             Z ee j                    ee j                  dۄ             Z ee j                    ee$j                  d܄             Z	 ee$j                    eeVj
                  d݄             Z eeVj
                    ee?j        j                  dބ             Z ee?j        j                  d߄             Z ee!j                  d             Z ee!j                  d             Z ee"j                  d             Z ee"j                  d             Z ee"j                  d             Z ee?j        j                  d             Z ee?j        j                   ee6j        =  ee6j                  d             Z ee6j                   d8dZeeQj        _        d ZeeQj        _        d Z d Z!e!eVj"        _        d Z#e#eVj"        _        d Z$e$eTj%        _        d Z&e&eTj%        _        e'j'        d             Z(d Z)d'dZ*d Z+e+eVj,        e<   d Z-e-e!j        e<   d9dZ.e.eTj/        e<   d Z0d  Z1d:dZ2d;dZ3e3eTj4        e<   d<d d dddddddd	Z5e'j6        d
             Z7e'j'        d             Z8d Z9d Z:eZ; G d de!j1                  Z< G d de!j                  Z=d Z>e'j        d             Z?e'j'        d             Z@d=dZAd>dZBe'j        d             ZCd ZDd ZEdS (?      )annotations)CallableHashableSequenceN)partial)prod)AnyTypeVarUnion)NamedShardingPartitionSpecMesh)ad_checkpoint)ad_util)callback)core)custom_derivatives)	debugging)dispatch)dtypes)linear_util)ops)pjit)prng)random)sharding_impls)source_info_util)traceback_util)util)Tracer)_shared_code_pmap_prepare_pmap)
laxparallelslicingwindowed_reductionsconvolutionfftlinalgspecialcontrol_flowann)
HashableFunctionHashablePartialunzip2unzip3as_hashable_functionmemoizepartition_listmerge_lists
split_list
subs_list2)flatten_fun_nokwargsshaped_abstractifyargnums_partial)batching)mlir)partial_eval)pxla)ad)tree_maptree_flattentree_unflattentree_structuretree_leaveskeystr)broadcast_prefixprefix_errors	PyTreeDefgenerate_key_pathsKeyPath) host_local_array_to_global_array global_array_to_host_local_arrayTfr   meshr   in_specsSpecs	out_specs	check_repboolautofrozenset[AxisName]c                *    t          | |||||          S )a  Map a function over shards of data.

  Note:
    ``shard_map`` is an experimental API, and still subject to change. For an
    introduction to sharded data, refer to :ref:`sharded-computation`. For a more
    in-depth look at using ``shard_map``, refer to `SPMD multi-device parallelism with shard_map`_.

  Args:
    f: callable to be mapped. Each application of ``f``, or "instance" of ``f``,
      takes as input a shard of the mapped-over arguments and produces a shard
      of the output.
    mesh: a ``jax.sharding.Mesh`` representing the array of devices over which
      to shard the data and on which to execute instances of ``f``. The names of
      the ``Mesh`` can be used in collective communication operations in ``f``.
      This is typically created by a utility function like
      :func:`jax.experimental.mesh_utils.create_device_mesh`.
    in_specs: a pytree with :class:`~jax.sharding.PartitionSpec` instances as leaves,
      with a tree structure that is a tree prefix of the args tuple to be mapped
      over. Similar to :class:`~jax.sharding.NamedSharding`, each ``PartitionSpec``
      represents how the corresponding argument (or subtree of arguments) should
      be sharded along the named axes of ``mesh``. In each ``PartitionSpec``,
      mentioning a ``mesh`` axis name at a position expresses sharding the
      corresponding argument array axis along that positional axis; not
      mentioning an axis name expresses replication. If an argument, or argument
      subtree, has a corresponding spec of None, that argument is not sharded.
    out_specs: a pytree with :class:`~jax.sharding.PartitionSpec` instances as leaves,
      with a tree structure that is a tree prefix of the output of ``f``. Each
      ``PartitionSpec`` represents how the corresponding output shards should be
      concatenated. In each ``PartitionSpec``, metioning a ``mesh`` axis name at
      a position expresses concatenation of that mesh axis's shards along the
      corresponding positional axis. Not mentioning a ``mesh`` axis name
      expresses a promise that the output values are equal along that mesh axis,
      and that rather than concatenating only a single value should be produced.
    check_rep: If True (default) enable additional validity checks and automatic
      differentiation optimizations. The validity checks concern whether any mesh
      axis names not mentioned in ``out_specs`` are consistent with how the outputs
      of ``f`` are replicated. Must be set False if using a Pallas kernel in ``f``.
    auto: (experimental) an optional set of axis names from ``mesh`` over which we
      do not shard the data or map the function, but rather we allow the
      compiler to control sharding. These names cannot be used in ``in_specs``,
      ``out_specs``, or in communication collectives in ``f``.

  Returns:
    A callable that applies the input function ``f`` across data sharded according to
    the ``mesh`` and ``in_specs``.

  Examples:
    For examples, refer to :ref:`sharded-computation` or `SPMD multi-device parallelism with shard_map`_.

  .. _SPMD multi-device parallelism with shard_map: https://jax.readthedocs.io/en/latest/notebooks/shard_map.html
  )
_shard_map)rL   rM   rN   rP   rQ   rS   s         Z/var/www/html/nettyfy-visnx/env/lib/python3.11/site-packages/jax/experimental/shard_map.py	shard_maprX   P   s    l 
AtXy)T	B	BB    Specs | Callable[[], Specs]c                D    t                     s#t          d  dt                      d          t          t                    s#t          d dt                     d                              j                  st          d dj                   t          t          j
                   t                    st          t          j                   t          j                   t          j         fd                        }|S )Nz>shard_map requires a callable for its first argument, but got 	 of type .zSshard_map requires a `jax.sharding.Mesh` instance for its second argument, but got zshard_map requires auto=z# to be a subset of mesh.axis_names=c            
     J   t          j                  }t          |           \  }}t          ||          \  }t	          | d           }n,# t
          $ r t          |           ^}} |d          d w xY wt          d t          |          D                       \  }}t          |||          \  }}t          ||||           t          t          t          |                    }t          fd            }	x}
rt          |||	          }	 t!          j        |g|R ||	|
d}n# t$          $ ru}|j        \  }t)                    sSt+          t,          j                     |          }t1          d |D                       r|dz  }t          |          d Y d }~n]d }~wt2          $ rM}|j        \  }t)                    s+t5                       |          }t          |          d Y d }~nd }~ww xY wt7                       |          S )	Nc                
    | d u S N xs    rW   <lambda>z-_shard_map.<locals>.wrapped.<locals>.<lambda>   s
    AI rY   is_leafzshard_map in_specsc              3  (   K   | ]\  }}|||fV  d S r`   ra   ).0iss      rW   	<genexpr>z._shard_map.<locals>.wrapped.<locals>.<genexpr>   s7       (: (:41a+,= *+A+8===(: (:rY   c                    t                    r&             } t          t          j        |            n} t	                       t                      g             j        z            }t          | |          }n,# t          $ r t          | |          ^}} |d          d w xY wt          t          t          |                    S )Nzshard_map out_specs)callable_check_specsSpecErrorTypeoutrA   object
num_leavesrE   
ValueErrorrF   tuplemap_canonicalize_spec)
out_specs_dummyout_specs_flate_rS   rP   out_trees        rW   out_names_thunkz4_shard_map.<locals>.wrapped.<locals>.out_names_thunk   s    	)		 Y[[
]&
D9999
XXZZ&((hhjj6K)KLLe,Z??>> 1 1 1j%00Aa%&&D01 3)>::;;;s   .A? ?)B(rM   in_namesr}   rQ   rewriterS   c              3  8   K   | ]}|t           uo|j         V  d S r`   )no_failshape)rh   fails     rW   rk   z._shard_map.<locals>.wrapped.<locals>.<genexpr>   s0      HH$t7"54:~HHHHHHrY   z In particular, for rank 0 outputs which are not constant over the mesh, add at least one (singleton) axis to them so that they can be concatenated using out_specs.)lu	wrap_initr@   r7   rE   rs   rF   r/   	enumerater9   _check_specs_vs_argsrt   ru   rv   r2   _efficient_transpose_rewriteshard_map_pbind
_SpecErrorargsrm   _spec_rank_errorro   rp   any	_RepError_inout_rep_errorrA   )r   fun	args_flatin_treein_specs_flatrz   r{   dyn_argnumsin_names_flatr}   r   out_flatfailsmsgr|   rS   rQ   rL   rN   rM   rP   s                 @rW   wrappedz_shard_map.<locals>.wrapped   s    ,q//C%d++Iw(g66MC)(D2E2EG G G . . .Hd++ea!A"##-. "( (: (:9];S;S (: (: (: ": ":K$S+yAANCD'8[-QZ[[[#0-@@AAM< < < < < < W< w T(dM?SSc(!
  $})Y  hh  ( ( (vfei   (}0!XXZZERRHH%HHHHH 	D
 C D# oo4'( ( ( ( (  ( ( (vfei   (q$

IuEEoo4'( ( ( ( ((
 ((**h///s1   A )A9D6 6
H
 A+F00H
=AHH
)rm   	TypeErrortype
isinstancer   issubset
axis_namesrs   rn   ro   inputrp   r   wrapsr   api_boundary)rL   rM   rN   rP   rQ   rS   r   s   `````` rW   rV   rV      s    
! 7
 6 6 6+/776 6 6 7 7 7	D$		 N
 M04M M?CDzzM M M N N N	t	'	' ;
 : : :(,: : ; ; ;}"Hd333	)		 5"It444:a==30 30 30 30 30 30 30 30 30  =30h 
.rY   .specr   return	AxisNamesc                b    t          | t                    rd t          |           D             S | S )Nc                N    i | ]"\  }}||t          |t                    r|n|f#S r`   )r   rt   )rh   ri   namess      rW   
<dictcomp>z&_canonicalize_spec.<locals>.<dictcomp>   sE     B B B5u/@ 
5%00>uuuh/@/@/@rY   )r   r   r   )r   s    rW   rv   rv      sH    m$$ B B%dOOB B B B KrY   ro   r   rp   
error_typespecsr	   Nonec                  	 | t           j        k    r|t          d          fdt          fdt	          |          D                       rd S | t           j        k    rdnd		fdt          |          D             }|st          |          D ]c\  }}|D ][}t          |t                    s|f}|D ]>}|v r8|                    d	 dt          |           d	t          |                      ?\dt          d
	 d dd                    |          z   d	 dz             t          d
	 dd                    |          z   d	 dz             )Nzshard_map in_specs argument must be a pytree of `jax.sharding.PartitionSpec` instances, but it was None.
Instead of `in_specs=None`, did you mean `in_specs=P()`, where `P = jax.sharding.PartitionSpec`?c                    t          | t                    sdS | D ]'}t          |t                    s|f}|D ]
}|v r  dS (dS )NFT)r   r   rt   )pr   namerS   s      rW   
check_specz _check_specs.<locals>.check_spec   st    a'' U  u%%   $4<<  4rY   c              3  .   K   | ]} |          V  d S r`   ra   )rh   r   r   s     rW   rk   z_check_specs.<locals>.<genexpr>   s+      331A333333rY   inrp   c                    g | ]H\  }}t          |t                    d  dt          |           d| dt          |          j         d	IS )  _specs is r\   , )r   PrD   r   __name__)rh   keyrc   prefixs      rW   
<listcomp>z _check_specs.<locals>.<listcomp>   su     
K 
K 
Kc1Aq9I9I
K
Nv
N
NVC[[
N
Na
N
N$q'':J
N
N
N 
K 
K 
KrY   r   r   z refers to z
shard_map z5_specs argument cannot refer to an axis marked auto (z	), but:



z

Check the z"_specs values passed to shard_map.zR_specs argument must be a pytree of `jax.sharding.PartitionSpec` instances, but:

)ro   r   r   allrC   rH   r   rt   appendrD   reprrs   join)
r   r   rS   msgsr   r   r   r   r   r   s
     `     @@rW   rn   rn      s<   =&&&5=
	23 3 3
	 	 	 	 	 	3333E 2 233333;VV!44444%&
K 
K 
K 
K*511
K 
K 
K$	 A$U++ Q QQ Q Q%%'' 	(% 	Q 	QDT\\KKOVOO6#;;OO4::OOPPP	QQ 	*V 	* 	*	* 	* 	*
++d

	@@ @ @	@A A A
 	:6 : : :D>> > >>	? 	? ?rY   c                      e Zd ZdS )NoFailNr   
__module____qualname__ra   rY   rW   r   r               rY   r   r   rG   r   Sequence[int]r   Sequence[P]xsr   c                &   t          t          |          }d t          ||          D             }t          d |D                       r=t	          |||          }t          t          j        | |||          }	t          |	          t          t          t          |                    }
fdt          ||
          D             }t          d |D                       r3t	          |||          }t          | |||          }	t          |	          d S )Nc                V    g | ]&\  }}t          |          |j        k    s|nt          'S ra   )lenndimr   )rh   r   as      rW   r   z(_check_specs_vs_args.<locals>.<listcomp>  sB     
4 
4 
4a q66QV##!! 
4 
4 
4rY   c              3  (   K   | ]}|t           uV  d S r`   r   rh   rL   s     rW   rk   z'_check_specs_vs_args.<locals>.<genexpr>  '      ((a'	((((((rY   c                    g | ]<\  }t          fd |                                D                       rnt          =S )c              3  p   K   | ]0\  }}j         |         t          fd |D                       z  V  1dS )c              3  2   K   | ]}j         |         V  d S r`   r   rh   nrM   s     rW   rk   z<_check_specs_vs_args.<locals>.<listcomp>.<genexpr>.<genexpr>  s)      $?$?qTZ]$?$?$?$?$?$?rY   N)r   r   )rh   dnsr   rM   s      rW   rk   z2_check_specs_vs_args.<locals>.<listcomp>.<genexpr>  sc       / /q" 71:$?$?$?$?B$?$?$? ? ?? / / / / / /rY   )r   itemsr   )rh   r   r   rM   s     @rW   r   z(_check_specs_vs_args.<locals>.<listcomp>  sw     
8 
8 
8a  / / / / / %/ / / / / <!!4;
8 
8 
8rY   c              3  (   K   | ]}|t           uV  d S r`   r   r   s     rW   rk   z'_check_specs_vs_args.<locals>.<genexpr>  r   rY   )ru   r8   zipr   _expand_failr   ro   r   rs   rt   rv   _spec_divisibility_error)rL   rM   r   rN   r   r   r   in_avalsr   r   r   s    `         rW   r   r     s8    #R(((
4 
4-22
4 
4 
4$((4((((( d33D
=.7Hd
K
KC
S//.>>??-
8 
8 
8 
8h66
8 
8 
8$ 	((4((((( d33D
"1dGXt
D
DC
S// rY   r   #Sequence[core.ShapedArray | NoFail]list[core.ShapedArray | NoFail]c                \    t           g| j        z  }t          ||          D ]
\  }}|||<   |S r`   )r   rr   r   )r   r   r   fail_ri   rL   s         rW   r   r     s@     -49w7I+I%+t$$  daE!HH	,rY   treer   strc                   t          |dt          |                    }| t          j        k    rd\  }}t	          ||          }nd| d}}g }	t          |||          D ]\  \  }
}\  }}d}| t          j        k    r||^}}|j        t          |j                  k     r@t          |j        
                                          |j                 }d| | d| d| d	}nt          |j        j                                                  d
         }|j        t          j        j        k    sJ d| | d|j        t          |j        j                  z
  dz    d| d|j         d	
}|	                    d| dt)          |
           d| dt          |           d| t)          |           | d|                                 d|j         d|j         dt          |           d           |	sJ t          |	          dk    r|	d         dd          g}	d| d| d| dd                    |	          z   dz   d | d!| d"| d#| d$	z   }t1          d% t          |||          D                       r	|d&| d'z  }|S )(Nr   )r   r   rp   z(*args) z, where  is bound to 's parameter '', is the index     component of 's varargs parameter 'z* r   r   z which has length z, but z has shape z, which has rank z (and z < )r      #shard_map applied to the function 'z' was given an zG_specs entry which is too long to be compatible with the corresponding zput value from the function:

r   zEntries in zQ_specs must be of length no greater than the number of axes in the corresponding z=put value.

Either revise the spec to be shorter, or modify 'z' so that its zputs have sufficient rank.c              3  2   K   | ]\  }\  }}|j          V  d S r`   )r   )rh   r{   avals      rW   rk   z#_spec_rank_error.<locals>.<genexpr>K  s.      KK<1iq$TYKKKKKKrY   z0

For scalar values (rank 0), consider using an z>_specs entry of `P()`, where `P = jax.sharding.PartitionSpec`.)getattrr   ro   r   _try_infer_args_iter_pathsidxr   	argumentslistkeys	signature
parametersvalueskindinspect	ParameterVAR_POSITIONALr   r   rD   	str_shortr   r   r   )r   rL   r   r   r   fun_namer   basebar   spec_keyr   fail_keyr   extraarg_keyr{   
param_nameparamr   s                       rW   r   r   $  s    Q
CFF++(=&&&LFD	D	!	!BBX...DF	$,7eU,K,K G G(x(4E](((R^kg	s2<((	(	(",++--..w{;
.D .' . . . .). . . R\,335566r:zW.=====FD F' F FK#bl&=">">>BF FF F7<zF F F 	KK	FV 	F 	F6(++ 	F 	F 	F 	Ft99	F 	F	F!!	F#(	F 	F59^^5E5E	F 	F )	F 	F ,09	F 	F :=T	F 	F 	FG G G G
 
+++YY!^^T!WQRR[MT
Cx 
C 
C
C 
C 
C 
C 
C ;;t
  &
&
8v 
8 
806
8 
8=E
8 
8 
8 
8 
8	
8# 	KK;tUE+J+JKKKKK G F F F F GC	*rY   c                   t          | |          }t          | dt          |                     }g }t          |||          D ]\  \  }}	\  }
}d}||
^}}|j        t          |j                  k     r>t          |j                                                  |j                 }d| d| d| d}nt          |j	        j
                                                  d         }|j        t          j        j        k    sJ d| d|j        t          |j	        j
                  z
  d	z    d
| d|j         d	}t#          |	          }|                                D ]\  }}|j        |         t)          fd|D                       z  rt          |          d	k    rd| nd|d          d}t          |          d	k    rdnd}t)          fd|D                       }|                    dt-          |
           d|                                 | dt-          |           d|	 d| d|j        |          d| d| d| d| d|j        |                     |sJ t          |          d	k    r|d         dd          g}d| d t1          j                                                   d!j         d"d#                    |          z   d#z   d$| d%z   }|S )&Nr   r   z, where argsr   r   r   r   r   r   r   r   c              3  2   K   | ]}j         |         V  d S r`   r   r   s     rW   rk   z+_spec_divisibility_error.<locals>.<genexpr>f  s)      88djm888888rY   zaxes zaxis 'r   'ztotal c              3  2   K   | ]}j         |         V  d S r`   r   r   s     rW   rk   z+_spec_divisibility_error.<locals>.<genexpr>i  s)      ,,A$*Q-,,,,,,rY   z* argsz
 of shape z corresponds to in_specsz
 of value z, which maps array axis z
 (of size z
) to mesh z (of zsize z), but z does not evenly divide r   r   z' was given argument arrays with axis sizes that are not evenly divisible by the corresponding mesh axis sizes:

The mesh given has shape  with corresponding axis names .

r   a  Array arguments' axis sizes must be evenly divisible by the mesh axis or axes indicated by the corresponding elements of the argument's in_specs entry. Consider checking that in_specs are correct, and if so consider changing the mesh axis sizes or else padding the input and adapting 'z' appropriately.)r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r  r   rv   r   r   r   r   rD   r  rt   r   r   )rL   rM   r   r   r   r  r	  r   r  r   r  r   r  r  r{   r  r  r   r   r   axistotalszr   s    `                      rW   r   r   P  s    q$"Q
CFF++(	$,7eU,K,K    (x(4E	~kg	s2<((	(	(",++--..w{;
. . .h . .). . . R\,335566r:zW.=====F F FK#bl&=">">>BF FF F7<zF F F t$$E 
  
 2	A8888R88888	8 	 "2ww{{|r|||0AA0A0A0AGGaKKR,,,,,,,,,VH%%  1A1A 5  &,X&6&6 BF %& 26*Q-       ')  24  z!}	 	  	  	 
  
+++YY!^^T!WQRR[MT
<x 
< 
< &+4:+<+<+>+>%?%?
< 
< &*_	
< 
< 
<
 ;;t

  &
&
G -5	
G 
G 
G

G# 
*rY   list[set | NoFail]c                   t          | dt          |                     }g }t          |||          D ]$\  \  }}\  }	t          |          }
t	          ||
          }t          |          dk    rd                    t          t          |                    }d                    t          t                              }d                    t          t          fd|D                                 }|                    dt          |           d| d| d| d	| 
           |\  }|                    dt          |           d| d
| d           &|sJ t          |          dk    r|d         dd          g}d| dt          |j                                                   d|j         dd                    |          z   dz   dz   }|S )Nr   r   ,c                    g | ]}|v|	S ra   ra   )rh   r   reps     rW   r   z$_inout_rep_error.<locals>.<listcomp>  s    GGGa!3,,,,,rY   z* out_specsr   zS which implies that the corresponding output value is replicated across mesh axes {z*}, but could only infer replication over {z&}, which is missing the required axes zS which implies that the corresponding output value is replicated across mesh axis 'z0', but could not infer replication over any axesr   r   r   z}' was given out_specs which require replication which can't be statically inferred given the mesh:

The mesh given has shape r  r  r   zCheck if these output values are meant to be replicated over those mesh axes. If not, consider revising the corresponding out_specs entries. If so, consider disabling the check by passing the check_rep=False argument to shard_map.)r   r   r   rv   _unmentionedr   r   ru   r   rD   rt   r   r  r   )rL   rM   r   r   r   r	  r   r  r   r  dstunmentionedneed_repgot_repdiff	need_rep_r   r  s                    @rW   r   r     sX   Q
CFF++(	$+6tUE+J+J K K'x3
T
"
"CtS))K
;!#c;//00hS#''gXXc#GGGG;GGGHHIId
kk7x(( 7 7d 7 77 7EL7 7 157 78 8 8 8 ji
kkJx(( J Jd J JJ J JK K K K 
+++YY!^^T!WQRR[MT
<x 
< 
< &+4:+<+<+>+>%?%?
< 
< &*_	
< 
< 
<
 ;;t

  &
&
2	
2# 
*rY   r   list[AxisName]c                f    d |                                 D             fd| j        D             S )Nc                    h | ]	}|D ]}|
S ra   ra   rh   r   r   s      rW   	<setcomp>z_unmentioned.<locals>.<setcomp>  %    555B"55Qa5555rY   c                    g | ]}|v|	S ra   ra   rh   r   name_sets     rW   r   z _unmentioned.<locals>.<listcomp>  s#    	:	:	:(9(9!(9(9(9rY   )r  r   rM   r   r/  s     @rW   r   r     s:    55%,,..555(	:	:	:	:T_	:	:	::rY   c                    t          |dg|j        z            }	  t          j        |           j        | S # t
          t          f$ r Y d S w xY wNF)rA   rr   r  r  r   r   rs   )rL   r   
dummy_argss      rW   r   r     sa    dUGdo$=>>*$7Q$j11
Z	    44s   7 AATlist[T | NoFail]1list[tuple[tuple[KeyPath, P], tuple[KeyPath, T]]]c                    t          | |          }t          |          }t          t          |          t          |                    }d }t          |||          }d t	          ||          D             S )Nc                    | d u pDt          |           t          u o.t          |           dk    ot          | d                   t          u S )Nr   r   )r   rt   r   r   rb   s    rW   rd   z_iter_paths.<locals>.<lambda>  s=    19TQ5 0 TSVVq[ TT!A$ZZST_ rY   re   c                <    g | ]\  }\  }}|
|t           u|||ffS r`   r   )rh   rj   r  	fail_datas       rW   r   z_iter_paths.<locals>.<listcomp>  sD     
9 
9 
9)A,AXy]y77 x#
$777rY   )rA   rH   rB   rE   r   )r   r   r   failuresfailures_augspecs_leaf	specs_augs           rW   r   r     s    D%(((#H--,.//1CE1J1JKK&	T	T$vx>>>)
9 
9L))
9 
9 
9 9rY   c                       e Zd ZdZddZd ZdS )ShardMapPrimitiveTr   lu.WrappedFunr   MaybeTracerrM   r   r   tuple[AxisNames, ...]r}   #Callable[[], tuple[AxisNames, ...]]rQ   rR   r   rS   rT   r   Sequence[MaybeTracer]c                  t          j        |          }	t          ||	j        |||||          \  }t	                    fd            }
t          |	j        |          }|	                    t          |||||
|||	  	        }             \  }}t          t           j	        t          j
        ||                    S )Nclosurec                 V                 }              \  }}|D ]} ||           } | S r`   ra   )	out_namesr{   xformstenv_todor}   s       rW   new_out_names_thunkz3ShardMapPrimitive.bind.<locals>.new_out_names_thunk  sC    !/##i(**ia ! !!AiLL		rY   r~   )r   find_top_traceprocess_env_traceslevelr1   ru   
full_raiseprocess_shard_mapr   
full_lowerapply_todos)selfr   rM   r   r}   rQ   r   rS   r   	top_tracerO  tracersoutstodosr{   rN  s       `          @rW   r   zShardMapPrimitive.bind  s    
 #D))I&sIOT8'6	7DR RMC /222     32 )&--G&&S'x+yd ' $ $D xzzHE1t 0 = =>>>rY   c                   t          |          }|                    d          }t          j        t          j        t
          j                  |d          }|                    d          t          fd          |d<   |g|fS )Njaxprra   rK  c                      S r`   ra   )axess   rW   rd   z3ShardMapPrimitive.get_bind_params.<locals>.<lambda>  s    T rY   rH  r}   )dictpopr   hashable_partialr   r   
eval_jaxprr-   )rW  params
new_paramsr]  subfunr_  s        @rW   get_bind_paramsz!ShardMapPrimitive.get_bind_params  s{    fJNN7##E do!>!>rJJF>>+&&D$4\\\\4$P$P$PJ !8ZrY   N)r   rB  r   rC  rM   r   r   rD  r}   rE  rQ   rR   r   rR   rS   rT   r   rF  )r   r   r   multiple_resultsr   rg  ra   rY   rW   rA  rA    s=        ? ? ? ?2         rY   rA  rX   rR  intr   c           	   '     K   |i fV }g g }
}		  fd|D             }|r$t          |t          j        d                    }nn|j        j                                        }t          |j        |          }|                    |||||||          \  }\  }}|		                    |           |
	                    |           |t          |	          t          |
          ffV  d S )NTc                l    g | ]0}t          |t          j                  r|j        j        k    .|1S r`   )r   r   r    _tracerR  )rh   rc   rR  s     rW   r   z&process_env_traces.<locals>.<listcomp>  sH     > > >Q*Q"<"< >MQX^e%;%; %;%;%;rY   z_trace.level)r   )maxop
attrgetterrl  mainwith_cur_sublevelru   rS  post_process_shard_mapr   rt   )rR  rM   r   r}   rQ   r   rS   r   rZ  r[  out_names_transformsrY  anstracetodoxforms   `               rW   rQ  rQ    s#      Rx$ "B	%'> > > >$ > > >G R]>::;;;ccJO--//Eu&&D66dHoy'4I ID-4	LL&&&' 	uU||U#7889999999rY   ru  pe.DynamicJaxprTraceprimcore.PrimitiverB  
in_tracersSequence[pe.DynamicJaxprTracer]r   rD  r}   rE  r   	frozensetc          	         d |D             }
t          t          t          |          ||
          } j        }t	          j                    5  t	          j        |j                                                  5  t          j
        |||          \  }}}\   d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          t          |          }t           |            |           t          t          t          |          |          }|r*t          |||          }t          | |            |           t          t          t           |           |            |          }t#          j                     fd|D             }t           j        |          }t           j        t           j        |                    }t           j        |          }i ft-          |          z  t/          |          z   }t	          j        |j                                                  5  t          j        |          }d d d            n# 1 swxY w Y   t3          ||t/           |                      ||||	          }t	          j        |j        |j                  }t          j        g ||||||          } j                            |           |S )Nc                    g | ]	}|j         
S ra   r   rh   rM  s     rW   r   z&_shard_map_staging.<locals>.<listcomp>  s    )))af)))rY   c                <    g | ]}t          j        |          S ra   )peDynamicJaxprTracer)rh   r   source_inforu  s     rW   r   z&_shard_map_staging.<locals>.<listcomp>  s(    QQQ!&ua==QQQrY   rM   r   rK  r]  rQ   r   rS   ) ru   r   _shard_avalrp  r   new_sublevelextend_axis_env_ndr   r   r  trace_to_subjaxpr_dynamic_check_shapedarray_check_names_in_names_to_rep
_check_rep_check_reps_unshard_avalr   currentgetvarinstantiate_constmakevarr   rt   convert_constvars_jaxprr`  filter_named_axis_effectseffectsr   new_jaxpr_eqnframeadd_eqn)ru  ry  rL   r{  rM   r   r}   rQ   r   rS   r   	in_avals_rp  r]  genavalsconsts
out_avals_in_repout_rep	out_avalsout_tracersinvars	constvarsoutvarsin_names_stagedrd  effseqnr  s   `                           @rW   _shard_map_stagingr    s    *)j)))('+t,,hAA)	$ S SD3DJ4D4D4F4FGG S S"$">q$	"R"RE8VRS S S S S S S S S S S S S S S S S S S S S S S S S S S S S S%x00*  *---w'..99& 2uf--Goo''111'-..0A0A:NN) (**+QQQQQyQQQ+u|Z((&%,E$;V D DEE){++'ECKK'%//9/tz//1122 . .&u--E. . . . . . . . . . . . . . .TO 1 122%#W4A A A& 
	't	G	G$
.9.v.v{	, 	,#+c	sH   ,B99B"B9"B&	&B9)B&	*B99B= B=.IIIr   core.AbstractValuecore.ShapedArrayc                >    t          | t          j                  sJ | S r`   )r   r   ShapedArrayr  s    rW   r  r  #  s     	D$*	+	++++	+rY   c                    t          |          t          j        v r)t          j        t          |                   | ||          S t          dt          |                     NzUnsupported aval type: )r   r   shard_aval_handlersNotImplementedErrorrM   r   r   s      rW   r  r  '  sS    	$ZZ4+++#DJJ/eTBBBBd4jjBBCCCrY   c                    t          |          t          j        v r)t          j        t          |                   | ||          S t          dt          |                     r  )r   r   unshard_aval_handlersr  r  s      rW   r  r  -  sS    	$ZZ4---%d4jj1$tDDD
DT

DD
E
EErY   c                     t          |t          j                  sJ |                    t	           fdt          |j                  D                                 S )Nc           	   3     K   | ]9\  }}|t          fd                     |d          D                       z  V  :dS )c              3  2   K   | ]}j         |         V  d S r`   r   r   s     rW   rk   z0_shard_shaped_array.<locals>.<genexpr>.<genexpr>7  s)      %N%Ndjm%N%N%N%N%N%NrY   ra   Nr   getrh   ri   r  rM   r   s      rW   rk   z&_shard_shaped_array.<locals>.<genexpr>7  ss       @ @ %2 %N%N%N%NUYYq"=M=M%N%N%N!N!NN @ @ @ @ @ @rY   r   r   r  updatert   r   r   r  s   `` rW   _shard_shaped_arrayr  4      	D$*	+	++++	U @ @ @ @ @)24:)>)>@ @ @ @ @ 
A 
A ArY   c                     t          |t          j                  sJ |                    t	           fdt          |j                  D                                 S )Nc           	   3     K   | ]9\  }}|t          fd                     |d          D                       z  V  :dS )c              3  2   K   | ]}j         |         V  d S r`   r   r   s     rW   rk   z2_unshard_shaped_array.<locals>.<genexpr>.<genexpr>>  s)      $M$MqTZ]$M$M$M$M$M$MrY   ra   Nr  r  s      rW   rk   z(_unshard_shaped_array.<locals>.<genexpr>>  ss       @ @ %2 $M$M$M$MEIIa<L<L$M$M$M M MM @ @ @ @ @ @rY   r  r  s   `` rW   _unshard_shaped_arrayr  ;  r  rY   c          	        ~t          |j        ||          D ]H\  }	}
}t          j        |	j        t          |||
j                            st          j        d          It          j        t          |j	        
                                                    5  t          j        |           d d d            n# 1 swxY w Y   |rot          t          t          |          |          }t          |||          }t          ||          D ]*\  }}t!          |||          st          j        d          +d |j        D             }t          t          t$          |          ||          }t          j        |j        |j                  }||fS )NzLshard_map argument avals not compatible with jaxpr binder avals and in_namesz7shard_map can't prove output is sufficiently replicatedc                    g | ]	}|j         
S ra   r  rh   rc   s     rW   r   z(_shard_map_typecheck.<locals>.<listcomp>V  s    555!qv555rY   )r   r  r   
typecompatr   r  JaxprTypeErrorr  rt   r   r   check_jaxprru   r   r  r  _valid_repeatsr  r  r  r  r   )r{   r]  rM   r   rK  rQ   r   rS   in_atomsvrc   in_namer  r  r  r!  out_avals_shardedr  r  s                      rW   _shard_map_typecheckr  F  s   
5<8<< C CmaG?16;tWaf#E#EFF C !B C C CC uTZ%5%5%7%78899  U               =)400(;;Fuf--G++ = =SD#s++ =! #< = = 	== 65u}555'-..	;LMM)		't	G	G$	Ds   B::B>B>set[AxisName]c                h    t          | j                  d |                                D             z
  S )Nc                    h | ]	}|D ]}|
S ra   ra   r*  s      rW   r+  z#_in_names_to_rep.<locals>.<setcomp>]  s%     H H HrR H H H H H HrY   )setr   r  )rM   r   s     rW   r  r  \  s.    	T_		 H HU\\^^ H H H	HHrY   r]  
core.Jaxprr  Sequence[RepType]c                  	 i 	d	fd}d	fd
}t          ||j        t          | j                  gt	          |j                  z             t          ||j        |           t          j        |          }|j        D ]}t          
                    |j        t          t          |j                            } || gt          ||j                  R i |j        }|j        j        rGt!          |          t          u r|gt	          |j                  z  n|}t          ||j        |           n ||j        d         |           t          j        |	|           t          ||j                  S )Nrc   	core.Atomr   RepTypec                N    t          |           t          j        u r|          nd S r`   r   r   Var)rc   envs    rW   readz_check_rep.<locals>.readc  s#    !WW((3q66d2rY   r  core.Varvalr   c                    || <   d S r`   ra   )r  r  r  s     rW   writez_check_rep.<locals>.writef  s    CFFFrY   r   )rc   r  r   r  )r  r  r  r  r   r   )ru   r  r  r   r   r  r   	last_usedeqns_check_rulesr  	primitiver   _rule_missingrd  rh  r   r  clean_up_dead_vars)
rM   r]  r  r  r  r  rz   ruler  r  s
            @rW   r  r  _  s   !##3 3 3 3 3 3      eU_s4?334s5?7K7KKLLLeU\6"""nU##): / /aAK)L)LMMDd4:#dAH--:::::G{# #.27mms.B.B	C	NN**g	%G$$$$eAIaL'"""AsI....	T5=	!	!!rY   r  r  r!  c                j    |d u p/t          t          | |                                        |          S r`   )r  r   r   )rM   r  r!  s      rW   r  r  w  s1    		BLs3344==cBBBrY   c                (    t          d|  d          )NzNo replication rule for z. As a workaround, pass the `check_rep=False` argument to `shard_map`. To get this fixed, open an issue at https://github.com/google/jax/issuesr  )ry  r{   __s      rW   r  r  z  s*    6 6 6 6	7 	7 7rY   c                  ~~d |j         D             }	d |j        D             }
t          t          t          | ||          || j        |	|          }t          j        |t          |j	                  |z
            }| j
                            |          }t          j        t          |j                                                            5  t#          j        d| j        |d ||	|
| j        g|R | j        t          t,          ||	          t          t,          ||
          d\  }}d d d            n# 1 swxY w Y   |                     |           t          t          t0          | ||          ||
| j        |          S )Nc                    g | ]	}|j         
S ra   r  rh   r  s     rW   r   z'_shard_map_lowering.<locals>.<listcomp>  s    ,,,!qv,,,rY   c                    g | ]	}|j         
S ra   r  r  s     rW   r   z'_shard_map_lowering.<locals>.<listcomp>  s    ...1...rY   )axis_context
shmap_body)dim_var_values	arg_namesresult_names)r  r  ru   r   
_xla_shardavals_inr   SPMDAxisContextr}  r   module_contextreplacer   r  rt   r   r   r;   call_lowering
name_stack	tokens_inr  _pspec_mhlo_attrsset_tokens_out_xla_unshard	avals_out)ctxr]  rM   r   rK  rQ   r   rS   in_nodesr  r  	in_nodes_new_axis_contextsub_ctx
out_nodes_
tokens_outs                   rW   _shard_map_loweringr    s   ,,u|,,,).....*'*c466#,X' ')#3
Ido&&-  &&4D&EE'uTZ%5%5%7%78899 D D!/cneT7ICMD$-D D>A>P'9==*IzBB	D D DJ
D D D D D D D D D D D D D D D Z   	W\3d33Y
]J
( 
( (s   AD44D8;D8c                    | j         j        }t          |t          j                  r|j        }nt          i           }t          |t          j        |          |          S )N)_manual_axes)	r  r  r   r   r  manual_axesr}  r   array_mapping_to_axis_resources)r  rM   r_  axis_ctxr  s        rW   _make_scoped_manual_shardingr    sa    ,(.899  &KKB--K	
N:4@@
  
  
   rY   r  mlir.LoweringRuleContextc                n   t          j        |t          |j                  |z
  |          }d |                                D             }t          | ||          }	t          j        |j        t          j	                  r)t          j        ||	          }	t          j        |          }|	                    |j                                                  }
|r!t#          t%          |j                            nt#                      }t'          j        | |||
|          }t'          j        | ||||          gS )Nc                $    i | ]\  }}|D ]}||S ra   ra   rh   ri   r   r   s       rW   r   z_xla_shard.<locals>.<dictcomp>  +    	<	<	<ea	<	<$	<	<	<	<rY   unspecified_dims)r=   manual_protor}  r   r   r  r   
issubdtypedtypeextendedr   physical_shardingr   physical_aval_to_xla_hlo_shardingr   to_protor  ranger;   wrap_with_sharding_opwrap_with_full_to_shard_op)r  rM   rS   r   aval_inaval_outrc   r  r_  r   shard_protounspecifiedsxs                rW   r  r    s   "7Ido,F,F,MtTT,	<	<u{{}}	<	<	<$#Ct44"w}fo66 *		)'2	6	6B ))G''55>>@@+,0;E',''(((cee+!#q';3>@ @ @"

)#r8\;
W
W	XXrY   c                l   d |                                 D             }t          | ||          }t          j        |j        t          j                  r)t          j        ||          }t          j	        |          }|r!t          t          |j                            nt                      }	t          j        |t          |j                  |z
  |          }
t#          j        | |||
|	          }|                    |j                                                  }t#          j        | ||||	          S )Nc                $    i | ]\  }}|D ]}||S ra   ra   r  s       rW   r   z _xla_unshard.<locals>.<dictcomp>  r  rY   r  )r   r  r   r  r  r  r   r  r   r  r  r  r   r=   r  r}  r   r;   r  r  r  wrap_with_shard_to_full_op)r  rM   rS   r   r!  r"  rc   r_  r   r$  r  r%  r#  s                rW   r  r    s   	<	<u{{}}	<	<	<$#Ct44"x~v77 ,		)(B	7	7B!(++H-1<E(-(()))suu+"7Ido,F,F,MtTT,!#q'<R]^^^"''66??AA+		(b(K)4
6 
6 6rY   c                    t          |t          j                  r4t          t	          | j        t          |j                                      S dS )Nr   )r   r   r  r   ru   r  r  r   )r   r   s     rW   r  r    s@    d&'' 1s59eDI..//000	rY   c               <   |	rt           ~~	t          t          t          |          ||          }t          t          t          |          |          }
t          j        t          ||          5 }t          |||
          \  }}t          j	                    5  t          j
        |j                                        |          5   |j        | }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   ~d d d            n# 1 swxY w Y   d |D             }t           |            |           |r!t          | |             |                       t          t            |                      }t          t          t"          ||          ||          S )N)rM   checkc           	     t    g | ]5}t          j        |j        d          d t          j        |                    6S r   )r   mapped_avalr   get_avalr  s     rW   r   z#_shard_map_impl.<locals>.<listcomp>  s7    OOOQt
At}Q/?/?@@OOOrY   )r  ru   r   _unmatch_specr  r   new_base_mainShardMapTrace_shmap_subtracer  r  r   r   call_wrappedr  r  _names_to_pspec_match_spec)ru  ry  r   r   rM   r   r}   rQ   r   rS   r  rp  r  rZ  r  pspecss                   rW   _shard_map_implr8    s=   	$$$
D	W]D))8T	:	:$w'..99&-d)DDD "3f55LC				 % %d5dj6F6F6H6H$OO % %St$d% % % % % % % % % % % % % % % % % % % % % % % % % % % % % %	              
 PO$OOO)  ),,, 4oo''333 1 122&	W[$	22FD	A	AAsZ   /(D
-C2CC2CC2"C#C2&D
2C6	6D
9C6	:D

DDc              '     K   |                                  }t          t          t          |          ||          }|i fV }t          |j        |          }t          d |D                       \  }}~~~~||fV  d S )Nc              3  2   K   | ]}|j         |j        fV  d S r`   r  r  r  s     rW   rk   z"_shmap_subtrace.<locals>.<genexpr>  s*      ==A!%======rY   )rq  ru   r   ShardMapTracerrS  r/   )	rp  r  in_valsrM  r{  rt  r  rZ  r  s	            rW   r3  r3    s      
!7>1--vw??*"n#AL#&&+=======-$S+grY   c                ~      rt                     dz   nd}d t           fdt          |          D              S )Nr   r   c                @    | t          |           dk    r| d         n| S )Nr   r   )r   )rM  s    rW   rd   z!_names_to_pspec.<locals>.<lambda>  s     Q]s1vv{{QqTT rY   c              3  T   K   | ]"}                      |                    V  #d S r`   )r  )rh   ri   r   unpacks     rW   rk   z"_names_to_pspec.<locals>.<genexpr>  s7      DD!		!--DDDDDDrY   )rm  r   r  )r   ndminrA  s   ` @rW   r5  r5    sM    !
(#e**q..q%AA&	DDDDDuU||DDD	EErY   srcrc   JaxTypec                n   t          j                    5  t          j        d          5   t          j        t          t          | t          |                                                              |          cd d d            cd d d            S # 1 swxY w Y   d d d            d S # 1 swxY w Y   d S r2  )	r   eval_contextjaxdisable_jitjitr.   _unmatchrt   r   )rM   rC  rc   s      rW   r0  r0    sa    K KCOE22 K KG37?8T53E3EFFGGJJK K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K Ks5   B*AB9B*B	B*B	B**B.1B.c                    t          t          |                    }t          | j                  } t	          t
          | |f|d          |          S NF)rQ   )r5  r`  r   r   rX   _add_singleton)rM   src_tuprc   rC  r!  s        rW   rJ  rJ    sI    W&&#	$/#	F>4#	F	F	Fq	I	IIrY   Sequence[AxisNames]avalsSequence[core.ShapedArray]c                    d t          | |          D             }t          d |D                       rt          |          d S )Nc                Z    g | ](\  }}|rt          |          |j        k     s|nt          )S ra   )rm  r   r   )rh   r   r   s      rW   r   z _check_names.<locals>.<listcomp>  sE     
) 
) 
)a 
5Q!&!!g 
) 
) 
)rY   c              3  (   K   | ]}|t           uV  d S r`   r   r   s     rW   rk   z_check_names.<locals>.<genexpr>  r   rY   )r   r   r   )r   rP  r   s      rW   r  r    sZ    
) 
)%''
) 
) 
)$((4(((((@
40@0@*@@@rY   c                      e Zd ZdS )r   Nr   ra   rY   rW   r   r     r   rY   r   c                      fdt          ||          D             }t          d |D                       rt          |          d S )Nc                J    g | ]\  }}t          ||          s|nt           S ra   )r  r   )rh   r   rrM   s      rW   r   z_check_reps.<locals>.<listcomp>  sB     
( 
( 
(a "$1--
:!!7 
( 
( 
(rY   c              3  (   K   | ]}|t           uV  d S r`   r   r   s     rW   rk   z_check_reps.<locals>.<genexpr>  r   rY   r   r   r   )rM   r   repsr   s   `   rW   r  r    sc    
( 
( 
( 
(%&&
( 
( 
($((4(((((?	$*???rY   c                      e Zd ZdS )r   Nr   ra   rY   rW   r   r     r   rY   r   c                    d t          ||          D             }t          d |D                       rt          |          d S )Nc                P    g | ]#\  }}|                     |          s|nt          $S ra   )r   r   )rh   r!  rC  s      rW   r   z _check_reps2.<locals>.<listcomp>  sA     
0 
0 
0c3 \\#&&
3##G 
0 
0 
0rY   c              3  (   K   | ]}|t           uV  d S r`   r   r   s     rW   rk   z_check_reps2.<locals>.<genexpr>   r   rY   rZ  )rM   	reps_destr[  r   s       rW   _check_reps2ra    sX    
0 
0i..
0 
0 
0$((4(((((?	$*???rY   pspecc           
     V   t          t          | ||          }t          j                    5  t	          j        d          5   t	          j        |t          | |                    |          cd d d            cd d d            S # 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NF)out_shardings)r.   _matchr   rF  rG  rH  rI  r   )rM   rQ   rb  rc   fns        rW   r6  r6    sg   vtY66" D DCOE22 D D@372]4%?%?@@@CCD D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D Ds5   B -B-BB
	
BB
	BB"%B"c                p    t          | j                  } t          t          | |f|d          |          S rL  )r   r   rX   _rem_singleton)rM   rQ   rb  rc   rC  s        rW   re  re    s6    	$/#	H>4#%	H	H	H	K	KKrY   c                F    |                      | j        dd                    S Nr   reshaper   rb   s    rW   rh  rh    s    aii444rY   c                (     | j         dg| j        R  S rj  rk  rb   s    rW   rM  rM    s    iai4AG4444rY   c                  p     e Zd ZU ded<   ded<    fdZd Zd Zd Zd	 Zd
 Z	d Z
d Zd Zd Zd Z xZS )r2  r   rM   rR   r+  c               P     t                      j        |  || _        || _        d S r`   )super__init__rM   r+  )rW  rM   r+  r   	__class__s       rW   rq  zShardMapTrace.__init__  s)    EGGdDIDJJJrY   c                P    t          | j        i |          }t          | d |          S r`   )r0  rM   r<  )rW  r  val_s      rW   purezShardMapTrace.pure  s'    B,,D$d+++rY   c                8    t          | |j        |j                  S r`   )r<  r  r  rW  tracers     rW   subliftzShardMapTrace.sublift  s    $
FJ777rY   c                ^   t          d |D                       \  }}t                              |          }|r || j        g|R i |}nt	          t
          |t          |                                          | j                  }t          j	                    5  t          j        d          5   t          j        |          | }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t                              |t          t          |                    }	| j        r |	| j        g|R i |nt#                      }
|j        rOt'          |
          t"          u r|
gt)          |          z  n|
}
t+          t          t,          |           |
|          S t-          | |
|          S )Nc              3  2   K   | ]}|j         |j        fV  d S r`   r;  r  s     rW   rk   z2ShardMapTrace.process_primitive.<locals>.<genexpr>!  *      ==aeQU^======rY   F)r/   eager_rulesr  rM   r.   _prim_applierrt   r   r   rF  rG  rH  rI  r  r   r  r+  r  rh  r   r   ru   r<  )rW  ry  rY  rd  r=  r  
eager_ruleout_valsrL   rep_ruler  s              rW   process_primitivezShardMapTrace.process_primitive   s   ==W=====OGV&&J (DI::::6::hh
-uV\\^^/D/Ddi
P
Pa ( ( 6 6 ( (371::w'( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (gmT&B&BCCH8<
Mhhty46444V444G C-1']]c-A-A	CMM))wg..BBB$222s6   C).CC)C	C)C	C))C-0C-c                (    t          d| d          )NzEager evaluation of `z` inside a `shard_map` isn't yet supported. Put a `jax.jit` around the `shard_map`-decorated function, and open a feature request at https://github.com/google/jax/issues !r  )rW  call_primitiver   rY  rd  s        rW   process_callzShardMapTrace.process_call0  s*    
	1 	1 	1 	12 2 2rY   c                     t          d          )NzEager evaluation of `pmap` inside a `shard_map` isn't yet supported.Put a `jax.jit` around the `shard_map`-decorated function, and open a feature request at https://github.com/google/jax/issues !r  )rW  map_primitiver   rY  rd  s        rW   process_mapzShardMapTrace.process_map7  s    
	FG G GrY   c               Z   |rd}t          |          ~~~t          d |D                       \  }}t          || j        |          \  }}	t	          j                    5   |j        | }
d d d            n# 1 swxY w Y   t          t          t          |            |	            |
          S )Nzcustom_jvp symbolic_zeros support with shard_map is not implemented; please open an issue at https://github.com/google/jax/issuesc              3  2   K   | ]}|j         |j        fV  d S r`   r;  r  s     rW   rk   z8ShardMapTrace.process_custom_jvp_call.<locals>.<genexpr>E  r|  rY   
r  r/   r3  rp  r   r  r4  ru   r   r<  )rW  ry  r   jvprY  symbolic_zerosr   r=  r  r  r  s              rW   process_custom_jvp_callz%ShardMapTrace.process_custom_jvp_call=  s     %4c  $$$c>==W=====OGV"3	6::LC				 , ,!!7+h, , , , , , , , , , , , , , ,w~t,,ggiiBBBs   A66A:=A:c                    J r`   ra   rW  r  r{   s      rW   post_process_custom_jvp_callz*ShardMapTrace.post_process_custom_jvp_callK      LrY   c                ^   |rd}t          |          ~~~~~t          d |D                       \  }	}
t          || j        |
          \  }}t	          j                    5   |j        |	 }d d d            n# 1 swxY w Y   t          t          t          |            |            |          S )Nzcustom_vjp symbolic_zeros support with shard_map is not implemented; please open an issue at https://github.com/google/jax/issuesc              3  2   K   | ]}|j         |j        fV  d S r`   r;  r  s     rW   rk   z8ShardMapTrace.process_custom_vjp_call.<locals>.<genexpr>W  r|  rY   r  )rW  ry  r   fwdbwdrY  	out_treesr  r   r=  r  r  r  s                rW   process_custom_vjp_callz%ShardMapTrace.process_custom_vjp_callN  s      %4c  $$$c3	>==W=====OGV"3	6::LC				 , ,!!7+h, , , , , , , , , , , , , , ,w~t,,ggiiBBBs   !A88A<?A<c                    J r`   ra   r  s      rW   post_process_custom_vjp_callz*ShardMapTrace.post_process_custom_vjp_call]  r  rY   c                   t          j                    5  t          j        d          5   t          j        fd                      cd d d            cd d d            S # 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NFc                 L    t           j                             j                  S r`   )rG  r#   
axis_indexr   )r  s   rW   rd   z2ShardMapTrace.process_axis_index.<locals>.<lambda>b  s    SW//
;; rY   )r   rF  rG  rH  rI  )rW  r  s    `rW   process_axis_indexz ShardMapTrace.process_axis_index`  s!   				 ? ?coe44 ? ?<SW;;;;<<>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?s4   A:A"	A:"A&	&A:)A&	*A::A>A>)r   r   r   __annotations__rq  ru  ry  r  r  r  r  r  r  r  r  __classcell__rr  s   @rW   r2  r2    s         ***+++    
, , ,8 8 83 3 3 2 2 2G G GC C C  C C C  ? ? ? ? ? ? ?rY   r2  c                  T    e Zd ZU ded<   ded<   d Zed             ZddZdd
ZeZ	dS )r<  r  r  rD  r  c                0    || _         || _        || _        d S r`   rl  r  r  rW  ru  r  r  s       rW   rq  zShardMapTracer.__init__i      DKDHDHHHrY   c                   t          j        | j                  }t          |t           j                  rs| j        t          | j        j        j	                  k    rLt          j
                    5  t          j        | j        d                   cd d d            S # 1 swxY w Y   d S t          j        |          }t          j        | j        j        j        d|          S Nr   )r   r/  r  r   ConcreteArrayr  r  rl  rM   r   rF  raise_to_shapedr.  size)rW  r   s     rW   r   zShardMapTracer.avaln  s    =""D4+,, >C(34444 * *}TXa[))* * * * * * * * * * * * * * * * * * !$''ddk.3Q===s   .BB!Br   c                    | S r`   ra   rW  s    rW   rU  zShardMapTracer.full_lowery      KrY   r   c                   t          j                    5  t          | j                  }d d d            n# 1 swxY w Y   | j        j        }dd                    t          t          |j	                             dd                    fdt          t          j        |j                  |          D                       S )N(r   z,)
c           
   3  B   K   | ]\  \  }}}d | d d| d| d	V  dS )zOn z at mesh coordinates z = z:
r  Nra   )rh   r   deviceblockr   s       rW   rk   z)ShardMapTracer.__str__.<locals>.<genexpr>  sk       O O MS&5 	LfKK:KK#KK%KKKO O O O O OrY   )r   rF  r   r  rl  rM   r   ru   r   r   r   npndenumeratedevices)rW  blocksrM   r   s      @rW   __str__zShardMapTracer.__str__|  s   				  DH~~f              ;D=TYYs38899===J99 O O O O$'t|(D(Df$M$MO O O O O Os   6::N)r   r<  r   r   
r   r   r   r  rq  propertyr   rU  r  __repr__ra   rY   rW   r<  r<  e  s         ,,,,,,  
 > > 8>   O O O O (((rY   r<  c                f      fd}t          |j                  } t          ||||d          | S )Nc                      j         t          t          |           i t                    }t	          t
          |          S r`   )r   ru   rh  r`  r?   rM  )r   rZ  
params_tupry  s     rW   applyz_prim_applier.<locals>.apply  s;    49c.$//D4
3C3CDDDND)))rY   F)r   r   rX   )ry  r  rM   r   r  r   s   ``    rW   r~  r~    sP    * * * * * * 
4?		$	25$dE	2	2D	99rY   zdict[core.Primitive, Callable]r}  r   Callable[..., Any]effectdebugging.DebugEffectc                   ~t          j                    5  t          t          t          |           }d d d            n# 1 swxY w Y   t          t          j        | j                  |          D ]\  \  }}} ||  g S r`   )r   rF  r   ru   r   r  r  r  )rM   r   r  r   
all_blocksr   r  r  s           rW   _debug_callback_eager_ruler    s     ' 'c$oo&J' ' ' ' ' ' ' ' ' ' ' ' ' ' '"2>$,#?#?LL  msFVHf	)s   >AAc               <    ~ ~|D ]}|t          d|           |S )Nzadevice_put with explicit device not allowed within shard_map-decorated functions, but got device )rs   )rM   srcsr  r   r  s        rW   _device_put_eager_ruler    sT    
D R Rf QHNQ Q R R R  
)rY   psum2c                    || z  S r`   ra   )r  	axis_sizes     rW   rd   rd     s
    y1} rY   r_  c               (    ~t          j        | ||dS Nr_  axis_index_groups)pbroadcast_pr   )ctsr_  r  r   s       rW   _psum2_transpose_ruler    s    
		Cd>O	P	P	PPrY   c                    t          |t                    s|fn|}|s| S t          |           \  }}t          j        ||d d}t          ||          S r  )r   rt   r@   r  r   rA   )rc   	axis_namer_  r   treedefyss         rW   
pbroadcastr    s\    '	599	H)y$	1HQ+"g"44@@@"		$	$$rY   r  c                    |S r`   ra   r_  r  r   s      rW   rd   rd     s    T rY   c                    |S r`   ra   r  s      rW   rd   rd     s    d rY   c                   |S r`   ra   )r  r_  r  rc   s       rW   rd   rd     s    a rY   c               n    t          d |D                       rt          t          j        | ||d}||fS )Nc              3  B   K   | ]}t          |          t          u V  d S r`   )r   ri  )rh   r  s     rW   rk   z&_pbroadcast_batcher.<locals>.<genexpr>  s-      ,,tds	,,,,,,rY   r  )r   r  r  r   )vals_indims_inr_  r  vals_outs        rW   _pbroadcast_batcherr    sQ    ,,t,,,,,G4G.Gd1BD D D(	7	rY   c                   t           r`   r  )r  r   
trace_typer  r  r_  groupss          rW   _pbroadcast_axis_batcherr    s    rY   c               &    t          j        | ||dS r  )psum2_pr   )r  r_  r  r{   s       rW   rd   rd     s    lCd>OPPP rY   _rewrite_rulesc                      fdS )Nc                :    t                               |           S r`   )r  
setdefault)rX  ry  s    rW   rd   z<lambda>.<locals>.<lambda>  s    .*C*CD!*L*L rY   ra   ry  s   `rW   rd   rd     s     L L L L rY   c                ^    t                               | t          t          |                     S r`   )r  r  r   _standard_rewrite_ruler  s    rW   rd   rd     s%    dG,BD$I$IJJ rY   c           	     v    t                               | t          t          | t          |                              S r`   )r  r  r   _no_rewriter  )r   s    rW   rd   rd     s*    aaa!I!IJJ rY   r  c                      fdS )Nc                :    t                               |           S r`   )r  r  )r  ry  s    rW   rd   z<lambda>.<locals>.<lambda>  s    <+B+B4+N+N rY   ra   r  s   `rW   rd   rd     s    NNNN rY   c                ^    t                               | t          t          |                     S r`   )r  r  r   _standard_checkr  s    rW   rd   rd     s     ((w/M/MNN rY   c                     | j         |i |} ||g|R i |}| j        r,t          |          t          u r|n|gt	          |          z  }n|g|g}}||fS r`   )r   rh  r   r   r   )	ry  r  rM   r  r   rd  r  r  out_rep_s	            rW   r  r    s    TY&v&&(D))))&))'	 /w--4//wwgYX5NHH"gYhH	8	rY   c                   	 |rt          j        | nt          |j                  		fdt          ||          D             } | j        |i |}| j        r	gt          |          z  n	g}| j        s|gn|}||fS )Nc           	     v    g | ]5\  }}|z
  r)t          |t          fd |D                                 n|6S )c              3  $   K   | ]
}|v|V  d S r`   ra   )rh   r   r  s     rW   rk   z4_standard_rewrite_rule.<locals>.<listcomp>.<genexpr>  s-      CCQ(1B1Bq1B1B1B1BCCrY   r  rt   )rh   rc   rC  r  s      rW   r   z*_standard_rewrite_rule.<locals>.<listcomp>  sh     E E E(.3cHn$:aCCCC#CCCCCDDD"#E E ErY   )r  intersectionr   r   r   rh  r   )
ry  rM   r  r   rd  args_	out_vals_r  r  r  s
            @rW   r  r    s    *0JSv&&c$/6J6J(E E E E25dF2C2CE E E%di)&)))+/+@PXJY''xj' $ 5Di[[9(	7	rY   c                    d |D             }|r,|d d         |dd          k    st          d|  d| d          |r|d         nd S )Nc                    g | ]}||S r`   ra   rh   rX  s     rW   r   z#_standard_check.<locals>.<listcomp>      0001!-Q---rY   r   r   z
Primitive z7 requires argument replication types to match, but got . Please open an issue at https://github.com/google/jax/issues and as a temporary workaround pass the check_rep=False argument to shard_mapr   	Exception)ry  rM   r  r  in_rep_s        rW   r  r    s     10000' QWSbS\WQRR[00
 P P P)/P P P Q Q Q 	(D(rY   c                     t          |           t          t          |                       t          |           t          t          |                      d S r`   )register_checkr   _standard_collective_checkregister_rewrite_standard_collective_rewriter  s    rW   register_standard_collectiver    sO    .w94@@AAA4!=tDDEEEEErY   c          	     H    ~~|||v rt          d|  d| d| d          |S )NzCollective z? must be applied to a device-varying replication type, but got & for collective acting over axis name r  r  )ry  rM   x_repr  rd  s        rW   r
  r
    sj    
F
]i5((
 P$ P P16P P&/P P P Q Q Q
 
,rY   c                    t          |t                    s|fn|}|\  }t          |          }||z  x}rt          |t          |                    } | j        |fd|i|}	|	g||z
  gfS )Nr  )r   rt   r  r  r   )
ry  rM   r  rc   r  rd  r  axis_name_setpbroadcast_axis_nameout_vals
             rW   r  r    s    ",Y">">MyllI)&%i..-*U22 31e01122ADIa779777'
U]*+	++rY   c                   J r`   ra   )r{   r_  r  r  s       rW   _psum_checkr  *  s    ,rY   c                    |t           t          |t                    s|fn|}t          |          fd|D             } fdt	          ||          D             }t          j        |||d}||fS )Nc                    g | ]}|z  S ra   ra   )rh   rX  axes_s     rW   r   z!_psum_rewrite.<locals>.<listcomp>4  s    '''1QY'''rY   c           	     v    g | ]4\  }t          |t          fd j        D                                 5S )c              3  *   K   | ]}|z  v 	|V  d S r`   ra   )rh   r   r  rC  s     rW   rk   z+_psum_rewrite.<locals>.<listcomp>.<genexpr>5  s1      NNQQ%#+=M=Mq=M=M=M=MNNrY   )r  rt   r   )rh   rc   rC  r  rM   s     @rW   r   z!_psum_rewrite.<locals>.<listcomp>5  sZ     , , ,q# aNNNNN$/NNNNNOO , , ,rY   r  )r  r   rt   r  r   r  r   )	rM   r  r_  r  r   r  r  r  r  s	   `       @rW   _psum_rewriter  .  s     "*=$="4//	9$T$
d))%'''''''', , , , ,T6**, , ,%L%d>OPPP'	'	rY   c                    t                    t          u sJ t          fd|D                       rt          d| d d          t           fd|D                       }fd|D             S )Nc              3  @   K   | ]}|t                    |z  V  d S r`   r  rh   rX  r_  s     rW   rk   z_psum2_check.<locals>.<genexpr>>  s-      881!-TQ----88rY   zNCollective psum must be applied to a device-varying replication type, but got r  . Please open an issue at https://github.com/google/jax/issues, and as a temporary workaround pass the check_rep=False argument to shard_mapc              3  H   K   | ]}|t          j                  n|V  d S r`   r  r   rh   rX  rM   s     rW   rk   z_psum2_check.<locals>.<genexpr>D  6      JJaT_%%%JJJJJJrY   c                4    g | ]}|t                    z  S ra   r  r   s     rW   r   z _psum2_check.<locals>.<listcomp>E  #    	(	(	(A!c$ii-	(	(	(rY   )r   rt   r   r  rM   r_  r  r  s   ``  rW   _psum2_checkr)  ;  s    	du				888888888 Q
 P17P P&*P P P Q Q Q
 JJJJ6JJJJJ&	(	(	(	(	(	(	((rY   c                    t                    t          u sJ t          fd|D                       st          d| d d          t           fd|D                       }fd|D             S )Nc              3  D   K   | ]}|d u pt                    |z  V  d S r`   r  r   s     rW   rk   z$_pbroadcast_check.<locals>.<genexpr>L  s6      88AQ$Y'#d))a-888888rY   zXCollective pbroadcast must be applied to a non-device-varying replication type, but got r  r!  c              3  H   K   | ]}|t          j                  n|V  d S r`   r#  r$  s     rW   rk   z$_pbroadcast_check.<locals>.<genexpr>S  r%  rY   c                4    g | ]}|t                    z
  S ra   r  r   s     rW   r   z%_pbroadcast_check.<locals>.<listcomp>T  r'  rY   )r   rt   r   r  r(  s   ``  rW   _pbroadcast_checkr.  I  s    	du					8888888	8	8 Q
 P17P P '+P P P Q Q Q JJJJ6JJJJJ&	(	(	(	(	(	(	((rY   c                   t          |          t          ur|fn|}t          | j                  t          |          z
  S r`   )r   rt   r  r   )rM   r  s     rW   _axis_index_checkr0  ^  s6    "&y//U":":yll	)	TZ3y>>	))rY   c               b    t          | ||          \  }}t          j        j        |d|i|}||fS Nr]  )_replication_rewrite_nomatchr   pjit_pr   rM   r  r]  r   kwargsjaxpr_r  r  s           rW   _pjit_rewriter8  e  s>    0ufEE/&'[t<6<V<<(	7	rY   c               .    t          | |j        |          S r`   r  r]  rM   r]  r  r6  s       rW   _pjit_checkr<  k  s    	D%+v	.	..rY   c                   t          j        |          }t          | ||          \  }}|j        |j        c}\   t          j        j        |d|i|}||fS r2  )r  close_jaxprr3  r]  r  r   remat_pr   r5  s           rW   _remat_rewriter@  p  s]    >%  &0vvFF/&'lFM)%"'EUEfEE(	7	rY   c               $    t          | ||          S r`   r  r;  s       rW   _remat_checkrC  x  s    	D%	(	((rY   c               $    t          | ||          S r`   rB  rM   
call_jaxprr  r6  s       rW   _core_call_checkrG  }  s    	D*f	-	--rY   c                    g S r`   ra   )rM   r  r{   s      rW   _debug_callback_rulerI    s    	)rY   c               @    t                      gt          |          z  S r`   r  r   rM   result_avalsr{   r  s       rW   _pure_callback_rulerN        
%%3|$$	$$rY   c               @    t                      gt          |          z  S r`   rK  rL  s       rW   _io_callback_rulerQ    rO  rY   c                     t          |          S r`   )r   )rM   r   r{   s      rW   _device_put_rulerS    s    	b/rY   c               @    t                      gt          |          z  S r`   rK  )rM   r  r{   r  s       rW   _custom_lin_rulerU    s    
%%3y>>	!!rY   c                   t          |||g          \  }}}t          | |j        |          }t          ||g          \  }}||k    st          d| d| d          |S )Nz=Scan carry input and output got mismatched replication types z and r!  )r5   r  r]  r  )	rM   r]  
num_consts	num_carryr  r{   carry_rep_inr  carry_rep_outs	            rW   _scan_checkr[    s    !&:y*ABB!\1tU[&11')55-		&	&
  )   0=      ! ! !
 
.rY   c                  t          |||g          \  }}}	t          d|z             D ]W}
g |||	}t          | ||          \  }
}t          ||g          \  }}t          t          j        ||          }||k    r n|}XJ d            d t          |||          D             }g ||}t          | |||          }t          j	        j
        j        ||||d|}||fS )Nr   FzFixpoint not reachedc           	     x    g | ]6\  }}|z
  r)t          |t          fd |D                                 n|7S )c              3  $   K   | ]
}|v|V  d S r`   ra   rh   r   r!  s     rW   rk   z+_scan_rewrite.<locals>.<listcomp>.<genexpr>  '      ==Aa==rY   r  rh   rc   rC  r!  s      @rW   r   z!_scan_rewrite.<locals>.<listcomp>  sj     
M 
M 
M"-!S#S3Y*Q=========
>
>
>
M 
M 
MrY   )r]  rW  rX  )r5   r  r3  ru   rn  and_r   _replication_rewrite_matchr+   loopsscan_pr   )rM   r  r]  rW  rX  r   rd  	const_reprY  xs_repr{   r  r  rZ  ys_repr7  r  s                    rW   _scan_rewriteri    s>   $.v
I7N$O$O!)\6Y 
) 
)a2	2L262G-dE7CCJAw&w<<M6}==M}$$e"ll(((((
M 
M14T671K1K
M 
M 
M$%m%f%'%dE7GDD&&+6jIQ QIOQ Q(	7	rY   c               b    t          | ||          \  }}t          j        j        |d|i|}||fS r2  )r3  r   closed_call_pr   )rM   r  rF  r   r6  	new_jaxprr  r  s           rW   _closed_call_rewriterm    s@    3D*fMM)W$dF)FvFF(	7	rY   c               .    t          | |j        |          S r`   r:  rE  s       rW   _closed_call_checkro    s    	D**F	3	33rY   c               .    t          | |j        |          S r`   r:  )rM   rF  jvp_jaxpr_thunkrW  r  r  s         rW   _custom_jvp_call_checkrr    s     
D**F	3	33rY   c          
     8    |rd}	t          |	          t           ||          \  }
}t          ||g          \  }g t          j         fd            }t          | fd          }t          j        j        ||
|||||d}rd         n|}||fS )NPlease open an issue at https://github.com/google/jax/issues and as a temporary workaround pass the check_rep=False argument to shard_mapc                     t          j         |   }t          |          \  }}                    |           |j        |j        fS r`   )r   ClosedJaxprr3  r   r]  r  )zeros	fwd_jaxpr
fwd_jaxpr_r  fwd_jaxpr_thunkr  rM   out_rep2s       rW   fwd_jaxpr_thunk_z8_custom_vjp_call_jaxpr_rewrite.<locals>.fwd_jaxpr_thunk_  sP     //5"9:I6tYPPJOOGZ...rY   c                      d         S r  ra   )r{  s   rW   rd   z0_custom_vjp_call_jaxpr_rewrite.<locals>.<lambda>  s    ! rY   )	fun_jaxprrz  r  rW  r  r  r   )	r  r3  r5   r  _memoize_rewrite_bwdr   custom_vjp_call_jaxpr_pr   )rM   r  r~  rz  r  rW  r  r  r   r   
fun_jaxpr_r  r{   r|  bwd_rZ  r  r{  s   `  `            @@rW   _custom_vjp_call_jaxpr_rewriter    s      #C c
"
""4T9fMM*g&:,//*!W(;/ / / / / / / ;/ 
c4!4!4!4!4g	>	>$		3	8z3Cy
Q 
Q 
Q$ $0HQKK'	wrY   c               .    t          | |j        |          S r`   r:  )rM   r~  r  r{   s       rW   _custom_vjp_call_jaxpr_checkr    s    	D)/6	2	22rY   c                   d |D             }|sJ |d d         |dd          k    sd}t          |          |d         gt          |j        j                  z  S )Nc                    g | ]}||S r`   ra   r  s     rW   r   z'_linear_solve_check.<locals>.<listcomp>  r  rY   r   r   zshard_map check_rep rewrite failed. Please open an issue at https://github.com/google/jax/issues and as a workaround pass the check_rep=False argument to shard_mapr   )r  r   solver  )rM   const_lengthsjaxprsr  r  r   s         rW   _linear_solve_checkr    sl    00000'	---	"	$	$3C C..
!*FL233	33rY   c                    |S r`   ra   )rM   r  y_reps      rW   
_tie_checkr    s    	,rY   batching.BatchTracer   Sequence[batching.BatchTracer]c
           
        t          d |D                       \  }
}t          d |D                       r |j        |g|
R |||||	dS t          d |D                       rt          t          j        || j        t          |                    \  }d t          ||          D             }| j
        Id |D             }t                    |z  rt          d          fdt          ||          D             }t          	          fd
            }t          ||||||	          } |j        |g|
R i |}t          t
          j        | t#          j                              }t'          ||                       S )Nc              3  2   K   | ]}|j         |j        fV  d S r`   )r  	batch_dimr  s     rW   rk   z#_shard_map_batch.<locals>.<genexpr>  s+      EEQQUAK0EEEEEErY   c              3  2   K   | ]}|t           j        u V  d S r`   r:   
not_mapped)rh   bdims     rW   rk   z#_shard_map_batch.<locals>.<genexpr>  s*      99$	$999999rY   r~   c              3  J   K   | ]}t          |t          j                  V  d S r`   )r   r:   
RaggedAxis)rh   r   s     rW   rk   z#_shard_map_batch.<locals>.<genexpr>  s/      ==Ax*	+	+======rY   c                6    g | ]\  fd D             S )c                L    i | ] }|t           j        uo|k    z   |         !S ra   r  rh   axr   r   s     rW   r   z/_shard_map_batch.<locals>.<listcomp>.<dictcomp>   sE     $ $ $ ("55A!r'BE"I $ $ $rY   ra   rh   r   r   s    @@rW   r   z$_shard_map_batch.<locals>.<listcomp>   sZ     L L L(0q$ $ $ $ $"$ $ $ L L LrY   c                J    h | ] }|                                 D ]	}|D ]}|
!S ra   r  rh   r   r   r   s       rW   r+  z#_shard_map_batch.<locals>.<setcomp>$  s:    III%ellnnIIbIIAIIIIIrY   z7vmap spmd_axis_name cannot appear in shard_map in_specsc                F    g | ]\  }}|t           j        uri ||in|S ra   r  rh   r   r   spmd_axis_names      rW   r   z$_shard_map_batch.<locals>.<listcomp>'  sV     E E E %A 129L0L0L,r,1^,,,E E ErY   rH  c                 F    t                                               S r`   )_batch_out_names)out_dimsr}   r  s   rW   rO  z-_shard_map_batch.<locals>.new_out_names_thunk)  s#    NHHJJ8I8IJJJrY   )r  )r/   r   r   r   r  r:   batch_subtracerp  rt   r   r  r  rs   r1   r`  r   BatchTracerr   r  ru   )ru  ry  r   r{  rM   r   r}   rQ   r   rS   r=  in_dimsnew_in_namesusedrO  re  r  make_tracerr  r  s         `           @@rW   _shard_map_batchr    s?    EE*EEEEE'79999999 149S 17 1 1%4	$41 1 1 1 	==W===== 
)#uz5>>JJ-#xL L47'4J4JL L L,'.II8IIID
>T! RPQQQE E E E),\7)C)CE E EL000K K K K K K 10K $79#$0 0 0* TYs3W333
33(,e$4$<$>$>@ @ @+	[(HHJJ	/	//rY   c                    ~~~~~~t          d |D                       \  }| j        fd}	t          t          | j                  }
||	|
ffS )Nc              3  >   K   | ]}|j         |j        |j        fV  d S r`   )r  r  r  r  s     rW   rk   z0_shard_map_batch_post_process.<locals>.<genexpr>9  sC       2 2 ! UAK? 2 2 2 2 2 2rY   c                                                     }t          t          t          j        |          |           S r`   )rq  ru   r   r:   r  )valsru  dimsmr  s     rW   rv  z+_shard_map_batch_post_process.<locals>.todo<  s5    !!Ewx+U33T4FFFrY   )r0   rp  r   r  r  )ru  r  rM   r   r}   rQ   r   rS   r  rv  out_names_transformr  r  r  s              @@@rW   _shard_map_batch_post_processr  6  s    
Hoy'4 2 2%02 2 2 2 2$dj!G G G G G G G   0%2FMM	)*	**rY   c                     d t          ||          D             } Id |D             }t                     |z  rt          d           fdt          ||          D             }|S )Nc                6    g | ]\  fd D             S )c                L    i | ] }|t           j        uo|k    z   |         !S ra   r  r  s     rW   r   z/_batch_out_names.<locals>.<listcomp>.<dictcomp>D  sE     # # # q 33?R@%) # # #rY   ra   r  s    @@rW   r   z$_batch_out_names.<locals>.<listcomp>D  sZ     I I I'/ua# # # # #!# # # I I IrY   c                J    h | ] }|                                 D ]	}|D ]}|
!S ra   r  r  s       rW   r+  z#_batch_out_names.<locals>.<setcomp>G  s:    JJJ%u||~~JJrJJ!AJJJJJrY   z8vmap spmd_axis_name cannot appear in shard_map out_specsc                F    g | ]\  }}|t           j        uri ||in|S ra   r  r  s      rW   r   z$_batch_out_names.<locals>.<listcomp>J  sP     > > >#b! /0x7J.J.J*R*>***> > >rY   )r   r  rs   )r  r  rK  
out_names_r  s   `    rW   r  r  C  s    I I36y$3G3GI I I*JJ9JJJD
>T! SQRRR> > > >'*:t'<'<> > >J	rY   c
                    t          d |D                       \  }
}d |D             }d |D             }t          |
|f          \  }}t          j        | j                  }t          j        |          \  }d t          ||          D             }t                    fd            }t          |g ||R ||||	          }t          j	        ||          \  }} |j
        |g|R i |}t           |            |          \  }}d t          ||          D             } fd	t          ||          D             S )
Nc              3  2   K   | ]}|j         |j        fV  d S r`   primaltangentr  s     rW   rk   z!_shard_map_jvp.<locals>.<genexpr>S  s+      DDqah	2DDDDDDrY   c                D    g | ]}t          |          t          j        uS ra   r   r>   Zeror  s     rW   r   z"_shard_map_jvp.<locals>.<listcomp>T  s%    FFF477"')FFFrY   c                L    g | ]!}t          |          t          j        ur|nd "S r`   r  r  s     rW   r   z"_shard_map_jvp.<locals>.<listcomp>U  s.    FFF477"'))aatFFFrY   c                    g | ]	\  }}||
S ra   ra   rh   r  nzs      rW   r   z"_shard_map_jvp.<locals>.<listcomp>Y  s!    EEEVR"EbEEErY   rH  c                 f                 } g | d t          |                        D             R S )Nc              3  $   K   | ]\  }}||V  d S r`   ra   r  s      rW   rk   z>_shard_map_jvp.<locals>.new_out_names_thunk.<locals>.<genexpr>^  s+      JJfb"rJrJJJJJJrY   r   )out_axr}   which_nz_outs    rW   rO  z+_shard_map_jvp.<locals>.new_out_names_thunk[  sB    _FKVKJJFLLNN(C(CJJJKKKrY   r~   c                    g | ]A\  }}|8t          j        t          j        |                                                    n|BS r`   )r>   r  r   r/  at_least_vspace)rh   r   rM  s      rW   r   z"_shard_map_jvp.<locals>.<listcomp>e  sW     ; ; ;Q AB	q))99;;<<<q ; ; ;rY   c                B    g | ]\  }}t          j        ||          S ra   )r>   	JVPTracer)rh   r   rM  ru  s      rW   r   z"_shard_map_jvp.<locals>.<listcomp>g  s+    	M	M	M1",ua
#
#	M	M	MrY   )r/   r@   r>   jvp_subtracerp  nonzero_tangent_outputsr   r1   r`  	traceabler   rA   )ru  r   rL   rY  rM   r   r}   rQ   r   rS   primalstangentswhich_nzr   r   f_jvptangent_in_namesrO  rd  r|   result
primal_outtangent_outr  s   `     `                @rW   _shard_map_jvpr  Q  s   DDGDDDDD'8FFXFFF(FFXFFF(233-$
/!UZ
(
(%2599%EEs8X'>'>EEE000L L L L L 10L T$Bh$B1A$B$B 3yd, , ,& L00/%;E3D333F33&*88::v>>*k; ; [99; ; ;+	M	M	M	MJ0L0L	M	M	MMrY   c                    ~~~~~~t          d |D                       \  }}	t          ||	f          \  }
d |	D             | j        fd}fd}|
||ffS )Nc              3  2   K   | ]}|j         |j        fV  d S r`   r  r  s     rW   rk   z._shard_map_jvp_post_process.<locals>.<genexpr>m  s+      HHqah	2HHHHHHrY   c                D    g | ]}t          |          t          j        uS ra   r  r  s     rW   r   z/_shard_map_jvp_post_process.<locals>.<listcomp>o  s%    :::Aa':::rY   c                    t          |           \  }}t          t          t          j                                                  ||          S r`   )rA   ru   r   r>   r  rq  )rc   r  r  r  r  s      rW   rv  z)_shard_map_jvp_post_process.<locals>.todoq  sA    &w22GXwr|Q%8%8%:%:;;WhOOOrY   c                B    g | d t          |           D             R S )Nc              3  $   K   | ]\  }}||V  d S r`   ra   )rh   r   r  s      rW   rk   zK_shard_map_jvp_post_process.<locals>.out_names_transform.<locals>.<genexpr>u  s+      KK2K!KKKKKKrY   r  )rK  tangents_nzs    rW   r  z8_shard_map_jvp_post_process.<locals>.out_names_transformt  s/    LYLKKY)D)DKKKLLLrY   )r/   r@   rp  )ru  r  rM   r   r}   rQ   r   rS   r  r  rp   rv  r  r  r  r  s                @@@rW   _shard_map_jvp_post_processr  j  s    
Hoy'4HHKHHHHH'8w122,#w:::::+j!P P P P P PM M M M M	t()	))rY   c
           	        *+,- d |D             }
t          j        |
          \  }}}t          j        ||          \  },t          |          *t	          t          t          |          ||          }t          j        | j        d          }t          |          }t          j
        |g |R g |R           \  }+t                    *+fd            }t          |g ,R ||||	          } |j        |g|R i |} +            \  }}}}}}t          d t          ||          D                       }t!          |t#          |          |z
  g          \  }}|j        rJ t          j        |                       \  }} |            -t'          |||||          }*,-fdt          ||          D             } g | R i ft#          |          z  z   g |R z   }t	           j        |          }!t	           j        |          }"d |D             }#t          ||||d||		          }$t	          t          t,          |          ||          }% fd
|%D             }&t/          j        |j        |j                  }'t          j        g |!|"|#R |&||$|'t9          j                              }(|&D ]	})|(|)_        
t          j        ||&|          S )Nc                    g | ]	}|j         
S ra   pvalr  s     rW   r   z+_shard_map_partial_eval.<locals>.<listcomp>{  s    &&&af&&&rY   FrH  c                                  \  } }}}}}t          j        | 	                      \  }}t          d t          | |          D                       }g |dif|z  R S )Nc              3  ,   K   | ]\  }}|d u o|d u V  d S r`   ra   rh   f1f2s      rW   rk   zC_shard_map_partial_eval.<locals>.known_out_names.<locals>.<genexpr>  s3      OOB"*+tOOOOOOrY   r   )r  r3   sumr   )
in_fwdout_fwd
out_knownsr{   r]  out_known_namesnum_res	all_namesauxr}   s
          rW   known_out_namesz0_shard_map_partial_eval.<locals>.known_out_names  s    /2suu,FGZE1*:7H7HIIAOO#fg:N:NOOOOOG;_;I07:;;;rY   r~   c              3  ,   K   | ]\  }}|d u o|d u V  d S r`   ra   r  s      rW   rk   z*_shard_map_partial_eval.<locals>.<genexpr>  s3      MMfb"d
)rTzMMMMMMrY   c                F    g | ]\  }}||         n||         ndiS r  ra   )rh   r  r  r  known_in_namesknown_out_names_s      rW   r   z+_shard_map_partial_eval.<locals>.<listcomp>  sV     B B B"("b &(^~b!!')~##9~B B BrY   c                :    g | ]}|                                 |S ra   is_knownr  s     rW   r   z+_shard_map_partial_eval.<locals>.<listcomp>  s%    <<<1qzz||<Q<<<rY   r  c                v    g | ]5}t          j        t           j                            |          d           6S r`   r  JaxprTracer
PartialValunknownrh   r   ru  s     rW   r   z+_shard_map_partial_eval.<locals>.<listcomp>  sG     % % % r}'<'<Q'?'?FF % % %rY   ) r  partition_pvalsr3   _all_mesh_namesru   r   r  trace_to_subjaxpr_nounits_fwd2rp  _promote_scalar_residualspartial_eval_wrapper_nounitsr1   r`  r   r  r   r5   r   r  r6   new_instantiated_constrS  r  r   r  r  r   new_eqn_reciper   r  reciper4   ).ru  r   rL   rY  rM   r   r}   rQ   r   rS   in_pvals	in_knownsr   	in_constsunk_in_namesin_avals_shardedf_knownr  known_paramsrp   r  r  r  r  r]  r  r  
out_constsnon_fwd_resunk_out_namesr{   res	res_namesconst_tracersenv_tracersunk_arg_tracers
unk_paramsr  r  r  r  rM  r  r  r  r  s.   `     `                                   @@@@rW   _shard_map_partial_evalr  y  s   &&g&&&(#%#5h#?#? )Xy!#!29h!G!G,d##)d33\8LL'5:u==!""!09,*,,. .,'3 000< < < < < < 10< 4*<N*<*<&5%D2 2 2, 	=9=====#?Bsuu<&':0%MMFG8L8LMMMMM'&sSXX-?,@AA*k_&z??3D3DEE-$_&&67Iz;GG#B B B B B B,/,@,@B B B) 9"S!114Dl4D4DD,e2C88-E$c**+<<<<</+5E#$0 0 0* '-..?PQQ)% % % %#% % %+		't	G	G$
JMJKJ/JJ%{J 0 8 : :	< 	<# &&a3	
K	<	<<rY   c                P   ~t                    d |D             }t          j        g |          \  d j        D             }	d t	          j                  D             t          |	          t          j        d |D                       \  }
g |
}| j        t          j	        j
                                                  5  t          j                  d d d            n# 1 swxY w Y   fd}fd}d |||ffS )Nc                :    g | ]}|                                 |S ra   r  r  s     rW   r   z8_shard_map_partial_eval_post_process.<locals>.<listcomp>  s%    888q1::<<8888rY   c                <    g | ]}t          |j        d d           S )r   T)r   r   r  s     rW   r   z8_shard_map_partial_eval_post_process.<locals>.<listcomp>  s)    
G
G
G!wqvw---
G
G
GrY   c                    g | ]=\  }}t          |j        d d          s t          j                            |d          n|>S )r   Tr   )r   r   rG  r#   	broadcast)rh   rc   r  s      rW   r   z8_shard_map_partial_eval_post_process.<locals>.<listcomp>  sV     	0 	0 	0Q ,3167D+I+I	P		1d	#	#	#q 	0 	0 	0rY   c                    g | ]	}|j         
S ra   r  r  s     rW   r   z8_shard_map_partial_eval_post_process.<locals>.<listcomp>  s    6O6O6O!qv6O6O6OrY   c           	                                          t          | t          |           t                    z
  g          \  }}t          j        |          }t          j                  }dift          |          z  i ft                    z  z   }t          |g R d          }t          t          t                              }fd|D             }	                                }	t          j                                        |	          }
t          j        j        j                  }t#          j        g ||R |t&          |||
          }|D ]	}||_        
t+          ||          S )Nr   F)r]  rM   r   rK  rQ   r   rS   c                v    g | ]5}t          j        t           j                            |          d           6S r`   r  r  s     rW   r   zF_shard_map_partial_eval_post_process.<locals>.todo.<locals>.<listcomp>  sG     ' ' ' >%)>)>q)A)A4HH ' ' 'rY   )r  )rq  r5   r   ru   r  rS  r`  r   r  _current_truncated_name_stackr   r  r  r   r  r  r   r  r  r   r  r4   )rp   r  res_r  r  staged_in_namesstaged_paramsr  r  r  sourcer  r  rM  ru  r  rS   r  r]  r7  rp  rM   r  r  out_names_unknownr  r   s                 @rW   rv  z2_shard_map_partial_eval_post_process.<locals>.todo  s   ""$$E!#C3s88(;'<==J4d;;Me&,,K9~'#d))3rec#hh6FFOvD?#8%6#8#8E!(t5 5 5M GM4002CZPPI' ' ' '%' ' 'K4466J%''//:/FFF)%-IID

:m:k::K'fF FC((S!((z;
;;;rY   c                `    t          |           \  }g |R dift                    z  z   S r  )r3   r   )rK  out_names_knownr  r  r-  r  s     rW   r  zA_shard_map_partial_eval_post_process.<locals>.out_names_transform  s=    )7
I)N)N&_1i.!2SXX!===rY   )r  r  tracers_to_jaxprr  r   _promote_scalar_residuals_jaxprr  rp  r   r  r   r   r  )ru  rY  rM   r   r}   rQ   r   rS   unk_tracerswhichr  rp   rv  r  r  r  r]  r7  rp  r  r  r-  r  s     `   ``      @@@@@@@@@rW   $_shard_map_partial_eval_post_processr4    s   d##)88G888+'K88/%c
G
Gu
G
G
G%	0 	0eo..	0 	0 	0#
)%
7
7%#%#56O6Ow6O6O6O#P#P *j&&3#	$tz//1122 / /'..F/ / / / / / / / / / / / / / /< < < < < < < < < < < < < < < <,> > > > > > > > $(	t()	))s   C==DDc               /     K   | |fV \  }\  }}}}}d t          |||j                  D             }t          ||          }d |D             }||||||ffV  d S )Nc                @    g | ]\  }}}|d u o|d u o|j         j         S r`   r   r   )rh   r  r  r  s       rW   r   z-_promote_scalar_residuals.<locals>.<listcomp>  sM     E E Er2q :9"*9QV\)9 E E ErY   c                t    g | ]5}t          |d d          s t          j                            |d          n|6S )r   ra   r#  )r   rG  r#   r$  r  s     rW   r   z-_promote_scalar_residuals.<locals>.<listcomp>  sP     % % % 3:!Wb2I2IP!!!T***q % % %rY   )r   r  r1  )	r   r6  r]  in_fwdsout_fwds	out_pvalsr  r  r3  s	            rW   r	  r	    s      AEv;M;M8%	8'8Y
CE E5?CCE E E%
)%
7
7%% %#% % %*9j#>>>>>>>rY   c                     t           j         fd            }d t           j                  D             }g |d  j        D             }t          j        ||          \   }}\    S )Nc                     t          | t          j                  g          \  }}d t          |          D             }t	          j        |g|R  S )Nc                :    g | ]\  }}|rt          |          n|S ra   rh  rh   rc   ws      rW   r   z@_promote_scalar_residuals_jaxpr.<locals>.fun.<locals>.<listcomp>  s.    
E
E
ETQ(>!q
E
E
ErY   )r5   r   r  r   r   rc  )res_and_argsr  r   r]  r3  s      rW   r   z,_promote_scalar_residuals_jaxpr.<locals>.fun  sX    <#eo*>*>)?@@IC
E
ESe__
E
E
EC?5#-----rY   c                ^    g | ]*\  }}|rt          j        d dd|j                  n|j        +S r   Nr   r   unmapped_avalr   )rh   r  rA  s      rW   r   z3_promote_scalar_residuals_jaxpr.<locals>.<listcomp>  sK     8 8 81a :;Ft!!T1af555 8 8 8rY   c                    g | ]	}|j         
S ra   r  r  s     rW   r   z3_promote_scalar_residuals_jaxpr.<locals>.<listcomp>  s    999qAF999rY   )r   r   r   r  r  r  trace_to_jaxpr_dynamic)r]  r3  r   	res_avalsr   r{   s   ``    rW   r1  r1    s    <. . . . . <.8 8u668 8 8):y:99EL999:(-c8<</%Ar	,rY   c                v    d |                                 D             fdt          |           D             S )Nc                    h | ]	}|D ]}|
S ra   ra   r*  s      rW   r+  z _unmentioned2.<locals>.<setcomp>  r,  rY   c                    g | ]}|v|	S ra   ra   r.  s     rW   r   z!_unmentioned2.<locals>.<listcomp>  s#    	@	@	@ax.?.?!.?.?.?rY   )r  r  r0  s     @rW   _unmentioned2rM    sB     65%,,..555(	@	@	@	@_T**	@	@	@@rY   c          
     ,   d fdt          ||           D             } fdt          |          D             }t          | |f          \  }	}
t          j        fd            }t	          j        |          \  }t          ||
          \  }}d t          ||           D             d t          |          D             z   }fd}t          j        |g|	R t          |          |||d}t           |            |          S )	Nc                    |dk    r| |z  n| S rj  ra   )rc   ys     rW   rd   z&_shard_map_transpose.<locals>.<lambda>  s    aAQ rY   c                "   g | ]\  }}t          |          t          j        u r(t          j        t          ||j                            nBr|n> |t          t          j        j        t          |                                        S ra   )
r   r>   r  r  r   r   ru   r   r  rM  )rh   r   rc   mb_divrM   r   s      rW   r   z(_shard_map_transpose.<locals>.<listcomp>  s     , , , "a 8<Aww"'7I7IRW[r1622333 I116!T#djnmD".E.EFFGGHH, , ,rY   c           	         g | ]J\  }}t          |          t          j        ur|n't          j        t          ||j                            KS ra   )r   r>   UndefinedPrimalr  r   )rh   r   rc   rM   s      rW   r   z(_shard_map_transpose.<locals>.<listcomp>  s^     
, 
, 
,b! Awwb000!!

[r16::
;
;
, 
, 
,rY   c                   t          t          t          j        |          |          \  }}t	          j        t	          j        
          t          t          j        |          d          \  }}}} t          j        |          | }t          j	        |j
        ddg ||R |           }fdt          	|          D             }|S )NFra   c                   g | ]\  }}t          |          t          j        u r(t          j        t          ||j                            n>r|n:t
          j                            |t          t          |                              S ra   )
r   r>   r  r  r   rG  r#   psumrt   rM  )rh   r   rc   rM   r   s      rW   r   z;_shard_map_transpose.<locals>.fun_trans.<locals>.<listcomp>  s     ) ) ) B 8<Aww"'7I7I27=r1622333 =QQW\\!U=r#:#:;;<<) ) )rY   )r3   ru   r>   is_undefined_primalr  partial_eval_jaxpr_nounitsr>  r   jaxpr_as_funbackward_passr]  r   )out_ctsr   r  undefsjaxpr_knownjaxpr_unknownr{   res_reshapedrp   r   r]  rM   r   s            rW   	fun_transz'_shard_map_transpose.<locals>.fun_trans  s     R%;T!B!BDIIKC')'D
us2#94@@%(I (I$K114$[1137L

UB(@,(@(@(@' C) ) ) ) ) 3'') ) )C JrY   c                N    g | ]"\  }}t          |          t          j        u |#S ra   r  rh   r   rc   s      rW   r   z(_shard_map_transpose.<locals>.<listcomp>  s.    GGGTQQrw0F0Fq0F0F0FrY   c                N    g | ]"\  }}t          |          t          j        u |#S ra   )r   r>   rT  rc  s      rW   r   z(_shard_map_transpose.<locals>.<listcomp>  s/    NNNTQDGG2;M,M,Mq,M,M,MrY   c                 b    t          d t                                  D                       S )Nc              3  $   K   | ]\  }}||V  d S r`   ra   )rh   r   r  s      rW   rk   zD_shard_map_transpose.<locals>.new_out_names_thunk.<locals>.<genexpr>  s+      JJ95"rJJJJJJJrY   )rt   r   )r   
nz_arg_ctss   rW   rO  z1_shard_map_transpose.<locals>.new_out_names_thunk  s0    JJHjjll(C(CJJJJJJrY   r~   )r   r@   r   r   r>   nonzero_outputsr7   r   r   rt   rA   )r\  r]  rM   r   rK  rQ   r   rS   r   all_argsr   ra  fun_trans_flatr|   r  rO  r   rR  rg  s    ```  `          @@rW   _shard_map_transposerk    s   ..&, , , , , , y'**, , ,'
, 
, 
, 
,8T**
, 
, 
,$ #GT?33(G<       < ,Y77)Z1)WEE.( HGSG,,GGGNNS4((NNNO K K K K K K  &*U<5H5H)Y  ( 


H	-	--rY   c                    d vr S |s S  fd}t          j         d         j                                                  5  t          j         d         |          }d d d            n# 1 swxY w Y   t           |          S )Nr]  c                >    | d         j         v r| fn
 |           S )NrM   r   )r   rd  substs    rW   shadowed_substz-_shard_map_axis_subst.<locals>.shadowed_subst,  s(    fVn222D77dCrY   rM   r]  )r   r  r   r   subst_axis_names_jaxprr`  )rd  rn  traversero  rl  s   ``   rW   _shard_map_axis_substrs  '  s    FM	 MD D D D D Dvf~399;;<< M M+F7O^LLIM M M M M M M M M M M M M M M	fI	&	&	&&s   A,,A03A0saveableCallable[..., pe.RematCases_]unks_inSequence[bool]inst_inr  core.JaxprEqnStuple[core.JaxprEqn, core.JaxprEqn, Sequence[bool], Sequence[bool], list[core.Var]]c                ,   |j         d         |j         d         c}t          j        j                                                  5  t          j        |||dd|           \  }}}}}	d d d            n# 1 swxY w Y   t          |j                  |	z
  }
t          j	        |          |
d          }t          |j        |
g          \  }}d t          |          D             fd|D             }d t          ||          D             }t          j        |j         d         j                                                  5  t          j        |dg|
z  |z             }t          |||          \  }}d d d            n# 1 swxY w Y   t          j        |j                  }t          j        |j                  }t#          ||j                  \  }}t#          ||j                  \  }t#          ||j                  \  }}t#          ||j                  \  }}t          j                    t)          ||t+          t,          j        |          ||||t1          |j         |          t1          |j         |          	  	        \  }}fd	t          |j        d |	         |          D             }t          j        |g ||j        ||j        |j                  }t;          ||||          }t          j        g ||||j        ||j        |j                  }t          |j                  t          |j                  k    sJ d
 t          |j        |          D             }|fdd |D             D             z  }||||||z   fS )Nr]  rM   Fc                4    i | ]\  }}t          |          |S ra   )id)rh   ri   r  s      rW   r   z3_partial_eval_jaxpr_custom_rule.<locals>.<dictcomp>A  s$    666$!QRUUA666rY   c                T    g | ]$}                     t          |                    %S ra   )r  r}  )rh   r  idx_maps     rW   r   z3_partial_eval_jaxpr_custom_rule.<locals>.<listcomp>B  s+    222AW[[A222rY   c                $    g | ]\  }}|d u o|d u S r`   ra   r  s      rW   r   z3_partial_eval_jaxpr_custom_rule.<locals>.<listcomp>C  s)    
H
H
HR2:$"*
H
H
HrY   Trp  c           	     \    g | ](\  }}| t          d i|j                            )S r-  )r  r   )rh   varrA  r  rM   newvars      rW   r   z3_partial_eval_jaxpr_custom_rule.<locals>.<listcomp>Q  sT     M M M3!MvvmD1i.#(CCDD M M MrY   c                R    g | ]$\  }}t          |          t          j        u  |"|%S ra   r  )rh   rc   insts      rW   r   z3_partial_eval_jaxpr_custom_rule.<locals>.<listcomp>[  s=     3 3 3GAtaDH$$T$ $$$rY   c                     g | ]
}|         S ra   ra   )rh   rL   out_binders_knowns     rW   r   z3_partial_eval_jaxpr_custom_rule.<locals>.<listcomp>]  s    SSS #SSSrY   c                    h | ]}||S r`   ra   )rh   ri   s     rW   r+  z2_partial_eval_jaxpr_custom_rule.<locals>.<setcomp>]  s    -R-R-RAAMaMMMrY   )rd  r   r  r   r   r  partial_eval_jaxpr_customr   r  _jaxpr_forwardingr5   r   r   prune_jaxpr_outputs_add_reshapesremove_named_axis_effectsr   r3   r  gensym_pe_custom_paramsru   rn  not_r`  r  r  r  r  r6   ) rt  rv  rx  r  r]  r^  jaxpr_stagedunks_outinst_outr  num_out_primalsr  out_varsres_varsr  r3  	ins_knownr{   
ins_stagedout_binders_stagedparams_knownparams_staged	residuals	eqn_knownfull_res
eqn_stagednew_instr  r  rM   r  r  s                               @@@@@rW   _partial_eval_jaxpr_custom_ruler  5  sw   
 
7#SZ%7+%tz//1122 V V
$UGWeUHUU ;Kx7V V V V V V V V V V V V V V V +,,w6/,,_-=-=>&!+"57HII(H66)H"5"5666'2222222'
H
H3vw3G3G
H
H
H%sz&17==??@@ P P(tf6NQV6VWWK -e[, O OKP P P P P P P P P P P P P P P .{DOLL+/doNN,44,)Q'#+>>Q #*55-!Z(3;??!;==&+<wBGX..&'5
3:[)))4
,+O+O+O,Q ,Q(,yM M M M M M !4XgX!>FFM M M)y*J,=*J	*J"}lK<O"0 0) 4EyQQ( 8( 8Z 8:L #} , 4coG G* 
Z		3|':#;#;	;	;	;	;3 3s3:w77 3 3 3(
SSSS-R-R-R-R-RSSSS(	J(Hy4H	HHs$    A88A<?A<1FF
Fc           	     p    d t           j        d t                                        D             t                    sfS j        sj        rJ t
          j         fd            }d j        D             }t          j        ||          \  }\   \   t
          j         fd            }d t          j        d t                                        D             }g |d j        t                     d          D             }t          j        ||          \  }\   \   fS )Nc                2    g | ]\  }}|o|j         j         S ra   r7  rh   rA  r  s      rW   r   z!_add_reshapes.<locals>.<listcomp>d  s>     F F F1 "afl" F F FrY   c                     t          j        dg| R  }t          |t          |          t	                    z
  g          \  }}d |D             }g ||S )Nra   c                >    g | ]}|j         st          |          n|S ra   )r   rM  r  s     rW   r   z0_add_reshapes.<locals>.known.<locals>.<listcomp>m  s,    
@
@
@qAG2>!
@
@
@rY   )r   rc  r5   r   r  )r   rp   	out_knownr  r^  r3  s       rW   knownz_add_reshapes.<locals>.knowni  sf    
/+r
1D
1
1
1Cc#hhU&;%<==NIs
@
@C
@
@
@CYrY   c                    g | ]	}|j         
S ra   r  r  s     rW   r   z!_add_reshapes.<locals>.<listcomp>o  s    111af111rY   c                     t          | t                    g          \  }}d t          |          D             }t          j        dg||R  S )Nc                :    g | ]\  }}|rt          |          n|S ra   r?  r@  s      rW   r   z1_add_reshapes.<locals>.staged.<locals>.<listcomp>u  s.    
G
G
GTQ(>!q
G
G
GrY   ra   )r5   r   r   r   rc  )r   r)  insr  r  r3  which_s       rW   stagedz_add_reshapes.<locals>.stagedr  sZ    4#e**..ID#
G
GSv5F5F
G
G
GC?<8c8C8888rY   c                ^    g | ]*\  }}|rt          j        d dd|j                  n|j        +S rD  rE  r  s      rW   r   z!_add_reshapes.<locals>.<listcomp>w  sQ     J J J1a :;Ft!!T1af555 J J JrY   c                    g | ]	}|j         
S ra   r  r  s     rW   r   z!_add_reshapes.<locals>.<listcomp>y  s    MMMqAFMMMrY   )	r   r  r   r   r  r   r   r  rH  )	r3  r^  r  r  r  r{   r  rI  r  s	   ```     @rW   r  r  b  s   F FE<#6{E

{#CDDF F F&	V6[,66"A<+AAAA<     <
 21k0111(4UHEE+q"b<9 9 9 9 9 9 <9J Jv|':;CJJ;'GHHJ J J)NyNMML,?E

,LMMMN(5fhGG,2r	l	""rY   c	                0   |d         }	t          |	          t          | |d                   \  }
t          ||d                   \  }
digt          |          z  z   t          |t	                    t	                              }t          ||d                   \  }
}fdt          ||          D             }||z   }t          ||d                   \  }
}t          |t	          |          t	          |          d          }||fS )	NrM   r   rK  r   )r   rK  c                F    g | ]\  }}||         n||         ndiS r  ra   )rh   r  r  r  in_names_knownr/  s      rW   r   z%_pe_custom_params.<locals>.<listcomp>  sU     B B B"("b &(^~b!!&(nr""9~B B BrY   F)r   rK  rQ   )r  r3   r  r`  rt   r   )rv  rx  kept_outs_knownkept_outs_stagedr  r  r3  r  r  rM   r{   new_params_knownr  r  out_names_stagednew_params_stagedr  r  r/  s                   @@@rW   r  r  ~  s_    
f	$d##)$Wl:.FGG.!%o|K7PQQ!_#9~&6U&CC/,~1F1F$)/$:$:< < < &g}Z/HII!_B B B B B B,/,@,@B B B) //&'7{9STT!=53I3I%*+;%<%<O O O	,i	77rY   tuple[AxisName, ...]c                    t           j        j        j        j        }fd|D             t          fd| j        D                       S )Nc                X    h | ]&}|j                             d d          xD ]}|'S )r  ra   )payloadr  )rh   r  r   r   s      rW   r+  z"_all_mesh_names.<locals>.<setcomp>  sR       ]&&'7<<<rI JI IIIIrY   c              3  $   K   | ]
}|v|V  d S r`   ra   )rh   r   r   s     rW   rk   z"_all_mesh_names.<locals>.<genexpr>  s-      EE4u3D3Dt3D3D3D3DEErY   )r   thread_local_statetrace_statetrace_stackstackrt   r   )rM   r  r   r   s     @@rW   r  r    sc    

!
-
9
?%   %   % 
EEEEEEE	E	EErY   used_outputs
list[bool]'tuple[list[bool], core.JaxprEqn | None]c                   |j         d         }t          j        |j                                                  5  t          j        |j         d         |           \  }}d d d            n# 1 swxY w Y   t          |          st          |           s|j        s|d fS t          ||j         d                   \  }}t          | |j         d                   \  }}t          |j         |t          |          t          |                    }t          j        |j        |j                  }	t          j        d t          |j        |          D             d t          |j        |           D             |j        ||	|j                  }
||
fS )NrM   r]  r   rK  )r]  r   rK  c                    g | ]	\  }}||
S ra   ra   )rh   r  r  s      rW   r   z"_shard_map_dce.<locals>.<listcomp>  s!    ???wq$$????rY   c                    g | ]	\  }}||
S ra   ra   )rh   rc   r  s      rW   r   z"_shard_map_dce.<locals>.<listcomp>  s!    AAAwq$DAAAArY   )rd  r   r  r   r   r  	dce_jaxprr   r  r3   r`  rt   r  r   r  r   r  r  r  r  )r  r  rM   r]  used_inputsr{   r   rK  re  r  new_eqns              rW   _shard_map_dcer    s   	F	$tz//1122 I Icj&9<HHE;I I I I I I I I I I I I I I I	[		  #l"3"3  EM   cj.DEEKAx!,
;0GHHLAycjh %i 0 02 2 2J)%-IID??#cj+66???AA#ck<88AAAz4: :G s   $A))A-0A-ra   )in_axesout_axesstatic_broadcasted_argnumsr  backendr  donate_argnumsglobal_arg_shapesc          	          t                    nt           ||          \   f	dfd}
fd}||
_        |
S )Nc                   	 t          | |
  
        }|j        D ]}t          j        |           t	          t          |          
f          }t          |j        ||j        |j	        
          \  }}}t          |j        |t          |                    }t          j        |d t          |j                  D                       }	|	||j        ||fS )Nc              3  $   K   | ]\  }}||V  d S r`   ra   )rh   ri   r  s      rW   rk   z-pmap.<locals>.infer_params.<locals>.<genexpr>  s+      KKfasKKKKKKKrY   )r  )r"   	flat_argsr   	check_argr   _get_devices_cached_shard_mapflat_funin_axes_flatout_axes_thunkrJ   r   rG  rI  r   donated_invarsr|   )r   r6  r   argrM   _pmappedrN   rP   flat_global_argsjitted_fr  r  r  r  donate_tuplerL   r  r  static_broadcasted_tuples             rW   infer_paramszpmap.<locals>.infer_params  s   a(,D"GWiv	O 	OA{  Q((9,77D$5	
D!.!*:I%G %G!Hh	7	T4>>+ +wKK	!2B(C(CKKKM M MH %qz4BBrY   c                      | i |\  }}}}} || }t          || |                      }t           |            |          S r`   )rK   rA   )	r   r6  r  r  r|   rM   rP   rZ  r  s	           rW   r   zpmap.<locals>.wrapped  s`    ,///X48%&D+D$		DDD((**d+++rY   c                 :     | i |\  }}}}} |j         | i |S r`   )lower)r   r6  r  r{   r  s       rW   r  zpmap.<locals>.lower  s9    '<888HaAq8>4*6***rY   )rt   r!   r  )rL   r  r  r  r  r  r  r  r  r  r   r  r  r  r  s   ```` ```    @@@rW   pmapr    s     &1E'NNNw'6G.7S 7S3)%|C C C C C C C C C C C C C, , , , ,+ + + + + '-	.rY   c           	         t          t          t          t                    |                    }fd}t	          | |          }t          |j        |||dt                                ||fS )Nc                 Z    t          t          t                                            S r`   )ru   r   _axis_to_spec)r  r  s   rW   rd   z#_cached_shard_map.<locals>.<lambda>  s#    c'-;;^^=M=MNN rY   F)rQ   rS   )rt   ru   r   r  _handle_reshapesrV   r4  r}  )r  rM   r  r  r  rN   rP   r   s      ``   rW   r  r    s|    3w}i88,GGHH(NNNNN)<@@#
S%tXy$9;;8 8 8
I
 rY   c              /     K   t          d t          |          t          |                     }|i fV }t          d t          |          t           |                                V  d S )Nc                6    || nt          j        | |          S N)r  )jnpsqueezerc   r  s     rW   rd   z"_handle_reshapes.<locals>.<lambda>  s    RZS[5L5L5L rY   c                6    || nt          j        | |          S r  )r  expand_dimsr  s     rW   rd   z"_handle_reshapes.<locals>.<lambda>  s    BJqqCOAB4O4O4O rY   )r?   r   )r  r  r   r6  rp   s        rW   r  r    s      	LLt**d7mm
- 
-$Bh#OOcD!1!122	4 	4 4 4 4 4 4rY   c                    t          |t                    rd g|z  | gz   }t          | S |t                      S t          |          r`   )r   ri  r   r   )r  r  r   s      rW   r  r    sJ    C FRK9+%Ee9	z33J
B--rY   c                    || j         t          j         |          }n!| j         t          j                     n| j         }t          j                    dk    r|d | j                 S |d | j                 S )N)r  r   )r  rG  process_countglobal_axis_sizelocal_axis_size)r   r  devss      rW   r  r    sw    QY.;w'''DDI-3;===19D1###$$	 q  	!!rY   c                  V    e Zd ZU ded<   ded<   d Zedd            Zdd	ZddZeZ	dS )RewriteTracerr  r  Valr  c                0    || _         || _        || _        d S r`   r  r  s       rW   rq  zRewriteTracer.__init__  r  rY   r   r  c                4    t          j        | j                  S r`   )r   r/  r  r  s    rW   r   zRewriteTracer.aval  s    ="""rY   c                    | S r`   ra   r  s    rW   rU  zRewriteTracer.full_lower  r  rY   r   c                *    t          | j                  S r`   )r   r  r  s    rW   r  zRewriteTracer.__str__  s    tx==rY   N)r   r  r   r  r  r  ra   rY   rW   r  r    s         
(((  
 # # # 8#      (((rY   r  c                  v     e Zd ZU ded<   ded<    fdZddZddZddZd Zd Z	d Z
d Zd Zd Zd Z xZS )RewriteTracer   rM   ri  dynac               P     t                      j        |  || _        || _        d S r`   )rp  rq  rM   r  )rW  rM   r  r   rr  s       rW   rq  zRewriteTrace.__init__  s)    EGGdDIDIIIrY   r   r  c                R    t          | t          | j        j                  |          S r`   r  r  rM   r   )rW  r  s     rW   ru  zRewriteTrace.pure!  s!    s49#788#>>>rY   rx  core.Tracerc                R    t          | t          | j        j                  |          S r`   r  rw  s     rW   liftzRewriteTrace.lift$  s!    s49#788&AAArY   c                8    t          | |j        |j                  S r`   )r  r  r  rw  s     rW   ry  zRewriteTrace.sublift'  s    vz6:666rY   c                   t                               |t          t          |                    }t	          d |D                       \  }}t          j        | j                  5   || j        |g|R i |\  }}d d d            n# 1 swxY w Y   t          t          t          |           ||          }	|j        r|	n|	d         S )Nc              3  2   K   | ]}|j         |j        fV  d S r`   r;  r  s     rW   rk   z1RewriteTrace.process_primitive.<locals>.<genexpr>,  *      AAquaenAAAAAArY   r   )r  r  r   r  r/   r   new_dynamicr  rM   ru   r  rh  )
rW  ry  r{  rd  r  r=  in_repsr  out_repsr  s
             rW   r  zRewriteTrace.process_primitive*  s   dGM4$@$@AADAAjAAAAAGW		$)	$	$ H H4	7GWGGGGGhH H H H H H H H H H H H H H HgmT22HhGGK/C;;[^Cs   $BB
B
c                b   t          d |D                       \  }}t          || j        t          |                    \  }}t	          j        | j                  5   |j        |g|R i |}d d d            n# 1 swxY w Y   t          t          t          |            |            |          S )Nc              3  2   K   | ]}|j         |j        fV  d S r`   r;  r  s     rW   rk   z,RewriteTrace.process_call.<locals>.<genexpr>3  r  rY   )r/   _rewrite_subtracerp  rt   r   r  r  r   ru   r   r  )	rW  r  rL   r{  rd  r=  r  r  r  s	            rW   r  zRewriteTrace.process_call2  s    AAjAAAAAGW#Aty%..AAKAx		$)	$	$ < <$$Q;;;;F;;h< < < < < < < < < < < < < < <w}d++XXZZBBBs   A::A>A>c                    J r`   ra   )rW  r  r  rd  s       rW   post_process_callzRewriteTrace.post_process_call9  r  rY   c               F   |rd}t          |          t          d |D                       \  }}t          || j        |          \  }}	t          || j        |dz            \  }}
t	          j        | j                  5   |j        ||g|R d|i}d d d            n# 1 swxY w Y   t          j	        |	|
          \  }}|s=||d t          |          dz           dz  k    sJ |d t          |          dz           }t          t          t          |           ||          S )Nrt  c              3  2   K   | ]}|j         |j        fV  d S r`   r;  r  s     rW   rk   z7RewriteTrace.process_custom_jvp_call.<locals>.<genexpr>B  *      >>quaen>>>>>>rY   r   r  )r  r/   r  rp  r   r  r  r   r   merge_linear_auxr   ru   r   r  )rW  ry  r   r  rY  r  r   r=  r  	out_reps1	out_reps2r  fstr  s                 rW   r  z$RewriteTrace.process_custom_jvp_call<  s    %c  $$$>>g>>>>>GW&sDIw??NC&sDIw{CCNC		$)	$	$ N N3MgMMMnMMhN N N N N N N N N N N N N N N'	9==MC /"53x==A#5"56:::::-3x==A--.hw}d++Xx@@@s   >BB!$B!c                    J r`   ra   )rW  r  jvp_was_runs      rW   r  z)RewriteTrace.post_process_custom_jvp_callM  r  rY   c                b    |rd}t          |          t          d |D                       \  }	}
t          | j        |
          \  }} fd|
D             }t          | j        |          \  }}t	          | j        ||
          }t          j         j                  5   |j	        |||g|	R ||d}d d d            n# 1 swxY w Y   t          j        ||          \  }}|s& |            \  }}t          ||j        g          \  }}t          t          t                      ||          S )Nrt  c              3  2   K   | ]}|j         |j        fV  d S r`   r;  r  s     rW   rk   z7RewriteTrace.process_custom_vjp_call.<locals>.<genexpr>W  r  rY   c                P    g | ]"}|t          j        j                  fD ]}|#S ra   )r  rM   r   )rh   rX  r_rW  s      rW   r   z8RewriteTrace.process_custom_vjp_call.<locals>.<listcomp>Y  s8    PPP!!S9M5N5N1OPP22PPPPrY   )r  r  )r  r/   r  rp  r  rM   r   r  r  r   r   r  r5   rr   ru   r   r  )rW  ry  r   r  r  rY  r  r  r   r=  r  r  fwd_in_repsr  r  r  r  r{   res_trees   `                  rW   r  z$RewriteTrace.process_custom_vjp_callP  s    %c  $$$>>g>>>>>GW&sDIw??NCPPPPwPPPK&sDI{CCNC
sDIy'
:
:C		$)	$	$ 9 93S 97 9 9i)79 9 9h9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 '	9==MC @IKKkax(*=)>??kaw}d++Xx@@@s   !CC	Cc                    J r`   ra   r  s      rW   r  z)RewriteTrace.post_process_custom_vjp_calle  r  rY   r  )rx  r  r   r  )r   r   r   r  rq  ru  r
  ry  r  r  r  r  r  r  r  r  r  s   @rW   r  r    s        ***)))    
? ? ? ?B B B B7 7 7 7D D DC C C  A A A"  A A A*      rY   r  c                    t          t          t                    |          }fd}t          | |          \  } }t	          | ||          S )Nc                 0     fd             D             S )Nc                J    g | ]}t          t          |                     S ra   )r  r   r   s     rW   r   zB_efficient_transpose_rewrite.<locals>.<lambda>.<locals>.<listcomp>l  s+    PPP#l43344PPPrY   ra   )rM   r}   s   rW   rd   z._efficient_transpose_rewrite.<locals>.<lambda>l  s$    PPPPoo>O>OPPP rY   )ru   r   r  $_efficient_transpose_rewrite_nomatch
_match_rep)r   rM   r   r}   r  out_reps_dstout_reps_srcs    ` `   rW   r   r   j  sY    ($//::'PPPPP,:3gNN#|	C|\	:	::rY   c              '    K   t          j                    }t          j        t          d| |          5 }|                                }t          t          t          |          ||          }|i fV }t          |j        |          }t          d |D                       \  }	}
~~~~~d d d            n# 1 swxY w Y   |	|
fV  d S )NTdynamicrM   r  c              3  2   K   | ]}|j         |j        fV  d S r`   r;  r  s     rW   rk   z7_efficient_transpose_rewrite_nomatch.<locals>.<genexpr>x  s*      DD1DDDDDDrY   )
r   dynamic_levelnew_mainr  rq  ru   r   r  rS  r/   )rM   r  r   lvlrp  rM  r{  rt  r  r  r  s              rW   r*  r*  p  s     #}\4dEEE .  AW]A..>>JB

CalC((KDDDDDDDHha[#. . . . . . . . . . . . . . . 	(s   A5B44B8;B8c              '     K   |i fV }t          |          r
 |            n|}t          |          r
 |            n|}t          | ||           d t          |||          D             }|V  d S )Nc           	     x    g | ]6\  }}|z
  r)t          |t          fd |D                                 n|7S )c              3  $   K   | ]
}|v|V  d S r`   ra   r_  s     rW   rk   z(_match_rep.<locals>.<listcomp>.<genexpr>  r`  rY   r  ra  s      @rW   r   z_match_rep.<locals>.<listcomp>  so     
K 
K 
K QS CF) *Q=========
>
>
>
K 
K 
KrY   )rm   ra  r   )rM   out_reps_src_out_reps_dst_r   rZ  r-  r,  s          rW   r+  r+  |  s      Rx$$,]$;$;N,$,]$;$;N,t\<000
K 
K$'lL$I$I
K 
K 
K$*****rY   core.ClosedJaxprSequence[set[AxisName]]out_rep_dstc                   t          j        t          t          j        |j        |j                            }t          || |          \  }}t          || ||          }t          j	        | j
                                                  5  t          j        ||j                  \  }}}\   d d d            n# 1 swxY w Y   t          j        ||          S r`   )r   r   r   r   rc  r]  r  r*  r+  r  r   r   r  rH  r   rv  )	rM   r]  r  r<  rL   r  r7  r{   r  s	            rW   rc  rc    s    	l74?EKFFGG!3AtVDD*!WD';//!tz//1122 I I5aHHFAvrI I I I I I I I I I I I I I I		&&	)	))s   	!B66B:=B:,tuple[core.ClosedJaxpr, list[set[AxisName]]]c                   t          j        t          t          j        |j        |j                            }t          || |          \  }}t          j        | j	        
                                          5  t          j        ||j                  \  }}}\   d d d            n# 1 swxY w Y   t          j        ||           |            fS r`   )r   r   r   r   rc  r]  r  r*  r  r   r   r  rH  r   rv  )rM   r]  r  rL   r  r7  r{   r  s           rW   r3  r3    s    
 	l74?EKFFGG!3AtVDD*!Wtz//1122 I I5aHHFAvrI I I I I I I I I I I I I I I		&&	)	)7799	44s   7!B$$B(+B(c              '    K   t          |          t          |          k    s&J t          |          t          |          f            |                                 }t          t          t          |          ||          }t          j        | j                  5  |i fV }d d d            n# 1 swxY w Y   t          |j        |          }t          d |D                       \  }}||fV  d S )Nc              3  2   K   | ]}|j         |j        fV  d S r`   r;  r  s     rW   rk   z$_rewrite_subtrace.<locals>.<genexpr>  s*      BBquaenBBBBBBrY   )
r   rq  ru   r   r  r   r  rR  rS  r/   )	rp  r  r=  rM  r{  rZ  r  r  r  s	            rW   r  r    s(     	WW	%	%	%Gc'll'C	%	%	%
!7=!,,gw??*
##    RD                             AL$''+BBkBBBBB(H(s   B--B14B1c                      fd}|S )Nc                 D   t          j                    }t          j        t          d|          5 }t	          t          j                  |                       \  }} |j        |  }~d d d            n# 1 swxY w Y   t          t           |            	|          S )NTr/  )
r   r2  r3  r  r  r   r   r4  ru   _match_replication)
r   r4  rp  r  
reps_thunkrp   r  r  rM   reps_dsts
         rW   new_bwdz_rewrite_bwd.<locals>.new_bwd  s    



C	|T3	G	G	G 4*2<+<+<dGGIINNdJDt$c
               !::<<3???s   :A88A<?A<ra   )r  rM   r  rF  rG  s   ```` rW   r  r    s=    @ @ @ @ @ @ @ @ 
.rY   c                      z
  r8t                               |t           fdD                       d           \  } z
  r)t          |t          fd D                                 }|S )Nc              3  $   K   | ]
}|v|V  d S r`   ra   )rh   r   rC  s     rW   rk   z%_match_replication.<locals>.<genexpr>  s'      #C#C!asllAllll#C#CrY   r  c              3  $   K   | ]
}|v|V  d S r`   ra   r_  s     rW   rk   z%_match_replication.<locals>.<genexpr>  s'      ;;!asllAllll;;rY   )r  r   rt   r  )rC  r!  rc   s   `` rW   rD  rD    s    3Y .	ae#C#C#C#Cs#C#C#CCC(, 
 
. 
.BA3Y =1e;;;;s;;;;;<<A	
(rY   )rL   r   rM   r   rN   rO   rP   rO   rQ   rR   rS   rT   )rL   r   rM   r   rN   rO   rP   rZ   rQ   rR   rS   rT   )r   r   r   r   )r   ro   r   r	   r   r   )rL   r   rM   r   r   rG   rN   rO   r   r   r   r   r   r   r   r   )r   rG   r   r   r   r   r   r   )r   ro   rL   r   r   rG   r   rO   r   r   r   r   )rL   r   rM   r   r   rG   r   rO   r   r   r   r   )rL   r   rM   r   r   rG   r   rO   r   r  r   r   )rM   r   r   r   r   r'  )r   rG   r   rO   r   r5  r   r6  )rR  ri  r   r	   )ru  rx  ry  rz  rL   rB  r{  r|  rM   r   r   rD  r}   rE  rQ   rR   r   rR   rS   r}  r   r|  )r   r  r   r  )rM   r   r   r   r   r  r   r  )rM   r   r   r   r   r  )rM   r   r]  r  r  r  r   r  )rM   r   r  r  r!  r   r   rR   )ry  rz  )r  r  )r   r   r   r  r   r   )r   r   r   r   )rM   r   rC  r   rc   rD  r   rD  )r   rO  rP  rQ  r   r   )
rM   r   rQ   rR   rb  r   rc   rD  r   rD  )r   r  r  r  )ru  r  ry  rz  r   rB  r{  r  rM   r   r   rD  r}   rE  rQ   rR   r   rR   rS   r}  r   r  )
rt  ru  rv  rw  rx  rw  r  ry  r   rz  )rM   r   r   r  )r  r  r  ry  r   r  r`   )
rM   r   r]  r:  r  r;  r<  r;  r   r:  )rM   r   r]  r:  r  r;  r   r>  (F  
__future__r   collections.abcr   r   r   enum	functoolsr   r  	itertoolsitmathr   operatorrn  typingr	   r
   r   numpyr  rG  	jax.numpyr  jax.shardingr   r   r   jax._srcr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   jax._src.corer    jax._src.apir!   r"   jax._src.laxr#   r$   lax_parallelr%   r&   r'   r(   r)   r*   r+   r,   jax._src.utilr-   r.   r/   r0   r1   r2   r3   r4   r5   r6   jax.api_utilr7   r8   r9   jax._src.interpretersr:   r;   r<   r  r=   r>   jax.tree_utilr?   r@   rA   rB   rC   rD   jax._src.tree_utilrE   rF   rG   rH   rI    jax.experimental.multihost_utilsrJ   rK   r   safe_mapru   
unsafe_mapsafe_zipr   
unsafe_zipregister_exclusion__file__rO   AxisNamer   r}  rX   rV   r`  ri  rt   r   rv   Enumro   rn   r   r   r   r   r   r   r   r   r   r4  r   rD  rC  	PrimitiverA  r   transformation_with_auxrQ  r  DynamicJaxprTracerT  r  r  r  r  r  r  r  r  r  r  r  custom_typechecksr  r  r  r  r  register_loweringr  r  r  r  r8  	EvalTracer3  r5  r0  rJ  r  r  r   r  r   ra  r6  re  rh  rM  Tracer2  r<  r~  r}  r  r  debug_callback_pr  device_put_pAxisPrimitiver  rh  def_implpsum_pimpldef_effectful_abstract_evalabstract_eval
_lowerings_reduction_batcherprimitive_batchers_batched_reduction_collectiveaxis_primitive_batchers_subst_all_names_in_paramaxis_substitution_rulesr  
deflinear2r  r  def_abstract_evalr  r  r  r  register_standard_rewriteregister_norewriter  r	  register_standard_checkr  r  r  r  r
  r  chain__dict__r  or   rd  cumsum_pcumlogsumexp_p	cumprod_pcummax_pcummin_pr   r  r  r)  r.  all_gather_pall_to_all_p
ppermute_preduce_scatter_paxis_index_pr0  r4  r8  r<  r?  r@  rC  call_prG  rI  pure_callback_prN  io_callback_prQ  rS  custom_lin_prU  re  r[  ri  rk  rm  ro  custom_jvp_call_prr  r  r  r  solveslinear_solve_pr  tie_pr  r  
BatchTracer  rr  r  r  JVPTracer  r  
JaxprTracer4  transformationr	  r1  rM  rk  primitive_transposesrs  r  partial_eval_jaxpr_custom_rulesr  r  r  r  	dce_rulesr  cacher  r  r  r  r  r  r  r   r*  r+  rc  r3  r  r  rD  ra   rY   rW   <module>r     sA   # " " " " " " 8 8 8 8 8 8 8 8 8 8                       & & & & & & & & & &     



       ; ; ; ; ; ; ; ; ; ; " " " " " "                   ' ' ' ' ' '                   & & & & & &                         # # # # # # % % % % % % # # # # # #                   9 9 9 9 9 9 9 96 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ S R R R R R R R R R * * * * * * & & & & & & 4 4 4 4 4 4 & & & & & & $ $ $ $ $ $@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @= = = = = = = = = = = = = =P P P P P P P P -Z-Z ! !( + + + 	  $))++5C 5C 5C 5C 5CnF F F FR eHcM**+	    	/GU+;<<&? &? &? &?P        
&((   (   * * * *X- - - -^" " " "H; ; ; ;
   GCLL	9 	9 	9 	9 GVO$"  "  "  "  "  "  "  " H  ,,: : : :*$ $ $ $J *<  &   D D D DF F F FA A A A
 .A ) *A A A A
 0E 4+ , Ht#
$  ( '; { #I I I I" " " "0C C C C7 7 7 7( ( ((  {$7 8 8 8     Y Y Y Y6 6 6 6   B B B" $3    F F F F
K K K KJ J J
A A A A
 " ! ! ! ! ! ! !@ @ @ !        	      @ @ @
D D D DL L L
 5 4 4 4 4 4R? R? R? R? R?DJ R? R? R?j    T[   B: : : /1 0 0 0 0    +EI& '   &<H! "
 $
W
%
%    $) * * *  # #L$7$E F F F  w0C D E E E'.w|/NPW'X'X G $	',
4g
,
,. . 	   ) GL2F;;  W %Q Q Q g, - - -% % % "t!,// $    AA B B B   JJ K K K  |%O%O P P P  
 -@ L )   2J   .GL2F;;  \ * lQ QR R R 24 3 3 3 3LL K K K K  02 1 1 1 1NNNN     	) 	) 	)F F F	 	 	, , , 
#,%%'')9)@)@)B)B%.5577"))++[-A-H-H-J-J,%%'')?)?)A)A,%%'')9)@)@)B)B-&&((#,*=*=*?*?/((**
, 
, 	! 	!A Z4>"" !Aa   


%|'9'H


&(:(C


%
'  A !A #$$  %$ ,%&&	 	 '&	 	) 	) 	)  7    
) 
) 
)  <        \6 7 7 7  \6 7 7 7  \4 5 5 5  \: ; ; ; )*** * +**  <, - - - $+  
 / / / -'((  )( %&&) ) '&) . . . 	*++  ,+  9- . . . ())% % *)%  8+ , , , &''% % ('%  8) * * * %&&  '&  8( ) ) )   " " ! "  2? # # # ")**
 
 +*
 ,$+,,  -,0 $$%%  &%
 "##4 4 $#4 "4554 4 654 $<==  >=8 ":;;3 3 <;3
 #2334 4 434  ,-< = = = 	    39   
#0 #0 #0 #0H )9  %
+ 
+ 
+ .K  *	 	 	N N N. !/ * * * &A "0= 0= 0=b #: .* .* .*^ (L $? ? ?
 
 
A A A A). ). ).T (<  $	' 	' 	' -B [ ))I )I )I )IV $  "; /# # #88 8 80F F F F       $ +[ "q1$&dd" " " " "J   
 4 4 4  " " " 
    DK   (N N N N N4: N N Nd; ; ; 	 	 	   * * * *	5 	5 	5 	5         rY   