
    Vpf}1             	      %   U d Z ddlmZ ddlmZmZmZ ddlmZ ddl	Z	ddl
Z
ddlZddlZddlZddlZddlmZmZ ddlZddlmZ ddlZddlZddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddl!m"Z" ddl#m$Z$ ddl#m%Z% ddl#m&Z& ddl#m'Z' ddl#m(Z( ddl#m)Z) ddl#m*Z* ddl#m+Z+ ddl#m,Z- ddl#m.Z. ddl#m/Z/ ddl#m0Z0 ddl#m1Z1 ddl#m2Z2 dd
l#mZ3 ddl#m4Z4 dd l#m5Z5 dd!l#m6Z6 dd"l7m8Z8 dd#l7m9Z9 dd$l:m;Z; dd%l:m<Z< dd&l=m>Z? ddl=mZ@ dd'l=mAZB dd(l=mCZD dd)l=mEZF dd*lGmHZH dd+lImJZJ ddlKZLddlMm"ZN dd,lOmPZP dd-lQmRZR 	 dd.lSmTZT n# eU$ r	 dd.lVmTZT Y nw xY wdd/lWmXZY dd0lZm[Z\ e4j]        Z]e9j^        Z^eZ_ej`        Z`d1Za ejb        d2          Zc ejb        d3          Zde5je        Zfe5jg        Zhd4 Zid5 ZjeZkelZmdd:Zn G d; d<          Zo eo            Zpi Zqd=erd><   i Zsd=erd?<   e jt        Ztd1au G d@ dAejv                  Zw ew            ZxddCZye	jz        dD             Z{ddFZ| ee'j}        dGH          ddIdJdJepddIdKddZ            Z~ G d[ d\          Z G d] d^e          Z G d_ d`e          ZddcZ ee'j}        ddH          ddeddf            ZddhZddlZddsZe	jz        ddu            Z	 dddyZdd}Z	 dddZdJd1ddddZe-j        dd            ZdJdddZddZd ZddZej        Zd Zd Z	 	 dddZdddZddZddZ G d de)j                  ZddZ G d de)j                  ZddZe)j        fD ]Z eee          eqe<   d eqe?j        j        <   g dZd eqe3j        <   eLj        eqe%j        <   ddZeeqe%j        <   ddddeqe*j        <   d eqe@j        <   ddZeeqe6j        <   ddZeeqej        <   ddZeeqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   ddZeesej        <   eLj
        j        eqej        <   d Zeeqej        <   eLj
        j        eqej        <   ddZeeqej        <   ddZeesej        <   ddZeesej        <   eLj
        j        eqej        <   d eqe@j        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <    ee@j        d1¦          esej        <    ee@j        d1¦          esej        <    ee@j        d1¦          esej        <    ee@j        d1¦          esej        <   dÄ Zeeqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   dĄ Zeeqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   eLj
        j        eqej         <   eLj
        j        eqej        <   eLj        eqej        <   dń Zeeqej        <   eLj
        j        eqej        <   eLj
        j	        eqej
        <   eeqej        <   eLj
        j        eqej        <   eLj
        j        eqej        <   dƄ Zeeqej        <   dǄ ZdȄ Zeeqej        <   eeqej        <   ddʄZdd˄Z eed1̦          esej        <    eedJ̦          esej        <   eLj        eLj        eLj        eLj        eLj        eLj        eLj         eLj!        iZ"d̈́ e"#                                D             Z$d΄ Z%dτ Z&e&eqej'        <   dЄ Z(dф Z)e)eqej*        <   d҄ Z+e+eqej,        <   ddӄZ-dԄ Z.e.eqej/        <   dddׄZ0 e0eLj1        j2        deLj3        ٦          eqej4        <    e0eLj1        j5        deLj6        ٦          eqej7        <    e0eLj1        j8        deLj
        j9        ٦          eqej:        <   eLj
        j;        eqej<        <   eLj
        j=        eqej>        <   dڄ Z?dۄ Z@e@eqejA        <   d܄ ZBd݄ ZCdބ ZDd߄ ZE e0eLj
        jF        dؐeB٦          eqejG        <    e0eLj
        jH        dؐeC٦          eqejI        <    e0eLj
        jJ        dؐeD٦          eqejK        <    e0eLj
        jL        dؐeE٦          eqejM        <   d ZN e0 eeNeLj
        jH                  dؐeC٦          eqejO        <    e0 eeNeLj
        jL                  dؐeE٦          eqejP        <   eLjA        jQ        eqejA        jR        <   d1dddZSeSeqejT        <   d ZUeUeqejV        <   d ZWeWesejX        <   d ZYeYeqejZ        <   d Z[ddZ\ddZ]e]esej^        <   ddZ_e_esej`        <   ddZaddZbebesejc        <   d Zdedeqe@je        <   d Zfefesejg        <   d Zheheseji        <   ddZjejesejk        <   d Zleleqejm        <   d Zneneqejo        <   d  Zpepeqejq        <   d Zr ereLjs                  eqejt        <    ereLju                  eqejv        <    e0 ereLjw                  dg ereLjx                  ٦          eqejy        <    e0 ereLjz                  dg ereLj{                  ٦          eqej|        <    ereLjx                  eqej}        <    ereLj{                  eqej~        <   dѐdZ eedJ          esej        <    eed1          esej        <    eLj        ed1          Z eLj        eLj
        jJ        d1          ZdҐdZeesej        <   d Zd ZdddZd Zd Z eeed d          esej        <    ee eedJ̦          ed          esej        <    ee eed1̦          ed          esej        <   eesej        <   dӐdZeesej        <   dԐd!Z eeFj        ej        d"#          esej        <    eeFj        ej        d$#          esej        <    eeFj        eJd%#          esej        <    eeFj        ej        d&#          esej        <    eeFj        ej        d'#          esej        <   d( Zeeqej        <    ee0dؐ)          d*             Zeesej        <   dՐd,Zeese2j        <   d֐d.Zeese2j        <   dאd0Zeese2j        <   d֐d1Zeese2j        <   dؐd3Zeese2j        <   d֐d4Zeese2j        <   dƐd5Zeese2j        <    e ee3j        dJ6          d1d78          esej        <   dِd:Zeeqej        <   dڐd=Zeeqej        <   d> Zeeqe2j        <   d? Zdǐd@Z ee0dg)          dېdC            Zeesej        <   dD Zeesej        <   dېdEZeesej        <   dȐdFZÐeesej        <   dG ZŐdH ZƐdȐdIZǐeesej        <   eesej        <   eesej        <   eesej        <   eesej        <   dܐdMZ͐eeqej        <   dݐdRZϐdݐdSZАeeqej        <    ee?j        dTU          esej        <    e ee$j        d1V          dJdW8          ese$j        <   dX eqe$j        <    eeNdY          reNj        eqe?j        <   dސd\Zڐeeqej        <   dߐdcZܐeeqej        <   ddhZސeeqej        <   dȐdiZeesej        <   dj ZeeqejA        j        <   	 dddoZeeqejA        j        <   ddrZeeqejA        j        <   ddvZeesejA        j        <   ddwZeesejA        j        <   dd~ZeesejA        j        <   dƐdZeesej        <   dƐdZeesejA        j        <   ddZeeqej        <   ddZeeqej        <   ddZeeqe;j        <   eeeldf         df         ZddZddZddZddZeese1j        <   ddZeese1j         <   ddZeese9j        <   ddZeeqe9j        <   ddZeeqe9j        <   d Zeeqej        <   d eqe@j	        <   d Z
 e
             dS (  z0Provides JAX and TensorFlow interoperation APIs.    )annotations)CallableIterableSequence)partialN)AnyUnion)logging)lax)custom_derivatives)random)numpy)	tree_util)sharding)export)impl_no_xla)xla)ad_checkpoint)ad_util)api)api_util)config)core)dispatch)dtypes)linear_util)op_shardings)sharding_impls)mesh)pjit)prng)source_info_util)util)shard_alike)_export)
shape_poly)ad)mlir)control_flow)linalg)slicing)windowed_reductions)
xla_client)	logaddexp)xla_data_pb2)attr_value_pb2)xla_sharding)ops)contextFz ^[A-Za-z0-9.][A-Za-z0-9_.\/>-]*$z[^A-Za-z0-9_.\/-]c                z    t                               d|           }t                              |          sd| }|S )N_.)_INVALID_SCOPE_CHARsub_VALID_SCOPE_REGEXmatch)name
scope_names     ^/var/www/html/nettyfy-visnx/env/lib/python3.11/site-packages/jax/experimental/jax2tf/jax2tf.py_sanitize_scope_namer>   o   s@    "&&sD11*		!	!*	-	- "!Z!!J	    c                     dS NT rB   r?   r=   <lambda>rC   x   s     r?   vTfValreturnboolc                    t          | t          j        t          j        f          rdS 	 t          j        d          5  t          j        |            d d d            n# 1 swxY w Y   dS #  Y dS xY w)NTCPUF)
isinstancetfTensorVariabledeviceconstant)rD   s    r=   	_is_tfvalrP      s    BIr{+,, 4	5		  k!nnn              455s.   A, AA, A##A, &A#'A, ,A1c                      e Zd ZdS )_DefaultNativeSerializationN)__name__
__module____qualname__rB   r?   r=   rR   rR      s        $r?   rR   z(dict[core.Primitive, Callable[..., Any]]tf_impltf_impl_with_avalsc                      e Zd Zd ZdS )_ThreadLocalStatec                h    d| _         d| _        d| _        d| _        d | _        d| _        d | _        d S )NTFrB    )
enable_xlainside_call_tf	shape_envinclude_xla_op_metadataconstant_cachetf_outer_name_scopecall_tf_concrete_function_listselfs    r=   __init__z_ThreadLocalState.__init__   sM    
 DO  D 35DN $)D 
 D  "D
 =AD'''r?   N)rS   rT   rU   re   rB   r?   r=   rY   rY      s(        !A !A !A !A !Ar?   rY   NameStack | strc                 (    t          j                    S N)r"   current_name_stackrB   r?   r=   _get_current_name_stackrj      s    		,	.	..r?   c               #  ~   K   t           j        } dt           _        	 d V  | t           _        d S # | t           _        w xY wrA   )_thread_local_stater]   )prevs    r=   r]   r]      sK       
	+$'+$.	EEE)-&&&&----s   . <list[Any] | Nonec                     t           j        S rh   rl   rb   rB   r?   r=   5get_thread_local_state_call_tf_concrete_function_listrq      s     
	;;r?   jax2tf_convert)tagrB   T)polymorphic_shapespolymorphic_constraintswith_gradientr\   native_serializationnative_serialization_platforms$native_serialization_disabled_checksfun_jaxr   rt   
str | Noneru   Sequence[str]rv   r\   rw   "bool | _DefaultNativeSerializationrx   Sequence[str] | Nonery   Sequence[DisabledSafetyCheck]c          	     R    s2t                      rt          j        d           nt          d          t          u rsdnt
          j        j        st          j        d           rst          d          srt          d          rlst          j        d           t          t          t          f          rt          d D                       st          d	           t                    t          j                    d fd}|S )a  Allows calling a JAX function from a TensorFlow program.

  See
  [README](https://github.com/google/jax/blob/main/jax/experimental/jax2tf/README.md)
  for more details about usage and common problems.

  Args:
    fun_jax: target JAX function to be called. Its arguments and return value
      should be JAX arrays, or nested standard Python containers
      (tuple/list/dict) thereof (pytrees).
    polymorphic_shapes: Specifies input shapes to be treated polymorphically
      during lowering.

      .. warning:: The shape-polymorphic lowering is an experimental feature.
        It is meant to be sound, but it is known to reject some JAX programs
        that are shape polymorphic. The details of this feature can change.

      It should be `None` (all arguments are monomorphic), a single PolyShape
      or string (applies to all arguments), or a tuple/list of the same length
      as the function arguments. For each argument the shape specification
      should be `None` (monomorphic argument), or a Python object with the
      same pytree structure as the argument.
      See [how optional parameters are matched to
      arguments](https://jax.readthedocs.io/en/latest/pytrees.html#applying-optional-parameters-to-pytrees).

      A shape specification for an array argument should be an object
      `PolyShape(dim0, dim1, ..., dimn)`
      where each `dim` is a dimension specification: a positive integer denoting
      a monomorphic dimension of the given size, or a string denoting a
      dimension variable assumed to range over non-zero dimension sizes, or
      the special placeholder string "_" denoting a monomorphic dimension
      whose size is given by the actual argument. As a shortcut, an Ellipsis
      suffix in the list of dimension specifications stands for a list of "_"
      placeholders.

      For convenience, a shape specification can also be given as a string
      representation, e.g.: "batch, ...", "batch, height, width, _", possibly
      with surrounding parentheses: "(batch, ...)".

      The lowering fails if it cannot ensure that the it would produce the same
      sequence of TF ops for any non-zero values of the dimension variables.

      polymorphic_shapes are only supported for positional arguments; shape
      polymorphism is not supported for keyword arguments.

      See [the README](https://github.com/google/jax/blob/main/jax/experimental/jax2tf/README.md#shape-polymorphic-conversion)
      for more details.

    polymorphic_constraints: a sequence of contraints on symbolic dimension expressions, of
      the form `e1 >= e2` or `e1 <= e2`.
      See more details at https://github.com/google/jax/blob/main/jax/experimental/jax2tf/README.md#user-specified-symbolic-constraints.
    with_gradient: if set (default), add a tf.custom_gradient to the lowered
      function, by converting the ``jax.vjp(fun)``. This means that reverse-mode
      TensorFlow AD is supported for the output TensorFlow function, and the
      value of the gradient will be JAX-accurate.
    enable_xla: if set (default), use the simplest conversion
      and use XLA TF ops when necessary. These ops are known to create issues
      for the TFLite and TFjs converters. For those cases, unset this parameter
      so the lowering tries harder to use non-XLA TF ops to lower the
      function and aborts if this is not possible. Cannot be set to `False`
      when using `native_serialization`.
      Starting with JAX 0.4.31 support for `enable_xla=False` is deprecated.
    native_serialization: serialize the JAX function natively to
      StableHLO with compatibility guarantees. This makes it easier to have
      confidence that the code executed when calling this function from
      TensorFlow is exactly the same as JAX would run natively.
      The DEFAULT_NATIVE_SERIALIZATION value defers to `False` if `enable_xla`
      is set to `False` or to the configuration flag
      `--jax2tf_default_native_serialization` otherwise.
      Native serialization cannot be used with `enable_xla=False`.
      Starting with JAX 0.4.31 support for non-native serialization is deprecated.
    native_serialization_platforms: In conjunction with
      `native_serialization`, specify the platform(s)
      for which to lower the code. Must be a tuple of
      strings, including a subset of: 'cpu', 'cuda', 'rocm', 'tpu'.
      The default (`None``), specifies the JAX default
      backend on the machine where the lowering is done.
    native_serialization_disabled_checks: In conjunction with
      `native_serialization`, disable the specified safety checks.
      See docstring of `DisabledSafetyCheck`.

  Returns:
    A version of `fun_jax` that expects TfVals as arguments (or
    tuple/lists/dicts thereof), and returns TfVals as outputs, and uses
    only TensorFlow ops and thus can be called from a TensorFlow program.
  z3jax2tf.convert with enable_xla=False is deprecated.z6jax2tf.convert with enable_xla=False is not supported.Fz=jax2tf.convert with native_serialization=False is deprecated.z;native_serialization is not supported with enable_xla=FalsezDpolymorphic_constraints are supported only with native serializationzusing native_serialization_platforms without native_serialization. The parameter will have no effect, since the same code is serialized for all platforms without native_serialization.c              3     K   | ]}|d v V  	dS ))cpucudarocmtpuNrB   .0ps     r=   	<genexpr>zconvert.<locals>.<genexpr>o  s<       9 9 33 9 9 9 9 9 9r?   znnative_serialization_platforms must be a sequence containing a subset of {'cpu', 'cuda', 'rocm', 'tpu'}. Got: args_tfrE   	kwargs_tfrF   c            
     L   t          j                    s5t          j        s)t	          ddt           j        j        j         z             t          s=t          j
        t          j                            t          j                             dadd}t!          j        ||           }t%          j        |	          t!          j        ||          }t%          j        |d           | |f}t!          j        |          \  }}t+          t-          t.          t1          t3          |                    |                    }rt5          
          nt7          |          	                                  d r#t          j        dfd            } || }	n)                    |          \  }
}dfd|
D             }	                                 n#                                  w xY wd |	D             }	t!          j         |	          }|S )Nz5convert must be used outside all JAX transformations.zTrace state: TarE   rF   jax.ShapeDtypeStructc                    t          j        |           }t          d |D                       }t          |           \  }}t	          j        ||          S )Nc              3  p   K   | ]1}t          |t          j        j        j                  r|j        n|V  2d S rh   )rJ   rK   compatv1	Dimensionvaluer   ds     r=   r   zRconvert.<locals>.converted_fun_tf.<locals>.jax_arg_spec_from_tf.<locals>.<genexpr>  sV       2 2  )BIL,BCCK177IJ2 2 2 2 2 2r?   )npshapetuple_tfval_to_tensor_jax_dtypejaxShapeDtypeStruct)r   tf_arg_shaper5   a_jax_dtypes       r=   jax_arg_spec_from_tfz?convert.<locals>.converted_fun_tf.<locals>.jax_arg_spec_from_tf  sb    Xa[[l 2 2$02 2 2 2 2l 2!44na !,<<<r?   )constraints)
args_specskwargs_specsrx   ry   )r   r   args_flat_tfr\   r   c                                          |           \  }}t          |          t          | ||          fS )N)implrv   r   r   r   
outs_avalsouts_tf)
run_fun_tfr   _make_custom_gradient_fn_tf)	r   r   r   r   rz   r   r   	outs_treerv   s	      r=   *converted_fun_flat_with_custom_gradient_tfzUconvert.<locals>.converted_fun_tf.<locals>.converted_fun_flat_with_custom_gradient_tf  sY     ,0??<+H+H
(':y..-$1!+,*!+%' ' '( (r?   zkThe jax2tf-converted function does not support gradients. Use `with_gradient` parameter to enable gradientsc                R    g | ]#}t           j                            |           $S ))inputmessage)rK   raw_opsPreventGradient)r   or   s     r=   
<listcomp>z5convert.<locals>.converted_fun_tf.<locals>.<listcomp>  s>     
 
 
 J&&Q&@@
 
 
r?   c                8    g | ]}t          j        |d           S )
jax2tf_out)rK   identity)r   xs     r=   r   z5convert.<locals>.converted_fun_tf.<locals>.<listcomp>  s$    GGGQBK<00GGGr?   )r   rE   rF   r   )r   rE   rF   rE   )!r   trace_state_cleanrl   r]   
ValueErrorthread_local_statetrace_statetrace_stack_has_registered_tf_source_pathr"   register_exclusionospathdirnamerK   __file__r   tree_mapr   symbolic_args_specstree_flattenr   mappreprocess_arg_tfrangelenNativeSerializationImplGraphSerializationImplbefore_conversioncustom_gradientr   after_conversiontree_unflatten)r   r   r   args_jax_specskwargs_jax_specscombined_args_tfr   args_kwargs_treer   outs_flat_tfr   r5   out_tfr   r   r   r   r   r\   rz   rw   ry   rx   ru   rt   rv   s                @@@@@r=   converted_fun_tfz!convert.<locals>.converted_fun_tfy  s    !## M,?,N M
A
K$1=I
K
KLM M M
 * ,)"'//"+*F*FGGG'+$
= 
= 
= 
= '(<gFFN+*+- - -J
 !)*>	JJ-$   L+%.%;<L%M%M"L"uS%6%677FFH HL  !$
l)G/S	U U Udd $
l#	! ! !d

'+i	 
			( 	( 	( 	( 	( 	( 	( 	( 	( 	( 
		( BA<P $ = =IG
 
 
 

 
 

 dGG,GGGL%i>>FMs   1A$G* *H )r   rE   r   rE   rF   rE   )allow_enable_xla_falsewarningswarnr   DEFAULT_NATIVE_SERIALIZATIONr   #jax2tf_default_native_serializationr   rJ   listr   allr   check_callable)	rz   rt   ru   rv   r\   rw   rx   ry   r   s	   ```````` r=   convertr      s   B 
 Q QmIJJJJOPPP999 N"#GM	 IMGI I I G* G
EG G G 
 "9 
N   $ K =m<= = =
 5e}EE 4 9 979 9 9 9 94 303 34 4 4 &++I%J%J"WY Y Y Y Y Y Y Y Y Y Y Y Yv 
r?   c                  .    e Zd ZdZd Zd ZddZdd
ZdS )SerializationImplz[Implementation details for jax2tf serialization.

  Abstract superclass for subclassing.
  c                    t           )zdCalled in the resulting TF function, before any other method.

    Useful to set any global context.NotImplementedErrorrc   s    r=   r   z#SerializationImpl.before_conversion  
     r?   c                    t           )zCalled in the resulting TF function, after conversion is done.

    Useful to restore any global context set up by `before_conversion`.r   rc   s    r=   r   z"SerializationImpl.after_conversion  r   r?   r   Sequence[TfVal]rF   Gtuple[Sequence[TfVal], Sequence[core.ShapedArray], tree_util.PyTreeDef]c                    t           )aC  Runs the resulting TF function.

    Args:
      args_flat_tf: a flat tuple of tf.Tensor arguments

    Returns: a tuple with:
      outs_tfs: a flat tuple of tf.Tensor results
      outs_avals: a flat tuple of JAX abstract values for the underlying JAX
        function.
      outs_tree: the PyTreeDef for the outputs
    r   )rd   r   s     r=   r   zSerializationImpl.run_fun_tf  s
     r?   -tuple[Callable, Sequence[core.AbstractValue]]c                    t           )z/Returns the VJP function, and the VJP in_avals.r   rc   s    r=   get_vjp_funzSerializationImpl.get_vjp_fun  s
     r?   Nr   r   rF   r   rF   r   )rS   rT   rU   __doc__r   r   r   r   rB   r?   r=   r   r     sd                      r?   r   c                  2    e Zd ZddZd Zd ZddZddZdS )r   rx   r~   ry   r   c               z    t          d||          | _        || _        || _        || _        || _        || _        d S )NT)rw   rx   ry   )dictconvert_kwargsrz   r   r   ry   rx   )rd   rz   r   r   rx   ry   s         r=   re   z NativeSerializationImpl.__init__  sT     D>\Dhj j jD DL DO$D0TD-*HD'''r?   c                    t           j        g t           _        fd}|| _        d g} t          j        | j        | j        | j        |          | j        i | j	        | _
        |d         J |d         | _        d S )Nc                       t           _        d S rh   rp   )_prev_func_lists   r=   _restore_contextzCNativeSerializationImpl.before_conversion.<locals>._restore_context  s    ;J888r?   )lowering_platformsdisabled_checks/_device_assignment_for_internal_jax2tf_use_onlyr   )rl   rb   r   r%   export_back_compatrz   rx   ry   r   r   exporteddevice_assignment)rd   r   _exported_device_assignmentr   s      @r=   r   z)NativeSerializationImpl.before_conversion  s    )HO9;6K K K K K -D#'&G.>A8S	  
 -
 +- -DM 'q)5558;Dr?   c                .    |                                   d S rh   r   rc   s    r=   r   z(NativeSerializationImpl.after_conversion      r?   r   r   rF   r   c                x    t          || j                  }|t          | j        j                  | j        j        fS rh   )_run_exported_as_tfr   r   	out_avalsout_tree)rd   r   resultss      r=   r   z"NativeSerializationImpl.run_fun_tf   s4     ",>>GE$-122DM4JJJr?   r   c           
         t          j        | j        | j        j        | j        j        | j        j        | j        j        | j        j        | j	        d          S )NTin_treein_avalsin_shardings_hlor  out_shardings_hlor   	apply_jit)
r%   _get_vjp_funrz   r   r  r	  r
  r  r  r   rc   s    r=   r   z#NativeSerializationImpl.get_vjp_fun&  sQ    (,(=)-)?151O*.-*A26-2Q262H*.0 0 0 0r?   N)rx   r~   ry   r   r   r   rS   rT   rU   re   r   r   r   r   rB   r?   r=   r   r     sv        I I I I< < <$  K K K K	0 	0 	0 	0 	0 	0r?   r   c                  2    e Zd ZddZd Zd Zdd
ZddZdS )r   r   r   r\   rG   c                   t          d          | _        || _        || _        || _        || _        t          |dd          }t          j        |d          }|| _	        || _
        d S )NF)rw   rS   unknownjax2tf)r   r   rz   r   r   r\   getattrr#   	wrap_name
name_stackr   )rd   rz   r   r   r   r\   fun_namer  s           r=   re   zGraphSerializationImpl.__init__2  sk     E:::DDL DO$D DOw
I66H(33J DO$Dr?   c                   t           j        t           j        t           j        fd}|| _        | j        t           _        dt           _        t          j                    t           _        t           j        rJ dt           j                     t          j	        | j
        | j        f          \  }| _        t          t          d |                    | _        t!          j        | j                  }t%          t'          t           j        | j        | j                  | j        | j        | j                  \  }}t/          ||          t           _        d S )Nc                 h     t           _        t           _        t           _        dt           _        d S NrB   )rl   r\   r_   ra   r^   )prev_enable_xlaprev_include_xla_op_metadataprev_tf_outer_name_scopes   r=   r   zBGraphSerializationImpl.before_conversion.<locals>._restore_contextE  s-    '6$4P10H-&(###r?   FzUnexpected shape environment c                N    t          j        t          j        |                     S rh   )r   raise_to_shapedget_aval)r   s    r=   rC   z:GraphSerializationImpl.before_conversion.<locals>.<lambda>S  s    d*4=+;+;<< r?   )r   )rl   r\   r_   ra   r   rK   get_current_name_scoper^   r   r   r   r   r  r   r   args_avals_flatr&   all_dim_vars_interpret_fun_jaxr    compute_dim_vars_from_arg_shapesr   r  zip)	rd   r   args_specs_flatdim_vars
dim_valuesr5   r  r  r  s	         @@@r=   r   z(GraphSerializationImpl.before_conversionA  sW   )4O#6#N 2F) ) ) ) ) ) )
 -D%)_"27/.0.G.I.I+",mm.mNaNk.m.mmmm$-$:	$+,%. %.!OT\ <<oNNP PD&t';<<H&
;$t|	E 	E 	E4/B BMJ
 %(*$=$=!!!r?   c                .    |                                   d S rh   r   rc   s    r=   r   z'GraphSerializationImpl.after_conversion\  r   r?   rF   r   c                    t          | j        | j                  \  }}t          ||| j        | j        d          \  }| _        || j         |            fS )NTfresh_constant_cache)flatten_fun_jaxrz   r  r#  r!  r  r   )rd   r   fun_flat_jaxout_tree_thunkr   s        r=   r   z!GraphSerializationImpl.run_fun_tf_  sc     $34<#N#N L.1d*!	 #  #  #GT_
 DO^^%5%555r?   r   c                    t          j        | j        | j        | j        dt          | j                  z  | j        dt          | j                  z  d d          S )Nrh   Fr  )r%   r  rz   r  r!  r   r   rc   s    r=   r   z"GraphSerializationImpl.get_vjp_funk  s]    
 (,)-)=183t?S;T;T1T*./29C<P<P2P26*/1 1 1 1r?   N)r   r   r\   rG   r   r   r  rB   r?   r=   r   r   1  sn        % % % %> > >6  
6 
6 
6 
61 1 1 1 1 1r?   r   valDTypec                4    t          |           \  }}|j        S )aX  Computes the TensorFlow dtype using JAX's typing rules.

  If the value is a tf.Tensor, it starts with its dtype. If the value is a
  constant it uses JAX to infer its dtype. The resulting dtype follows the
  JAX type inference rules, and depends on the value of the
  JAX_ENABLE_X64 flag.

  See README.md for how 64-bit values are treated.
  )r   dtype)r1  tvalr5   s      r=   dtype_of_valr6  z  s     's++'$	r?   jax2tf_eval_polymorphic_shapes)rt   c                    d fd}|S )a  Evaluates the output shape in presence of shape polymorphism.

  This is done without lowering or executing the function, same as for
  `jax.eval_shape`.

  Args:
    fun_jax: target JAX function to be called. Its arguments and return value
      should be JAX arrays, or nested standard Python containers
      (tuple/list/dict) thereof (pytrees).
    polymorphic_shapes: Specifies input shapes to be treated polymorphically
      during shape evaluation. See discussion for `jax2tf.convert`.

      .. warning:: The shape-polymorphic lowering is an experimental feature.

  Returns: a function that takes `jax.ShapeDtypeStruct`s (or any values
    with `.shape` and `.dtype` attributes) corresponding to the inputs for
    `fun_jax`, and returns a tuple with:

      * the jax.ShapeDtypeStruct corresponding to the result, as for
       `jax.eval_shape`. The shape may contain symbolic dimension expressions.
      * the value that can be passed to `polymorphic_shapes` for a subsequent
        call to `jax2tf.eval_polymorphic_shape`, or `jax2tf.convert`.

  For example:

  >>> import jax
  >>> from jax.experimental import jax2tf
  >>> from jax import numpy as jnp
  >>>
  >>> f = lambda A, x: jnp.sin(jnp.dot(A, x))
  >>> A = jax.ShapeDtypeStruct((2000, 3000), jnp.float32)
  >>> x = jax.ShapeDtypeStruct((3000, 1000), jnp.float32)
  >>> out_spec, out_poly_shape = jax2tf.eval_polymorphic_shape(f, polymorphic_shapes=["a, b", "b, c"])(A, x)
  >>> print(out_spec.shape)
  ("a", "c")
  >>> print(out_poly_shape)
  (a, c)
  >>> res_spec, res_poly_shape = jax2tf.eval_polymorphic_shape(lambda x: x.T, polymorphic_shapes=[out_poly_shape])(out_spec)
  >>> print(res_poly_shape)
  (c, a)
  rF   r   c                     t          j        |           }t          j        g|R  }t	          j        d |          }||fS )Nc                *    t          | j                  S rh   strr   )rs    r=   rC   zKeval_polymorphic_shape.<locals>.do_eval_polymorphic_shape.<locals>.<lambda>  s    QW r?   )r   r   r   
eval_shaper   r   )r   args_poly_specsres_poly_specres_polymorphic_shaperz   rt   s       r=   do_eval_polymorphic_shapez9eval_polymorphic_shape.<locals>.do_eval_polymorphic_shape  sU    0&( (ON7=_===M%./E/E}UU///r?   )rF   r   rB   )rz   rt   rB  s   `` r=   eval_polymorphic_shaperC    s0    Z0 0 0 0 0 0 0 
#"r?   tuple[Callable, Callable]c                (     d fd}|fdfS )a  Wraps the function to take a (flat) list of positional args.

  jax2tf works better and is simpler when the JAX function takes and returns
  just a tuple of values (no pytrees, no kwargs). This is in part because
  jax.vjp does not support kwargs and we can only set
  tf.custom_gradient on functions with flat arguments and results

  Returns:
     * the wrapped JAX function taking and returning a flat list of arguments
     * a thunk that can be called after the wrapped function has been called
       to return the output pytree.
  Nc                     t          j        |           \  }} |i |}t          j        |          \  }}|k    sJ ||S rh   )r   r   r   )	args_flat_jax	tree_argstree_kwargstree_resres_flat_jaxr  rz   r  out_tree_refs	         r=   r.  z%flatten_fun_jax.<locals>.fun_flat_jax  sd    &5g}MMI{w	1[11H&3H==L(<8#;#;#;#;Lr?   c                      S rh   rB   )rL  s   r=   rC   z!flatten_fun_jax.<locals>.<lambda>  s    | r?   rB   )rz   r  r.  rL  s   `` @r=   r-  r-    sF     ,       
++++	++r?   arg_idxintarg_tfc                    t          |          s%d| dt          |           d}t          |          t          |          \  }}t	          j        |d|            }|S )zwPre-processes the TF args.

  Returns: a tuple with the pre-processed TF arg, the TF shape, and the
      JAX dtype.
  z	Argument z	 of type zN of jax2tf.convert(f) should be NumPy array, scalar, tf.Variable, or tf.Tensorjax2tf_arg_)rP   type	TypeErrorr   rK   r   )rN  rP  msgr5   s       r=   r   r     s|     
6		 ?v ? ?V ? ? ?C
C.. )00)&!;v6W6677&	-r?   r   r   r   r   Sequence[core.ShapedArray]r   c               *    dddfd}|S )a  Prepares the TF function to be used with tf.custom_gradient.

  Args:
    impl: the serialization implementation details
    with_gradient: whether to include a tf.custom_gradient
    args_specs, kwargs_specs: the jax.ShapeDtypeArrays for the args and kwargs
    args_tf: the flattened TF arguments of the primal function
    outs_avals: the flattened output JAX abstract values of the primal function
    outs_tf: the flattened TF outputs of the primal function
  N)	variablesout_cts_flat_tfrE   c           	        | rt          d|           t          j        d          5  d
d}t          t	          ||                    }t          	          |z   }
                                \  }}t          d |D                       } t          |f|d	
j        | }d d d            n# 1 swxY w Y   |S )NzmUnexpected variables used in forward pass. This should not happen for first-order differentiation. variables=
jax2tf_vjpout_ct_avalcore.ShapedArrayr   rE   c                    | | S t          j        |j                  t          j        k    sJ d|             t          j        |t                    S )Nz
out_ct_tf=r4  )r   primal_dtype_to_tangent_dtyper4  r   float0rK   
zeros_like_tf_np_dtype_for_float0)	out_ct_tfr\  r   s      r=   
fix_out_ctzC_make_custom_gradient_fn_tf.<locals>.grad_fn_tf.<locals>.fix_out_ct  s[    
  
1+2CDDUUUWfZcWfWfUUU }V+BCCCCr?   c              3  >   K   | ]}t          |j                  V  d S rh   r;  )r   r   s     r=   r   zB_make_custom_gradient_fn_tf.<locals>.grad_fn_tf.<locals>.<genexpr>  s>       % % 	AG% % % % % %r?   )rv   rt   )r\  r]  r   rE   )r   rK   
name_scoper   r   r   r   r   )rX  rY  re  out_cts_fixed_flat_tfvjp_args_flat_tffun_vjp_jaxvjp_in_avalsvjp_polymorphic_shapesin_cts_flatr   r   r   r   rv   s            r=   
grad_fn_tfz/_make_custom_gradient_fn_tf.<locals>.grad_fn_tf  sf        
|	$	$ 2 2
D 
D 
D 
D $C
OZQX$Y$YZZw*??"&"2"2"4"4k<$ % %% % %    G#1  
	  !1	2k+2 2 2 2 2 2 2 2 2 2 2 2 2 2 2: s   A>B44B8;B8)rY  rE   rB   )	rz   r   rv   r   r   r   r   r   rn  s	    ``  ``` r=   r   r     sQ    &  & & & & & & & & & & & &P 
r?   extra_name_stackc              #     K   | rt          j        |           nt          j                    }|5  d V  d d d            n# 1 swxY w Y   d S rh   )r"   extend_name_stack
contextlibnullcontext)ro  name_ctxs     r=   _extended_name_stackru  ,  s       
$01ABBB!##   
 
	EEE
 
 
 
 
 
 
 
 
 
 
 
 
 
 
&s   ?AA
args_avalsr,  6tuple[tuple[TfVal, ...], tuple[core.ShapedArray, ...]]c                   t          j        t                    5 }t          t	          j        |           ||          }t          |          5  t          j                    5  t          |||          }d d d            n# 1 swxY w Y   ~d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          j
        |          S )Nr+  )r   new_base_mainTensorFlowTrace_interpret_subtracelu	wrap_initru  new_sublevel%_call_wrapped_with_new_constant_cacher#   unzip2)rz   r   rv  ro  r,  mainsubtrace_funout_valss           r=   r#  r#  6  s    /** d&r|G'<'<dJOOL	.	/	/   ] ]1,G[] ] ] 	] ] ] ] ] ] ] ] ] ] ] ] ] ] ]                              
X		sY   3B/B!B 4B BBBBB/B	B/B	 B//B36B3r   r   export.Exportedc           	     D   j         }d }t          t          || |                    } t          d j        D                       }t          d j        D                       }fdt	          j                   D             }fdt	          |           D             }j        }	 t          j        }	n# t          $ r d}	Y nw xY w|	r |	            }
nd}
||
k    rt          d|
 d	| d
          t          |||t          j        d t          j        D             ng d          }t          d j        D                       |d<   |dk    r"t          d j        D                       |d<   n+|dk    r%t                                          j        v rd|d<   t#          j        d          r#t#          j        ddt)          |                     j        |d<   g }j        D ]"}|                    j        |                    #t          t          t3          t4          t7          j                              ||                    } t          j        | fi |}t          j        Jt          j        D ]=}t6          j        j                                         !                    |j"                   >tG          t          t3          t4          t7          j                              |j$                            }t          t          ||j                            }|S )z]Runs the `exported` as an XlaCallModule TF op.

  Returns: the flattened tuple of results.
  c                    t          d t          |j        | j                  D                       sJ || f            t          |j                  }||j        k    rt          j        | |          S | S )Nc              3  P   K   | ]!\  }}t          j        |          ||k    V  "d S rh   r   is_constant_dim)r   d_avald_vals      r=   r   z>_run_exported_as_tf.<locals>._convert_value.<locals>.<genexpr>U  sN       0 0 65&v..0v 0 0 0 0 0 0r?   )r   r%  r   _to_tf_dtyper4  rK   cast)r1  avalconversion_dtypes      r=   _convert_valuez+_run_exported_as_tf.<locals>._convert_valueS  s     0 0$'
CI$>$>0 0 0 0 0 = =26= = = $DJ//4:%%WS*+++jr?   c              3  R   K   | ]"}t          d  |j        D                       V  #dS )c              3  F   K   | ]}t          j        |          r|nd V  d S rh   r  r   s     r=   r   z0_run_exported_as_tf.<locals>.<genexpr>.<genexpr>a  sJ       % % %a((2AAd % % % % % %r?   N)r   r   r   out_avals     r=   r   z&_run_exported_as_tf.<locals>.<genexpr>`  s`       * *   % %^% % % % %* * * * * *r?   c              3  >   K   | ]}t          |j                  V  d S rh   )r  r4  r  s     r=   r   z&_run_exported_as_tf.<locals>.<genexpr>e  s,      TTXL00TTTTTTr?   c                ,    g | ]\  }}|j         v |S rB   module_kept_var_idx)r   ir  r   s      r=   r   z'_run_exported_as_tf.<locals>.<listcomp>g  s*    kkkgahNjIjIjTIjIjIjr?   c                ,    g | ]\  }}|j         v |S rB   r  )r   r  atfr   s      r=   r   z'_run_exported_as_tf.<locals>.<listcomp>h  s*    fffvq#AIeDeDesDeDeDer?   N   zUXlaCallModule from your TensorFlow installation supports up to serialization version z) but the serialized module needs version z5. You should upgrade TensorFlow, e.g., to tf_nightly.c                0    g | ]}|j         j        j        S rB   )function_def	signaturer;   )r   concrete_fns     r=   r   z'_run_exported_as_tf.<locals>.<listcomp>  s1        
"
,
1  r?   F)versionToutSoutfunction_listhas_token_input_outputc              3  >   K   | ]}|                                 V  d S rh   )upperr   s     r=   r   z&_run_exported_as_tf.<locals>.<genexpr>  s*      (O(Oq(O(O(O(O(O(Or?   	platformsc              3  4   K   | ]}t          |          V  d S rh   )r<  )r   dcs     r=   r   z&_run_exported_as_tf.<locals>.<genexpr>  s<       13 13 	B13 13 13 13 13 13r?   r      rB   zXlaCallModule %smoduleskip_replicated_sharding)%r	  r   r   r  	enumeratecalling_convention_versiontfxla%call_module_maximum_supported_versionAttributeErrorr   r   rl   rb   r  disabled_safety_checksDisabledSafetyCheckplatformr
   
vlog_is_onvlogr<  mlir_module_serializedr  appendr
  r   _shard_valuerK   executing_eagerlycall_moduler   r   get_default_graph_add_function_recursive_inference_functionr   r  )r   r   rv  r  out_shapes_tf	out_typeskept_args_avalskept_args_flat_tfr  get_max_supported_versionmax_supported_versioncall_module_attrskept_in_shardingsr  resr  s    `              r=   r  r  I  s     *	 	 	 s><DDEE, * * (* * * * *-
 TTASTTTTT)kkkk83D)E)Ekkk/ffff<)@)@fff/'% % K	 % % % $%  5577$$$
<4< <%< < <    =I 0O    PR #   $)(O(OH<N(O(O(O#O#OK \\+0 13 13113 13 13 ,3 ,3'(( !||		%	%	'	'8+J	J	J)++& @L&,=(>(>??? ( ?H ' ; ;aX6q9::::)+)=)?)?	A 	A 	A,. ./ /, 	,<<*;<<# 7C*I  il$$&&>>

)    	S242F2H2HJ J Jh02 2 	3 	3# 	c.#x'9::;;#	*s   )B6 6CCfunlu.WrappedFunin_vals(Sequence[tuple[TfVal, core.ShapedArray]]c                   	 t           j        }|ri t           _        n1|!t          |                                          nt                      } | j        | }|s5|3t
          r,t          |                                          |z
  }|D ]}||= |t           _        nH# |s5|3t
          r,t          |                                          |z
  }|D ]}||= |t           _        w xY w|S rh   )rl   r`   setkeyscall_wrapped_WRAP_JAX_JIT_WITH_TF_FUNCTION)r  r  r,  prev_constant_cacheprev_constant_cache_keysr  newly_added_keysks           r=   r  r    s2   =-<  o+-((DWDc%8%=%=%?%?!@!@!@ilinin'"  ! #'& 	(0557788;SS # #!"")<&& ! #'& 	(0557788;SS # #!"")<&<<<<	/s   AB AC!multiple_resultswith_physical_avalsro  impl_jaxc                     d
 fd	}|S )a%  Convert the JAX implementation of a primitive.

  Args:
    impl_jax: typically the impl-rule for a primitive, with signature
      `(*args_jax: JaxVal, **kwargs) -> Sequence[JaxVal]`. This function implements
        a primitive in terms of other primitives.
    multiple_results: whether `impl_jax` returns a sequence of results.
    extra_name_stack: additional element to add to the name stack for the
      converted ops.

  Returns:
     a function with signature `(*args_tf: TfVal, _in_avals, _out_aval, **kwargs)
     -> Sequence[TfVal]`.
  r   rE   	_in_avalsrV  	_out_avalr]  rF   r   c                    
r$t          t          |           } t          |          }	fd}t          |||           \  }}	r|n|d         S )Nc                 "     | i }r|n|gS rh   rB   )args_jaxresults_jaxr  kwargsr  s     r=   impl_multiple_results_jaxzH_convert_jax_impl.<locals>.wrapped_tf.<locals>.impl_multiple_results_jax  s(    Hh1&11k,?[[;-?r?   r   )r   _jax_physical_avalr#  )r  r  r   r  r  
results_tfr5   ro  r  r  r  s      `   r=   
wrapped_tfz%_convert_jax_impl.<locals>.wrapped_tf  s      0()44i$Y//i@ @ @ @ @ @ @ '!7I MJ *<::z!}<r?   )r   rE   r  rV  r  r]  rF   r   rB   )r  r  r  ro  r  s   ```` r=   _convert_jax_implr    s;    &= = = = = = = = =$ 
r?   r  core.MainTracer	  c              '    K   t          | t          j                              t          fdt	          ||          D                       }|i fV }t          j        |          }t          d |D                       }|V  d S )Nc              3  @   K   | ]\  }}t          ||          V  d S rh   TensorFlowTracer)r   r1  r  traces      r=   r   z&_interpret_subtrace.<locals>.<genexpr>  sG       / /
#t uc4((/ / / / / /r?   c              3  2   K   | ]}|j         |j        fV  d S rh   )r1  r  r   ts     r=   r   z&_interpret_subtrace.<locals>.<genexpr>  s*      11QUAFO111111r?   )rz  r   cur_sublevelr   r%  r   
full_raise)r  r	  r  
in_tracersoutsout_tracersout_vals_with_avalsr  s          @r=   r{  r{    s       $ 1 3 3
4
4% / / / /7H--/ / / / /* 2~		$	%
D!!  11[11111 r?   r+  jaxprcore.ClosedJaxprc               b    t          t          j        |           || j        ||          \  }}|S )zEvaluates a Jaxpr with tf.Tensor arguments.

  This is most often used as the body of a tf.function, or tf.switch_case,
  in which case it should use a fresh constant cache.
  The output is a sequence of TfVal, suitable for use with TF.
  r+  )r#  r   jaxpr_as_funr	  )r  ro  r,  r   r   r5   s         r=   _interpret_jaxprr    s@     "$"3E":":")5>;K7KM M M*'1 
.r?   r  r]  c                    t          j        |           }t          |j                  t          | j                  k    r*|j        dt          | j                           | j        k    sJ || f            |S )a  Converts JAX avals from logical to physical, if relevant.

  JAX might have avals whose logical vs physical shape/dtype may
  differ, and only the physical view is expected to possibly
  relate to TF. TF impl rules should operate on the physical form.

  A JAX logical aval might even correspond, in principle, to several
  physical avals, but we don't support those here. Instead we assert
  there is only one and return it.
  N)r   physical_avalr   r   )r  r  s     r=   r  r    so     $T**-
m!
"
"c$*oo
5
5

.s4:.
/4:
=
=
=PT@U
=
=
=	r?   c                P    t          t          j        d|                     j        S r  )r  r   ShapedArrayr4  r_  s    r=   _jax_physical_dtyper   +  s     	D,R77	8	8	>>r?   tuple[int | None, ...]c                f    t          |           } t          t          d | j                            S )zIGenerate a TF shape, possibly containing None for polymorphic dimensions.c                2    t          j        |           rd n| S rh   )r   is_symbolic_dim)r   s    r=   rC   z#_aval_to_tf_shape.<locals>.<lambda>4  s    V%;A%>%>ETTA r?   )r  r   r   r   r  s    r=   _aval_to_tf_shaper  0  s:     
D	!	!$	sEE:  
  
   r?   c                    	 t          |           } n# t          $ r Y nw xY w| t          j        k    rt          } t
          j                            |           S rh   )r   rT  r   ra  rc  rK   as_dtype)	jax_dtypes    r=   r  r  ;  sb    	#I..II	 	 	 	 	D	 &-'I			I	&	&&s    
c                z    t          j        | j                  }|t           j        vrt	          d| d          |S )Nzdtype zR is not a valid JAX array type. Only arrays of numeric types are supported by JAX.)r   canonicalize_dtypeas_numpy_dtype_jax_dtype_setrT  )tf_dtypedts     r=   _to_jax_dtyper  I  sY      !899"v$$$
 OR O O O P P P	)r?   r	  DType | Nonetuple[TfVal, DType]c                v   t          | t          j        t          j        f          rK|pt	          | j                  }t          |          }|| j        k    rt          j        | |          |fS | |fS |pt          j	        |           j        }t          |           |f}|o%t          j        |           dk    ot          j        du}|r$t          j                            |d          \  }}nd}|t          |          }|t           j        k    r,t          j        t          j        |           |j                  } t+          | d          rCt!          j        | j        t           j                  r| j        j                            |           } t          j        | |          }|r| |ft          j        |<   ||fS )a<  Converts a scalar, ndarray, or tf.Tensor to a tf.Tensor with proper type.

  If `jax_dtype` is missing, uses JAX typing rules.
  See README.md for details regarding 64-bit values.

  Args:
    val: a scalar, ndarray, tf.Tensor, or tf.Variable
    jax_dtype: an optional dtype to use. If missing, uses JAX type inference
      rules for constants.
    memoize_constants: whether to memoize TF constants. We can't do this
      everywhere, we may be outside of a conversion scope.

  Returns:
    a tuple with a tf.Tensor with the type as needed by JAX, and the JAX type.
     N)NNr4  r_  )rJ   rK   rL   rM   r  r4  r  r  r   abstractifyidr   sizerl   r`   getr   ra  zerosr   r  hasattr
issubdtypeextended_rulesphysical_constconvert_to_tensor)r1  r	  memoize_constantsr  	const_key
do_memoizer5   tf_vals           r=   r   r   S  s   $ bi-.. %5]3955I#I..39$$WS*++Y66)^7S_S117I C)$I $kq(8k=P=_gk=kJ %488LQQiaf~%i00	fm	#	#hrx}}&6&EFF	g		 36#4SY#P#P 3i--c22#C/?@@@f	 F9<f*959r?   r   Sequence[shape_poly.DimSize]c                4   t          t          d |                     sJ d|              |'t          t          j        | |                    j        } t          j        |           rt          d | D                       S t          j	        t          j                  \  }}t          t          t          j        | |          |t          j                    gt!          |          z  d          \  }}t          d t#          | |          D                       S )Nc                
    | d uS rh   rB   r   s    r=   rC   z_eval_shape.<locals>.<lambda>  s
    1D= r?   z3Argument shape should be a valid JAX shape but got c              3  4   K   | ]}t          |          V  d S rh   rO  r   s     r=   r   z_eval_shape.<locals>.<genexpr>  s(      ''AQ''''''r?   r[   c              3  p   K   | ]1\  }}t          j        |          rt          j        |          n|V  2d S rh   )r   r  operatorindex)r   r   d_tfs      r=   r   z_eval_shape.<locals>.<genexpr>  sY       ; ;1d %)$8$;$;Ex~a    ; ; ; ; ; ;r?   )r   r   r  r   r  r   is_constant_shaper   r#   r  rl   r^   r#  r   evaluate_shapedim_value_avalr   r%  )r   r4  r'  r(  shape_values_tfr5   s         r=   _eval_shaper2    s1    
S((%00	1	1 E ECECCE E E
t/u==>>DE	E"" (''''''''%8%BCC(J)d!5(334&(()C
OO;RA A/1 
 ; ;!%99; ; ; 
; 
; ;r?   r   c                b    | j                                         r| S t          j        | |          S rh   )r   is_fully_definedrK   ensure_shape)r   r   s     r=   _ensure_tf_shape_if_dynamicr6    s/    W H	E	"	""r?   c                    d
dt          | j                  t          |          k    r.t          fdt          | j        |          D                       sJ d| d| j                     d	S )zeAsserts that shape matches x.shape in the known dimensions and has
  dimension polynomials elsewhere.xd
int | Nonesdr   c                f    t          j        |          r| |k    S t          j        |          sJ dS rA   )r   r  r   r  )r8  r:  s     r=   	check_onez2_assert_matching_abstract_shape.<locals>.check_one  s:    B 2Xo#B'''''Tr?   c              3  6   K   | ]\  }} ||          V  d S rh   rB   )r   r8  r:  r<  s      r=   r   z2_assert_matching_abstract_shape.<locals>.<genexpr>  sG       1 1"b iB 1 1 1 1 1 1r?   zShape z does not match x.shape N)r8  r9  r:  r   )r   r   r   r%  )r   r   r<  s     @r=   _assert_matching_abstract_shaper>    s        ag,,#e**
$
$
 1 1 1 1//1 1 1 1 1 %
$ 6U55AG55 %
$
$
$
$r?   c                  >    e Zd ZdZddgZdd	Zed
             Zd ZdS )r  a  Tracer class that boxes a TF value and a JAX abstract value.

  In addition to the TF value we carry the JAX abstract value because
  there are some cases when it cannot be recovered from the value:
  when we are converting with polymorphic shapes or when the JAX aval
  has a custom element type. In these cases the shape of the value may
  have dimensions set to `None`, or it may only correspond to the JAX
  "physical" (TF/lowering-compatible) shape, so the JAX abstract value
  may contain more precise information.

  When the value has a partially-known shape, the dimensions marked as `None`
  must correspond to non-constant dimensions in the abstract value.

  See README.md for details.
  r1  _avalr  rz  rE   r  core.AbstractValuec                (   || _         || _        t          | j                  }t          |t          j        t          j        f          r|j        }t          j	        j
        rt          |j                  t          |          k    sJ d|j         d|            |j        t          j        k    r8t          |j                  |j        k    sJ d|j         d|j                     n7|j        t!          |j                  k    sJ d|j         d|j                     t#          |j        |          D ]\  }}|*t%          j        |          sJ d|j         d|             1t%          j        |          s||k    sJ d|j         d|             a	 t)          t+          |g                    }n# t,          t.          f$ r Y w xY w||k    sJ d|j         d| d| d| d	            t1          ||j        d	          d
         | _        d S )Nz_aval.shape=z different rank than val_shape=z	expected z == z. Found z != r6   Tr   r   )_tracer@  r  rJ   rK   rL   rM   r   r   enable_checksr   r   r4  r   ra  r  r  r%  r   r  rO  r2  rT  KeyErrorr   r1  )	rd   r  r1  r  	phys_aval	val_shapeaval_dimval_dimaval_ints	            r=   re   zTensorFlowTracer.__init__  so   DKDJ"4:..I#	2;/00 u)i			# u9?##s9~~5557xio7x7xlu7x7x555?fm++io..#);;;=i=i=i^a^g=i=i;;;;M#)$<$<<<<>j)/>j>j_b_h>j>j<<<!$Y_i!@!@ 	u 	uHg_)(33aa5a5a5aV_5a5aaaaa)(33 uw&&&(TIO(T(T(T(T&&&&[(4455hhx(   hw&&&(tIO(t(t(t(t\d(t(tjq(t(t(t&&&&)#*3/<@B B BBCEDHHHs   F66G
	G
c                    | j         S rh   )r@  rc   s    r=   r  zTensorFlowTracer.aval  s
    :r?   c                    | S rh   rB   rc   s    r=   
full_lowerzTensorFlowTracer.full_lower  s    Kr?   N)r  rz  r1  rE   r  rA  )	rS   rT   rU   r   	__slots__re   propertyr  rN  rB   r?   r=   r  r    sm         " g) E  E  E  ED   8    r?   r  	primitivecore.Primitiveparamsr   source_infosource_info_util.SourceInfoxla_client.OpMetadatac          	     *   t          |j                  dz   t          j        | |          z   }t	          j        |          }t          j        | j        |t          j
        |r|j        ndt          j                              |r|j        nd           S )N/r[   op_typeop_namesource_filesource_line)r<  r  r   str_eqn_compactr"   
user_framer-   
OpMetadatar;   r(   get_canonical_source_file	file_nameTracebackCaches
start_line)rQ  rS  rT  eqn_strframes        r=   _make_op_metadatarg    s     '((3.#Iv667'

%k
2
2%		2$,EOO"d.B.D.DF F(-7E$$4
9 
9 
9 9r?   c                  t    e Zd ZdZd dZd!dZd"d	Zd#dZd$dZd%dZ	d Z
d Zd Zd Zd Zd Zd Zd&dZdS )'rz  a  Trace class that underlies the jax2tf transformation.

  We are going to ensure that jax2tf.convert is never nested inside other
  transformations. This is sufficient for intended use cases (converting
  fully-transformed JAX code). It also simplifies our job because we do not have
  to handle situations where we apply primitives on a mix of TF values and
  JAX tracers from an outer transformation. E.g., for addition both the TF
  values
  and the JAX tracers have an override and they get confused if they see values
  from the other world.

  Hence a TFT trace does not interact with non-TFT traces at lower-level. For
  higher-order control-flow primitives we invoke recursively
  _interpret_fun on the body of the conditional, which will create a nested TFT.

  We do want to allow transformations nested inside a TensorFlowTrace (TFT), but
  those will introduce their own MainTrace, and any operations involving those
  will be done on those traces, i.e., not a concern for TFT.
  r1  rE   rF   r  c                4   t          |d          r+|                                }t          |t                    r|S t	          |d          \  }}t          | |t          j        t          j        |          |t          j
        |                              S )zoLifts a non-Tracer into the TensorFlowTracer.

    This function may be called by way of trace.full_raise.
    __jax_array__TrC  )	weak_type)r  rj  rJ   r  r   r   r  r   r   r   is_weakly_typed)rd   r1  r#  r	  s       r=   purezTensorFlowTrace.pure  s    
 sO$$ c	C)	*	* 
23$OOOFIfd&rx}}i171G1L1LN N NO O Or?   core.Tracerc                    J rh   rB   rd   r1  s     r=   liftzTensorFlowTrace.lift(  s	     Lr?   c                8    t          | |j        |j                  S rh   )r  r1  r@  rp  s     r=   subliftzTensorFlowTrace.sublift.  s     D#'39555r?   rQ  rR  tracersSequence[TensorFlowTracer]c                J                          |          \  t          d |D                        |j        i \  }d |D             dfd}t                      }d                    d |j        D                       }t          j        rt          j         d| }|                    d          s|dz   }t          j
        t          |                    5  t          j        rt          |t          j                              }t!          j        |j        |j        |j        |j        	          }	t-          j                                        d
t3          j        |	                                          i          5   |            }
d d d            n# 1 swxY w Y   n
 |            }
d d d            n# 1 swxY w Y   |j        r fdt;          |
          D             }nt=           |
          }t>          j         j!        r|j        r]t;          |          D ]K\  }}|j"        #                                |#                                k    sJ | d|j"         d|             Ln"|j"        k    sJ | d|j"         d             |S )Nc              3  $   K   | ]}|j         V  d S rh   r  r  s     r=   r   z4TensorFlowTrace.process_primitive.<locals>.<genexpr>7  s$      2K2Ka162K2K2K2K2K2Kr?   c                    g | ]	}|j         
S rB   r1  r  s     r=   r   z5TensorFlowTrace.process_primitive.<locals>.<listcomp>>  s    777!777r?   rF   rE   c                 .    r  dS  i S )Nr  r  rB   )rv  r   r   impl_needs_avalsr  rS  s   r=   invoke_implz6TensorFlowTrace.process_primitive.<locals>.invoke_impl?  sN    	 (t   	  	 tW''''r?   rX  c                    g | ]	}|j         
S rB   r;   )r   ss     r=   r   z5TensorFlowTrace.process_primitive.<locals>.<listcomp>L  s    ???af???r?   )rT  rY  _XlaOpMetadatar  c                8    g | ]\  }}t          ||          S rB   r  r   rD   r   rd   s      r=   r   z5TensorFlowTrace.process_primitive.<locals>.<listcomp>j  s9       a 4A
&
&  r?   z: out.aval = z; expected )rF   rE   )$get_primitive_implr   abstract_evalrj   joinstackrl   ra   endswithrK   rg  r>   r_   rg  r"   currentr/   r`  rZ  r[  r\  r]  tf_opsr  _attr_scoper0   	AttrValueSerializeToStringr  r%  r  r   rE  r   r  strip_weak_type)rd   rQ  rt  rS  r5   r}  ri   scopeop_metadataop_metadata_protoval_outoutr   expected_avalrv  r   r   r|  r  s   `  `          @@@@@r=   process_primitivez!TensorFlowTrace.process_primitive3  s    "44Y??D
-22K2K72K2K2K-K-KJ *)):@@@KHa77w777G( ( ( ( ( ( ( ( ( ( ( 122 HH??&8&>???@@E . C$8BB5BBe>># cke	+E22	3	3    		4  '	64D4L4N4NP P P(3''#/#/	
 
 
 %''33~7#5577 9  9  9 :; ; 	" 	"  KMM'	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"
 +--                             " ! 6   '8,,  cc
 T7H55c ! 
		# 
 #C 2 2 	M 	MA}''))]-J-J-L-LLLLKKKKMKK MLLL	M x8###FFsxFFHFF $## Js7   *B)GF*G*F.	.G1F.	2GGGcall_primitiver  r  c                d    |j         sJ d |D             }t          d |D                       }t          | j        |          }d }t	          |          5  t          j                    5   |j        | }	d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y    fd|	D             S )Nc                    g | ]	}|j         
S rB   ry  r  s     r=   r   z0TensorFlowTrace.process_call.<locals>.<listcomp>  s    444qQU444r?   c              3  $   K   | ]}|j         V  d S rh   r  r  s     r=   r   z/TensorFlowTrace.process_call.<locals>.<genexpr>  s$      -F-Faf-F-F-F-F-F-Fr?   c                8    g | ]\  }}t          ||          S rB   r  r  s      r=   r   z0TensorFlowTrace.process_call.<locals>.<listcomp>  s*    >>>TQT1a((>>>r?   )r  r   r{  r  ru  r   r~  r  )
rd   r  r  rt  rS  valsavalsinterpreted_funro  vals_outs
   `         r=   process_callzTensorFlowTrace.process_call~  sM   ****44G444D(--F-Fg-F-F-F(F(FE)#ty%@@O	.	/	/ 7 7 7 7/?/67 7 7 7 7 7 7 7 7 7 7 7 7 7 77 7 7 7 7 7 7 7 7 7 7 7 7 7 7 ?>>>X>>>>s6   B+B6BB	B	B	
BB Br  c                \    t          d D                       }| j        dfd}||fS )Nc              3  $   K   | ]}|j         V  d S rh   ry  r  s     r=   r   z4TensorFlowTrace.post_process_call.<locals>.<genexpr>  s$      ,,1,,,,,,r?   r  r   c                    t          t          j                              fdt          |           D             S )Nc                B    g | ]\  }}t          ||j                  S rB   )r  r  )r   rD   
out_tracerr  s      r=   r   zCTensorFlowTrace.post_process_call.<locals>.todo.<locals>.<listcomp>  s;       a 5!Z_
5
5  r?   )rz  r   r  r%  )r  r  r  r  s    @r=   todoz/TensorFlowTrace.post_process_call.<locals>.todo  sQ    dD$5$7$788e   "455   r?   )r  r   )r   r  )rd   r  r  rS  r  r  r  s     `   @r=   post_process_callz!TensorFlowTrace.post_process_call  sX    
 ,,,,,,,D9D       :r?   c                     t          d          )Nprocess_mapr   )rd   map_primitivefrt  rS  s        r=   r  zTensorFlowTrace.process_map  s    
m
,
,,r?   c                     t          d          )Npost_process_mapr   )rd   r  r  rS  s       r=   r  z TensorFlowTrace.post_process_map  s    
0
1
11r?   c               J    ~~|                      t          j        ||i           S rh   r  r   call_p)rd   primr  jvprt  symbolic_zeross         r=   process_custom_jvp_callz'TensorFlowTrace.process_custom_jvp_call  s%     	^T[#w;;;r?   c                    J rh   rB   rd   r  r5   s      r=   post_process_custom_jvp_callz,TensorFlowTrace.post_process_custom_jvp_call      Lr?   c                N    ~~~~|                      t          j        ||i           S rh   r  )rd   r  r  fwdbwdrt  	out_treesr  s           r=   process_custom_vjp_callz'TensorFlowTrace.process_custom_vjp_call  s)    
 	S)^T[#w;;;r?   c                    J rh   rB   r  s      r=   post_process_custom_vjp_callz,TensorFlowTrace.post_process_custom_vjp_call  r  r?   c                    J rh   rB   )rd   r5   __s      r=    post_process_custom_vjp_call_fwdz0TensorFlowTrace.post_process_custom_vjp_call_fwd  r  r?   r   tuple[Callable, bool]c                (   t           j        s 	 t          |         dfS # t          $ r Y nw xY w	 t          |         dfS # t          $ rJ 	 t
          |         dfcY S # t          $ r*}d}t          |                    |                    |d }~ww xY ww xY w)NTFz7TensorFlow interpretation rule for '{}' not implemented)rl   r\   tf_impl_no_xlarF  rV   rW   r   format)rd   r   errrU  s       r=   r  z"TensorFlowTrace.get_primitive_impl  s     ) a $&&   :QZ : : ::!!$d**** : : :G!#**Q--00c9::s9    
**= 
BAB
B#%BBBN)r1  rE   rF   r  )r1  rn  rF   r  )r1  r  rF   r  )rQ  rR  rt  ru  rF   r  )r  rR  r  r  rt  ru  )r  rR  r  ru  )r   rR  rF   r  )rS   rT   rU   r   rm  rq  rs  r  r  r  r  r  r  r  r  r  r  r  rB   r?   r=   rz  rz    s        &O O O O   6 6 6 6
I I I IV
? 
? 
? 
?   $- - -2 2 2< < <  < < <    : : : : : :r?   rz  r   c                    J d|              )NFz!Encountered unexpected primitive rB   )r   argsr  s      r=   _unexpected_primitiver    s    77A77777r?   c                $    t          | g|R dd dS )NF)r,  ro  r  )r  r  s     r=   rC   rC     s6    )II I+04I I I r?   ):clzigamma_grad_arandom_gamma_grad	polygamma
reduce_xorschurclosed_callunreachablebintgetslicefull_to_shardshard_to_fullpure_callback	run_stateforinspect_shardingio_callbackbroadcast_to	shard_map global_array_to_host_local_array host_local_array_to_global_arraycall_exportedzeta	after_all
all_to_allcheckcreate_tokencustom_transpose_callcustom_vmap_callinfeedlinear_calloutfeedpmax_ppminppermutepsumpsum2
pbroadcastpmaxpgatherreduce_scatter
axis_index
all_gatherlu_pivots_to_permutationxla_pmapgeqrfhouseholder_product
hessenbergtridiagonaleigh_jacobiplatform_indexassert_consumed_valueconsume
ragged_dotcholesky_updatebitcastrepeatrollc                    | S rh   rB   r'  s    r=   rC   rC     s    A r?   yc                D    t           j                            | |          S )Nr   r  )rK   r   AddV2r
  s     r=   _addr    s    			A		#	##r?   )devicessrcsc                    |S rh   rB   )r  r  xss      r=   rC   rC     s    b r?   c                    | S rh   rB   r'  s    r=   rC   rC      s     r?   r  c                      t          |           S rh   r   r  r5   s     r=   _shard_aliker  "  s    	tr?   c                   | j         j        r`t          | j                  }t          j        | |          }t          j                            |          }t          j        || j                   S t          j                            |           S rh   )r4  is_unsigned_UNSIGNED_TO_SIGNED_TABLErK   r  mathnegative)r   signed_dtypex_signed
res_signeds       r=   _negr  &  sk    W ,QW5Lwq,''H!!(++J7:qw'''7Ar?   c                $   | j         j        rft          j        t          j                            | d          t          j        d| j                   t          j        d| j                             S t          j                            |           S )Nr   r_  r  )r4  r  rK   wherer  equalrO   signr'  s    r=   _signr#  2  sq    W 8
aR[!':::
AQW%%%' ' ' 7<<??r?   r  r  c          	     0   |t           j        j        u rt          |           }| |z  } t          j                            |           }| |z  } t          j                            | t	          j        t          j
        d          | j                            }|t	          j        |t	          j        t          j
        d          | j                  t          j                            |                     |z   z  S t          j                            |           S )Ng      ?r  )r   RoundingMethodAWAY_FROM_ZEROr#  rK   r  floorr!  rO   r   arrayr4  r   round)operandrounding_methodr  r  r"  r'  conds          r=   _roundr-  A  s     *999>>DtOGGMM'""EuG7=="+bhsmmW]"K"KLLD
r{28A;;>>w''	) 	)+0	12 2 7==!!!r?   c                v    | j         }t          j        t          j                            |           |          S )Nr'  )r4  rK   r  r   PopulationCount)r   
orig_dtypes     r=   _population_countr1  U  s.    w*	++a+00*	=	==r?   c                \    | j         j        st          j                            |           n| S rh   )r4  r  rK   r  absr'  s    r=   _absr4  ^  s"     w2	9Q9r?   c               
   t           j                            | t          |j                            } t           j                            |t          |j                            }t           j                            | |          S rh   )rK   r   r  r  r4  r  pow)r   r  r  r  s       r=   _powr7  f  sV    innQY_5566!innQY_5566!	Q		r?   c                  |dk    rAt          j        t          j        d| j        d          t	          |j                            S |dk     }|r| }d }|dk    rZ|dz  r$|| nt           j                            ||           }|dz  }|dk    r t           j                            | |           } |dk    Z|rt           j                            |          n|S )Nr   r  rB   )r4  r   )	rK   r  rO   r4  r2  r   r  multiply
reciprocal)r   r  r  r  is_reciprocalaccs         r=   _integer_powr=  o  s     !VV?
AQWB///Y_1M1MO O Oa%- 	
A#	A1u ;AA"'"2"23":":c!GA1uu
'

1a
 
 a 	
A %2	:		C	 	 	 s:r?   c           
         t           j                            t           j                            t           j                            t          j        d| j                            |                     S )N   )rK   r  expr9  logrO   r4  r'  s    r=   rC   rC     sF    GKK  R[AG-D-D!E!EqIIJJ r?   r  c           	         |j         j        s| j         j        rt          j        t          j        t          j        t          j        i                    | j                   }t          j        d|          }t          j        d|          }t          j	        ||          }| t          j
                            ||| z  z   t          j
                            ||z  | | z  z             z            z  S t          j
                            | |          S Nr   r  )r4  
is_complexrK   	complex64float32
complex128float64r  rO   complexr  rA  sqrtatan2)r  r   r  complex_component_dtypezerooner  s          r=   _atan2rP    s    W 
17- 
lBJmRZ 
c!'ll  ;q122D
+a0
1
1C

4A2QQYQUQU](C(CCDDDD7==Ar?   c                    t           j                            |           t           j                            t           j                            |           d          z  S )NgUUUUUU?)rK   r  r"  r6  r3  r'  s    r=   _cbrtrR    s5    	a27;;rw{{1~~s;;	;;r?   c                   | j         t          j        k    rt          j        | t          j                  S | j         t          j        k    rt          j        | t          j                  S t          j                            |           S rh   )	r4  rK   rG  r  rF  rI  rH  r  conj)r   r  s     r=   _conjrU    s\     W
71bl###w"*71bm$$$7<<??r?   c                   t          |           } t          |          }t          j        t          j        |         t          j                  t          j                  }fdt          t          |                    D             }t          j        t          j        t          j        ||          |          |           S )Nr_  c                $    g | ]}|k    rd ndS )r  rB   )r   r  	dimensions     r=   r   z_iota.<locals>.<listcomp>  s%    FFFqQ)^^rrFFFr?   )	r  r2  rK   r   r  int32r   r  reshape)r4  r   rY  shape_tfvec	vec_shapes     `   r=   _iotar_    s    
u

% (
),bh77rxHHH#FFFFE#e**4E4EFFF)	C!;!;XFF	N	NNr?   c           	        | j         j        rt          j                            | |          }t          j                            t          j        t          |           t          |                    t          j        t          j                            | |          d                    }t          j	        ||dz   |          S t          j        
                    | |          S rD  )r4  
is_integerrK   r  floordivlogical_and	not_equalr#  floormodr   truediv)lhsrhsquotientselects       r=   _divrk    s    Y %wS))HW  
U3ZZs,,
RW%%c3//335 5F 8FHqL(3337??3$$$r?   c                    t          |           t          j                            t	          |           t	          |                    z  S rh   )r#  rK   r  re  r4  )rg  rh  s     r=   _remrm    s1    	sbg&&tCyy$s))<<	<<r?   is_minc                  t          j        | j        j        t          j                  rO t          t          t          j	        |rt          j        nt          j                  d          | |||          S | j        j        t          j        k    r. |rt          j        j        nt          j        j        | |          S  |rt          j        j        nt          j        j        | |          S )N)lax_cmp_pick_xFrB  r{  )r   r  r4  r  r   complexfloatingr  r   lax_internal_minmax_complex_loweringr   ltgtbool_rK   r  rc  
logical_orminimummaximum)r   r  rn  r  r  s        r=   _minmaxrz    s     qw-r/ABB B 5)/;svvSV	= 	= 	=      !"1	YP P P P 	w))A6ABGrw/A1aHHH:v:BGOO27?AqAAAr?   c                  t          | j                  dk    rt          |j                  dk    sJ d| j         d|j                     t          j        dt	          | j                            }t          | ||||g|          S )Nr   zx: z, y: rB   )rn  r  r  )r   r   r   r  r  r4  rz  )r   r  rn  r  s       r=   _minmax_scalarr|    s     
QW		s17||q0002O2O2Oag2O2O000		"mAG44	5	5$	Af $<4
9 
9 
9 9r?   )rn  c                    i | ]\  }}||	S rB   rB   )r   r  us      r=   
<dictcomp>r    s    PPPdaQPPPr?   c                d   | j         j        r| j         |j         k    sJ | j         }t          |         }t          j        | |          } t          j        ||          }t          j                            | |          }t          j        ||          S t          j                            | |          S rh   )r4  r  r  rK   r  bitwiseright_shift)r   r  r0  r  r  s        r=   _shift_right_arithmetic_rawr    s    W 	(7agJ,Z8L
<  A
<  A
*
 
 A
&
&C73
###:!!!Q'''r?   c                    d| j         j        z  }t          j        t	          | |          ||dz
            }t          | |          S )N   r  )r4  r  rK   r   _shift_in_boundsr  )r   r  x_bitsclamp_ys       r=   _shift_right_arithmeticr  ,  sD    
 qw|&H%a++Q
;;'	$Q	0	00r?   c                d   | j         j        r t          j                            | |          S | j         |j         k    sJ | j         }t
          |         }t          j        | |          } t          j        ||          }t          j                            | |          }t          j        ||          S rh   )r4  r  rK   r  r  _SIGNED_TO_UNSIGNED_TABLEr  )r   r  r0  unsigned_dtyper  s        r=   _shift_right_logical_rawr  9  s    W 	$:!!!Q'''7agJ.z:N
>""A
>""A
*
 
 A
&
&C73
###r?   c                    t          j        t          | |          t          | |          t          j        |                     S rh   )rK   r   r  r  rb  r
  s     r=   _shift_right_logicalr  F  sA    
 
q!6q!<<bmA>N>N
P 
P Pr?   c                    t          j        t          | |          t           j                            | |          t          j        |                     S rh   )rK   r   r  r  
left_shiftrb  r
  s     r=   _shift_leftr  R  sH    
 
q!bj33Aq992=;K;K
M 
M Mr?   c                0   d| j         j        z  }|j         j        r%t          j        |t
          |j                            n|}t          j                            ||          }t          j                            |d          }t          j	        ||          S )Nr  r   )
r4  r  r  rK   r  r  r  lessgreater_equalrc  )r   r  r  y_compy_lt_x_bitsy_ge_0s         r=   r  r  ^  s    qw|& 120CK27	"17	+- - -IJ 	VV,,+7  ++&	V	,	,,r?   c                    | j         t          j        k    rt          j        |           S t          j                            |           S )a  Computes bitwise not with support for booleans.

  Numpy and JAX support bitwise not for booleans by applying a logical not!
  This means that applying bitwise_not yields an unexpected result:
    jnp.bitwise_not(jnp.array([True, False]))
    >> Array([False,  True], dtype=bool)

  if you assume that booleans are simply casted to integers.
    jnp.bitwise_not(jnp.array([True, False]).astype(np.int32)).astype(bool)
    >> Array([True,  True], dtype=bool)
  )r4  rK   rG   logical_notr  invertr'  s    r=   _notr  j  s9     W>!:Qr?   argnumsSequence[int]c                Z     t           j                                      d fd}|S )a  Computes functions with some bool args and bool results using int8.

  This is needed because some TF ops do not work for bool args, e.g.,
  inequalities, min/max.

  Args:
    f: a TF callable to wrap. It will be called with non-boolean arguments.
    argnums: the positional arguments that may be booleans.
    boolean_f: [Optional] a TF callable compatible with boolean
      arguments.

  Returns: a TF callable that can take a mix of boolean positional arguments
    (in the positions specified by `argnums`) and some non-boolean positional
    arguments. If there are no boolean arguments, just calls `f`. Otherwise,
    it calls `boolean_f` if defined. Otherwise, casts the boolean
    arguments to `int8`, calls `f`, then casts the result to `bool`.
  r  rE   c                      fdD             }t           j        |vr 
 i |S t          |          dk    s
J |            	d k    r 	 i |S fdt                     D             }d|v r]d fdt          |d                   D             }t           j                            |d                   }t          |||          } 
|i |}t           j                            d	 |          S )
Nc                *    h | ]}|         j         S rB   r_  )r   r  r  s     r=   	<setcomp>z7handle_boolean_args.<locals>.wrapper.<locals>.<setcomp>  s    333aDGM333r?   r  c                `    g | ]*\  }}|v rt          j        |t           j                  n|+S rB   )rK   r  int8)r   r  r   r  s      r=   r   z8handle_boolean_args.<locals>.wrapper.<locals>.<listcomp>  sH     1 1 11 ./'\\bga)))q 1 1 1r?   r  c                x    | j         t          j        k    sJ t          j        | j        t          j                  S rh   )r4  r   rv  r   r  r   r  r  s    r=   	cast_avalz7handle_boolean_args.<locals>.wrapper.<locals>.cast_aval  s.    :))))#DJ888r?   c                8    g | ]\  }}|v r |          n|S rB   rB   )r   r  r  r  r  s      r=   r   z8handle_boolean_args.<locals>.wrapper.<locals>.<listcomp>  sA       !T "#giiooo4  r?   r  r{  c                @    t          j        | t           j                  S rh   )rK   r  rG   )r   s    r=   rC   z6handle_boolean_args.<locals>.wrapper.<locals>.<lambda>  s    rwq"'/B/B r?   )rK   rG   r   r  nestmap_structurer   )r  r  argnum_types	args_cast_in_avals_cast_out_aval_castr  r  r  	boolean_fr  s   `      @r=   wrapperz$handle_boolean_args.<locals>.wrapper  sq   33337333L	wl""Q !###\###	d		y$)&)))1 1 1 1 )$1 1 1	&  9 9 9    &vk':;;  . 700F;<OPP..J J J&a%f%%w$$%B%BCHHHr?   r  rE   )rK   r  flatten)r  r  r  r  s   ``` r=   handle_boolean_argsr    sQ    $ GOOG$$'I I I I I I I I8 
.r?   )r   r  )r  r  c                x   t          j        | j        j        t          j                  s| S t          j        | j        j        t          j                  sJ t          j        | j        j                  j        }t          j
        |         }t          j        |         }t          j        | |          }t          j        t          j                            ||dz
            |          }t          j        t          j                            |d          |          }t          j                            ||          S Nr  )r   r  r4  r  r   inexactfloatingfinfobitsrr  _INT_DTYPES_UINT_DTYPESrK   r  r  r  bitwise_xor)r   nbitsr  r  signed	sign_masksign_magnitude_masks          r=   _total_order_adjustmentr    s    		1712:	>	> H		1712;	?	???? ,qw-
.
.
3%)%0,,U3.:a&&&j//	BBNSS)
2:#9#9)Q#G#GVV			(;	<	<<r?   c                   t          j        | j        j        t          j                  rt          t          j        	                    |           t          j        	                    |                    oIt          t          j        
                    |           t          j        
                    |                    S t          j                            t          |           t          |                    S rh   )r   r  r4  r  r   rq  _total_order_equalrK   r  realimagr!  r  r
  s     r=   r  r    s    qw-r/ABB ybgll1oorw||A??xDVWYW^WcWcdeWfWfhjhohthtuvhwhwDxDxx	.q113J13M3M	N	NNr?   c                P    t          j        | t          j        |                    S rh   rK   rc  r  r
  s     r=   rC   rC     s    bnQq0A0ABB r?   c                P    t          j        t          j        |           |          S rh   r  r
  s     r=   rC   rC     s    2>".*;*;Q?? r?   c                F    t          j        t          | |                    S rh   )rK   r  boolean_lessr
  s     r=   rC   rC     s    |Aa7H7H(I(I r?   c                F    t          j        t          | |                    S rh   )rK   r  boolean_greaterr
  s     r=   rC   rC     s    R^OAa4H4H%I%I r?   c                N     | t          |          t          |                    S rh   )r  )r,  r   r  s      r=   _total_order_condr    s&    	%a((*A!*D*D	E	EEr?   )rk  r   c               p   | j         j        }t          j        |t          j                  r>t          j        |t          j                  st          j                            |           } t          j        |t          j	                  rt          j        |t          j	                  sct          j        |t          j                  sD|t          j
        k    s4t          |           }|t          j                            || z            z  } t          j                            | t          |                    S rh   )r4  r  r   r  r   rq  rK   r  r  r  rv  r#  r'  r  r  )r*  	new_dtyperk  r   	old_dtyper"  s         r=   _convert_element_typer    s    m*)	2#566 $

Ir'9
:
:$gll7##G	2;// 3Y4438>8I
R'9) 9)3,5,A,A>>DRW]]4'>222G	i!8!8	9	99r?   c                `    | j         |k    r| S t          j        | t          |                    S rh   )r4  rK   r  r  )r*  r  s     r=   _bitcast_convert_typer    s.    ]iN	G\)44	5	55r?   c               
   t          |d         j        |d         j                  }t          j        ||          }t          j                            t          j        | |          |          } t          j        || |          S r  )r2  r   r4  rK   r  r  rx  clip_by_value)minvalr*  maxvalr  r  op_shape_tf_vals         r=   _clampr    sf    	! 2IaL4FGG/?6?33&7??2?6?CCVLL&		'66	2	22r?   c                h    t          j        |t          j        | t           j                            S Naxis)rK   concatr  rZ  )rY  operandss     r=   _concatenater    s%    	8"')RX">">	?	?	??r?   c                   t          | t          j                  sJ | \  }}}t          j                    }|d         |_        |d         |_        |d         |_        |d         |_        |d         |_	        |d         |_
        |j                            |dd                    |j                            |dd                    |j                            |dd                    |S )zFConverts a ConvDimensionNumbers to an XLA ConvolutionDimensionNumbers.r   r  r?  N)rJ   r   ConvDimensionNumbersr/   ConvolutionDimensionNumbersinput_batch_dimensioninput_feature_dimensionoutput_batch_dimensionoutput_feature_dimensionkernel_output_feature_dimensionkernel_input_feature_dimensioninput_spatial_dimensionsextendkernel_spatial_dimensionsoutput_spatial_dimensions)dimension_numberslhs_specrhs_specout_specprotos        r=   %_conv_general_dimension_numbers_protor    s    	%s'?	@	@@@@!2(Hh

2
4
4% (%"*1+%!)!%#+A;% *21+%')1!%& ''555!((!""666!((!""666	,r?   	precision*None | tuple[PrecisionType, PrecisionType]c                    | dS t          j                    }|j                            | d         j                   |j                            | d         j                   |S )z-Convert an integer to an XLA.PrecisionConfig.Nr   r  )r/   PrecisionConfigoperand_precisionr  r   )r  r  s     r=   _precision_config_protor  '  s^     4

&
(
(%  1!3444  1!3444	,r?   r  lax.ConvDimensionNumbersfeature_group_countbatch_group_count*tuple[PrecisionType, PrecisionType] | Nonepreferred_element_typec          
        t          |          t          |          t          |	          df	d}t          j        |d         j        t          j                  rl|
Jt          j        |
t          j                  sJ |
t          j        k    rt          j        nt          j	        }nd}t          j                            |           t          j                            |           }}t          j                            |          t          j                            |          }} |t          ||          ||          } ||t          j                            ||          |          } ||t          ||          |          }t          j        t          j                            ||          t          ||                    S  || ||
          S )z;Implementation of lax.conv_general_dilated_p using XlaConv.r  r  c                  	 t          d t          j                            d          d d         D                       }|dk    r-d D             }t	          j        | ||	|d          }n@dk    rt          d	          d
 D             }t	          j        | ||	|d          }t          |
          }t          rt          j	        |          }|S )Nc              3  4   K   | ]}t          |          V  d S rh   r)  )r   rD   s     r=   r   z:_conv_general_dilated.<locals>.gen_conv.<locals>.<genexpr>C  s(      EE!s1vvEEEEEEr?   r6   r?  )r?  r  c                ,    g | ]}t          |          S rB   r2  r   s     r=   r   z;_conv_general_dilated.<locals>.gen_conv.<locals>.<listcomp>G      444qKNN444r?   T)r  r  precision_configr  use_v2r  zlThe batch_group_count parameter for conv requires TF version at least 2.8.0. You may want to use tf-nightly.c                ,    g | ]}t          |          S rB   r  r   s     r=   r   z;_conv_general_dilated.<locals>.gen_conv.<locals>.<listcomp>U  r  r?   )r  r  r  r  )
r   rK   __version__splitr  convr   r6  r  stop_gradient)rg  rh  r  
tf_version
padding_tfr  r  dnums_protor  lhs_dilationout_tf_shapepaddingprecision_config_protorhs_dilationwindow_stridess         r=   gen_convz'_conv_general_dilated.<locals>.gen_convB  s(   EEr~';';C'@'@!'DEEEEEJV 54G444jJ
sNJl
1-1!7  cc 
a		>? ? 	? 54G444jJ
sNJl
11!7  c &c<
8
8C% "S!!cJr?   r   N)r  r  )r  r  r  r   r  r4  rq  rH  rI  rG  rK   r  r  r  r  subtractrJ  )rg  rh  r  r  r  r  r  r  r  r  r  r  r  r  preferred_float_etlhs_reallhs_imagrhs_realrhs_imagk1k2k3r  r  r  s     ```` ``             @@@r=   _conv_general_dilatedr   3  s    #9--,56GHH+29==             J ]9Q<%r'9:: 6)]123EFFFFF."-??"**RZ   c**BGLL,=,=hHc**BGLL,=,=hH	$x**H6H	I	IB	(BG,,Xx@@$
& 
&B	(D8446H	I	IB:bg&&r2..R===8C4555r?   c                  t          | |d         ||d         |          \  } }}|\  \  }}	\  }
}t          j                    }|j                            |           |j                            |	           |j                            |
           |j                            |           t          |          }t          j
        | ||||d          } ||          }t          rt          j        |          }|S )zAImplementation of lax.dot_general_p in terms of tf.linalg.einsum.r   r  T)r  r  )$_dot_general_convert_to_common_dtyper/   DotDimensionNumberslhs_contracting_dimensionsr  rhs_contracting_dimensionslhs_batch_dimensionsrhs_batch_dimensionsr  r  dot_generalr  rK   r  )rg  rh  r  r  r  r  r  convert_resultlhs_contractingrhs_contracting	lhs_batch	rhs_batchr  r  r  s                  r=   _dot_generalr.  }  s   & B1sIaL)5 5#sN @Q<$?O&<y)022+(//@@@(//@@@")))444")))44429==		3	 	 	# 	s##  

3

C	*r?   rg  lhs_avalrh  rhs_avalr  -tuple[TfVal, TfVal, Callable[[TfVal], TfVal]]c                    |j         |j         k    rSt          j        ||          }||j         k    rt          | |          } ||j         k    rt          ||          }fd}nd }| ||fS )Nr  c                0    t          | j                  S Nr3  )r  r4  )r  r  s    r=   rC   z6_dot_general_convert_to_common_dtype.<locals>.<lambda>  s    !6shn!U!U!U r?   c                    | S rh   rB   )r  s    r=   rC   z6_dot_general_convert_to_common_dtype.<locals>.<lambda>  s     r?   )r4  r   result_typer  )rg  r/  rh  r0  r  common_dtyper)  s       `  r=   r"  r"    s     ^x~%%%h99Lx~%%!#>>>cx~%%!#>>>cUUUUNN$_N
sN	##r?   c               ,   |d         j         }|d         j        }dgt          |          z  }t          |          D ]\  }}	||         ||	<   t	          j        | t          ||                    }
t	          j        |
t          ||                    S )Nr   r  r_  )r   r4  r   r  rK   r[  r2  r  )r*  r   broadcast_dimensionsr  r  op_shaper4  add_1s_shaper  broadcast_dim_iwith_1ss              r=   _broadcast_in_dimr?    s     q\(
A,
%s5zz!,%&:;; 0 0a$,QKL!!JwL F F FGG'	+e5"A"A"A	B	BBr?   c                    t          j        | t           j                  rt          t	          j        t          j        d|                     S )Nr   r_  )r   r  r  r   rK   rO   r   r(  r_  s    r=   _emptyrA    s>    ufo.. 
	RXau---	.	..r?   c                   |&t          j        t          j        |                     }t          ||d         j                  }t          j        t          j        | |          |          S Nr   )rK   r   rankr2  r4  r[  	transpose)r*  	new_sizes
dimensionsr  r  new_sizes_tfs         r=   _reshaperI    sU    "''**++JY	!(:;;,	BL*55|	D	DDr?   c                   t          |d                   }|j        }t          fdt          |          D                       }t	          ||j                  }t          j        | |          S )Nr   c              3  *   K   | ]\  }}|v	|V  d S rh   rB   )r   r  r   rG  s      r=   r   z_squeeze.<locals>.<genexpr>  s1      MM$!Q*9L9LA9L9L9L9LMMr?   )r  r   r   r  r2  r4  rK   r[  )r*  rG  r  r  op_avalr;  	new_shapenew_shape_tfs    `      r=   _squeezerO    sh    y|,,'](MMMM)H"5"5MMMMM)Y66,	G\	*	**r?   c                   t          j        t          t          |                    \  }}}t	          j        | ||||          }t          |t          |                    }t          rt          j
        |          }|S rh   )r#   unzip3r   r2  r  padr6  r  r  rK   r  )	r*  padding_valuepadding_configr  r  lowhighinteriorr  s	            r=   _padrX    sp     C^$D$DEE#tX	'=#tX>>##C):9)E)EFF##  

3

C	*r?   c               ,    t          j        | |          S rh   )rK   reverse)r*  rG  s     r=   _revr[    s    	GZ	(	((r?   c                      j         t          j        k    rLt          |          dk    sJ t          |          dk    r|n!t          j         |d         |d                   S  fd d|          S )Nr?  r  r   c           
     (   t          |          dk    sJ t          |          dk    r|d         S t          |          dz  }t          j        t          j        | |z              | |d |                    |||d                              S )Nr   r  r?  )r   rK   r   r  )offsetcasesmid_selectwhichs      r=   ra  z_where.<locals>._select		  s    u::>>>>
5zzQ1Xo
e**/C8BGE6C<00GFE$3$K00GCstt--/ / /r?   )r4  rK   rG   r   r   )rb  r_  ra  s   ` @r=   _whererc  	  s    
[BGu::????JJ!OO55%q58)L)LL/ / / / / / 
E		r?   c               .    t          j        | |          S )N)perm)rK   rE  )r*  permutations     r=   
_transposerg  	  s    	gK	0	0	00r?   c                      fdS )Nc                     | |          S r  rB   )r*  axesfuncs     r=   rC   z<lambda>.<locals>.<lambda>	  s    $$wT2J2J2J r?   rB   )rk  s   `r=   rC   rC   	  s    JJJJ r?   r*  rj  index_dtypec                    | rd}t           j        }t          j        }nd}t           j        }t          j        } t          t          t          j        ||          d|          |||||          }	|	S )NargminargmaxFr  ro  )rl  rj  r  r  )	r   rt  rr  _get_min_identityru  _get_max_identityr  r   _compute_argminmax)
rn  r*  rj  rl  r  r  ro  value_comparatorget_identityr  s
             r=   
_argminmaxrv  -	  s    
  2v1LLv1L	)l-/?NN'	) 	) 	) !	 	 	# 
*r?   	autographtangentsselect_primwindow_dimensionsr  base_dilationwindow_dilationr  Sequence[tuple[int, int]]c
                P   |j         t          j        j                  j        d}
|
k    sJ dz  |
k    }d |r?t
          j                 t
          j        dz           fdfdfd}nt          ddz   d|
 d	          t          j	        u st          j
        u s
J             fd
}t          j	        u rt          j         nt          j        fd}t          || ||           |||||||	
  
        } ||          S )N@   r?  c                P    t          j        t          j        |          |           S rh   )rK   rO   r   r(  )r4  r   s     r=   rC   z(_select_and_gather_add.<locals>.<lambda>a	  s    2;rx{{E:: r?   c                   t          |           } t          |          }t          |           } t          |          }t          j                            |                      } t          j                            | |          S r5  )r  r  rK   r  r  
bitwise_or)r   bconstdouble_word_dtyper  
word_dtypes     r=   packz$_select_and_gather_add.<locals>.packh	  s    
:
.
.a
:
.
.a
->
?
?
?a
->
?
?
?a
*

55):E#B#B
C
CaZ""1a(((r?   c                    | j         k    sJ t          |                      }t          t          |                    S r5  )r4  r  r  r  )r  str  r  r4  r  r  s     r=   fstz#_select_and_gather_add.<locals>.fstq	  sZ    W)))))55):E#B#BCCb"
j
9
9
95B B Br?   c                B    t          t          |                     S r5  )r  r  )r  r4  r  s    r=   sndz#_select_and_gather_add.<locals>.sndx	  s*    "
Z
8
8
8%A A Ar?   zTODO: need to pack z* bits but this platform can only go up to z bits.c                    t                    }t          t          j                  | |            |                    ||           S rh   )rV   r   
select_n_p)r   r  rb  r  rz  s      r=   reducerz'_select_and_gather_add.<locals>.reducer	  sB    K E3>"55QQ#8#8!Q???r?   c                D                 d                    S rC  rB   )r   r  r4  initr  s    r=   rC   z(_select_and_gather_add.<locals>.<lambda>	  s)    DDud!3!3UU5!__EE r?   )r{  r  r  r|  r}  r  r  )r4  r   r  r  r  rr  r  r   r   ge_ple_pr   inf_specialized_reduce_window)ry  r*  rz  r{  r  r|  r}  r  r  r  max_bitsdouble_word_reductionr  r  init_identityr  r  r  r4  r  r  r  r  r  s     `             @@@@@@@@r=   _select_and_gather_addr  O	  s    -%
,u+
,
,
1% (	(				!)x/
:
:% *51J$1%!)<) ) ) ) ) ) ) )B B B B B B B B BA A A A A A A
 ceaiccS[ccc   
	 	 K38$;$;$;[$;$;$;@ @ @ @ @ @  38++"&$EEEEEEE-"
d7H)#!%
	 
	 
	# 
S/r?   c
           
     H   t          j        d| j                  }
t          j        |d                              |
|
          }t          |t           j        t           j        f          st          j        || j                  }t          |          }t          |          }t          |          }t          |          }d |D             }t          j        | |||||||          }t          |t          |	                    }t          rt          j        |          }|S )NrB   r_  Frw  c                ,    g | ]}t          |          S rB   r  r   s     r=   r   z)_common_reduce_window.<locals>.<listcomp>	  s    0001A000r?   )base_dilationswindow_dilationsr  )rK   
TensorSpecr4  functionget_concrete_functionrJ   rL   rM   rO   r2  r  reduce_windowr6  r  r  r  )r*  init_valr  r{  r  r  r|  r}  r  r  o_spec
reducer_fnwindow_dimensions_tfwindow_strides_tfwindow_dilation_tfbase_dilation_tfr  r  s                     r=   _common_reduce_windowr  	  s*    =7=111&{      5 5ff E E  
Hry"+6	7	7 4{8W]33H$%677!.11"?33 //00000*%)	 	 	# 	$C):9)E)EFF##  

3

C	*r?   c        	        :    t                    dk    s
J d            t          j        |	t          |	          dz  g          \  }
}t          |
          dk    rt          d          d fd
}t	          |
d         |d         ||||||||d         
  
        fS )a|  TensorFlow implementation of reduce_window.

  Args:
    operands: N dimensional arrays containing elements of type T
    init_values: starting values of the reduction
    jaxpr: the jaxpr corresponding to the reduction function
    consts: the constants associated with jaxpr.
    window_dimensions: array of integers for window dimension values
    window_strides: array of integers for window stride values
    padding: array of pairs of integers for padding values
    base_dilation: array of integers for base dilation values
    window_dilation: array of integers for window dilation values

  Returns:
    The reduced operand.
  r   z+Reduction computation cannot have constantsr?  r  z.jax2tf does not support variadic reduce_windowarg1rE   arg2rF   c                \    t          j                  }t          || |d           \  }|S Nro  r   ClosedJaxprr  )r  r  closed_jaxprr  constsr  s       r=   r  z_reduce_window.<locals>.reducer	  s2    #E622LL$tLLLDCJr?   r  rE   r  rE   rF   rE   )r   r#   
split_listr   r  )r  r  r{  r  r  r|  r}  r  r  r  r  init_valuesr  s   ``           r=   _reduce_windowr  	  s    & 
V			H			/$Ta0@AA(K]]a
N
O
OO      
  [^W 1>7 -	 )!. . 
0 0r?   r  c               N    t          | ||j                  | |||||||	
  
        S )a  Wraps the TensorFlow reduce window operation based on a reducer and an

  identity function defining the initial value of the reduction depending on
  the dtype of the operand.

  Args:
    reducer: reduction function of type TfVal -> TfVal -> TfVal
    identity: function that takes a TensorFlow dtype as a parameter and returns
      the starting value of the reduction.
    operand: N dimensional array containing elements of type T
    window_dimensions: array of integers for window dimension values
    window_strides: array of integers for window stride values
    padding: array of pairs of integers for padding values
    base_dilation: array of integers for base dilation values
    window_dilation: array of integers for window dilation values
    name: the name of the specialized reduce window primitive for which this
      conversion function is called. This information may help to choose a
      different conversion path (optional)

  Returns:
    The reduced operand.
  )r  r4  )r  r   r*  r{  r  r  r|  r}  r  r  r;   s              r=   r  r  	  s:    D 
w(?(?0.',oy(
* 
* *r?   c                f   | j         }| t          j        k    st          j        |t
          j                  r |t
          j                   S t          j        |t
          j                  rt          j	        |          j
        S t          j        |t
          j                  sJ |  d            dS )Nz has no defined max identityF)r  rK   bfloat16r   r  r   r  r  integeriinfominrv  r  numpy_tf_dtypes     r=   rr  rr  
  s    *. 1."* M M>26'"""44 <''++" " O O'/$M$M$MO O O5r?   c                d   | j         }| t          j        k    st          j        |t
          j                  r |t
          j                  S t          j        |t
          j                  rt          j	        |          j
        S t          j        |t
          j                  sJ |  d            dS )Nz has no defined min identityT)r  rK   r  r   r  r   r  r  r  r  maxrv  r  s     r=   rq  rq  
  s    *. 1."* M M>"&!!!44 <''++" " O O'/$M$M$MO O O4r?   c                    dS rC  rB   r'  s    r=   rC   rC    
  s     r?   reduce_window_sumreduce_window_minreduce_window_maxr  computationrG  c                   ~ j         rJ t          |          dz  dk    sJ t          |          dz  }||d          }|d|         }t          d |D             dz            }dfd}	 t          j        |	d	          j        | }
t          j        ||||

          }t          rt          d |D                       }|S )Nr?  r   c                B    g | ]}t          j        d |j                  S )rB   )rK   r  r4  )r   ops     r=   r   z_reduce.<locals>.<listcomp>>
  s&    LLLBBM"bh77LLLr?   r  rE   rF   c                 (    t          g| R dd i}|S )Nro  r  )r  r  r  s     r=   reducer_computationz$_reduce.<locals>.reducer_computation@
  s%    
5
?4
?
?
?$
?
?CJr?   Frw  )dimensions_to_reducer  c              3  >   K   | ]}t          j        |          V  d S rh   rK   r  r   r  s     r=   r   z_reduce.<locals>.<genexpr>L
  s-      773!#&&777777r?   r  rE   rF   rE   )	r  r   r   rK   r  r  r  variadic_reducer  )r  r  rG  r  r  r  nr_operands	init_valsreducer_arg_specr  xla_reducer_computationr  s    `          r=   _reducer  /
  s     \	X	a				 H"+{||$)am$(LL)LLLqPQQ     
9bk%!# # ##8:JL  
	x4>'>
@ 
@ 
@$ $ 877$77777D	+r?   lax_reduce_fnlax_reduce_window_fnr<  c                    t          t          j        |           t          t          j        |          fd}t	          |d|          S )Nc                 >    t           j        j        } |rn| i |S rh   )r   "jax2tf_associative_scan_reductionsr   )r  r  associativeassociative_scanr  s      r=   
_call_implz_cumred.<locals>._call_impl^
  s0     ;AK>>PPPPr?   Frp  )r   lax_control_flowr  cumred_reduce_window_implr  )r  r  ro  r  r  r  s       @@r=   _cumredr  V
  sw     ->NN02F -Q Q Q Q Q Q 
5;K
 
 
 r?   cummax)r  r  ro  cummincumlogsumexpcumsumcumprodc
                     t          d          )Nz0TODO: jax2tf can not convert _select_and_scatterr   )
r*  source
init_valueselect_jaxprselect_constsscatter_jaxprscatter_constsr{  r  r  s
             r=   _select_and_scatterr  
  s     	NOOOr?   )r  c          
        t          j        d|j                  }t          j        t          |         d                              ||          }	t                              ||          }
t          j        ||||| ||	|
          }t          |t          |                    }t          rt          j        |          }|S )NrB   Frw  )rK   r  r4  r  rV   r  _add_fnr  select_and_scatterr6  r  r  r  )r  r*  rz  r{  r  r  r  r  r  	select_fn
scatter_fnr  s               r=   _select_and_scatter_addr  
  s     xGM***k'+&%888NN
j" "  ,,ZDD* *;^!(&*i!+	- 	-# 	$C):9)E)EFF##  

3

C	*r?   seedsc               N    d	d}t          |ddd          } || |||          S )
Nr  rE   c               .    t          j        | |          S )N)r   )r!   random_seed_impl_base)r  r   s     r=   impl_wrapperz'_random_seed_impl.<locals>.impl_wrapper
  s    %e$7777r?   FTrandom_seedr  )r   r  r  r  rE   r  )r  r   r  r  r  converted_impls         r=   _random_seed_implr  
  s^    8 8 8 8 %U$& & &. 
$)y
B 
B 
B Br?   r  c               \    |\  d	fd}t          |ddd          } || |||          S )
Nr  rE   c               R    t          j        j        j        | j        |          S )N)r   )r!   random_split_impl_baser4  _implndim)r  r   	keys_avals     r=   r  z(_random_split_impl.<locals>.impl_wrapper
  s1    &tY^5B B B Br?   FTrandom_splitr  )r   r  r  r  rE   r  )r  r   r  r  r  r  r  s         @r=   _random_split_implr  
  sv    *)B B B B B B %U%' ' '. 

%9	
C 
C 
C Cr?   msgsc               ^    |\  }d
fd}t          |ddd          } || |||	          S )Nr  rE   r	  c                P    t          j        j        j        | |j                  S rh   )r!   random_fold_in_impl_baser4  r  r   )r  r	  r  s     r=   r  z*_random_fold_in_impl.<locals>.impl_wrapper
  s(    (tT9?< < <r?   FTrandom_fold_inr  r{  r  rE   r	  rE   r  )r  r	  r  r  r5   r  r  r  s          @r=   _random_fold_in_implr  
  sn    ,)Q< < < < < < %U') ) ). 

DI
< 
< 
< <r?   c               f    |\  d	fd}t          |ddd          } || ||          S )
Nr  rE   c                T    t          j        j        j        | j                  S )N)	bit_widthr   )r!   random_bits_impl_baser4  r  r  )r  r  r  r  r   s     r=   r  z'_random_bits_impl.<locals>.impl_wrapper
  s0    %tY^5* * * *r?   FTrandom_bitsr  )r  r   r  r  r  r  )r  r  r   r  r  r  r  r  s    ``    @r=   _random_bits_implr  
  s~    *)* * * * * * * *
 %U$& & &. 
	"+y
B 
B 
B Br?   base_arrc                   | S rh   rB   )r  r   r  r  s       r=   _random_wrap_implr  
  s    	/r?   c                   | S rh   rB   )r  r  r  s      r=   _random_unwrap_implr  
  s    	+r?   c                j     t          t          t          j        d          dd          || |d}|S )NF)use_rolled_loopsTthreefryrp  r{  )r  r   r!   _threefry2x32_lowering)r  r  r  r  s       r=   _threefry2x32_jax_implr  
  sR    	:d)EBBBj	: 	: 	: 9		; 	; 	;# 
*r?   )use_vmaprandom_gammarp  keyc               <   | j         t          t          j                  k    }|rAt	          j        | d          } t          j        | t          t          j                            } t          |          }|t          j        j        k    rt          j        j        j        }nZ|t          j        j        k    rt          j        j        j        }n.|t          j        j        k    rt          j        j        j        }nJ t          j        |j        | |t          |                    \  }}|rAt          j        |t          t          j                            }t	          j        |d          }t,          r(t	          j        |          }t	          j        |          }||fS )N)r?  r?  Fr_  )   )r4  r  jnpuint32rK   r[  r  bitcast_convert_typeuint64r2  r   RandomAlgorithmRNG_THREE_FRYr   	AlgorithmTHREEFRY
RNG_PHILOXPHILOXRNG_DEFAULTAUTO_SELECTrng_bit_generatorr   r  r  )	r"  r   r4  	algorithmis_uint32_keyr\  algorithm_tfnew_keyr  s	            r=   _rng_bit_generatorr6  
  sY   )|CJ777- D
*S&
!
!C

$S,sz*B*B
C
CC(#%3339&/LLC'2229&-LLC'3339&2LLL*<+=sH1=e1D1DF F F.7C ((,sz2J2JKKGj$''G#  w''G

3

C	#r?   r  r  c               p    t          |          }t          j                            || || j                  S )N)r  r  r4  )r2  rK   r   uniformr4  )r  r  r   r\  s       r=   _rng_uniformr9    s0    (			8F6		V	VVr?   c                    d }d }d } fdt          t                               D             }t          j        || |          } ||          } |t          j                            |d                    }||fS )Nc                    | |z   S rh   rB   r
  s     r=   r  z_iota_2x32_shape.<locals>._add  s    q5Lr?   c                    t          j        |           sUt          j        t	          | f          d         |j                  } t          j        | t          j        |                    } | |z  S rC  )r   r  rK   r  r2  r4  r  r   r
  s     r=   _mulz_iota_2x32_shape.<locals>._mul!  sY    "" *
'+qd##A&
0
0a
/!RXa[[
)
)aq5Lr?   c                Z    t          j        | t          t          j                            S rh   )rK   r  r  r%  r&  r  s    r=   _cast32z!_iota_2x32_shape.<locals>._cast32&  s    "'"l3:&>&>???r?   c                H    g | ]}t          t          j        |           S ))r4  r   rY  )r_  r%  r(  )r   rY  r   s     r=   r   z$_iota_2x32_shape.<locals>.<listcomp>'  s:     / / / sz)DDD / / /r?       )r   r   r!   bcast_iotas_to_reshaped_iotarK   r  r  )r   r  r=  r@  iotascounts	counts_lo	counts_his   `       r=   _iota_2x32_shaperH    s        
 @??/ / / /!#e**--/ / /%,T4FF&gfoo)gbj,,VR8899)	I	r?   c                   t          j                    }|j                            |j                   |j                            |j                   |j                            |j                   | sJ t          |           dz
  |_        |S r  )r/   GatherDimensionNumbersoffset_dimsr  collapsed_slice_dimsstart_index_mapr   index_vector_dimindices_shaper  r  s      r=   _gather_dimensions_protorQ  1  s    

-
/
/%,8999##$5$JKKK0@AAA	}--1%	,r?   c                    | j         t          j        k    r4| j         t          j        k    rt          j        | t          j                  S | S rh   )r4  rK   rZ  int64r  r'  s    r=   _maybe_cast_to_int64rT  ;  s8    WQW0071bh	
(r?   slice_sizes
core.Shapec                 	 |t           j        j        k    r5t          t          j        d          }
 |
| ||||||	j        |	
  
        S |d         }t          |          }t          j	        |j
        t          j                  r}t          |          j        t          |j                  d         }	fdt          t          |                    D             }|                    g |j        |R           }g ||R }t#          |j        |          }t%          |          }t'          j        | ||||          }t+          |t-          	                    }t.          rt1          j        |          }|S )z$Tensorflow implementation of gather.FrB  )r  rU  unique_indicesindices_are_sorted
fill_valueoutput_shaper  r  r   Nc                >    g | ]}t          j                  |z   S rB   )r   r   )r   r  r  s     r=   r   z_gather.<locals>.<listcomp>T  s(    WWWC	0014WWWr?   )rK  )r   GatherScatterModeFILL_OR_DROPr  lax_slicing_gather_fillr   rT  r   r  r4  r  r  r   r   _replacerK  rQ  r2  r  gatherr6  r  r  rK   r  )r*  start_indicesr  rU  rY  rX  moderZ  r  r  gather_fill_fnoperand_avalopaque_shapetrailing_offset_dimsr  slice_sizes_tfr  s            `       r=   _gatherrj  A  s    
S"///&{'?8=? ? ?N>2C-*_	Y	P P P P 1,&}55-|)6?;; 0%l339#l>P:Q:Q:R:RSLWWWWeCDUDU>V>VWWW)22K'3K6JKK 3 M M/K/,//K
"=#68I
J
J%{++.WmUN'	) 	)##C):9)E)EFF##  

3

C	*r?   c           
        |dgt          |          z  }t          t          t          t	          |          t	          |          t	          |                              }| |         }t          |t          |                    }|S r  )r   r   r   slicer2  r6  r  )r*  rc  limit_indicesstridesr  r  slicesr  s           r=   _slicerp  e  s    _cC&&&G	%]++[-G-G
g

   ! !& 	# 	$C):9)E)EFF#	*r?   c               
   t          t          j        |                    }|d         }t          j        |j        t          j                  rtt          |          j        t          |j                  d          }g ||R }t          j
        |t          j        t          |          f|j                  gd          }t          |          }t          j        | ||          }t          rt          j        |          }|S )Nr   r_  r  )size_indices)rT  rK   r  r   r  r4  r  r  r   r   r  r  r2  r  dynamic_slicer  r  )	r*  rU  r  r  rc  rf  rg  ri  r  s	            r=   _dynamic_slicert  v  s    'rx'>'>??-1,|)6?;; &%l339#l>P:Q:Q:R:RSL/K/,//KI}bhL8I8I7K=J=P/R /R /R S#$& & &M {++.G]PPP##  

3

C	*r?   c                  t          t          j        |                    }|d         }t          j        |j        t          j                  rmt          |          j        t          |j                  d          }t          j
        |t          j        t          |          f|j                  gd          }t          j        | ||          }t          rt          j        |          }|S )Nr   r_  r  )rT  rK   r  r   r  r4  r  r  r   r   r  r  r  dynamic_update_slicer  r  )r*  updater  r  rc  rf  rg  r  s           r=   _dynamic_update_slicerx    s     'rx'>'>??-1,|)6?;; &%l339#l>P:Q:Q:R:RSLI}bhL8I8I7K=J=P/R /R /R S#$& & &M 	"7FMBB##  

3

C	*r?   c                   t          j                    }|j                            |j                   |j                            |j                   |j                            |j                   | sJ t          |           dz
  |_        |S r  )r/   ScatterDimensionNumbersupdate_window_dimsr  inserted_window_dimsscatter_dims_to_operand_dimsr   rN  rO  s      r=   _scatter_dimensions_protor~    s    

.
0
0%!!"3"FGGG##$5$JKKK$++46 6 6	}--1%	,r?   c                    |S rh   rB   r
  s     r=   rC   rC     s    a r?   c          	        ~>rJ t          j        t          t          j        d| j        j                            \  |t          j        j	        k    r2t          t          j        d          } || ||||	|	d                   }t                    dk    s
J d            t          |j        |          }dfd}t!          j        d| j                  }t!          j        |d                              ||          }t)          j        | |||||          }t,          rt!          j        |          }|S )NrB   FrB  r  )dnumsr  r  r   z(Update computation cannot have constantsr  rE   r  rF   c                \    t          j                  }t          || |d           \  }|S r  r  )r  r  r  r  update_constsupdate_jaxprs       r=   update_computationz$_scatter.<locals>.update_computation  s2    #L-@@LL$tLLLDCJr?   r_  rw  )rY  r  )rr  _reduction_jaxpr_scatter_reduction_computationr   r  r4  r  r   r]  CLIPr  r_  _clamp_scatter_indicesr   r~  r   rK   r  r  r  r  scatterr  r  )r*  scatter_indicesupdatesr  r  r  rY  rX  rd  r  r  clip_fnr  r  r  xla_update_computationr  s      ``            r=   _scatterr    s    "."?&W]9::#< #<L- 
S"''' B168 8 8Gg'1By|5 5 5O 
]		q	 	 	 "L	 	 	 
#O$9;L
M
M%      
 =7=111&k$!# # ##8#8#H#H  	+	- 	- 	-# $  

3

C	*r?   r,  branchesSequence[core.ClosedJaxpr]c               f   fdt          |          D             }t          t          t          j        d          |                    }t          |          dk    r?t          j        t          j        | t          j	                  |d         |d                   S t          j
        | |          S )Nc           	     J    g | ]\  }}t          t          |gR d d| di S )ro  branch__fun)r   r  )r   r  r  r  s      r=   r   z_cond.<locals>.<listcomp>  sb        !U  2 2 2 200002 2  r?   r,  r?  r  r   )r  r   r   r"   rq  r   rK   r,  r  rG   switch_case)r,  r  r  branches_tfs     ` r=   _condr    s         ))	  + S);FCC   +]]a7275"'**KNKNKKK>%---r?   cond_nconsts
cond_jaxprbody_nconsts
body_jaxprc                
  
 t          j        || |g          \  
}}j        d         j        rt	          || ||dS d
fd}t          t          |g|R ddi}d	 |j        D             }	t          j        ||||	
          S )Nr   )r  r  r  r  r  rE   rF   c                 0    t          g| R ddi\  }|S )Nro  z
while/condr  )r  predcond_constsr  s     r=   cond_tf_funcz_while.<locals>.cond_tf_func  s=    Z <+ < < < <.:< <ED Kr?   ro  
while/bodyc                P    g | ]#}t          j        t          |                    $S rB   )rK   TensorShaper  )r   r  s     r=   r   z_while.<locals>.<listcomp>  s;     = = =# n%6y%A%ABB = = =r?   )shape_invariantsr  )	r#   r  r  r   _batched_cond_whiler   r  rK   
while_loop)r  r  r  r  r  body_consts
init_carryr  body_tf_funcr  r  s    `        @r=   _whiler    s    )-
\<(** **&+{J!" 	!!          ): 8 8 8 8*68 8,= ='1';= = =	|\:(8
: 
: 
: :r?   c                    t          j        || |g          \  }t          g|R ddi\  }d
d}dfd	}t          j        |||g|R           ^}	}
|
S )a  Interprets a while_loop with a batched condition.

  A batched while has a conditional that returns a tensor of booleans, and
  a body that returns a list of tensors whose leading dimensions match those
  of the conditional tensor.

  We need to turn it into a while with scalar boolean conditional. We will
  expand the loop carry to include a prefix with the current tensor boolean
  condition. We prepend to the loop the first calculation of the tensor boolean
  condition. The loop condition will use a "reduce_any" to calculate a scalar
  boolean from the tensor boolean condition. The end of the loop body will
  compute the new carry using a "tf.where", and we compute the new tensor
  boolean condition.
  ro  zwhile/body_predpred_brE   carryrF   c           
         t          j        | t          t          t	          | j                                                }|S r  )rK   
reduce_anyr   r   r   r   )r  r  r  s      r=   new_cond_tf_funcz-_batched_cond_while.<locals>.new_cond_tf_func,  s5    =d5V\1B1B+C+C&D&DEEEDKr?   r   c                     t          g|R ddi}d	 fd	}t          t          |||j                            }t          	g|R dd
i\  }|g|R S )Nro  r  new_crE   cc_avalr]  rF   c                "   t          t          |          j        t          t	          t          j                                      j        t          j        |j        t          j
                            }t          j        || |          S )N)r   r:  r  r  )r?  r  r   r   r   r   r  r   r  r   rv  rK   r   )r  r  r  pred_b_bcastr  r  s       r=   select_one_carryzG_batched_cond_while.<locals>.new_body_tf_func.<locals>.select_one_carry5  su    &
"6**0#E#fl*;*;$<$<==($V\28<<> > >l XlE1---r?   	body_pred)r  rE   r  rE   r  r]  rF   rE   )r  r   r   r  )
r  r  	new_carryr  selected_carrynext_pred_br  r  r  r  s
   `     r=   new_body_tf_funcz-_batched_cond_while.<locals>.new_body_tf_func0  s    !1* "Q{ "Q38"Q "Q "QCO"Q "QI. . . . . . . '+3/?ES]Sg+h+h&i&iN#J B B~ B B B5@B BLK).)))r?   )r  rE   r  rE   rF   rE   )r  rE   r  rE   rF   r   )r#   r  r  rK   r  )r  r  r  r  r  r  init_pred_br  r  r5   	res_carryr  r  s    ` `       @@r=   r  r    s    " *.
\<(** **&+{J "* F{ FZ F F F3DF F,+   * * * * * * * * *& - 02B!, :z : :< <-!i	r?   scanr  )is_gpu_platform
checkpointc                   | S rh   rB   )r   r;   s     r=   rC   rC   V  s    1 r?   optimization_barrierr  tuple[TfVal, TfVal]c                   d } || j                   }t          j        |          s6t          |f          d         }t	          j        |t          j                  }n|}|rjt          j                            t          j	                            | |          |d          \  }}t          j	                            || j                   |fS t          j                            | |d          S )Nc                    | t           j        t           j        t           j        fv rt           j        S | t           j        t           j        fv rt           j        S | t           j        u rt           j	        S d S rh   )
rK   rG   uint8uint16r&  r  int16rZ  float16rG  )r  s    r=   promote_tf_dtypez _top_k.<locals>.promote_tf_dtype_  sW    BGRXry111YBGRX&&&Xo2:Z4r?   r   T)r  sorted)
r4  r   r  r2  rK   r  rZ  r  top_kr   )r*  r  r  r  k_tfvaluesindicess          r=   _top_kr  \  s       &%gm44		a	 	  tQD74""DDD 7gmm
	w 011T$ $ H HOFG9>>&'-00'997==D=666r?   reduction_dimensionrecall_targetfloatis_max_kreduction_input_size_overrideaggregate_to_topkc                    t          |f          d         }|r$t          j                            | |||||          S t          j                            | |||||          S rC  )r2  rK   r  approx_max_kapprox_min_k)r*  r  r  r  r  r  r  r  s           r=   _approx_top_kr  y  st     
aT		1	$ 37/BM = 13 3 3 7/BM = 13 3 3r?   rY  	is_stablenum_keystuple[TfVal, ...]c                   dcxk    rt          |          k    sn J d| cxk    rt          |d         j                  k     s*n J d|  dt          |d         j                               g }g |D ]r}t          j        d|j                  }|                    ||g           t          j        dt          |j                            }                    ||g           sdfd
} t          j	        |d          j
        | }	t          j        || ||	          }
t          rt          d |
D                       }
|
S )Nr  r   zInvalid z
 for ndim rB   r_  tf_argsrE   rF   c                      t          t          j        d          | t          j        dt
          j                  dS )NFrB  rB   )r  r  r  )r  rr  _sort_lt_comparatorr   r  r   rv  )r  comparator_jax_in_avalsr  s    r=   lexicographic_comparatorz'_sort.<locals>.lexicographic_comparator  sU    B(5B B B-&r2844   r?   Frw  )rY  r  
comparatorc              3  >   K   | ]}t          j        |          V  d S rh   r  r  s     r=   r   z_sort.<locals>.<genexpr>  s-      ==cB$S))======r?   )r  rE   rF   rE   )r   r   rK   r  r4  r  r   r  r  r  r  r  variadic_sortr  r   )rY  r  r  r  comparator_specr  r  o_avalr  xla_comparator_computationr  r  s     `        @r=   _sortr    s   	
h	'	'	'	'#h--	'	'	'	'	'	'	
i 
 
 
 
#qk  
 
 
 
 
=	==S!):%;%;==
 
 
 *,/46 5 5b]2RX...FFF+,,,b-"9"9::F""FF#34444      9bk*!# # ##8/K  +	- - -'
 $ >==W=====G	.r?   c          
     8   t          t          t          j        g d                    \  }}}}|t          j        j        t          j        j        t          j        j        g|t          j        j	        t          j        j
        t          j        j        g|t          j        j        t          j        j        t          j        j        g|t          j        j        t          j        j        t          j        j        gi}	|	|         t%          |          dz
           }
|||fv rW|\  }|j        }|t%          |           d         |d         dz
  dz  fz   }||k    rt)          |
t+          |                    }
 |
|           }t-          |t/          |                    S )N)r   r  r?  r  r  rX  r?  )
fft_length)r   r   r-   FftTyperK   signalfftfft2dfft3difftifft2difft3drfftrfft2drfft3dirfftirfft2dirfft3dr   r   r   r2  r6  r  )r   fft_typefft_lengthsr  r  FFTIFFTRFFTIRFFTtf_funcstf_funcx_avalx_shapeexpected_lengthsr  s                  r=   _fftr    sX     J$6 E EFF#tT5	BIM29?BIO<
RY^RY-ry/?@
RY^RY-ry/?@biory0")2CD	( Xs;//!34'$ IVlGK 0 00256	qA:  &&&K,D,DEEEg

#	$S*;I*F*F	G	GGr?   c                D    t           j                            | |          S )N)full_matrices)rK   r*   qr)r*  r
  s     r=   _qrr    s    	g]	;	;;r?   r
  
compute_uvsubset_by_indextuple[int, int] | Nonec                   |;|dt          | j        d         | j        d                   fk    st          d          t          j                            | ||          }|s|fS |\  }}}||t          j                            |          fS )Nr   rX  "subset_by_index is not implemented)r  r   r   rK   r*   svdadjoint)r*  r
  r  r  resultr  r~  rD   s           r=   _svdr    s     	QGM"$5w}R7H I IJ	J	J
B
C
CC9==-<<&	 7N'!Q	
Ary  ##	##r?   compute_left_eigenvectorscompute_right_eigenvectorsc                   |r|rd}t          |          |s"|s t          j                            |           fS |r,t	          t          j                            |                     S t          j                            t          j                            |                     \  }}t          j                            |          }||fS )NzxConversion of eig is not implemented when both compute_left_eigenvectors and compute_right_eigenvectors are set to True.)	r   rK   r*   eigvalsr   eigr  r  rT  )r*  r  r  rU  wHvlwHHs          r=   _eigr    s     #= 
C c
"
""% )C Ig&&((! w''(((Y]]29,,W5566FB
',,r

C9r?   lowersort_eigenvaluesr   c           	     &   ~| j         d         dk    r8| t          j        | t          |d         j         d d                             }}nC|st          j                            |           } t          j                            |           \  }}t          j        t          j        t          j	        t          j
        i                    | j                  }|"|d| j         d         fk    st          d          |t          j        ||          }||fS )NrX  r   r  )r   rK   r[  r2  r*   r  eighrF  rG  rH  rI  r  r4  r   r  )	r*  r   r!  r  r  r  rD   w	cast_types	            r=   _eighr&    s     ]2!BJwIaL4Fss4K(L(LMMqAA +	!!'**g9>>'""DAqlBJmRZ C  
!_GM"<M8N%N%N
B
C
CC
9A	
A+r?   c                P     t          t          j        d          | ||          S )Nr|  r  r{  )r  
lax_linalg
_lu_python)r*  r  r  s      r=   _lur*  "  s3    	H	:04	H	H	Hi
9 
9 
9 9r?   r   r  	left_sidetranspose_aconjugate_aunit_diagonalc                  |rZ|\  }	}
t          |	j                  }t          j                            | t          j        |d d         | j                            } |skt          | j                  }t          t          |dz
                      |dz
  |dz
  gz   }t          j
        | |          } t          j
        ||          }| }| j        t          j        t          j        fv r'|r|r|s!|rt          j                            |           } t          j                            | |||          }|st          j
        ||          }|S )NrX  r_  r?  r  )r   r  )r2  r   rK   r*   set_diagonesr4  r   r   r   rE  rF  rH  r  rT  triangular_solve)r   r  r+  r   r,  r-  r.  r  r  a_avalr5   a_shaperD  transpose_dimensionsr  s                  r=   _triangular_solver6  *  sB     DIFA&,''G
	1bggcrcl!'BBBCCA	 qw<<DdQh00D1HdQh3GG
Q,--A
Q,--AIE Wr}--- K   
',,q//a9%%a%%MM&	 8\&"677F	-r?   c                J     t          t          j        d          || |||dS )Nlinear_solver  )const_lengthsjaxprsr  r  )r  r  _custom_linear_solve_impl)r9  r:  r  r  r  s        r=   _linear_solver<  F  sE    
<	+E,:
< 
< 
<!
 
 
 r?   c                H     t          t          j        dd          || |dS )NFtridiagonal_solverp  r{  )r  r(  _tridiagonal_solve_jax)r  r  r  rS  s       r=   _tridiagonal_solver@  R  sG    
A	:<,1,?
A 
A 
A 
 
 
 r?   
call_jaxprjvp_jaxpr_thunk
num_constsc                (    ~~t          | g|R dddS )N
custom_jvpFro  r,  r  )rA  rB  rC  r  s       r=   _custom_jvp_callrG  ]  s8     z	* 
6t 
6 
6l/4
6 
6 
6 6r?   	fun_jaxprc                $    t          | g|R dddS )N
custom_vjpFrF  r  )rH  r  r5   s      r=   _custom_vjp_call_jaxprrK  i  s3     
) 
6d 
6 
6\/4
6 
6 
6 6r?   c                      t          d          )NzAcan't apply forward-mode autodiff (jvp) to a custom_vjp function.)rT  r  s     r=   _custom_linrM  s  s      	 	 r?   .tensorpartition_dimensionsPartitionsOrReplicatedc                    |t          j        | d          S t          j        |          }t	          j        |                              |          }t          j        | |d          S )a  Like TPUMPStrategy.experimental_split_to_logical_devices.

  For jax2tf purposes we want to avoid needing to thread the `strategy` object
  through the generated computation. It seems that the original function needs
  the strategy object only for error checking, which we assume is done upstream
  by JAX.

  Args:
    tensor: Input tensor to annotate.
    partition_dimensions: A list of integers, with one integer per tensor
      dimension, specifying in how many parts the dimension should be split. The
      product of integers must equal the number of devices per replica.
    use_sharding_op: whether to use a sharding op, or not.

  Returns:
    an annotated tensor.
  NTuse_sharding_op)r1   	replicater  prodr   aranger[  tile)rN  rO  num_partition_splitstile_assignments       r=   split_to_logical_devicesrZ  }  sl    * !!&$????#788I233;; /		6?D	I	I	IIr?   r  sharding.Shardingxla_client.HloSharding | Nonec                b    t          j        |           rd S |                     |j                  S rh   )r   is_unspecified_to_xla_hlo_shardingr  )r  r  s     r=   (_xla_compatible_sharding_to_hlo_shardingr`    s2     "1%% 4	
				*	**r?   r:  r  c               t   || S |                                 }|rt          j        |          r| S |j        su|j        rnt          t          j        t          j	        |j
                                                |j                                      |j                  j                  }n|j        }t          j        t#          |j                  |j
        ||j        |j                  }t+          j                    rt/          d          t1          j        |                              | d          S )zApply sharding to a TfVal.N)rS  tile_assignment_dimensionstile_assignment_devicesreplicate_on_last_tile_dimlast_tile_dimszA jit function with sharded arguments or results must be used under a `tf.function` context. See https://github.com/google/jax/blob/main/jax/experimental/jax2tf/README.md#support-for-partitioning for a discussion)r  TrR  )to_protor   is_op_sharding_replicatedrc  iota_reshape_dimsr   r   rV  r  rU  rb  r[  rE  iota_transpose_permflatr/   
OpShardingrO  rS  rd  re  
tf_contextr  r   r1   Shardingapply_to_tensor)r1  r:  r  sharding_prototadxla_sharding_protos         r=   r  r    sM    ZJ;;==. ,^<<J
 
01

*1 
	$)NEFFGG	1	2	2	>5	6	6		 CC 
0C 1=0G~"##!/!J!!/!J#21 1 1 !## C
	BC C C 
	%7	8	8	8	H	H	4 
I 
! 
! !r?   in_shardingsSequence[sharding.Sharding]out_shardingsresource_envmesh.ResourceEnvr;   keep_unusedinlinec                   ~t          t          ||
          }t          t          t          t          t
          j                   ||                    }t          | g|R t          j	        |d          dd}t          t          ||          }t          t          t          t          t
          j                   ||                    }t          |          S )Nr  r    FrF  )
r   r`  r   r   r  rl   r\   r  r#   r  )r  rr  rt  
in_layoutsout_layoutsru  donated_invarsr;   rw  rx  r  r  r  in_hlo_shardingssharded_argsr  out_hlo_shardingssharded_resultss                     r=   _pjitr    s    >A,lI?G ?G"'	',/B/M+MO O O
 " "## ##, U :\ : :/3~dF/K/K38: : :' @C,mY@H @H%*	',/B/M+MO O O
$& &&' &'/ 
		r?   argr   c               R    t          ||d                   }t          | |d          S )Nr   Fr  )r`  r  )r  r   ru  r  r  r  hlo_shardings          r=   _pjit_sharding_constraintr    s5     :(IaLQQ,	c</4
6 
6 
6 6r?   r  c                   t          j        |           |         }|j        t          |j                  k    rt	          ||j        |j                  S |S )N)r  rk  )rK   r   r4  r  r  rk  )r  rY  r  r  dim_tfs        r=   _dimension_size_jax2tfr    sY    8B<<	"&\\)/2222 9?+4+>@ @ @ @ Mr?   dimshape_poly.DimSizec                *    t          | f          \  }|S rh   r  )r  r  s     r=   _dim_as_value_jax2tfr    s    '&	-r?   error_messagec               Z    t           j                            | d |j        |            g S )NT)r   )rK   	debuggingassert_equalr  )assert_whatr  error_message_inputss      r=   _shape_assertion_jax2tfr    s=     , M "67  9 9 9 
)r?   c               0    t          j        | ||          S )N)exponent_bitsmantissa_bits)r  reduce_precision)r   r  r  s      r=   _reduce_precisionr    s#    		.;
= 
= 
= =r?   c                    |S rh   rB   r
  s     r=   rC   rC     s    1 r?   c                    t          j                    } t          j                    t          j                    f| _        t          j                    t          j                    g| _        dt          j                    i| _        t          | j                  }t          | j                  }t          | j                  |t          usJ |t          usJ t          usJ t          j
                            |d d            t          j
                            |d d            t          j
                            d fd           dS )	z/Registers TF custom container types as pytrees.r   c                $    t          |           d fS rh   r  r?  s    r=   rC   z._register_checkpoint_pytrees.<locals>.<lambda>1  s    &+Bii%6 r?   c                     t          |          S rh   r  r5   r  s     r=   rC   z._register_checkpoint_pytrees.<locals>.<lambda>2  s    eBii r?   c                $    t          |           d fS rh   r  r?  s    r=   rC   z._register_checkpoint_pytrees.<locals>.<lambda>4  s    uRyy$>O r?   c                     t          |          S rh   )r   r  s     r=   rC   z._register_checkpoint_pytrees.<locals>.<lambda>5  s    488 r?   c                    t          |                                           t          |                                           fS rh   )r   r  r  r  s    r=   rC   z._register_checkpoint_pytrees.<locals>.<lambda>9  s)    qxxzz""E!&&((OO4 r?   c                6     t          | |                    S rh   )r%  )r  r  dict_wrappers     r=   rC   z._register_checkpoint_pytrees.<locals>.<lambda>:  s    LLQ,, r?   N)rK   Moduler   r  r  rS  r   r   r   r   r   register_pytree_node)mtuple_wrapperlist_wrapperr  s      @r=   _register_checkpoint_pytreesr     sB   ikk! 
bikk"!#	bikk"!#	bikk!#qs))-ac,ac, 
e	#	#	#	#	T	!	!	!	!	T	!	!	!	!-$$] 57 578O8OQ Q Q -$$\3O3O%;%;= = = -$$44,,,,. . . . .r?   )rD   rE   rF   rG   )rF   rf   )rF   rn   )rz   r   rt   r{   ru   r|   rv   rG   r\   rG   rw   r}   rx   r~   ry   r   rF   r   )r1  rE   rF   r2  )rz   r   rF   r   )rz   r   rF   rD  )rN  rO  rP  rE   rF   rE   )
r   r   rv   rG   r   r   r   rV  r   r   )ro  r{   )F)rz   r   r   r   rv  rV  ro  r{   r,  rG   rF   rw  )r   r   r   r  rF   r   )r  r  r  r   r,  rG   rF   r  )r  r   ro  r{   rF   r   )r  r  r	  rV  r  rE   )
r  r  r   rE   ro  r{   r,  rG   rF   r   )r  r]  rF   r]  )r  r]  rF   r  )NF)r1  rE   r	  r  rF   r  rh   )r   r$  rF   r   )r   rE   )r   rE   r   r$  )rQ  rR  rS  r   rT  rU  rF   rV  )r   rR  )r   rE   r  rE   rF   rE   r  )r   rE   rF   rE   )r  rV  r  r]  )r  rO  r  rV  r  r]  )r   rE   r  rE   rn  rG   r  rV  r  r]  rF   rE   )r   rE   r  rE   rn  rG   rF   rE   )r  r  )r  r  )r  r  r  rO  r  rO  r  r  r  r  r  rV  r  r]  )r  r  r  r  r  rV  r  r]  )rg  rE   r/  r]  rh  rE   r0  r]  r  r]  rF   r1  )rn  rG   r*  rE   rj  r  rl  r2  r  rV  r  r]  )ry  rE   r*  rE   rz  rR  r{  r  r  r  r|  r  r}  r  r  r~  r  rV  r  r]  )r  rE   r  r   r  r  rG  r  r  rV  r  r]  rF   r   )r  r   r  r   ro  r<  r  r  r  )r  rE   )r"  rE   rF   r   )r  rE   r  rE   rF   rE   )rU  rV  r  rV  r  r]  )r,  rE   r  rE   r  r  rF   r   )r  rE   r  rO  r  r  r  rO  r  r  rF   r   )r*  rE   r  rO  rF   r  )r*  rE   r  rO  r  rO  r  r  r  rG   r  rO  r  rG   rF   r  )
r  rE   rY  rO  r  rG   r  rO  rF   r  )r*  rE   r
  rG   r  rG   r  r  )r*  rE   r  rG   r  rG   )r*  rE   r   rG   r!  rG   r  r   )r*  rE   )r   rE   r  rE   r+  rG   r   rG   r,  rG   r-  rG   r.  rG   r  rV  r  r]  )
r  rE   rA  r  rB  r   rC  rO  rF   r   )r  rE   rH  r  rF   r   )r  rE   rF   r   )rN  rE   rO  rP  )r  r[  r  r]  rF   r\  )r1  rE   r:  r\  r  rG   rF   rE   )r  rE   r  r  rr  rs  rt  rs  ru  rv  r;   r<  rw  rG   rx  rG   r  rV  r  rV  rF   rE   )r  rE   r   r[  ru  rv  r  rV  r  r]  rF   rE   )r  rE   )r  r  )r  r<  (  r   
__future__r   collections.abcr   r   r   	functoolsr   rr  r  r+  r   re	threadingtypingr   r	   r   abslr
   r   r   r   r   r   r   r%  r   r   r   jax.experimental.jax2tfr   jax.interpretersr   jax._srcr   r   r   r   r   r   r   r   r   r|  r   r   r   r    r!   random_internalr"   r#   r$   jax._src.exportr%   r&   jax._src.interpretersr'   r(   jax._src.laxr)   r  rr  r*   r(  r+   r_  r,   lax_windowed_reductionsjax._src.libr-   jax._src.numpy.ufuncsr.   
tensorflowrK   !tensorflow.compiler.tf2xla.pythonr  tensorflow.compiler.xlar/   tensorflow.core.frameworkr0   +tensorflow.python.compiler.xla.experimentalr1   ModuleNotFoundError1tensorflow.compiler.xla.experimental.xla_shardingtensorflow.python.frameworkr2   r  tensorflow.python.eagerr3   rl  	NameStack	PolyShaper2  r  r  compiler9   r7   safe_mapr   safe_zipr%  r>   r   rE   rO  PrecisionTyperP   rR   r   rV   __annotations__rW   r  r   localrY   rl   rj   contextmanagerr]   rq   api_hookr   r   r   r   r6  rC  r-  r   r   ru  r#  r  r  r  transformationr{  r  r  r   r  rv  rc  r  r  r   r2  r6  r>  Tracerr  rg  Tracerz  r  r  
unexpectedloopseval_jaxpr_ptf_not_yet_implrandom_clone_pr  stop_gradient_pr  add_jaxvals_pdevice_put_pcopy_pr  shard_alike_pr  neg_pr#  sign_pr'  floor_pceilceil_pr-  round_p	nextafternextafter_pr1  population_count_p	is_finiteis_finite_pr4  abs_pr7  pow_pr=  integer_pow_pr@  exp_pexp2_pexpm1expm1_prA  log_plog1plog1p_ptantan_ptanhtanh_psinsin_psinhsinh_pcoscos_pcoshcosh_p	acos_implacos_p	asin_implasin_p	atan_implatan_plogistic_impl
logistic_prP  atan2_pacoshacosh_patanhatanh_pasinhasinh_prK  sqrt_prsqrtrsqrt_prR  cbrt_plgammalgamma_pdigamma	digamma_pigammaigamma_pigammac	igammac_pbetaincregularized_incomplete_beta_perferf_perfcerfc_perfinv	erf_inv_p
bessel_i0ebessel_i0e_p
bessel_i1ebessel_i1e_prJ  	complex_prU  conj_pr  real_pr  imag_padd_pr  sub_pr9  mul_pr_  iota_prk  rm  div_prem_prz  r|  max_pmin_pr  r  r  r  rZ  r&  rS  r(  r  itemsr  r  r  shift_right_arithmetic_pr  r  shift_right_logical_pr  shift_left_pr  r  not_pr  r  r  rw  or_pbitwise_andrc  and_pr  logical_xorxor_pr!  eq_prd  ne_pr  r  eq_to_pr  r  boolean_greater_or_equalboolean_less_or_equalgreatergt_pr  lt_pr  r  
less_equalr  r  lt_to_ple_to_pcholesky
cholesky_pr  convert_element_type_pr  bitcast_convert_type_pr  clamp_pr  concatenate_pr  r  r   conv_general_dilated_pr.  dot_general_pr"  r?  broadcast_in_dim_prA  empty_prI  	reshape_prO  	squeeze_prX  pad_pr[  rev_prc  r  rg  transpose_paxes_to_axis
reduce_sumreduce_sum_preduce_prodreduce_prod_p
reduce_maxr  reduce_max_p
reduce_min
reduce_allreduce_min_preduce_or_preduce_and_prv  argmin_pargmax_pr  r  _ge_fnr  select_and_gather_add_pr  r  r  rr  rq  reduce_window_sum_preduce_window_min_preduce_window_max_preduce_window_pr  reduce_pr  _reduce_window_maxr  cummax_p_reduce_window_minr  cummin_p_reduce_window_logaddexpcumlogsumexp_p_reduce_window_sumaddcumsum_p_reduce_window_prodmul	cumprod_pr  select_and_scatter_pr  select_and_scatter_add_pr  random_seed_pr  random_split_pr  random_fold_in_pr  random_bits_pr  random_wrap_pr  random_unwrap_pr  threefry2x32_p_gamma_implrandom_gamma_pr6  rng_bit_generator_pr9  rng_uniform_prH  iota_2x32_shape_prQ  rT  rj  gather_prp  slice_prt  dynamic_slice_prx  dynamic_update_slice_pr~  r  r  	scatter_pscatter_min_pscatter_max_pscatter_mul_pscatter_add_pr  cond_pr  r  while_p
_scan_implscan_premat_expansionremat_pname_pr  r  optimization_barrier_pr  top_k_pr  approx_top_k_pr  sort_pr  fft_pr  qr_pr  svd_pr  eig_pr&  eigh_pr*  lu_pr6  triangular_solve_pr<  linear_solve_pr@  tridiagonal_solve_prG  custom_jvp_call_prK  custom_vjp_call_jaxpr_prM  custom_lin_pr   rP  rZ  r`  r  r  pjit_pr  sharding_constraint_pr  dimension_size_pr  dim_as_value_pr  shape_assertion_pr  reduce_precision_ptie_pr  rB   r?   r=   <module>r     sM    7 6 6 " " " " " " 8 8 8 8 8 8 8 8 8 8             				 				                        



       " " " " " "                               / / / / / /             " " " " " "                                           & & & & & & ! ! ! ! ! ! # # # # # #                   . . . . . . % % % % % %                   # # # # # # & & & & & & $ $ $ $ $ $ & & & & & & 9 9 9 9 9 9 , , , , , , - - - - - - / / / / / / G G G G G G # # # # # # + + + + + +     ; : : : : : 0 0 0 0 0 0 4 4 4 4 4 4MFFFFFFF M M MLLLLLLLLM 6 5 5 5 5 5 9 9 9 9 9 9 &	 	0  "'   RZ CDD  bj!566 
m
m   &  		 	 	 	       ::<<  57 6 6 6 6 @B  A A A A +  "' "A "A "A "A "A	 "A "A "AH (')) / / / / . . .< < < < 		 0111 .257"&#GcCGRTe e e e e 21eN$ $ $ $ $ $ $ $N20 20 20 20 20/ 20 20 20hF1 F1 F1 F1 F1. F1 F1 F1R    		 @AAA #'4# 4# 4# 4# 4# BA4#r, , , ,6   (: : : :x 	 	 	 	 "'    &o o o oh HM    4 (,*/59% % % % % %P    " 37         ? ? ?
        ( ' ' '   :>167 7 7 7 7t; ; ; ; ;&# # # #6 6 6 6&; ; ; ; ;t{ ; ; ;z9 9 9 9C: C: C: C: C:dj C: C: C:J8 8 8 8 ;- C CJ 5zBB'*I I + ,
= = =~ +6+& '#%#3  $ $ $ $ "& 59!H!H!H!H *{    %1! "    	     
 w} gl
 " " " "  #) 3; 7, > > >
 #4 7, : : : :
 	     !% 39 ; ; ; ;& )5 3$ %W[	  K  K w} W[	 w} W[	 gl
 W[	 gl
 W[	 gl
 !2!2U"4 "4 "4 3: !2!2U"4 "4 "4 3: !2!2U"4 "4 "4 3:  &7%6&8 &8 &8 3> "    w} w} w} gl
 w} < < < 
     -/W_) *W[	 gl
  G. G.     
 gl
 gl
 	 W%	 W%	 O O O 
 % % %= = = 	 	 B B B B9 9 9 9 !( > > > 39  ' = = = 39  GRXHbiHbiHbi	  QP.G.M.M.O.OPPP 

( 
( 
(1 1 1 )@$ %
$ 
$ 
$P P P &:! "M M M ( 	- 	- 	- 	-     $ 	 0 0 0 0 0f ('
(=vY[Yfggg (()?[][ijjj	 (()?[][b[nooo	 GM G% = = =.O O O
 * BB??II II ''Sbccc ''fP\]]] ''(=vYqrrr ''(:FVklll F F F +*773Dbgl+S+S]co{||| **773DbgFX+Y+Yci  vK  L  L  L !#!3
  <A4 
: 
: 
: 
: 
: '<" #6 6 6 '<" #3 3 3 #) 3; @ @ @ *   "	 	 	 	D6 D6 D6 D6N 2G 3- .' ' ' 'T )5 3$ %$ $ $ $(C C C C  .? 3) */ / / !' E E E %- 3= !+ + + %- 3= !	 	 	 	 !% 39 ) ) ) 	   " ! 1 1 1 & JJ )L77 )\".99 //,r}sL'') ) )  0/,r}sL'') ) )  (<66 (L77    4 $+7:t#<#< 3<  #*7:u#=#= 3<   "+de
,
,
,	RW*e	<	<	<G G G GT 3I 3. /  <!0 !0 !0^ %)%* %* %* %* %*P	 	 		 	 	 G&kk$& & & 3* + G&GN4000$& & & 3* + G&GN5111$& & & 3* +
 +9 3& '   @ $+ 3<     ( $+70C'$ $ $ 3<   $+70C'$ $ $ 3<   *10I#*% *% *% 3% & $+70C'$ $ $ 3<   %,G0D'%  %  %  3= !P P P %8  ! 		f---  .-  4K 3/ 0	B 	B 	B 	B *; 4% &C C C C += 4& '< < < < -A 4( )B B B B *; 4% &    *; 4% &    ,? 4' (    +A 4& ' ->,=GO'$777^-= -= -= 6( )
   6 $6  W W W W *     #3       		qc***   +*@ $+ 3<     #) 3;    & +9 3& '     2G 3- .   "0 ( ( ( (V %- 3= !(0 3$ %(0 3$ %(0 3$ %(0 3$ %. . . .* 
 : : : :<0 0 0 0f   "3!2" " " 3: 
 GGM9,13 3 3 &*%1	3 3 3 =( ) !5 4  75()) P5:5O'

127 7 7 74  3 3 3 3 , % % % %P 
 H H H H. !% 39 < < < 
  /3	$ $ $ $ $& !
    * !
    4 ). 3:$ %9 9 9 9
 '* 3:? #   2 5F 3:0 1    *7 3% &    6H 3:1 26 6 6 6 1A, -6 6 6 6 7M2 3   
 '  uS#X45 J J J J:+ + + +)! )! )! )!X       @ #( 4; 6 6 6 6 2K 4- .    3I :. /    &:
! "    )@
$ %= = = #4 ,n . . .:       s   E EE