
    Vpfh                    V   d dl mZ d dlmZmZ d dlZd dlmZ d dlZd dlm	Z	 d dl
Z
d dlZd dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d dlmZ d dlm Z! d dl"m#Z$ d dl"m%Z& d dl'm(Z( d dl)m*Z* d dl+m,Z,m-Z-m.Z.m/Z/ d dl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8  ej9        e:            ej9        e:           e5Z;e6Z< ej=        e>          Z?dydZ@dydZAdydZBeBZCdyd ZDd! ZE ejF        d"          ZGd# ZHd$ ZId% ZJd& ZKd' ZL e
jM        e@eAeBeBeDeDeEeHeIeJeLeK(          ZNe*d)dd*d+dzd3            ZOeOZPd4 ZQ G d5 d6          ZRd{d<ZSd{d=ZT e8eT          ZUe8d>             ZVd|d@ZWd|dAZXdB ZY ejF        dC          ZZd)eZ_[        eZj\        dD             Z]eZj^        dE             Z_dF Z`e`eja        eZ<   ejb        c                    e$jd                   dG Zeeee!jf        eZ<   e8d}dK            ZgdL Zh ee!ji        dHeh          e!jj        eZ<   dM Zkekejl        eZ<   d~dSZme8dT             ZndU Zo eeod          ejp        eZ<   eoejq        eZ<   dd[Zrere!js        eZ<   dyd\Ztd]d^ddaZuddbZvddfZwddgZxddhZyd]d^ddiZz ej{        eZez            ej{        eZ eezd)^          djk           dl Z|dm Z}dn Z~ ejF        do          Zd)e_[        e\                     eej        e                     e                    e|            ej{        ee}           dp ZeG\                    dq            eG                    dr            ds Zeeja        eG<    ej{        eGdt            du Zeej        eG<    ej2        eO          d]d)d*ddvddx            ZdS )    )annotations)CallableSequenceN)partial)Any)ad_util)api)config)core)dispatch)dtypes)linear_utileffects)source_info_util)traceback_util)flatten_funshaped_abstractify)ad)batching)mlir)partial_eval)lax)convolution)hlo)api_boundary)tree_flattentree_unflattentree_structurekeystr)unzip2wraps
split_listpartition_listsafe_mapsafe_zipmerge_listsweakref_lru_cachereturnboolc                     dS )NT ___s     V/var/www/html/nettyfy-visnx/env/lib/python3.11/site-packages/jax/_src/ad_checkpoint.pyeverything_saveabler1   ;   s    	    c                     dS NFr,   r-   s     r0   nothing_saveabler5   ?   s    	r2   c                6    | t           j        t          j        hv S N)lax_internaldot_general_plax_convolutionconv_general_dilated_p)primr.   r/   s      r0   dots_saveabler=   C   s    	,,!8: 
: :r2   c                P    | t           j        u r|d         \  \  }}\  }}|s|sdS dS )Ndimension_numbersTF)r8   r9   )r<   r.   paramslhs_brhs_bs        r0   dot_with_no_batch_dims_saveablerC   I   sA    	\'''#$78FQNUE  T	r2   c                      fd}|S )Nc                    | t           j        u r+|d         \  \  }}\  }}|s|st          j                  S t          j        S )Nr?   srcdst)r8   r9   peOffloadable	Recompute)r<   r.   r@   rA   rB   offload_dstoffload_srcs        r0   policyz.offload_dot_with_no_batch_dims.<locals>.policyR   sW    |)))%&9:fq!nue @5 @~+;????<r2   r,   )rM   rL   rN   s   `` r0   offload_dot_with_no_batch_dimsrO   Q   s)          
-r2   namec                 0     t                       fd}|S )z@Save any values (not just named ones) excluding the names given.c                .    | t           u r
|d         vS dS )NrP   Tname_pr<   r.   r@   names_not_to_saves      r0   rN   z0save_anything_except_these_names.<locals>.policya   s!    v~~F^#4444r2   	frozensetrV   rN   s   ` r0    save_anything_except_these_namesrZ   ^   s2     122     
-r2   c                 0     t                       fd}|S )z2Save only named values, excluding the names given.c                .    | t           u r
|d         vS dS NrP   FrS   rU   s      r0   rN   z(save_any_names_but_these.<locals>.policyj   s!    v~~F^#4445r2   rW   rY   s   ` r0   save_any_names_but_theser^   g   s2     122     
-r2   c                 0     t                       fd}|S )z7Save only named values, and only among the names given.c                .    | t           u r
|d         v S dS r]   rS   )r<   r.   r@   names_which_can_be_saveds      r0   rN   z%save_only_these_names.<locals>.policys   s!    v~~F^7775r2   )set)ra   rN   s   ` r0   save_only_these_namesrc   p   s2     !9::     
-r2   c                     t                      t                                                   }|rt          d  d d|            fd}|S )NzThe names should be exclusive and should not intersect in `names_which_can_be_saved` and `names_which_can_be_offloaded`. Got names_which_can_be_saved=z, names_which_can_be_offloaded=z and the intersection=c                    | t           u r|d         v rt          j        S | t           u r |d         v rt          j                  S t          j        S )NrP   rF   )rT   rI   SaveablerJ   rK   )r<   r.   r@   names_which_can_be_offloadedra   rL   rM   s      r0   rN   z1save_and_offload_only_these_names.<locals>.policy   sX    v~~&.,DDD[v~~&.,HHH^====<r2   )rb   intersection
ValueError)ra   rg   rM   rL   rh   rN   s   ````  r0   !save_and_offload_only_these_namesrj   y   s     !!9::!$%A!B!B)667STT, )
	(%=	( 	( *F	( 	( &		( 	() ) )        
-r2   c                      fd}|S )Nc                4     | g|R i |p | g|R i |S r7   r,   )r<   argsr@   policy_1policy_2s      r0   rN   z'save_from_both_policies.<locals>.policy   sB    8D*4***6**Mhht.Md.M.M.Mf.M.MMr2   r,   )rn   ro   rN   s   `` r0   save_from_both_policiesrp      s/    N N N N N N 
-r2   )r1   r5   r=   checkpoint_dots dots_with_no_batch_dims_saveable"checkpoint_dots_with_no_batch_dimsrO   rZ   r^   rc   rp   rj   Tr,   prevent_cserN   static_argnumsfunr   ru   rN   Callable[..., bool] | Nonerv   int | tuple[int, ...]c               b     t                     t           fd                        }|S )a  Make ``fun`` recompute internal linearization points when differentiated.

  The :func:`jax.checkpoint` decorator, aliased to :func:`jax.remat`, provides a
  way to trade off computation time and memory cost in the context of automatic
  differentiation, especially with reverse-mode autodiff like :func:`jax.grad`
  and :func:`jax.vjp` but also with :func:`jax.linearize`.

  When differentiating a function in reverse-mode, by default all the
  linearization points (e.g. inputs to elementwise nonlinear primitive
  operations) are stored when evaluating the forward pass so that they can be
  reused on the backward pass. This evaluation strategy can lead to a high
  memory cost, or even to poor performance on hardware accelerators where memory
  access is much more expensive than FLOPs.

  An alternative evaluation strategy is for some of the linearization points to
  be recomputed (i.e. rematerialized) rather than stored. This approach can
  reduce memory usage at the cost of increased computation.

  This function decorator produces a new version of ``fun`` which follows
  the rematerialization strategy rather than the default store-everything
  strategy. That is, it returns a new version of ``fun`` which, when
  differentiated, doesn't store any of its intermediate linearization points.
  Instead, these linearization points are recomputed from the function's saved
  inputs.

  See the examples below.

  Args:
    fun: Function for which the autodiff evaluation strategy is to be changed
      from the default of storing all intermediate linearization points to
      recomputing them. Its arguments and return value should be arrays,
      scalars, or (nested) standard Python containers (tuple/list/dict) thereof.
    prevent_cse: Optional, boolean keyword-only argument indicating whether to
      prevent common subexpression elimination (CSE) optimizations in the HLO
      generated from differentiation. This CSE prevention has costs because it
      can foil other optimizations, and because it can incur high overheads on
      some backends, especially GPU. The default is True because otherwise,
      under a :func:`~jax.jit` or :func:`~jax.pmap`, CSE can defeat the purpose
      of this decorator.
      But in some settings, like when used inside a :func:`~jax.lax.scan`, this
      CSE prevention mechanism is unnecessary, in which case ``prevent_cse`` can
      be set to False.
    static_argnums: Optional, int or sequence of ints, a keyword-only argument
      indicating which argument values on which to specialize for tracing and
      caching purposes. Specifying arguments as static can avoid
      ConcretizationTypeErrors when tracing, but at the cost of more retracing
      overheads. See the example below.
    policy: Optional, callable keyword-only argument. It should be one of the
      attributes of ``jax.checkpoint_policies``. The callable takes as input a
      type-level specification of a first-order primitive application and
      returns a boolean indicating whether the corresponding output value(s) can
      be saved as residuals (or instead must be recomputed in the (co)tangent
      computation if needed).

  Returns:
    A function (callable) with the same input/output behavior as ``fun`` but
    which, when differentiated using e.g. :func:`jax.grad`, :func:`jax.vjp`, or
    :func:`jax.linearize`, recomputes rather than stores intermediate
    linearization points, thus potentially saving memory at the cost of extra
    computation.

  Here is a simple example:

  >>> import jax
  >>> import jax.numpy as jnp

  >>> @jax.checkpoint
  ... def g(x):
  ...   y = jnp.sin(x)
  ...   z = jnp.sin(y)
  ...   return z
  ...
  >>> jax.value_and_grad(g)(2.0)
  (Array(0.78907233, dtype=float32, weak_type=True), Array(-0.2556391, dtype=float32, weak_type=True))

  Here, the same value is produced whether or not the :func:`jax.checkpoint`
  decorator is present. When the decorator is not present, the values
  ``jnp.cos(2.0)`` and ``jnp.cos(jnp.sin(2.0))`` are computed on the forward
  pass and are stored for use in the backward pass, because they are needed
  on the backward pass and depend only on the primal inputs. When using
  :func:`jax.checkpoint`, the forward pass will compute only the primal outputs
  and only the primal inputs (``2.0``) will be stored for the backward pass.
  At that time, the value ``jnp.sin(2.0)`` is recomputed, along with the values
  ``jnp.cos(2.0)`` and ``jnp.cos(jnp.sin(2.0))``.

  While :func:`jax.checkpoint` controls what values are stored from the
  forward-pass to be used on the backward pass, the total amount of memory
  required to evaluate a function or its VJP depends on many additional internal
  details of that function. Those details include which numerical primitives are
  used, how they're composed, where jit and control flow primitives like scan
  are used, and other factors.

  The :func:`jax.checkpoint` decorator can be applied recursively to express
  sophisticated autodiff rematerialization strategies. For example:

  >>> def recursive_checkpoint(funs):
  ...   if len(funs) == 1:
  ...     return funs[0]
  ...   elif len(funs) == 2:
  ...     f1, f2 = funs
  ...     return lambda x: f1(f2(x))
  ...   else:
  ...     f1 = recursive_checkpoint(funs[:len(funs)//2])
  ...     f2 = recursive_checkpoint(funs[len(funs)//2:])
  ...     return lambda x: f1(jax.checkpoint(f2)(x))
  ...

  If ``fun`` involves Python control flow that depends on argument values,
  it may be necessary to use the ``static_argnums`` parameter. For example,
  consider a boolean flag argument::

    from functools import partial

    @partial(jax.checkpoint, static_argnums=(1,))
    def foo(x, is_training):
      if is_training:
        ...
      else:
        ...

  Here, the use of ``static_argnums`` allows the ``if`` statement's condition
  to depends on the value of ``is_training``. The cost to using
  ``static_argnums`` is that it introduces re-tracing overheads across calls:
  in the example, ``foo`` is re-traced every time it is called with a new value
  of ``is_training``. In some situations, ``jax.ensure_compile_time_eval``
  is needed as well::

    @partial(jax.checkpoint, static_argnums=(1,))
    def foo(x, y):
      with jax.ensure_compile_time_eval():
        y_pos = y > 0
      if y_pos:
        ...
      else:
        ...

  As an alternative to using ``static_argnums`` (and
  ``jax.ensure_compile_time_eval``), it may be easier to compute some values
  outside the :func:`jax.checkpoint`-decorated function and then close over them.
  c                    t          
|           \  }} t          | |f          \  }}d |D             }t          ||t          |                    \  }}}t	          j        g ||R |dd}	t          ||	          S )Nc                ,    g | ]}t          |          S r,   )r   .0xs     r0   
<listcomp>z1checkpoint.<locals>.fun_remat.<locals>.<listcomp>>  s!    999!"1%%999r2   Fjaxprru   differentiatedrN   )_remat_static_argnumsr   _trace_to_jaxprtupleremat_pbindr   )rm   kwargsfun_	args_flatin_treein_avalsr   constsout_treeout_flatrw   rN   ru   rv   s             r0   	fun_rematzcheckpoint.<locals>.fun_remat9  s     'sNDAAJD$%tVn55Iw99y999H-dGU8__MME68| -	-- -#(kV- - -H (H---r2   )r"   r   )rw   ru   rN   rv   r   s   ```` r0   
checkpointr      sS    b 	::. . . . . . . < :. 
r2   c                d   t          |          t          u r|f}nAt          |          t          u rt          d |D                       st	          d|           t          fd|D                       st          d|           |s| fS t                    }t          fd|D                       }g g }}t                    D ]A\  }}||v r#|	                    t          |                     ,|	                    |           Bt          | |t          |          |          }	|	|fS )Nc              3  B   K   | ]}t          |          t          u V  d S r7   )typeint)r~   ds     r0   	<genexpr>z(_remat_static_argnums.<locals>.<genexpr>P  s,      771Q3777777r2   zvthe `static_argnums` argument to `jax.checkpoint` / `jax.remat` must be an int, tuple of ints or, bool, but got value c              3  n   K   | ]/}t                     |cxk    ot                    k     nc V  0d S r7   lenr~   r   rm   s     r0   r   z(_remat_static_argnums.<locals>.<genexpr>U  sM      AAQc$iiZ1((((s4yy((((AAAAAAr2   zthe `static_argnums` argument to `jax.checkpoint` / `jax.remat` can only take integer values greater than or equal to `-len(args)` and less than `len(args)`, but got c              3  <   K   | ]}|t                    z  V  d S r7   r   r   s     r0   r   z(_remat_static_argnums.<locals>.<genexpr>^  s-      DDa#d))mDDDDDDr2   )r   r   r   all	TypeErrorri   r   rX   	enumerateappendWrapHashably_dyn_args_fun)
rw   rv   rm   nargsstatic_argnums_dyn_argsstatic_argsir   new_funs
     `       r0   r   r   L  s   	.S  $&NN  E))7777777 *
 2!/2 2 3 3 3 
AAAA.AAA	A	A *
 ) ') ) * * *
 
 9
d))%DDDD^DDDDD/bK(oo  daO[//Q@@@@
//!



#k0B0BEJJ'	(	r2   c                  @    e Zd ZU ded<   ded<   ded<   d Zd Zd	 Zd
S )r   r   valr   hashr*   hashablec                    || _         	 t          |          | _        d| _        d S #  t          |          | _        d| _        Y d S xY w)NTF)r   r   r   id)selfr   s     r0   __init__zWrapHashably.__init__k  sI    DHs))didmmmS''didmmmms	   & Ac                    | j         S r7   )r   )r   s    r0   __hash__zWrapHashably.__hash__s  s
    9r2   c                    t          |t                    r,| j        r|j        r| j        |j        k    S | j        |j        u S dS r4   )
isinstancer   r   r   )r   others     r0   __eq__zWrapHashably.__eq__u  sJ    %&& %	 %5> %x59$$x59$$5r2   N)__name__
__module____qualname____annotations__r   r   r   r,   r2   r0   r   r   f  s^         
((()))...        r2   r   frozenset[int]r   tuple[WrapHashably, ...]r   r   c                |    t          d |D                       rt          | |||          S t          | |||          S )Nc              3  T   K   | ]#}t          |j        t          j                  V  $d S r7   )r   r   r   Tracerr}   s     r0   r   z _dyn_args_fun.<locals>.<genexpr>  s0      ==AAE4;	'	'======r2   )any_dyn_args_fun_uncached_dyn_args_fun_cached)rw   rv   r   r   s       r0   r   r     sJ    ======= K!#~{EJJJ	c>;	F	FFr2   c                      fd}|S )Nc                     t                    t          |           cfdt                    D             } |i |S )Nc                ^    g | ])}|v rt                    j        nt                    *S r,   )nextr   )r~   r   	dyn_args_rv   static_args_s     r0   r   z;_dyn_args_fun_uncached.<locals>.new_fun.<locals>.<listcomp>  sJ     = = =*+ ,-+>+>l##''9oo= = =r2   )iterrange)	r   r   	full_argsr   r   rw   r   rv   r   s	      @@r0   r   z'_dyn_args_fun_uncached.<locals>.new_fun  sl    ";//hL)= = = = = =/4U||= = =I3	$V$$$r2   r,   )rw   rv   r   r   r   s   ```` r0   r   r     s5    % % % % % % % %
 
.r2   c                   t          t          j        |           |          \  }}t          j        | ||dd          }	 t          j        |||          \  }}}\   na# t          j        $ rO}	|	j        \  }
d|
vr |
dz   dz   }t          j        	                    t          j                  }|f|_        |d d }	~	ww xY wt          j
        |          | |            fS )NTr   zfor checkpointz

zConsider using the `static_argnums` parameter for `jax.remat` or `jax.checkpoint`. See the `jax.checkpoint` docstring and its example involving `static_argnums`:
https://jax.readthedocs.io/en/latest/_autosummary/jax.checkpoint.html
)r   lu	wrap_initrI   
debug_infotrace_to_jaxpr_dynamicr   ConcretizationTypeErrorrm   __new__convert_constvars_jaxpr)rw   r   r   flat_funr   debugr   r.   r   emsgnew_msgnew_es                r0   r   r     s    "2<#4#4g>>(H
-Whl
C
C%4XxOOE1fbb		%   6DCs""Fl	G (001MNNEEJ
T 
	#E	*	*FHHJJ	>>s   A B:+A
B55B:$list[tuple[core.AbstractValue, str]]c                   
 t          ||f          \  } fd
 t          j        
fdd          | }t          |t                    sJ |\  }|j        }fd}t          |j                  t          |          k    sJ t          j	         |dd          }t          j
        |          }	t          ||	          S )Nc                 :    t          |           \  } } | i |S r7   )r   )rm   r   fr   s     r0   f_zsaved_residuals.<locals>.f_  s+    !'400LD&1dfr2   c                 4    t          j        g| R  d         S )N   )r	   	linearize)rm   r   s    r0   <lambda>z!saved_residuals.<locals>.<lambda>  s    S]2%=%=%=%=a%@ r2   T)return_shapec                 "    t                     S r7   )r   )	out_shapes   r0   r   z!saved_residuals.<locals>.<lambda>  s    ^I.. r2   saved_residuals)r   r	   
make_jaxprr   r   r   r   invarsrI   r   arg_info_all_saved_residuals)r   rm   r   	in_leavesoutjaxpr_r   r   dbgarg_infor   r   r   s   `         @@@r0   r   r     s    #T6N33)W     	*@@@@$(	* 	* 	*+4	6#	C		&)
,%....(	U\		c)nn	,	,	,	,
a(D2CDD#_S!!(	%	*	**r2   c           	        d | j         D             }d | j         D             }g }|D ]}|                    |j        df           | j        D ]"}||v r|                    |j        df           #t	          | j                  D ]I\  }}||v r@| ||         \  }}	d| t          |	           }
nd}
|                    |j        |
f           Jd | j        D             }| j        D ]}t          j	        |j
                  }
|j         D ]}||v r|j        t          u s||v r8||         x}r.|                    |j        d|j        d	          d
|
 f           Pt          |j                  dk    r.|                    |j        d|j        d	          d
|
 f           |                    |j        d|j        j         d|
 f           Ìt!          |          t!          | j                   k    sJ |S )Nc                F    g | ]}t          |t          j                  |S r,   r   r   Literalr}   s     r0   r   z$_saved_residuals.<locals>.<listcomp>  )    JJJAjDL.I.IJaJJJr2   c                F    h | ]}t          |t          j                  |S r,   r   r}   s     r0   	<setcomp>z#_saved_residuals.<locals>.<setcomp>  r   r2   zfrom a literalzfrom a constantzfrom the argument z(from the argument at flattened index {i}c                D    i | ]}|j         t          u |j        D ]}||S r,   )	primitiverT   r   )r~   r   vs      r0   
<dictcomp>z$_saved_residuals.<locals>.<dictcomp>  sC     # # #AK6,A,A -B,A 1,A,A,A,Ar2   znamed 'rP   z' from xla_callzoutput of jitted function 'z
output of z from )outvarsr   aval	constvarsr   r   r    eqnsr   	summarizesource_infor   rT   r@   strrP   r   )r   r   res_litsres_varsresultsr   r   r   arg_namearg_pathrG   
named_varseqns                r0   r   r     s|   JJJJJ(JJJJJ(' / /aNNAF,-....? 2 2aH}}nnaf/0111%% $ $daH}}		%a[(?8?VH-=-=??8nnafc]#### #ej # # #* Z Q Qc

$S_
5
5C[ 	Q 	Q	
h=F""a:oo*Q-;O3o
..!&"LCJv,>"L"Ls"L"LM
N
N
N
N:--
..!&(
68J ( ("%( () * * * * ..!&"Ns}/A"N"N"N"NO
P
P
P	Q 
WU]++	+	+	+	+	.r2   c                    t          | g|R i |D ]-\  }}t          |                    d           d|            .d S )NT)short_dtypes )r   print	str_short)r   rm   r   r   rG   s        r0   print_saved_residualsr    sd    "16t666v66 8 8idC	T^^^..
6
6
6
677778 8r2   remat2c                .    ~~~t          j        | dg|R  S Nr,   r   
eval_jaxprr   ru   r   rN   rm   s        r0   
remat_implr    s#    >6		*T	*	*	**r2   c                :    ~~~~d | j         D             | j        fS )Nc                    g | ]	}|j         
S r,   r   r~   r   s     r0   r   z'remat_abstract_eval.<locals>.<listcomp>  s    	(	(	(Q!&	(	(	(r2   )r   r   r  s        r0   remat_abstract_evalr    s(    
K	(	(%-	(	(	(%-	77r2   c                   |j         rJ d |D             }t          j        t          j        |          |d          \  }}d |D             }	t          j        |j                  }
t          j        g |j	        | |	R |
|||d}t          |t          |j                  g          \  }t                    fdt          ||          D             }||fS )Nc                D    g | ]}t          |          t          j        uS r,   r   r   Zeror~   ts     r0   r   zremat_jvp.<locals>.<listcomp>  s%    ???a,???r2   Fc                H    g | ]}t          |          t          j        u| S r,   r  r!  s     r0   r   zremat_jvp.<locals>.<listcomp>  s*    IIIAT!WWGL-H-Ha-H-H-Hr2   r   c                v    g | ]5\  }}|rt                    nt          j                            |          6S r,   )r   r   r   
from_value)r~   pnzout_tangents_s      r0   r   zremat_jvp.<locals>.<listcomp>  sO     9 9 9a *,K$}%%%1H1H1K1K 9 9 9r2   )r   r   	jvp_jaxprrI   close_jaxprr   r   r   r   r   r#   r   r   r   zip)primalstangentsr   ru   r   rN   in_nonzeros
jaxpr_jvp_out_nznonzero_tangents	jaxpr_jvpoutsout_primalsout_tangentsr(  s                 @r0   	remat_jvpr6    s'   _??h???+|BN5$9$9;NN*fIIIII()9::)	 
M
M"
M%5
M 
M=FnV
M 
M 
M$  *$U]1C1C0DEE+}}%%-9 9 9 9";779 9 9,	l	""r2   c          	     L    |j         rJ t          j                            |j                  }|rt	          d|           |d         pt
          }d |D             }t          j        ||dgt          |          z  dd|          \  }}}	}
}t          |
|	          \  }}t          j
        ||          \  }}t          ||g          \  }}dgt          |	          t          |	          z
  z  |z   }t          j
        ||          \  }}t          |          }t          ||          }t          d |D                       \  }}t          ||          \  }}t          j        |dg|R  }t          |t          |          |z
  g          \  }}t#           j        |          }t          ||          \  }}|t#           j        |          z   } fd|j        D             }t+          ||d	          }t          j        ||t.          ||j        t1          j                              }t4          j        j        rt:          j        nt:          j        }t@          !                    |          r	 t          ||          \  }} t          | |j"        |d                    \  }!}|j        t          |j                  |z
  d          }"tG          |$                    |"
          d           }#tA          j%        |ddtM          |!          z  z   dt          |!          z  z   dtM          |!          z  tM          |#          z  z   dtM          |#          z  z   dt          |#          z  z   gd |!D             d |#D             R   n#  Y nxY w|D ]	}$||$_'        
tQ          |	||          S )Nz?Effects not supported in partial-eval of `checkpoint`/`remat`: rN   c                8    g | ]}|                                  S r,   )is_knownr!  s     r0   r   z&remat_partial_eval.<locals>.<listcomp>  s#    333aQZZ\\!333r2   TFc              3  V   K   | ]$}|j                                         |j         V  %d S r7   )pvalr9  r!  s     r0   r   z%remat_partial_eval.<locals>.<genexpr>*  s5      FFAAFOO4E4EFFFFFFFr2   r,   c                    g | ]:}t          j        t           j                            |j                  d           ;S r7   )rI   JaxprTracer
PartialValunknownr   )r~   r   traces     r0   r   z&remat_partial_eval.<locals>.<listcomp>3  sI     7 7 7 ~eR]-B-B16-J-JDQQ 7 7 7r2   )r   r   r   zremat-decorated function zsaving inputs with shapes:
z  %s
zand zsaving these intermediates:
z  %s from %s
c                @    g | ]}|j                                         S r,   )r   r  r  s     r0   r   z&remat_partial_eval.<locals>.<listcomp>I  s&    999!&""$$999r2   c                J    g | ] \  }}|                                 |fD ]}|!S r,   )r  )r~   aselts       r0   r   z&remat_partial_eval.<locals>.<listcomp>J  s6    KKK&1a8JKK#KKKKr2   ))r   r   remat_allowed_effectsfilter_not_inNotImplementedErrorr5   rI   partial_eval_jaxpr_customr   r$   	dce_jaxprr#   sum_insert_reduce_precisionr!   r   r  mapnew_instantiated_constinstantiate_constr   dictnew_eqn_reciper   r   currentr
   log_checkpoint_residualsvalueloggingWARNINGDEBUGloggerisEnabledForr   r   replacelogr*   reciper'   )%r@  r   tracersr@   disallowed_effectsrN   in_unknownsjaxpr_knownjaxpr_stagedout_unknownsout_instnum_resr.   out_inst_unknownjaxpr_unknownin_used_stagedused_resout_used_knownin_used_knownjaxpr_known_
in_consts_	in_consts
out_consts
out_knowns	residualsres_tracerstracers_stagedin_jaxpr_tracersout_jaxpr_tracers
new_paramsr]  	log_level
staged_unk
res_invarsres_outvarsbody_resr"  s%   `                                    r0   remat_partial_evalr|    s%   _4BB5=QQ !
	 	  	 ! ! ! (//&337333+"
tfs;'7'77vO O =+|\8W
 'x>>!"$,|=M"N"N-'	BB(N 6S..\1B1BBChN.!|KHH+}MM'
 *+w??, FFFFFFF-!Zz::,!Y|R<)<<<*$Z#j//'2I1JKK*i E0)<<+$^W==!^ 3u'>#O#OO7 7 7 7 - 57 7 7F-EEE*-/@'')>-5779 9&
 "(!@!FYgooGM)## $^[AAma$Z1Eghh1OPPmj!'K,?(@(@7(J(K(KLk!+"5"5k"5"J"JDQQhj+.j1A1AAB3z??*+ j)))DNN:; 0$x..@	A
 !3x==01M :9j999M LKHKKKM M M M M
d//aahh 
\:/@	A	AAs   .DN Nr   
core.Jaxprre  c           
     B   | j         t          | j                   |z
  d          }d | j        D             }| j        d d          | j        d d          | j        d d          }}}|D ]t          j        t          j                  r)t          j
        j        j        t          j                  sL|vrQt          t          j                  sJ t          j        j        j                  t          j        j        j                  }t#          |j        |j                  }|v s|v rh|v r|n|}	t          j        ggt*          j        |t/                                }
|	|	                              <   |                    d|
           5fdt5          |          D             \  \  }}|j        t*          j        k    r|j        |k    rw|                    fd|j         D                       }t          j        ggt*          j        |t/                      |j        |j                  }
|||<   |                    |dz   |
           |                     |||          }t@          j!        j"        ot          j#        |           |S )	Nc                Z    h | ](}|j         D ]}t          |t          j                  |)S r,   )r   r   r   Var)r~   r   r   s      r0   r   z+_insert_reduce_precision.<locals>.<setcomp>W  s8    RRRQahRR*Q:Q:QRqRRRRr2   )exponent_bitsmantissa_bitsr   c              3  8   K   | ]\  }}|j         v ||fV  d S r7   rA  )r~   r   r   r   s      r0   r   z+_insert_reduce_precision.<locals>.<genexpr>j  s1      NNDAqqAI~~!Q~~~~NNr2   c                $    g | ]}|k    r|nS r,   r,   )r~   v_newvarr   s     r0   r   z,_insert_reduce_precision.<locals>.<listcomp>n  s;     )@ )@ )@-/ 02QwwF )@ )@ )@r2   rA  r   )r   r   r   )$r   r   r   r   r   r   r   r   UnshapedArrayr   
issubdtypedtypenpinexactr  suffixfinforQ  nexpnmantnew_jaxpr_eqnr8   reduce_precision_prb   indexinsertr   r   r@   r[  r  ctxr
   enable_checksrU  check_jaxpr)r   re  r  	used_varsr   r   r   r  r@   lstnew_eqneqn_idxr
  replace_eqn	new_jaxprr  r   s                  @@r0   rM  rM  T  s   ]3u}--7889(RR%*RRR)!LOU_QQQ-?AAAT)& & &aqvt122 aflBJ77	a"""""Xah''FL&&E
%+FFFFI~~fiIIVc"(QC8&#%%I Ig c#))A,,
kk!WNNNNIdOONNNonw
-<:
:
:
*

KK )@ )@ )@ )@ )@36;)@ )@ )@K A Ak"(QC8&#%%
/37$ $g "d7m
kk'!)W%%%%mm6YTmJJ)<!1)!<!<	r2   c                 4    | ^ }}}|t          |d          fS )NT)r   )rQ  )rm   r.   params_knownparams_stageds       r0   (remat_partial_eval_custom_params_updaterr  y  s&    $(!1lM	tM$???	??r2   c                  |j         rJ d |D             }d | D             }t          t          j        |          ||          \  }}|j        |j        }	}t          j        |          }t          || f          \  }
}t          j	        g |	|
R d|i|}t          |          t          |          cfd|D             }t          d           t          d           cxu rn J |S )Nc                6    g | ]}t          j        |          S r,   )r   is_undefined_primalr}   s     r0   r   z#remat_transpose.<locals>.<listcomp>  s#    ===Qr%a((===r2   c                D    g | ]}t          |          t          j        u S r,   r  r~   cts     r0   r   z#remat_transpose.<locals>.<listcomp>  s%    :::BtBxx7<':::r2   r   c                    g | ]O}t          j        |          sd n6t                    rt          j        |j                  nt                    PS r7   )r   r  r   r   r   r   )r~   r   
in_cts_nz_	in_zeros_s     r0   r   z#remat_transpose.<locals>.<listcomp>  se     ! ! ! .q11 JDD$(OOIGL   j9I9I! ! !r2   )r   transpose_jaxprrI   r*  r   r   r   r   r   r   r   r   )out_ctsr   
in_primalsr@   	in_linear	out_zerostransposed_jaxpr_in_zerostransposed_jaxprr   rm   r.   	in_cts_nzin_ctsr  r  s                 @@r0   remat_transposer    s?   _==*===)::':::) /nUY	!3 !3X.46G6NF/0@AA*g.//'$lLFLTLLL1ALVLL)y//4>>*i! ! ! ! !! ! !& 
j$		4	4#8#8	@	@	@	@	@	@	@	@	@	-r2   core.ClosedJaxprr  bool | Sequence[bool]r  #tuple[core.ClosedJaxpr, list[bool]]c                   t          |          t          u r|ft          | j                  z  }t          |          t          u r|ft          | j                  z  }t          | t          |          t          |                    S r7   )r   r*   r   r   	out_avals_transpose_jaxprr   )r   r  r  s      r0   r  r    st     
)__s5>222I	)__s5?333I	%y!1!153C3C	D	DDr2   c                4   	 d t           j                  D             d t           j                  D             z   }d 	t          j        	 fd            }t          j        ||          \  }}}\   t          j        ||          }|	j	        fS )Nc                    g | ]	\  }}||
S r,   r,   )r~   rD  lins      r0   r   z$_transpose_jaxpr.<locals>.<listcomp>  s!    KKKWQsKqKKKr2   c                    g | ]	\  }}||
S r,   r,   )r~   rD  zeros      r0   r   z$_transpose_jaxpr.<locals>.<listcomp>  s!    LLLWQtLqLLLr2   c                     d S r7   r,   r,   r2   r0   r   z"_transpose_jaxpr.<locals>.<lambda>  s     r2   c                    t          | t                    t                    z
  g          \  }}t          |          fdt	          j                  D             }t          d           J t          j        d          5  t          j
        t          j        t          j                            |d          \  }}}d d d            n# 1 swxY w Y   t          |          fdt	          j                  D             }t          d           J d |j        D             }t#          j        |d|||          }	d |	D             x}
_        t)          |
|	          \  }}|S )Nc                    g | ]R\  }}|rt           j                            |          n+t           j                            t	                              SS r,   )rI   r>  r?  knownr   )r~   r   r  ins_iters      r0   r   z8_transpose_jaxpr.<locals>.transposed.<locals>.<listcomp>  s`     > > >D# 03 4%%d+++##DNN33> > >r2   rematted_computationFc                `    g | ]*\  }}|rt          j        |          nt                    +S r,   )r   r   r   )r~   r   r  out_cts_iters      r0   r   z8_transpose_jaxpr.<locals>.transposed.<locals>.<listcomp>  sO     B B B4 &*Aw|D!!!tL/A/A B B Br2   c                @    g | ]}t          j        |j                  S r,   )r   UndefinedPrimalr   r  s     r0   r   z8_transpose_jaxpr.<locals>.transposed.<locals>.<listcomp>  s%    GGG"$QV,,GGGr2   c                D    g | ]}t          |          t          j        u S r,   r  r  s     r0   r   z8_transpose_jaxpr.<locals>.transposed.<locals>.<listcomp>  s%    "M"M"M488w|#;"M"M"Mr2   )r#   r   rL  r   r+  r   r   r   extend_name_stackrI   trace_to_jaxpr_nounitsr   r   r   jaxpr_as_funr  r   r   backward_passin_cts_zeror$   )r   ins_flatout_cts_flatin_pvals	lin_jaxprr.   r   r  
dummy_argsr  r  r  r  r  cellin_linr   r  s               @@r0   
transposedz$_transpose_jaxpr.<locals>.transposed  s   '	CKK#f++4M3NOOHl H~~H> > > >!$U^V!<!<> > >H $'''		+,B	C	C C C6
,t(//
0
0(EC CiFC C C C C C C C C C C C C C C
 %%LB B B B!$U_i!@!@B B BGd##+++GGi6FGGGJi
GLLF #N"Mf"M"M"MMHt!(F33LIqs   ?CCC)
r+  r   r  r   r   rI   r   r   ClosedJaxprr  )
r   r  r  r   r  r  r.   r   r  r  s
   ```      @r0   r  r    s    KKENV ? ?KKKLLEOY ? ?LLLM(	$<       <4 &(%>z8%T%T"Q%&7@@	4+	++r2   c          	     H   |j         rJ t          j        t          j        |          ||t          j        gt          |j                  z  || |          \  }}	|j        |j	        }}
|rt          j
        |
          }
d |	D             }t          j        g ||R d|
i||fS )N)	axis_namespmd_axis_name	main_typec                    g | ]}|rd nd	S )r   Nr,   )r~   bs     r0   r   zremat_vmap.<locals>.<listcomp>  s!    44411aa$444r2   r   )r   r   batch_jaxpr_axesrI   r*  zero_if_mappedr   r   r   r   r   r   r   )r  	axis_sizer  r  rm   dimsr   r@   jaxpr_batched_out_batchedjaxpr_batchedr   out_dimss                r0   
remat_vmapr    s    _ ( 9nUY#em"4"44.I!O !O !O.+ ).0E- >.}==M44444(		Dv	D	D	D	DM	DV	D	Dh	NNr2   used_outputs
list[bool]r
  core.JaxprEqn'tuple[list[bool], core.JaxprEqn | None]c           	        t          j        |j        d         |           \  }}t          |j        |          }t	          |          s't	          |           st          |j                  r|d fS t          j        d t          |j	        |          D             d t          |j
        |           D             |j        ||j        |j        |j                  }||fS )Nr   )r   c                    g | ]	\  }}||
S r,   r,   r~   r   useds      r0   r   zremat_dce.<locals>.<listcomp>  s!    ???wq$$????r2   c                    g | ]	\  }}||
S r,   r,   r  s      r0   r   zremat_dce.<locals>.<listcomp>  s!    AAAwq$DAAAAr2   )rI   rK  r@   rQ  r   _has_effectsr   r  r+  r   r   r   r  r  )r  r
  r  used_inputsrv  r  s         r0   	remat_dcer    s    <
7(;\JJ)[CJi000*
k

  3|#4#4  9$%% ??#cj+66???AA#ck<88AAAz9#4coswP PG r2   c                4    t          d | D                       S )Nc                F    h | ]}t          |t          j                  |S r,   )r   r   NamedAxisEffect)r~   r   s     r0   r   z_has_effects.<locals>.<setcomp>  s*    MMMQAt7K)L)LMqMMMr2   )r*   r   s    r0   r  r    s    	MM'MMM	N	NNr2   F)is_gpu_platformr   r  c                    | j         rJ |r-|r+t          j        j        rt          }n|rt
          }nt          }nd } t          j        |d          |d| iS )Nc                (    t          j        | dg|R  S r  r  r   rm   s     r0   r   z!remat_expansion.<locals>.<lambda>  s    DOE2,M,M,M,M r2   r   rP   r   )	r   r
   remat_opt_barrierrU  $_remat_translation_using_opt_barrier_remat_translation_using_while_remat_translation_using_condr	   
named_call)r   ru   r   r  rm   r.   translation_rules          r0   remat_expansionr    s     _ N N% 7=	 776MM	<(|	<	<	<d	P%	P	PPr2   c                F    t          |          }t          j        | dg|R  S r  )_optimization_barrierr   r  r  s     r0   r  r    s)    	t	$	$$		*T	*	*	**r2   r   core.AbstractValuer   c                   | t           j        u rt          j                    S t	          | t           j        t           j        f          r1t          j        t          j        | j	                  | j
                  S t          |           r7   )r   abstract_tokenr8   create_tokenr   ShapedArrayDShapedArray	broadcastemptyr  shaperi   r  s    r0   _dummy_liker    sl    	T   $&&&$)4+<=>> !,"4TZ"@"@$*MMM
T

r2   c                    ddl m} t          d  j        D                       }t	          j        d          t          t          t          |                    |f}d } fd}|                    |||          }|d         S )Nr   control_flowc              3  $   K   | ]}|j         V  d S r7   r  r  s     r0   r   z1_remat_translation_using_while.<locals>.<genexpr>
  $      22qAF222222r2   c                    | \  }}}t          j        t          j        d          t          j        d          d          }||k     S )Nr      r,   r  )r8   rng_uniformr  int32)carrycounterr.   unifs       r0   condz,_remat_translation_using_while.<locals>.cond  s>    MGQ#BHQKK!BGGGDT>r2   c                `    | \  }}}t          j        dg|R  }|dz   t          |          |fS )Nr,   r   )r   r  r   )r  r  r.   rm   r  r   s        r0   bodyz,_remat_translation_using_while.<locals>.body  s?    GQoeR/$///GaKw..r2   r   )	jax._src.laxr  r   r   r  r  rN  r  
while_loop)r   rm   lax_control_flow	avals_out
carry_initr  r  	carry_ress   `       r0   r  r    s     <;;;;;22EM22222)U3{I#>#>??F*  
/ / / / /
 ))$jAA)	1r2   c                $    ddl m} t          d  j        D                        fd}fd}t	          j        t          j        d          t          j        d          d          } |j        |t          j        d	          k     ||g|R  S )
Nr   r  c              3  $   K   | ]}|j         V  d S r7   r  r  s     r0   r   z0_remat_translation_using_cond.<locals>.<genexpr>!  r  r2   c                 D    t          t          j        dg| R            S r  )r   r   r  )rm   r   s    r0   
remat_compz1_remat_translation_using_cond.<locals>.remat_comp#  s%    2T222333r2   c                 H    t          t          t                              S r7   )r   rN  r  )rm   r"  s    r0   
dummy_compz1_remat_translation_using_cond.<locals>.dummy_comp%  s    [),,---r2   r   r,   r  r  )	r  r  r   r   r8   r  r  float32r  )r   rm   r!  r(  r*  r  r"  s   `     @r0   r  r    s     <;;;;;22EM22222)4 4 4 4 4. . . . . 
	!"*Q--Ab	I	I	I$			tbjmm3Z	Sd	S	S	SSr2   c          	        |r|rt           j        j        s) t          j        t
          d          | g|R |||||dS t          t          j        | j                  }t          j	        |          }t          j        |          }	t          j        |	j        |          }
n|}
t          j        | j        || j                            d          | j        dg|
R d| j        i\  }}|                     |           |S )NT)multiple_results)r   ru   r   rN   r  r   r,   dim_var_values)r
   r  rU  r   	lower_funr  rN  aval_to_ir_typeavals_inflatten_ir_valuesr   OptimizationBarrierOpunflatten_ir_values_like_typesr  jaxpr_subcompmodule_context
name_stackextend	tokens_inr.  set_tokens_out)r  r   ru   r   rN   r  rm   	arg_types	flat_args
barrier_op
jaxpr_argsr3  
tokens_outs                r0   _remat_loweringr@  +  s<       #) +CT^OdCCC
++ +!{')+ + + +
 D(#,77I&t,,I*955J4)% %JJ J'	%!6!6|!D!D	mRI$I I I585GI I$
 Z   	+r2   gpu)platformc                     | S r7   r,   )rm   s    r0   #_optimization_barrier_abstract_evalrD  H  s    	+r2   c                    t          t          j        | j                  }t          j        |          }t          j        |          }t          j        |j        |          S r7   )	rN  r   r0  r1  r2  r   r3  r4  r  )r  rm   barrier_typesr<  r=  s        r0   #_optimization_barrier_lowering_rulerG  K  sL    d*CL99-$T**)(33*		,Z-?	O	OOr2   c                ^    t          |           \  }}t          |t          j        |           S r7   )r   r   optimization_barrier_pr   )argr<  treedefs      r0   r  r  Q  s-    #C(()W	!7!<i!H	I	IIr2   optimization_barrierc                :    t                               | |          S Nr  rT   r   r   rP   s     r0   checkpoint_namerQ  ^  s    	QT	"	""r2   c                   | S r7   r,   rP  s     r0   r   r   a  s    1 r2   c                   | S r7   r,   rP  s     r0   r   r   b  s    A r2   c               P    | |c\  }\  }t                               ||          |fS rN  rO  )r,  r-  rP   r   xdots        r0   name_jvprV  d  s,    8-$1	QT	"	"D	((r2   c                   |gS r7   r,   )r  r   rP   s      r0   r   r   i  s    s r2   c               P    | |c\  }\  }t                               ||          |fS rN  rO  )rm   r  rP   r   r   s        r0   name_batcherrY  k  s,    T*$1	QT	"	"A	%%r2   )concreteru   rv   rN   rZ  c               N    |rd}t          |          t          | |||          S )Na  The 'concrete' option to jax.checkpoint / jax.remat is deprecated; in its place, you can use its `static_argnums` option, and if necessary the `jax.ensure_compile_time_eval()` context manager.

For example, if using `concrete=True` for an `is_training` flag:

  from functools import partial

  @partial(jax.checkpoint, concrete=True)
  def foo(x, is_training):
    if is_training:
      return f(x)
    else:
      return g(x)

replace it with a use of `static_argnums`:

  @partial(jax.checkpoint, static_argnums=(1,))
  def foo(x, is_training):
    ...

If jax.numpy operations need to be performed on static arguments, we can use the `jax.ensure_compile_time_eval()` context manager. For example, we can replace this use of `concrete=True`
:
  @partial(jax.checkpoint, concrete=True)
  def foo(x, y):
    if y > 0:
      return f(x)
    else:
      return g(x)

with this combination of `static_argnums` and `jax.ensure_compile_time_eval()`:

  @partial(jax.checkpoint, static_argnums=(1,))
  def foo(x, y):
    with jax.ensure_compile_time_eval():
      y_pos = y > 0
    if y_pos:
      return f(x)
    else:
      return g(x)

See https://jax.readthedocs.io/en/latest/jep/11830-new-remat-checkpoint.html
rt   )rI  r   )rw   rZ  ru   rv   rN   r   s         r0   checkpoint_wrapperr\  q  sE      .#,\CZ c
"
""	C[#1
3 
3 
3 3r2   )r)   r*   )
rw   r   ru   r*   rN   rx   rv   ry   r)   r   )rw   r   rv   r   r   r   r   r   )r)   r   )r   r}  re  r   r)   r}  )r   r  r  r  r  r  r)   r  )r  r  r
  r  r)   r  )r   r}  ru   r*   r   r*   r  r*   )r   r}  )r   r  r)   r   )r   r}  ru   r*   r   r*   )rw   r   rZ  r*   ru   r*   rv   ry   rN   rx   r)   r   )
__future__r   collections.abcr   r   	functoolsr   rV  typingr   typesnumpyr  jax._srcr   r	   r
   r   r   r   r   r   r   r   r   jax._src.api_utilr   r   jax._src.interpretersr   r   r   r   rI   r  r   r8   r   r:   jax._src.lib.mlir.dialectsr   jax._src.traceback_utilr   jax._src.tree_utilr   r   r   r    jax._src.utilr!   r"   r#   r$   r%   r&   r'   r(   register_exclusion__file__rN  r+  	getLoggerr   rY  r1   r5   r=   rq   rC   rO   	PrimitiverT   rZ   r^   rc   rj   rp   SimpleNamespacecheckpoint_policiesr   rematr   r   r   r   r   r   r   r   r  r   r-  def_implr  def_effectful_abstract_evalr  r6  primitive_jvpsrG  add_typeInOutFeedEffectr|  custom_partial_eval_rulesrM  r  call_partial_eval_custom_rulepartial_eval_jaxpr_custom_rulesr  reducing_transposesr  r  r  axis_primitive_batchersspmd_axis_primitive_batchersr  	dce_rulesr  r  r  r  r  r  r@  register_loweringrD  rG  r  rI  apply_primitivedef_abstract_evalrQ  rV  rY  primitive_batchersr\  r,   r2   r0   <module>r     s"	   # " " " " " . . . . . . . .                                                           & & & & & &       % % % % % % # # # # # # = = = = = = = = $ $ $ $ $ $ * * * * * * & & & & & & 4 4 4 4 4 4 , , , , , , 7 7 7 7 7 7 * * * * * * 0 0 0 0 0 0 S S S S S S S S S S S SE E E E E E E E E E E E E E E E E E E E $  #H - - - ! !( + + +		8	$	$      : : : :        
		        ,   ,e++%!%D'F#A%E5/3&GI I I " 594879[ [ [ [ [ [z 	  4       6G G G G    )()?@@  ? ? ?.+ + + +$' ' ' 'R8 8 8 $.
"
" + + + 	$8 8 %$8# # # ' '    & &|'C D D DDB DB DBJ ); W %" " " "H@ @ @ GB,g46 6  "7 +    #2 w E E E E ", ", ",HO O O -4GJ,E,E   )1; %g .        "W O O O O
 CHQ Q Q Q Q Q"+ + + +
      .T T T T& BG     2  w 0 0 0  w N N N %' ' ' '  P P PJ J J ((>?? *.  '   GH$&<==? ? ?  ( ()L M M M  -:< < <# # # $$ % % %   -- . . .) ) ) % &   v:: ; ; ;& & & '3 F #  ,.)-83 83 83 83 83 83 83 83r2   