
    Vpf                        U d dl mZ 	 d dlmZmZ d dlZd dlmZmZ d dl	Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZmZ d dlm	Z d dlmZ d dlmZ d dlm Z! d dl"m#Z#m$Z$m%Z% d dl&m'Z'm(Z(m)Z) d dl*m+Z+ d dl,m-Z- d dl.m/Z/ d dl0m1Z1m2Z2 i Z3de4d<   i Z5de4d<   ej6        ej7        ej8        ej9        ej:        gZ;ej<        ej=        ej>        ej?        ej@        ejA        ejB        ejC        ejD        ejE        ejF        ejG        ejH        ejI        ejJ        gZKejL        ejM        ejN        ejO        ejP        ejQ        ejR        ejS        ejT        ejU        ejV        ejW        ejX        ejY        gZZde4d<   d Z[eZ\eZ] G d d          Z^ G d de          Z_d  Z`d! Zadud&Zbdvd(Zcdvd)Zddwd-Zedxd/Zf G d0 d1ejg                  Zh G d2 d3eji                  Zjejk        d4             Zldyd6Zmd7 Zndzd=Zod> Zpd? Zqd{dAZrdB ZsdC ZteKD ]"Zu eteud@D          e3eu<    eteud@D          e5eu<   #e;D ]"Zu eteudED          e3eu<    eteudED          e5eu<   #dF Zvejw        e/jx        ejy        dG ejz        e/j{        ej|        e/j}        ej~        dH ej        e/j        ej        e/j        ej        e/j        ej        e/j        i	Ze                                D ]\  ZZ evee          e3e<   ej|        e/j        ejw        e/j        ejy        dI iZe                                D ]\  ZZ evee          e5e<   dJ Zee3ej        <   ee5ej        <   dK Zee3ej        <   dL Zee3ej        <   dM Zee3ej        <   dN Zee3ej        <   dO Zee3ej        <   dP Zee3ej        <   dQ Zee3ej        <   dR ZdS Zee3ej        <   dT Zee3ej        <   dU ZdV Zee3ej        <   dW Zee3ej        <   dX Zee3e/j        <   ee5e/j        <   dY Zee3ej        j        <   ee5ej        j        <   dZ Zd[ Zd\ Z	 	 d|d]Zd^ Zd_ Zi d`edaedbedc erej                  dd erej                  de erej                  df erej                  dg er eej                            dh erej                  di er eej                            dj erej                  dk er eej                            dl erej                  dm er eej                            dn erej                  do er eej                            dpdq  er eej                            ee erej                   erej                   erej                   erej                   erej                   erej                  dr	Zeà                                D ]\  ZZ ee1eeŦ           	 	 d|dsZ erej                   er eej                            edtZeà                                D ]\  ZZ ee2eeŦ           dS )}    )annotations)CallableSequenceN)Any
NamedTuple)lax)config)core)lift_jvp)linear_util)pjit)sharding_impls)bcoo_multiply_densebcoo_multiply_sparse)flatten_fun_nokwargs)pytree)partial_eval)tree_flattentree_maptree_unflatten)safe_mapsafe_zip
split_list)_check_tree_and_avals)	lax_numpy)sparse)BCOOBCSRzdict[core.Primitive, Callable]sparse_rules_bcoosparse_rules_bcsrzlist[core.Primitive]_densifying_primitivesc                `    | t           v rt          d|  d          t          d|  d          )Nsparse rule for z is not implemented because it would result in dense output. If this is your intent, use sparse.todense() to convert your arguments to dense matrices.z is not implemented.)r!   NotImplementedError)	primitives    a/var/www/html/nettyfy-visnx/env/lib/python3.11/site-packages/jax/experimental/sparse/transform.py_raise_unimplemented_primitiver'   z   sY    (((
 ^ ^ ^ ^ _ _ _ 	NyNNNOOO    c                  b    e Zd ZU dZded<   ddZdd
ZddZddZddZ	d Z
ddddddddZdS )SparsifyEnva  Environment for sparse jaxpr evaluation.

  The environment is essentially a collection of buffers and/or tracers
  that may be shared between one or more SparsifyValue objects, which
  represent sparse or dense arrays via indices into the list of buffers.
  zlist[Array]_buffers c                .    t          |          | _        d S N)listr+   )selfbufss     r&   __init__zSparsifyEnv.__init__   s    JJDMMMr(   arrArrayreturnintc                    | j                             t          j        |                     t	          | j                   dz
  S )N   )r+   appendjnpasarraylen)r0   r3   s     r&   _pushzSparsifyEnv._push   s7    MS))***t}!!r(   spvalueSparsifyValuec                R    |j         t          d          | j        |j                  S )z4Get the data buffer associated with a SparsifyValue.Nz8Internal: requested data from spvalue with data_ref=None)data_refRuntimeErrorr+   r0   r>   s     r&   datazSparsifyEnv.data   s*    STTT=)**r(   c                R    |j         t          d          | j        |j                  S )z7Get the indices buffer associated with a SparsifyValue.Nz>Internal: requested indices from spvalue with indices_ref=None)indices_refrB   r+   rC   s     r&   indiceszSparsifyEnv.indices   s*    "YZZZ=,--r(   c                R    |j         t          d          | j        |j                  S )z;Get the BCSR indptr buffer associated with a SparsifyValue.Nz=Internal: requested indices from spvalue with indptr_ref=None)
indptr_refrB   r+   rC   s     r&   indptrzSparsifyEnv.indptr   s*    !XYYY=+,,r(   c                l    t          t          j        |          |                     |                    S )z2Add a new dense array to the sparsify environment.)r?   npshaper=   )r0   rD   s     r&   densezSparsifyEnv.dense   s&    $D)9)9:::r(   NF)rA   rF   rI   indices_sortedunique_indicesc               |   ||J |                      |          }n||t          | j                  k     sJ ||J |                      |          }n||t          | j                  k     sJ ||J |                      |          }n||t          | j                  k     sJ t          ||||||	          S )z3Add a new sparse array to the sparsify environment.NrO   rP   )r=   r<   r+   r?   )
r0   rM   rD   rG   rJ   rA   rF   rI   rO   rP   s
             r&   r   zSparsifyEnv.sparse   s     D!!hh!hT]1C1C&C&C&C&C   JJw''kk$s4=7I7I)I)I)I)I::f%%jj		#dm,,,,,,+z(6~W W W Wr(   )r,   )r3   r4   r5   r6   )r>   r?   r5   r4   )NNN)__name__
__module____qualname____doc____annotations__r2   r=   rD   rG   rJ   rN   r   r,   r(   r&   r*   r*      s              " " " "+ + + +. . . .- - - -; ; ;W4D!%W W W W W W Wr(   r*   c                      e Zd ZU ded<   ded<   dZded<   dZded<   dZd	ed
<   dZd	ed<   ed             Z	d Z
d Zd Zd ZdS )r?   ztuple[int, ...]rM   z
int | NonerA   NrF   rI   Fzbool | NonerO   rP   c                *    t          | j                  S r.   )r<   rM   r0   s    r&   ndimzSparsifyValue.ndim   s    tz??r(   c                    | j         d uS r.   rF   rZ   s    r&   	is_sparsezSparsifyValue.is_sparse   s    4''r(   c                    | j         d u S r.   r]   rZ   s    r&   is_densezSparsifyValue.is_dense   s    t##r(   c                <    |                                  o| j        d u S r.   r^   rI   rZ   s    r&   is_bcoozSparsifyValue.is_bcoo   s    >>74 77r(   c                <    |                                  o| j        d uS r.   rb   rZ   s    r&   is_bcsrzSparsifyValue.is_bcsr   s    >>;t ;;r(   )rS   rT   rU   rW   rF   rI   rO   rP   propertyr[   r^   r`   rc   re   r,   r(   r&   r?   r?      s          +    * %.%%%% %.%%%%  8( ( ($ $ $8 8 8< < < < <r(   r?   c                :    t          | t          t          f          S r.   )
isinstancer   r   args    r&   <lambda>rk      s    ZdD\:: r(   c                ,    t          | t                    S r.   )rh   r?   ri   s    r&   rk   rk      s    *S-88 r(   spenvargsr   r5   c                <      fd}t          ||t                    S )zHConvert a pytree of (sparse) arrays to an equivalent pytree of spvalues.c                Z   t          | t                    r3                    | j        | j        | j        | j        | j                  S t          | t                    r9                    | j        | j        | j        | j	        | j        | j                  S 
                    |           S )NrR   )rh   r   r   rM   rD   rG   rO   rP   r   rJ   rN   )rj   rm   s    r&   array_to_spvaluez,arrays_to_spvalues.<locals>.array_to_spvalue   s    #t 	\\#)SXs{),);),);  = = = 
C		 \\#)SXs{CJ),);),);  = = = [[r(   is_leaf)r   _is_sparse_obj)rm   rn   rq   s   `  r&   arrays_to_spvaluesru      s4    

 
 
 
 
 
"D.	A	A	AAr(   spvaluesc                <      fd}t          ||t                    S )zHConvert a pytree of spvalues to an equivalent pytree of (sparse) arrays.c                   |                                  rJt                              |                               |           f| j        | j        | j                  S |                                 r^t                              |                               |           	                    |           f| j        | j        | j                  S                     |           S )N)rM   rO   rP   )
rc   r   rD   rG   rM   rO   rP   re   r   rJ   )r>   rm   s    r&   spvalue_to_arrayz,spvalues_to_arrays.<locals>.spvalue_to_array   s     	!5::g&&g(>(>?g6L!(!79 9 9 9 
		 !5::g&&g(>(>W@U@UVg6L!(!79 9 9 9 ZZ   r(   rr   r   _is_spvalue)rm   rv   ry   s   `  r&   spvalues_to_arraysr|      s4    

! 
! 
! 
! 
! 
"Hk	B	B	BBr(   c                <      fd}t          ||t                    S )zHConvert a pytree of spvalues to an equivalent pytree of abstract values.c                                         |           }t          j        | j        |j        |j        j                  S r.   )rD   r
   ShapedArrayrM   dtypeaval	weak_type)r>   rD   rm   s     r&   spvalue_to_avalz*spvalues_to_avals.<locals>.spvalue_to_aval  s2    ::gDGM4:ty7JKKKr(   rr   rz   )rm   rv   r   s   `  r&   spvalues_to_avalsr   	  s8    
L L L L L 
/8[	A	A	AAr(   objnamestrc                j    t          | |          sJ t          | |          }t          | |           |S r.   )hasattrgetattrdelattrr   r   vals      r&   popattrr     s<    	d		T#	#t	*r(   r   c                L    t          | |          rJ t          | ||           d S r.   )r   setattrr   s      r&   
setnewattrr     s0    S$	#tSr(   c                  H    e Zd ZddZed             Zed             Zd ZdS )	SparseTracertrace
core.Tracec               "    || _         || _        d S r.   )_spvalue_trace)r0   r   r>   s      r&   r2   zSparseTracer.__init__"  s    DMDKKKr(   c                v    t          | j        j        d          st          d          | j        j        j        S )Nrm   +Internal: main does not have spenv defined.)r   r   mainrB   rm   rZ   s    r&   rm   zSparseTracer.spenv&  s7    4;#W-- HFGGG;!!r(   c                D    t          | j        | j        g          d         S )Nr   )r   rm   r   rZ   s    r&   r   zSparseTracer.aval,  s    TZ$-99!<<r(   c                    | S r.   r,   rZ   s    r&   
full_lowerzSparseTracer.full_lower0  s    Kr(   N)r   r   )rS   rT   rU   r2   rf   rm   r   r   r,   r(   r&   r   r   !  sm            " " 8"
 = = 8=    r(   r   c                  :    e Zd ZddZddZddZd ZddZd ZdS )SparseTracer   r   c                    t          | j        d          st          d          t          | j        j        |g          \  }t          | |          S Nrm   r   r>   r   r   rB   ru   rm   r   r0   r   r>   s      r&   purezSparseTrace.pure4  P    49g&& HFGGG!$)/C599HGg....r(   core.Tracerc                    t          | j        d          st          d          t          | j        j        |g          \  }t          | |          S r   r   r   s      r&   liftzSparseTrace.lift:  r   r(   r   c                8    t          |j        |j                  S )Nr   )r   r   r   )r0   r   s     r&   subliftzSparseTrace.sublift@  s    
CL9999r(   c                    t           j        d          d |D             }t          d |D                       r9|t          vrt	          |           t          |         gd |D             R i |}n3 |j        fd|D             i |}t          |j        r|n|g          }t           j        d           t           fd|D                       }|j        r|n|d         S )Nrm   c                    g | ]	}|j         
S r,   r   .0ts     r&   
<listcomp>z1SparseTrace.process_primitive.<locals>.<listcomp>E  s    ,,,q
,,,r(   c              3  >   K   | ]}|                                 V  d S r.   )r^   )r   r>   s     r&   	<genexpr>z0SparseTrace.process_primitive.<locals>.<genexpr>F  s.      
7
777
7
7
7
7
7
7r(   c              3  $   K   | ]}|j         V  d S r.   r   r   s     r&   r   z0SparseTrace.process_primitive.<locals>.<genexpr>I  s$      :W:W!1::W:W:W:W:W:Wr(   c              3  B   K   | ]}                     |          V  d S r.   rD   )r   r>   rm   s     r&   r   z0SparseTrace.process_primitive.<locals>.<genexpr>K  s/      !N!N'%**W"5"5!N!N!N!N!N!Nr(   c              3  :   K   | ]}t          |           V  dS )r   Nr   r   r>   r0   s     r&   r   z0SparseTrace.process_primitive.<locals>.<genexpr>N  s0      XXT7;;;XXXXXXr(   r   )
r   r   anyr   r'   bindru   multiple_resultsr   tuple)	r0   r%   tracersparamsrv   out_spvaluesout_bufsout_tracersrm   s	   `       @r&   process_primitivezSparseTrace.process_primitiveC  s-   DIw''E,,G,,,H

7
7h
7
7
777 g	+	+	+&y111&y1%b:W:Ww:W:W:Wbbb[abbll!N!N!N!NX!N!N!NYRXYYh'9;U/exx\d[efflty'5)))XXXX<XXXXXK#4H;;+a.Hr(   flu.WrappedFunc                    t           j        d          }t          d |D                       }|j        }t	          | j        |          \  }}	t          |d                   rt          d          t          |t          d |D                                 } |j        |g|R i |}
t           j        dt          |
                      fd |	            D             S )Nrm   c              3  $   K   | ]}|j         V  d S r.   r   r   s     r&   r   z+SparseTrace.process_call.<locals>.<genexpr>S  s$      11AQZ111111r(   donated_invarsz(sparsify does not support donated_invarsc              3     K   | ]}d V  dS FNr,   )r   bufs     r&   r   z+SparseTrace.process_call.<locals>.<genexpr>X  s"      .H.Hu.H.H.H.H.H.Hr(   )r   c                2    g | ]}t          |           S r   r   r   s     r&   r   z,SparseTrace.process_call.<locals>.<listcomp>[  s&    NNNGLw///NNNr(   )r   r   r   r+   sparsify_subtracer   r$   dictr   r   r*   )r0   call_primitiver   r   r   rm   rv   in_bufsfunr   bufs_outs   `          r&   process_callzSparseTrace.process_callQ  s    DIw''E1111111HnG)!TYAAC
6"#$$ L JKKK&.H.H.H.H.H)H)HIIIF"~"3;;;;F;;Hty';x#8#8999NNNN||~~NNNNr(   c                   ~~~ |j         | S r.   )call_wrapped)r0   r%   r   jvpr   symbolic_zeross         r&   process_custom_jvp_callz#SparseTrace.process_custom_jvp_call]  s    33W%%r(   N)r   r   )r   r   )r   r   )r   r   )	rS   rT   rU   r   r   r   r   r   r   r,   r(   r&   r   r   3  s        / / / // / / /: : : :I I I
O 
O 
O 
O& & & & &r(   r   c              '     K   t          | dt          |                     |                                 fd|D             }|i fV }fd|D             }t          | d          j        }|d |D             fV  d S )Nrm   c                2    g | ]}t          |           S r   r   )r   r>   r   s     r&   r   z%sparsify_subtrace.<locals>.<listcomp>f  s&    MMMUG444MMMr(   c                :    g | ]}                     |          S r,   )
full_raise)r   outr   s     r&   r   z%sparsify_subtrace.<locals>.<listcomp>h  s'    666#  %%666r(   c                    g | ]	}|j         
S r,   r   )r   r   s     r&   r   z%sparsify_subtrace.<locals>.<listcomp>j  s    5553#,555r(   )r   r*   with_cur_sublevelr   r+   )r   rv   r1   
in_tracersouts
out_tracesbuffersr   s          @r&   r   r   b  s      T7K--...

 
 
"
"%MMMMHMMM*2~		$6666666*D'""+'55*555555555r(   list[ArrayOrSparse]c                :   t          j        t                    5 }t                      }t	          ||          }|j        }t          | ||          \  }} |j        | }t          |          }~d d d            n# 1 swxY w Y   t          | |                      S r.   )	r
   new_mainr   r*   ru   r+   r   r   r|   )	wrapped_funrn   r   rm   rv   r   r   r   r   s	            r&   sparsify_funr   l  s    }[!! TMME!%..HnG)+tXFFCs)H!!E               
E<<>>	2	22s   AA::A>A>c                F     t          j                    fd            }|S )z+Implementation of sparsify() using tracers.c                     t          | t                    \  }}t          t          j                  |          \  }}t          ||          }t           |            |          S )Nrr   )r   rt   r   lu	wrap_initr   r   )rn   	args_flatin_treer   out_treer   r   s         r&   _wrappedz'_sparsify_with_tracer.<locals>._wrappedy  s]    %dNCCCIw0c1B1BGLLK
{I
.
.C((**c***r(   )	functoolswraps)r   r   s   ` r&   _sparsify_with_tracerr   w  s8    ?3+ + + + +
 
/r(   jaxpr
core.JaxprconstsSequence[Array]Sequence[SparsifyValue]c                   i dfd}dfd
}dfd}t          || j        |           t          || j        |           | j        D ]n}|j        }t          ||j                  }	t          d |	D                       r4|t          vrt          |           t          |         g|	R i |j        }
nt          d |	D                       r4|t          vrt          |           t          |         g|	R i |j        }
n |j
        fd|	D             i |j        }|j        r|n|g}g }
t          ||j                  D ]]\  }}t          |t          j                  r|
                    d            5|
                                        |                     ^t          ||j        |
           pt          || j                  S )Nvar	core.Atomr5   r?   c                    t          | t          j                  r                    | j                  S t          | t          j                  sJ |          S r.   )rh   r
   LiteralrN   r   Var)r   envrm   s    r&   readzeval_sparse.<locals>.read  sL    #t|$$ [[!!!TX&&&&&Xor(   core.Varar4   Nonec                p    t          | t          j                  rd S                     |          | <   d S r.   )rh   r
   DropVarrN   )r   r  r  rm   s     r&   write_bufferz!eval_sparse.<locals>.write_buffer  s3    #t|$$ f{{1~~CHHHr(   c                R    t          | t          j                  rd S |J || <   d S r.   )rh   r
   r  )r   r  r  s     r&   writezeval_sparse.<locals>.write  s2    #t|$$ f===CHHHr(   c              3  >   K   | ]}|                                 V  d S r.   )re   r   r   s     r&   r   zeval_sparse.<locals>.<genexpr>  s*      
+
+S3;;==
+
+
+
+
+
+r(   c              3  >   K   | ]}|                                 V  d S r.   )rc   r  s     r&   r   zeval_sparse.<locals>.<genexpr>  s*      --sS[[]]------r(   c              3  B   K   | ]}                     |          V  d S r.   r   )r   r   rm   s     r&   r   zeval_sparse.<locals>.<genexpr>  s-      ??UZZ__??????r(   )r   r   r5   r?   )r   r  r  r4   r5   r  )r   r  r  r?   r5   r  )r   	constvarsinvarseqnsr%   r   r    r'   r   r   r   r   r   outvarsrh   r
   r  r9   rN   )r   r   rv   rm   r  r	  r  eqnpriminvalsr   r   r   outvarr  s      `          @r&   eval_sparser    sW    )+#            
      <&111
5%,)))Z & &c=DdCJ''F

+
+F
+
+
+++ '	&	&	&&t,,,d#EAFAAAcjAAcc	--f---	-	- '	&	&	&&t,,,d#EAFAAAcjAAcc???????N3:NNh!2B
hc!(CK88 ' '+#vfdl++ 	'
**T




**U[[%%
&
&
&
&UCK%%%%	$	&	&&r(   c                     d
 fd	}|S )Nrm   r*   rv   r?   r   r   r5   0tuple[Sequence[SparsifyValue], pytree.PyTreeDef]c                r   t          |t                    \  }}t          | |          }t          t	          j        |          |          \  }}t          j        ||          \  }}	}
\   t          ||
||           }t          |	          t          |          k    rt          d          | |            fS )Nrr   zkInternal: eval_sparse does not return expected number of arguments. Got {result} for avals {out_avals_flat})r   r{   r   r   r   r   petrace_to_jaxpr_dynamicr  r<   	Exception)rm   rv   r   spvalues_flatr   in_avals_flatr   r   r   out_avals_flatr   resultr   s               r&   wrappedzsparsify_raw.<locals>.wrapped  s     *(KHHHM7%e];;M0a1H1H'RRK(*(A+}(](]%E>62u==F
>c&kk)) @ A A A88::r(   )rm   r*   rv   r?   r   r   r5   r  r,   )r   r#  s   ` r&   sparsify_rawr$    s(          
.r(   c                d    t          |           t          j        |           fd            }|S )z5Implementation of sparsify() using jaxpr interpreter.c                     t                      }t          ||           } |g|R i |\  }}t          ||          }t          ||          S r.   )r*   ru   r|   r   )rn   r   rm   rv   spvalues_outr   r   f_raws          r&   r#  z+_sparsify_with_interpreter.<locals>.wrapped  s^    MME!%..H"U5>8>>>v>>L(
UL
1
1C(C(((r(   )r$  r   r   )r   r#  r(  s     @r&   _sparsify_with_interpreterr)    sC    
q//%?1) ) ) ) ) 
.r(   Fc                B    |rt          |           S t          |           S )a  Experimental sparsification transform.

  Examples:

    Decorate JAX functions to make them compatible with :class:`jax.experimental.sparse.BCOO`
    matrices:

    >>> from jax.experimental import sparse

    >>> @sparse.sparsify
    ... def f(M, v):
    ...   return 2 * M.T @ v

    >>> M = sparse.BCOO.fromdense(jnp.arange(12).reshape(3, 4))

    >>> v = jnp.array([3, 4, 2])

    >>> f(M, v)
    Array([ 64,  82, 100, 118], dtype=int32)
  )r   r)  )r   
use_tracers     r&   sparsifyr,    s(    *  ) ###%a(((r(   c                    |                                 s|j        r|S t          | |          }|                    |j        d          }t          | |          S )z;Return an spvalue representation with deduplicated indices.F)nseremove_zeros)r`   rP   r|   sum_duplicatesr.  ru   )rm   r>   r3   s      r&   _ensure_unique_indicesr1    s^     71 N5'**#swU;;#	E3	'	''r(   c                      fd}|S )Nc                v   t          |          dk    sJ |d         }st          | |          }|                     |          } j        |fi |}|d                                         r5|                     |j        ||j        |j        |j	        |j
                  }n|                     |          }|fS )Nr8   r   )rF   rI   rO   rP   )r<   r1  rD   r   r^   r   rM   rF   rI   rO   rP   rN   )	rm   rv   kwargsr>   r   buf_outout_spvaluelinearr  s	          r&   funcz'_zero_preserving_unary_op.<locals>.func  s    x==AqkG 7 'ug66g
**W

Cdi&&v&&G{ %LL-4-@,3,>070F070F	 ! H Hkk KK$$k>r(   r,   )r  r7  r8  s   `` r&   _zero_preserving_unary_opr9    s)         $ 
+r(   )r7  Tc                      fd}|S )Nc                d     t          | |          i |}t          | j        r|n|g          S r.   )r|   ru   r   )rm   rv   kwdsr"  r  	sparse_ops       r&   _sparse_rulez+_standard_sparse_rule.<locals>._sparse_rule  s@    Y*5(;;DtDDFet/D%RVV6(SSSr(   r,   )r  r=  r>  s   `` r&   _standard_sparse_ruler?    s0    T T T T T T 
r(   c                 &    t          j        | fi |S r.   )r   bcoo_concatenater  ks     r&   rk   rk   !      V%<Q%D%D!%D%D r(   c                 D    t          j        | d         | dd          fi |S )Nr   r8   )r   bcoo_dynamic_slicerB  s     r&   rk   rk   $  s(    v'@1qu'R'RPQ'R'R r(   c                 &    t          j        | fi |S r.   )r   bcsr_concatenaterB  s     r&   rk   rk   1  rD  r(   c                   |dk    r t          dt          j         d| d           t          t          j        d          | g|R d|iS )Nr   r#   z with non-positive exponent z is not implemented because it would result in dense output. If this is your intent, use sparse.todense() to convert your argument to a dense array.Fy)r$   r   integer_pow_pr9  )rm   rJ  rv   s      r&   _integer_pow_sparserL  7  s}    !VV
 h1B h h`a h h h i i i 
=	"3#4e	<	<U	SX	S	S	SQR	S	SSr(   c          	        t          |          }t          | |          }|d         j        t          j        |d         |          }t          fd|D                       }|d         j        j        dz
  }|d         j        j        d         }|d |         t          t          |                    k    }|||z   d          t          t          ||z   t                                        k    }	||||z            t          t          |||z                       k    }
i }|r|	r|d         j	        |d<   n
|j
        |d<   |r|
r|d         j        |d<   n
|j        |d	<   |j        |d
<   |j        |d<    | j        |fi |}|fS )Nr   )permutationc              3  (   K   | ]}|         V  d S r.   r,   )r   irM   s     r&   r   z$_transpose_sparse.<locals>.<genexpr>F  s'      22E!H222222r(      rA   rD   rF   rG   rO   rP   )r   r|   rM   r   bcoo_transposerG   r[   ranger<   rA   rD   rF   rO   rP   )rm   rN  rv   rn   mat_transposed	out_shapen_batchn_sparsebatch_dims_unchangeddense_dims_unchangedsparse_dims_unchangedr<  r>   rM   s                @r&   _transpose_sparser\  A  s   k""+	E8	,	,$
q'-%(akJJJ.2222k22222)GO 1$'!W_"2&(%hwh/5w3H3HH%g&8&9&9:eE'T\J\^abg^h^hDiDi>j>jj&ww/A'ABeERY[bem[mLnLnFoFoo 
$ '2 '{+D!&DL  -3 -"1+1D$,DO)8$)8$EL++d++'
r(   c                   |\  }}t          j        |j        |j                  }|                                r.|                                r|j        |j        k    rt	          d          |j        |j        k    rt          j        |                     |          |                     |                    }t          j	        j
        r$|j        |j        k    sJ |j        |j        k    sJ |                     |j        ||j        |j        |j                  }n|                     |          j        |                     |          j        k    s6|                     |          j        |                     |          j        k    rt	          d          t          j        |                     |          |                     |          g|                     |          j        dz
            }t          j        |                     |          |                     |          g|                     |          j        dz
            }|                     |j        ||          }n|                                r||}}|                                r|                                sJ |j        |k    rt	          d          t%          | ||f          \  }}	|                                |	z   }
|                     |
          }|fS )Nz5Addition between sparse matrices of different shapes.rF   rO   rP   zGAddition between sparse matrices with different batch/dense dimensions.rQ  )	dimensiona  Addition between a sparse array X and a dense array Y is not implemented when the output shape is larger than Y.shape. This is to prevent silent densification of a large sparse array. If this is your intent, you can explicitly cast the sparse array to a dense matrix.)r   broadcast_shapesrM   r^   r$   rF   addrD   r	   enable_checksvaluerO   rP   r   rG   r[   concatenater`   r|   todenserN   )rm   rv   XYrV  out_datar6  out_indices
X_promoted
Y_promotedr   s              r&   _add_sparserl  b  s   	$!Q"17AG44)[[]] #q{{}} #w!' WXXX}%%A

166h			# 41#333331#33333LL(010@010@ ! B Bkk 
q			%--"2"2"7	7	75::a==;MQVQ[Q[\]Q^Q^Qc;c;c ijjjOU]]1%5%5u}}Q7G7G$HTYTaTabcTdTdTilmTmnnnk%**Q--A!?5==YZK[K[K`cdKdeeehLL(K@@kk{{}} a;;==)QZZ\\)))w)	#$ $ $
 01v>>J






+C++c""K
r(   c                    |\  }}|                                 r@|                                 r,t          | |gt          t          j                 | |          R  S t          d          )Nz+Subtraction between sparse and dense array.)r^   rl  r   r   neg_pr$   )rm   rv   rf  rg  s       r&   _sub_sparsero    se    	$!Q[[]] Mq{{}} MuaI"3CI">ua"H"HIIII
K
L
LLr(   c                L   |\  }}|                                 r|                                 r|j        |j        k    r|j        rt          j        j        r$|j        |j        k    sJ |j        |j        k    sJ t          j        | 	                    |          | 	                    |                    }| 
                    |j        ||j        |j        d          }nt          | |          \  }}t          ||          }| 
                    |j        |j	        |j                  }ny|                                 r||}}t          | |          }t          || 	                    |                    }| 
                    |j        ||j        |j        |j                  }|fS )NTr^  )r^   rF   rP   r	   rb  rc  rO   r   mulrD   r   rM   r|   r   rG   r   )	rm   rv   rf  rg  rh  r6  rj  rk  mats	            r&   _mul_sparsers    s   	$!Q[[]] @q{{}} @}%%!*:%			# 41#333331#33333A

166hLL(010@04 ! 6 6kk  2%BBj* Z88cLLCHckBBkk{{}} a#E1--J":uzz!}}==H,,qwam./.>./.>  @ @K r(   c                    |\  }}|                                 rt          d          t          | |          }t          |d|                     |          z            }|                     |j        ||j        |j        |j	                  }|fS )NzDivision by a sparse array is not implemented because it would result in dense output. If this is your intent, use sparse.todense() to convert your arguments to a dense array.g      ?r^  )
r^   r$   r|   r   rD   r   rM   rF   rO   rP   )rm   rv   rf  rg  rj  rh  r6  s          r&   _div_sparseru    s    	$!Q[[]] F
EF F F "%++* R%**Q---?@@(QWhAM-.-=-.-=  ? ?+ r(   c                  |\  }t          | |          }t          j        ||          }|j        }|dk    r-|                     |j                                                  }n!|                     ||j        |j                  }|fS )N)axesr,   )r|   r   bcoo_reduce_sumrM   rN   rD   sumrG   )rm   rw  rv   rf  rj  rr  rV  r6  s           r&   _reduce_sum_sparserz    s{    "!!%++*z555#i)"__++chllnn--KK,,y#(CK@@K
r(   c          
         t          | |          \  }}	t          j        ||	||||||          }
t          | |
f          S )N)dimension_numbersslice_sizesrP   indices_are_sortedmode
fill_value)r|   r   bcoo_gatherru   )rm   r|  r}  rP   r~  r  r  rn   operandstart_indicesr"  s              r&   _gather_sparse_ruler    sX    -eT::'=g}HY*5n1C#'J@ @ @& 
EF9	-	--r(   c                   
 d t           j        
 fd            }t           |          }t          |          \  }
d |D             }t	          j        ||          \  }}}	\   t	          j        ||	          }J |fS )Nc                     t          |           }t          	|          }t          j        j        |	          }t          	|          }t          |          \  }|S r.   )r   ru   r  r   r   r|   r   )
r   rn   rv   r"  r   out_flatr   r   r   rm   s
         r&   r#  z _sparsify_jaxpr.<locals>.wrapped  s`     '9--D!%..HelHeDDF
UF
+
+C%c**HhOr(   c                Z    g | ](}t          j        t          j        |                    )S r,   )r
   raise_to_shapedget_avalr   rj   s     r&   r   z#_sparsify_jaxpr.<locals>.<listcomp>  s-    NNNS$T]3%7%788NNNr(   )r   r   r|   r   r  r  ClosedJaxpr)rm   r   rv   r#  rn   r   
avals_flatsp_jaxpr_r   r   r   s   ``        @@r&   _sparsify_jaxprr    s    
 '+(<       < 
E8	,	,$#D)))WNNINNN*5gzJJ(Avr^Hf--(					8	r(   c                  t          |||g          \  }}}t          | |g||R  \  }	}
t          | |g||R  \  }}t          t          | |                    \  }}
t          t          | |                    \  }}
t          t          | |                    \  }}
t	          j        j        g |||R t          |          |	t          |          |d}t          | t          ||                    S )N)cond_nconsts
cond_jaxprbody_nconsts
body_jaxpr)
r   r  r   r|   r   while_pr   r<   ru   r   )rm   r  r  r  r  rv   cond_const_spvaluesbody_const_spvaluesinit_val_spvaluescond_sp_jaxprr  body_sp_jaxprr   cond_constsbody_consts	init_valsr  s                    r&   _while_sparser    s<   @J|\*A, A,=*,= %UJa9LaO`aaa-+E:h@ShVghhh- 25:M N NOO.+q 25:M N NOO.+q08IJJKK,)Q[ W{ W[ W9 W W+.{+;+;+.{+;+;W W W( 
E>(H#E#E	F	FFr(   c       
           t          |          rt          d          t          | |g|R  \  }}t          t	          | |                    \  }}t          d |D                       }|t          d t          t          |          t          |          z
            D                       z   }|t          d t          t          |j                  t          |          z
            D                       z   }|t          d t          t          |          t          |          z
            D                       z   }|t          d t          t          |j                  t          |          z
            D                       z   }t          j
        j        ||||||||||	|
d
}t          | t          ||                    S )Nz#sparse xla_call with donated_invarsc              3     K   | ]}d V  dS r   r,   r  s     r&   r   z_pjit_sparse.<locals>.<genexpr>  s"      443444444r(   c              3  .   K   | ]}t           j        V  d S r.   r   UNSPECIFIEDr   r  s     r&   r   z_pjit_sparse.<locals>.<genexpr>  s9       & &
  & & & & & &r(   c              3  .   K   | ]}t           j        V  d S r.   r  r  s     r&   r   z_pjit_sparse.<locals>.<genexpr>  s9       ( (
  ( ( ( ( ( (r(   c              3     K   | ]}d V  d S r.   r,   r  s     r&   r   z_pjit_sparse.<locals>.<genexpr>  s3       " "d" " " " " "r(   c              3     K   | ]}d V  d S r.   r,   r  s     r&   r   z_pjit_sparse.<locals>.<genexpr>  s3       $ $d$ $ $ $ $ $r(   )
r   in_shardingsout_shardings
in_layoutsout_layoutsresource_envr   r   keep_unusedinline)r   r$   r  r   r|   r   rT  r<   	out_avalsr   pjit_pr   ru   r   )rm   r   r  r  r  r  r  r   r   r  r  rv   sp_call_jaxprr   r   r  r  s                    r&   _pjit_sparser    s    	 E
C
D
DD+E5D8DDD-0AABB,)Q44)44444.  & &S^^c,&7&7788& & & ! ! ,  % ( (S011C4F4FFGG( ( ( # # - E " "#i..3z??:;;" " "   * e $ $#m566[9I9IIJJ$ $ $   + [!#  ( 
E>(H#E#E	F	FFr(   c              #  x   K   t          | |          D ]&\  }}|                                r||gn|gE d {V  'd S r.   )r   r^   )rv   r   r>   params       r&   _duplicate_for_sparse_spvaluesr  2  sg       622 B Bngu!(!2!2!4!4Au~~5'AAAAAAAAB Br(   c          	     $   t          |||g          \  }}}|rt          d          t          | |g|||R  \  }	}
t          t	          | |                    \  }}
t          t	          | |                    \  }}t          t	          | |                    \  }}t          |                    d          ||g          \  }}}g t          ||          t          ||          t          ||          R }t          j        j	        g |||R |	|t          |          t          |          d|}t          ||d t          |                             }t          ||t          |          d                    }t          | ||z             S )Nz#sparse rule for scan with x values.r7  )r   r7  
num_consts	num_carry)r   r$   r  r   r|   popr  r   scan_pr   r<   r   ru   )rm   r   r  r  rv   r   const_spvaluescarry_spvaluesxs_spvaluesr  r  r   carry
carry_treexsxs_treeconst_linearcarry_linear	xs_linear	sp_linearr   	carry_outxs_outs                          r&   _scan_sparser  6  s   0:z9%1' 1'-..+ E C
D
DDu]~]]Q\]]]+(A-e^DDEE)&!"#5e^#L#LMM%/{CCDD+"g +5
JJx:y1+3 +3',i=#NLAA=#NLAA= $K;;= =)
 	
 	P 	P% 	P" 	P 	PHY#&v;;#e**	P 	PHN	P 	P#Z[c%jj[)9::)'3s5zz{{#344&	E9v#5	6	66r(   c               f    t           fd|D              \  }}t          d|d         |d         j        |d         |d         j                   t          t	           |gR                     \  }}t          j        j        |d|i|}	t          |d         |	          }
t           |
          S )Nc              3  4   K   | ]}t          |gR  V  d S r.   )r  )r   r   operandsrm   s     r&   r   z_cond_sparse.<locals>.<genexpr>T  sI        7  7$) !0u Hx H H H  7  7  7  7  7  7r(   z(sparsified true_fun and false_fun outputr   r8   branches)
zipr   r  r   r|   r   cond_pr   r   ru   )rm   predr  r  r   sp_branchestreedefsrn   r  r  r   s   `  `       r&   _cond_sparser  S  s      7  7  7  7  7-5 7  7  7 8+xB [^%= [^%=? ? ? +ED3D83D3DEEFF'$Z_dC[CFCC(x{H--#	E3	'	''r(   c               t    ~t          | |                                          }|                     |          fS r.   )r|   re  rN   )rm   r>   treer   s       r&   _todense_sparse_ruler  `  s5    
5'**2244#
++c

	r(   c                v    |                     d          |                     d          }|                     d          }t           g|R  \  }}t          j         fd            }t	          ||          }t           |          }	t          j        j        j	        ||g|	R i |}
t           |
          S )N
call_jaxprjvp_jaxpr_thunkr  c                 |    t          |           }t          j        j        |          }t	          |          S r.   )ru   r  r   r   r|   )arrssparrsr   r  rm   s      r&   r   z$_custom_jvp_sparse_rule.<locals>.funm  s;    t,,F
j&
(965
I
ICeS)))r(   )r  r  r   r   r   r|   jaxcustom_derivativescustom_jvp_call_pr   ru   )rm   rv   r   r  r  r  r   r   r   r  outvalsr  s   `          @r&   _custom_jvp_sparse_ruler  h  s    zz,''*JJ011/zz,''*+E:IIII-<* * * * * <* 	_--#eX..&"49#sVVVVVvVV'	E7	+	++r(   c                >     t          fd          |           S )zSum array along axis.c                     | j         i S r.   )ry  xrn   r4  s    r&   rk   z_sum.<locals>.<lambda>  s    EAE42622 r(   r,  r0   rn   r4  s    ``r&   _sumr    s)    	322222	3	3D	9	99r(   c                >     t          fd          |           S )z;Returns an array containing the same data with a new shape.c                     | j         i S r.   )reshaper  s    r&   rk   z_reshape.<locals>.<lambda>  s    IAIt6v66 r(   r  r  s    ``r&   _reshaper    s)    	766666	7	7	=	==r(   c                >     t          fd          |           S )z-Copy the array and cast to a specified dtype.c                     | j         i S r.   )astyper  s    r&   rk   z_astype.<locals>.<lambda>  s    HAHd5f55 r(   r  r  s    ``r&   _astyper    s)    	655555	6	6t	<	<<r(   c                     t          t          j        t          j        |||||                    |           }t          |t                    s,t          j        |          dk    rt          j	        |          }|S )Nidxr~  r  rP   r  r   )
r,  r   partialr   _rewriting_takerh   r   rL   size	fromdenser3   r  r~  rP   r  r  r"  s          r&   _bcoo_rewriting_taker    s    F8I%3;M	nE E E F FFIK K& 
FD	!	! $bgfoo&:&:^F##F	-r(   c                j     t           fdt           j        d                   D                       S )Nc              3  (   K   | ]}|         V  d S r.   r,   )r   rP  r3   s     r&   r   z_sparse_iter.<locals>.<genexpr>  s'      22c!f222222r(   r   )iterrT  rM   )r3   s   `r&   _sparse_iterr    s4    	2222eCIaL11222	2	22r(   c                      fdS )Nc                     ||           S r.   r,   )r  br   s     r&   rk   z<lambda>.<locals>.<lambda>  s    AAaGG r(   r,   )r   s   `r&   rk   rk     s    ++++ r(   r  r  ry  __abs____neg____pos__
__matmul____rmatmul____mul____rmul____truediv____rtruediv____add____radd____sub____rsub____pow__c                :     t          fd          |           S )Nc                .    t          j        |           S r.   )r:   powerr  rJ  s    r&   rk   z<lambda>.<locals>.<lambda>  s    SYq!__ r(   r  r  s    `r&   rk   rk     s$    =(#<#<#<#<==a@@ r(   )	__rpow____getitem____iter____gt____ge____lt____le____eq____ne__c                z     t          t          j        t          j        |||||                    |           }|S )Nr  )r,  r   r  r   r  r  s          r&   _bcsr_rewriting_taker    sW    F8I%3;M	nE E E F FFIK K& 
-r(   )r  r  r  )rm   r*   rn   r   r5   r   )rm   r*   rv   r   r5   r   )r   r   r   r   r5   r   )r   r   r   r   r   r   )rn   r   )
r   r   r   r   rv   r   rm   r*   r5   r   )F)FFNN)
__future__r   collections.abcr   r   r   typingr   r   numpyrL   r  r   jax._srcr	   r
   jax._src.custom_derivativesr   r   r   r   r   jax.experimental.sparse.bcoor   r   	jax.numpyr:   jax._src.api_utilr   jax._src.libr   jax._src.interpretersr   r  jax.tree_utilr   r   r   jax.utilr   r   r   jax._src.lax.control_flowr   jax._src.numpyr   jax.experimentalr   jax.experimental.sparser   r   r   rW   r    conj_pcopy_pimag_prn  real_p(_zero_preserving_linear_unary_primitivesabs_pasin_pasinh_patan_patanh_pbessel_i1e_pexpm1_plog1p_psign_psin_psinh_psqrt_ptan_ptanh_pconvert_element_type_p!_zero_preserving_unary_primitivesacos_pacosh_pbessel_i0e_pcos_pcosh_peq_pexp_pge_pgt_ple_plt_plog_pne_pxor_pr!   r'   r4   ArrayOrSparser*   r?   rt   r{   ru   r|   r   r   r   Tracerr   Tracer   transformation_with_auxr   r   r   r  r$  r)  r,  r1  r9  _primr?  broadcast_in_dim_pbcoo_broadcast_in_dimconcatenate_pconv_general_dilated_pbcoo_conv_general_dilateddot_general_pbcoo_dot_generaldynamic_slice_p	reshape_pbcoo_reshaperev_pbcoo_revslice_p
bcoo_slice	squeeze_pbcoo_squeeze_BCOO_STANDARD_PRIMITIVESitemsr  	bcoo_implbcsr_dot_generalbcsr_broadcast_in_dim_BCSR_STANDARD_PRIMITIVES	bcsr_implrL  rK  r\  transpose_prl  add_pro  sub_prs  mul_pru  div_prz  reduce_sum_pr  gather_pr  r  r  r  r  r  r  r  r  r  r  	todense_pr  r  r  r  r  r  r  r  
_swap_argsabsnegativepositivematmulmultiplydividera  subtractr  greatergreater_equalless
less_equalequal	not_equal_bcoo_methodsmethodimplr   r  r,   r(   r&   <module>r     s   # " " " " " " D / . . . . . . .     " " " " " " " "     



                   0 0 0 0 0 0 & & & & & &       # # # # # # R R R R R R R R       2 2 2 2 2 2       4 4 4 4 4 4 @ @ @ @ @ @ @ @ @ @ 3 3 3 3 3 3 3 3 3 3 ; ; ; ; ; ; $ $ $ $ $ $ # # # # # # . . . . . . . .57  7 7 7 757  7 7 7 7 ***)*, ( )*+*+++*)**)*% !& *+)*()(((()()1     "P P P 	=W =W =W =W =W =W =W =W@< < < < <J < < <2 ;:88B B B B&C C C C&B B B B          4;   $-& -& -& -& -&$* -& -& -&^ 6 6 6	3 	3 	3 	3  4' 4' 4' 4'l  "
 
 
) ) ) )<( ( (  * / L LE66uUKKKE66uUKKKE5 K KE66uTJJJE66uTJJJE   &6DDf>V,RR-$)V_+v -$
  16688 C COD)11$	BBD V,&6DD  16688 C COD)11$	BBDT T T (; ## $': ## $  > &7 #/ "" " "H  + #) M M M  + #)   4  + #)     + #) 	 	 	 '9 #" #. . . #6 #,   :G G G  "/ #+ 'G 'G 'GR ". $+ B B B7 7 76 !- #* 	( 	( 	( !- #*   
 '; &" #&: &" #, , , ?V #(: ;>U #(: ;: : :> > >= = = MR15	 	 	 	3 3 3 ,+

GX  XXcg	
 XXcl## XXcl## $$ **SZ0011 XXcl## hhzz#,//00 #*%% ((::cj1122 XXcg hhzz#'**++ XXcl##  hhzz#,//00!" @@#$ hzz#),,--%HS[!!HS&''HSXHS^$$HSYHS]##5  : "''))  LFD	'$ MR15    $$**SZ0011%  "''))  LFD	'$ r(   