
    Vpf*                       d Z ddlmZ ddlZddlmZmZ ddlZddlZddl	Z	ddl
Z
ddlmZmZ ddlZddlmZ ddlmZ ddlmZ dd	lmZ eZeZd
 Z G d de          Z ej        dd          dd            Zi Z G d dej                   Z! e e! ej"        de#          d                    Z$ G d de
j%                  Z& e&            Z'dS )z$Definitions of Mesh and ResourceEnv.    )annotationsN)HashableSequence)Any
NamedTuple)config)
xla_bridge)util)
xla_clientc                Z    d                     t          d | D                                 S )N, c              3  "   K   | ]
}d | d V  dS )`N ).0as     M/var/www/html/nettyfy-visnx/env/lib/python3.11/site-packages/jax/_src/mesh.py	<genexpr>zshow_axes.<locals>.<genexpr>'   s*      11q(a(((111111    )joinsorted)axess    r   	show_axesr   &   s+    	611D11111	2	22r   c                      e Zd ZU ded<   ddZedd            Zedd            Zed	             Zed
             Z	d Z
dS )ResourceEnvMeshphysical_meshmeshc                    t          |j                  | j        t          | j        j                  z
  z  }|rt	          dt          |                     |                     |          S )NzfCannot update the mesh of the current resource environment. The new mesh shadows already defined axes )r   )set
axis_namesresource_axesr   
ValueErrorr   _replace)selfr   overlaps      r   	with_meshzResourceEnv.with_mesh-   sv    $/""d&83t?Q?\;];]&]^G 0 /#G,,/ / 0 0 0 ==t=,,,r   returnset[ResourceAxisName]c                4    t          | j        j                  S N)r    r   r!   r%   s    r   physical_resource_axesz"ResourceEnv.physical_resource_axes5   s    t!,---r   c                    | j         S r+   )r-   r,   s    r   r"   zResourceEnv.resource_axes9   s    &&r   c                    | j         j        S r+   )r   shaper,   s    r   r0   zResourceEnv.shape=   s    ##r   c                $    | j         j        j        S r+   )r   
local_meshr0   r,   s    r   local_shapezResourceEnv.local_shapeA   s    (..r   c                    d                     d | j        j                                        D                       }d| dS )Nr   c              3  ,   K   | ]\  }}d | d| V  dS 'z': Nr   r   kvs      r   r   z'ResourceEnv.__repr__.<locals>.<genexpr>F   sK       D D!QA!D D D D D Dr   zResourceEnv(mesh=Mesh(z)))r   r   r0   items)r%   	mesh_reprs     r   __repr__zResourceEnv.__repr__E   s[    		 D D"&"4":"@"@"B"BD D D D DI1I1111r   N)r   r   )r(   r)   )__name__
__module____qualname____annotations__r'   propertyr-   r"   r0   r3   r=   r   r   r   r   r   *   s         - - - - . . . 8. ' ' ' 8' $ $ 8$ / / 8/2 2 2 2 2r   r      F)max_sizetrace_context_in_keyglobal_meshr   process_indexintr(   c                   | j         r| S  t          j        fdt          g          | j                  }g }t          | j        j                  D ]}t          j        t          t          | j        j                            |          }|
                    |d          }t          j        |          }t          t          j        |                    t          t          j        |                    }	}|                    t!          ||	dz                        t          |          }
||
                                         st%          d          t'          | j        |
         | j                  S )Nc                    | j         k    S r+   )rG   )drG   s    r   <lambda>z!_get_local_mesh.<locals>.<lambda>P   s    =0 r   otypesF)keepdims   zWhen passing host local inputs to pjit or xmap, devices connected to a single host must form a contiguous subcube of the global device mesh)emptynp	vectorizebooldevicesrangendimr
   tuple_deletetupleanyflatnonzerorH   minmaxappendsliceallr#   r   r!   )rF   rG   is_local_devicesubcube_indicesaxis
other_axeslocal_slicesnonzero_indicesstartendsubcube_indices_tuples    `         r   _get_local_meshrj   K   sx    ABL0000$A A AALATV V//K',-- 2 2d"5{/B/G)H)H#I#I4PPJ #&&zE&BBLn\22ORVO,,--s26/3J3J/K/K3E5a00111100
 
.	/	3	3	5	5 
	   
k!"78+:P	Q	QQr   c                      e Zd ZU dZded<   ded<   d  fdZd	 Zd
 Zd Z fdZ	d Z
d Zed             Zej        d             Zed             Zed             Zej        d             Zed             Zd Zed             Zej        d             Zej        d             Zej        d             Zej        d             Zej        d             Zd Zej        d             Zd Zej        d             Z xZ S )!r   aa  Declare the hardware resources available in the scope of this manager.

  In particular, all ``axis_names`` become valid resource names inside the
  managed block and can be used e.g. in the ``in_axis_resources`` argument of
  :py:func:`jax.experimental.pjit.pjit`. Also see JAX's multi-process programming
  model (https://jax.readthedocs.io/en/latest/multi_process.html)
  and the Distributed arrays and automatic parallelization tutorial
  (https://jax.readthedocs.io/en/latest/notebooks/Distributed_arrays_and_automatic_parallelization.html)

  If you are compiling in multiple threads, make sure that the
  ``with Mesh`` context manager is inside the function that the threads will
  execute.

  Args:
    devices: A NumPy ndarray object containing JAX device objects (as
      obtained e.g. from :py:func:`jax.devices`).
    axis_names: A sequence of resource axis names to be assigned to the
      dimensions of the ``devices`` argument. Its length should match the
      rank of ``devices``.

  Examples:

    >>> from jax.experimental.pjit import pjit
    >>> from jax.sharding import Mesh
    >>> from jax.sharding import PartitionSpec as P
    >>> import numpy as np
    ...
    >>> inp = np.arange(16).reshape((8, 2))
    >>> devices = np.array(jax.devices()).reshape(4, 2)
    ...
    >>> # Declare a 2D mesh with axes `x` and `y`.
    >>> global_mesh = Mesh(devices, ('x', 'y'))
    >>> # Use the mesh object directly as a context manager.
    >>> with global_mesh:
    ...   out = pjit(lambda x: x, in_shardings=None, out_shardings=None)(inp)

    >>> # Initialize the Mesh and use the mesh as the context manager.
    >>> with Mesh(devices, ('x', 'y')) as global_mesh:
    ...   out = pjit(lambda x: x, in_shardings=None, out_shardings=None)(inp)

    >>> # Also you can use it as `with ... as ...`.
    >>> global_mesh = Mesh(devices, ('x', 'y'))
    >>> with global_mesh as m:
    ...   out = pjit(lambda x: x, in_shardings=None, out_shardings=None)(inp)

    >>> # You can also use it as `with Mesh(...)`.
    >>> with Mesh(devices, ('x', 'y')):
    ...   out = pjit(lambda x: x, in_shardings=None, out_shardings=None)(inp)
  z
np.ndarrayrU   ztuple[MeshAxisName, ...]r!    np.ndarray | Sequence[xc.Device]str | Sequence[MeshAxisName]c                `   t          |t          j                  st          j        |          }t          |t                    r|f}t          |          }|j        t          |          k    r(t          d|j         dt          |           d          ||j	        t          |j
                  f}t                              |d           }||S t                                          |           }|                                |_        d|j        j        _        ||_        |t          |<   |S )NzMesh requires the ndim of its first argument (`devices`) to equal the length of its second argument (`axis_names`), but got devices.ndim == z and len(axis_names) == .F)
isinstancerR   ndarrayarraystrrY   rW   lenr#   r0   flat_mesh_object_dictgetsuper__new__copyrU   flags	writeabler!   )clsrU   r!   keyvalr%   	__class__s         r   ry   zMesh.__new__   s    grz** "!!g*c"" !=jz""J|s:&&3$\3 3 !$J3 3 34 4 4 w}eGL&9&9
:C


T
*
*C
j77??3D<<>>DL#(DL  DO!cKr   c                <    t          |           | j        | j        ffS r+   )typerU   r!   r,   s    r   
__reduce__zMesh.__reduce__   s    JJt788r   c                    t          |t                    sdS t          |           t          |          k    rdS | j        |j        k    o)| j        j        |j        j        k    o| j        |j        k    S )NFT)rp   r   idr!   rU   r0   _internal_device_list)r%   others     r   __eq__zMesh.__eq__   sv    eT"" U 
$xx2e99TOu// FL%-"55F&%*EEGr   c                    t          | d          s+t          | j        | j        | j        j        f          | _        | j        S )N_hash)hasattrhashr!   r   rU   r0   r   r,   s    r   __hash__zMesh.__hash__   sD    4!! M?D68J
KM Mdj:r   c                    t          | |          r)t          | |          |k    rd S t          d| d          t                                          ||           d S )NzCannot reassign attributes (z) of immutable mesh objects)r   getattrRuntimeErrorrx   __setattr__)r%   namevaluer   s      r   r   zMesh.__setattr__   ss    tT 	t			%	% 	
J
J
J
J   
GGe$$$$$r   c                   t           j        d                             |           }t           j                            |           |t           _        t          j        t          d t           j        D                                  | S )Nc              3  <   K   | ]}|j         j        |j         V  d S r+   r   rQ   r   ts     r   r   z!Mesh.__enter__.<locals>.<genexpr>   J       #A #Aq*+/*?#A1? #A #A #A #A #A #Ar   mesh_context_manager)thread_resourcesstackr'   r^   env
jax_configupdate_thread_local_jit_staterY   )r%   new_envs     r   	__enter__zMesh.__enter__   s    $R(22488G!!'***"," #A #A<L<R #A #A #A A AB B B B Kr   c                    t           j                                         t           j        d         t           _        t	          j        t          d t           j        D                                  dS )Nr   c              3  <   K   | ]}|j         j        |j         V  d S r+   r   r   s     r   r   z Mesh.__exit__.<locals>.<genexpr>   r   r   r   F)r   r   popr   r   r   rY   )r%   exc_type	exc_value	tracebacks       r   __exit__zMesh.__exit__   s       +1"5," #A #A<L<R #A #A #A A AB B B B 5r   c                    t          j        d t          j        | j        | j        j                  D                       S )Nc              3  $   K   | ]\  }}||fV  d S r+   r   r   r   sizes      r   r   zMesh.shape.<locals>.<genexpr>   sF       #N #ND$ 
t#N #N #N #N #N #Nr   )collectionsOrderedDictr
   safe_zipr!   rU   r0   r,   s    r   r0   z
Mesh.shape   sN    " #N #N-9KLL#N #N #N N N Nr   c                x    t          d t          j        | j        | j        j                  D                       S )Nc              3  $   K   | ]\  }}||fV  d S r+   r   r   s      r   r   z#Mesh.shape_tuple.<locals>.<genexpr>   sF       N ND$ 
tN N N N N Nr   )rY   r
   r   r!   rU   r0   r,   s    r   shape_tuplezMesh.shape_tuple   sK     N N-9KLLN N N N N Nr   c                X    t          j        | j                                                  S r+   )mathprodr0   valuesr,   s    r   r   z	Mesh.size   s     9TZ&&(()))r   c                "    | j         j        dk    S )Nr   )rU   rW   r,   s    r   rQ   z
Mesh.empty   s    <!!r   c                F    | j         j        t          | j                  k    S r+   )rU   r   rt   local_devicesr,   s    r   is_multi_processzMesh.is_multi_process   s    <D$6 7 777r   c                N    |                      t          j                              S r+   )_local_meshxbrG   r,   s    r   r2   zMesh.local_mesh  s    B,..///r   c                "    t          | |          S r+   )rj   )r%   rG   s     r   r   zMesh._local_mesh  s    4///r   c                    dS )NTr   r,   s    r   _is_jax_device_meshzMesh._is_jax_device_mesh  s	     4r   c                j    | j         rJ  t          j        d t          g          | j                  S )Nc                    | j         S r+   )r   )rK   s    r   rL   z!Mesh.device_ids.<locals>.<lambda>  s    !$ r   rM   )rQ   rR   rS   rH   rU   r,   s    r   
device_idszMesh.device_ids  s5    z52<u555dlCCCr   c                *    t          | j                  S r+   )r    r   r,   s    r   _local_devices_setzMesh._local_devices_set  s    t!"""r   c                4    t          | j        j                  S r+   )rY   rU   ru   r,   s    r   _flat_devices_tuplezMesh._flat_devices_tuple  s    "###r   c                4    t          j        | j                  S r+   )xc
DeviceListr   r,   s    r   r   zMesh._internal_device_list  s    =1222r   c                4    t          | j        j                  S r+   )r    rU   ru   r,   s    r   _flat_devices_setzMesh._flat_devices_set!  s    t| !!!r   c                z    d                     d | j                                        D                       }d| dS )Nr   c              3  ,   K   | ]\  }}d | d| V  dS r6   r   r8   s      r   r   zMesh.__str__.<locals>.<genexpr>&  s2      FF41aQ1FFFFFFr   zMesh())r   r0   r;   )r%   mesh_strs     r   __str__zMesh.__str__%  sA    yyFF4:3C3C3E3EFFFFFH8r   c                :    | j         rdS d| j        d| j        dS )Nz"Mesh(device_ids=[], axis_names=())zMesh(device_ids=z, axis_names=r   )rQ   r   r!   r,   s    r   _reprz
Mesh._repr)  s/    z 211RdoRRdoRRRRr   c                    | j         S r+   )r   r,   s    r   r=   zMesh.__repr__/  s
    :r   c                .    d | j         j        D             S )Nc                V    g | ]&}|j         |j                                         k    $|'S r   )rG   client)r   rK   s     r   
<listcomp>z&Mesh.local_devices.<locals>.<listcomp>4  s<     < < <!!("8"8":"::: :::r   )rU   ru   r,   s    r   r   zMesh.local_devices2  s'    < <t|( < < < <r   )rU   rl   r!   rm   )!r>   r?   r@   __doc__rA   ry   r   r   r   r   r   r   rB   r0   	functoolscached_propertyr   r   rQ   r   r2   r   r   r   r   r   r   r   r   r   r=   r   __classcell__)r   s   @r   r   r   k   s        0 0d &&&&     69 9 9	G 	G 	G  	% 	% 	% 	% 	%     N N 8N
 N N N
 * * 8* " " 8" 8 8 8 0 0 800 0 0   8 D D D # # # $ $ $ 3 3 3 " " "   S S S
   < < < < < < <r   r   )dtypec                      e Zd Zd ZdS )_ThreadResourcesLocalStatec                D    t           g| _        | j        d         | _        d S )Nr   )	EMPTY_ENVr   r   r,   s    r   __init__z#_ThreadResourcesLocalState.__init__<  s    DJz"~DHHHr   N)r>   r?   r@   r   r   r   r   r   r   :  s#            r   r   )rF   r   rG   rH   r(   r   )(r   
__future__r   r   collections.abcr   r   
contextlibr   r   	threadingtypingr   r   numpyrR   jax._srcr   r   r	   r   r
   jax._src.libr   r   MeshAxisNameResourceAxisNamer   r   cacherj   rv   ContextDecoratorr   rQ   objectr   localr   r   r   r   r   <module>r      s   + * " " " " " "     . . . . . . . .              " " " " " " " "     ) ) ) ) ) ) % % % % % %       ) ) ) ) ) )  3 3 32 2 2 2 2* 2 2 2B Su555R R R 65R8  J< J< J< J< J<:& J< J< J<Z KXRXb777<<==	        .-//   r   