Vpf ndZddlmZddlZddlmZmZmZmZm Z ddl m Z m Z ddl Z ddlZddlZddlmZmZmZmZmZmZddlZddlZddlmZmZddlmZdd lmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.dd lm/Z/dd lm0Z0dd lm1Z1ddlm2Z2ddlm3Z3ddlm4Z4ddlm5Z5ddlm6Z6ddlm7Z7ddlm8Z8ddlm9Z9ddlm:Z:ddlm;Z;ddlmm?Z?m@Z@mAZAddlBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTddlUmVZWddlXmYZYddlXmZZ[ddlXm\Z\ddl]m^Z^dd l_m`Z`maZadd!lbmcZcmdZddd"lemfZfdd#lmgZgdd$lhmiZimjZjmkZkmlZlmmZmmnZndd%lmoZodd&lpmqZqdd'lpmrZrdd(lpmsZsdd)lpmtZudd*lpmvZvdd+lpmwZwe:jxeye e6jzd,-Z{eZ|e[j}Z}ed.e/Z~ed0Zed1ZejecZZekecZZd2Zd3Zd4Ze0jeee0jeee6jZe7je7jddddd5ddd5df ddIZedddKZ dddTZ ddd\Z ddd^Zd_Zd`Ze edaZdbZe edaZ dddcZddgZdhZ dddiZeZe edjZe edjZ dddkZdlZdmZdnZdoZdpZdqZ ddd{Zd|Z ddddLddddLdd}ddZGddeZddZdZdZGddeZ ddddLddddLdddZejZ dddZdddZed5dddZeddZd5dddZdZdZddZed5dLdddZedLdddZd5dLdddZd5dddZdLdddZddZe dddZe dddZ dddZddZe ddZ dddddZddÄZddĄZdńZddƄZGdDŽdȦZdɄe1je<efddʄZdd˜dd̈́ZeddτZdЄZdфZd҄ZddӄZdԄZdS(a\JAX user-facing transformations and utilities. The transformations here mostly wrap internal transformations, providing convenience flags to control behavior and handling Python containers of arguments and outputs. The Python containers handled are pytrees (see tree_util.py), which include nested tuples/lists/dicts, where the leaves are arrays. ) annotationsN)Callable GeneratorHashableIterableSequence)partial lru_cache)AnyLiteral NamedTupleTypeVaroverloadcast)contextmanager ExitStack) linear_util)stages)tree_map tree_flattentree_unflattentree_structuretree_transpose tree_leavesPartial PyTreeDef all_leaveskeystrbroadcast_prefix prefix_errorsgenerate_key_pathstree_flatten_with_path)api_util)config)core)dispatch)effects)array) basearray)dtypes)sharding_impls)sharding_specs)source_info_util)traceback_util)pjit) xla_bridge) eval_jaxpr ShapedArray ConcreteArray) flatten_funflatten_fun_nokwargsflatten_fun_nokwargs2argnums_partialargnums_partial_except flatten_axesdonation_vectorrebase_donate_argnums _ensure_index_ensure_index_tupleshaped_abstractifyapply_flat_fun_nokwargscheck_callable debug_info result_paths flat_out_axesdebug_info_finalfun_sourceinfo)lax)jax_jit) xla_client)pmap_lib)Sharding) PmapShardingTransferToMemoryKind)Layout AutoLayout) api_boundary) tree_util)unzip2safe_mapsafe_zip wrap_namewraps split_list)util)ad)batching)mlir) partial_eval)pxla)xlaT) canonicalizeF)boundTUcg}t|D]6}t|dr$|d|jD7 t jt jj|dS#t$rJtj j stj j sJtd|j|i|dYdSwxYw)zAHook function called by the C++ jit/pmap to perform NaN checking.addressable_shardscg|] }|j S)data).0shards L/var/www/html/nettyfy-visnx/env/lib/python3.11/site-packages/jax/_src/api.py z'_nan_check_posthook..msFFFUejFFFziInvalid nan value encountered in the output of a C++-jit/pmap function. Calling the de-optimized version.rN)rhasattrextendrdr& check_specialr/pjit_pnameFloatingPointErrorr$ debug_nansvalue debug_infsprint _cache_miss)funargskwargsoutputbuffersleafs rj_nan_check_posthookr~hs '&!!HHdt)**H nnFFd.EFFFGGG( 4;+W55555 (((   "=f&7&====  8999COT$V$$Q''''' (s $A00ACCctjdstjdrttj_dSdtj_dS)Njax_debug_nansjax_debug_infs)r$_readr~rG global_state post_hook_s rj_update_debug_special_globalrxsU \"##,v|4D'E'E,':G$$$'+G$$$rlcttjddsttjddrtt j_dSdt j_dS)NrFr)getattrr$_thread_local_stater~rGthread_local_staterrs rj"_update_debug_special_thread_localr~s` f(*:EBB2 f(*:EBB2-@G  ***-1G  ***rlFrxrstatic_argnumsint | Sequence[int] | Nonestatic_argnamesstr | Iterable[str] | Nonedonate_argnumsdonate_argnames keep_unusedbooldevicexc.Device | Nonebackend str | Noneinlineabstracted_axes Any | Nonereturnpjit.JitWrappedc Dtj||||||||| | || d S)aSets up ``fun`` for just-in-time compilation with XLA. Args: fun: Function to be jitted. ``fun`` should be a pure function. The arguments and return value of ``fun`` should be arrays, scalar, or (nested) standard Python containers (tuple/list/dict) thereof. Positional arguments indicated by ``static_argnums`` can be any hashable type. Static arguments are included as part of a compilation cache key, which is why hash and equality operators must be defined. JAX keeps a weak reference to ``fun`` for use as a compilation cache key, so the object ``fun`` must be weakly-referenceable. in_shardings: optional, a :py:class:`Sharding` or pytree with :py:class:`Sharding` leaves and structure that is a tree prefix of the positional arguments tuple to ``fun``. If provided, the positional arguments passed to ``fun`` must have shardings that are compatible with ``in_shardings`` or an error is raised, and the compiled computation has input shardings corresponding to ``in_shardings``. If not provided, the compiled computation's input shardings are inferred from argument shardings. out_shardings: optional, a :py:class:`Sharding` or pytree with :py:class:`Sharding` leaves and structure that is a tree prefix of the output of ``fun``. If provided, it has the same effect as applying corresponding :py:func:`jax.lax.with_sharding_constraint`s to the output of ``fun``. static_argnums: optional, an int or collection of ints that specify which positional arguments to treat as static (trace- and compile-time constant). Static arguments should be hashable, meaning both ``__hash__`` and ``__eq__`` are implemented, and immutable. Otherwise they can be arbitrary Python objects. Calling the jitted function with different values for these constants will trigger recompilation. Arguments that are not array-like or containers thereof must be marked as static. If neither ``static_argnums`` nor ``static_argnames`` is provided, no arguments are treated as static. If ``static_argnums`` is not provided but ``static_argnames`` is, or vice versa, JAX uses :code:`inspect.signature(fun)` to find any positional arguments that correspond to ``static_argnames`` (or vice versa). If both ``static_argnums`` and ``static_argnames`` are provided, ``inspect.signature`` is not used, and only actual parameters listed in either ``static_argnums`` or ``static_argnames`` will be treated as static. static_argnames: optional, a string or collection of strings specifying which named arguments to treat as static (compile-time constant). See the comment on ``static_argnums`` for details. If not provided but ``static_argnums`` is set, the default is based on calling ``inspect.signature(fun)`` to find corresponding named arguments. donate_argnums: optional, collection of integers to specify which positional argument buffers can be overwritten by the computation and marked deleted in the caller. It is safe to donate argument buffers if you no longer need them once the computation has started. In some cases XLA can make use of donated buffers to reduce the amount of memory needed to perform a computation, for example recycling one of your input buffers to store a result. You should not reuse buffers that you donate to a computation; JAX will raise an error if you try to. By default, no argument buffers are donated. If neither ``donate_argnums`` nor ``donate_argnames`` is provided, no arguments are donated. If ``donate_argnums`` is not provided but ``donate_argnames`` is, or vice versa, JAX uses :code:`inspect.signature(fun)` to find any positional arguments that correspond to ``donate_argnames`` (or vice versa). If both ``donate_argnums`` and ``donate_argnames`` are provided, ``inspect.signature`` is not used, and only actual parameters listed in either ``donate_argnums`` or ``donate_argnames`` will be donated. For more details on buffer donation see the `FAQ `_. donate_argnames: optional, a string or collection of strings specifying which named arguments are donated to the computation. See the comment on ``donate_argnums`` for details. If not provided but ``donate_argnums`` is set, the default is based on calling ``inspect.signature(fun)`` to find corresponding named arguments. keep_unused: optional boolean. If `False` (the default), arguments that JAX determines to be unused by `fun` *may* be dropped from resulting compiled XLA executables. Such arguments will not be transferred to the device nor provided to the underlying executable. If `True`, unused arguments will not be pruned. device: This is an experimental feature and the API is likely to change. Optional, the Device the jitted function will run on. (Available devices can be retrieved via :py:func:`jax.devices`.) The default is inherited from XLA's DeviceAssignment logic and is usually to use ``jax.devices()[0]``. backend: This is an experimental feature and the API is likely to change. Optional, a string representing the XLA backend: ``'cpu'``, ``'gpu'``, or ``'tpu'``. inline: Optional boolean. Specify whether this function should be inlined into enclosing jaxprs. Default False. Returns: A wrapped version of ``fun``, set up for just-in-time compilation. Examples: In the following example, ``selu`` can be compiled into a single fused kernel by XLA: >>> import jax >>> >>> @jax.jit ... def selu(x, alpha=1.67, lmbda=1.05): ... return lmbda * jax.numpy.where(x > 0, x, alpha * jax.numpy.exp(x) - alpha) >>> >>> key = jax.random.key(0) >>> x = jax.random.normal(key, (10,)) >>> print(selu(x)) # doctest: +SKIP [-0.54485 0.27744 -0.29255 -0.91421 -0.62452 -0.24748 -0.85743 -0.78232 0.76827 0.59566 ] To pass arguments such as ``static_argnames`` when decorating a function, a common pattern is to use :func:`functools.partial`: >>> from functools import partial >>> >>> @partial(jax.jit, static_argnames=['n']) ... def g(x, n): ... for i in range(n): ... x = x ** 2 ... return x >>> >>> g(jnp.arange(4), 3) Array([ 0, 1, 256, 6561], dtype=int32) F)use_resource_env)r/make_jit) rx in_shardings out_shardingsrrrrrrrrrs rjjitrs;V  \=.//Ve 5 5 55rldisablec#lKtj|5dVddddS#1swxYwYdS)aContext manager that disables :py:func:`jit` behavior under its dynamic context. For debugging it is useful to have a mechanism that disables :py:func:`jit` everywhere in a dynamic context. Note that this not only disables explicit uses of :func:`jit` by the user, but will also remove any implicit JIT compilation used by the JAX library: this includes implicit JIT computation of `body` and `cond` functions passed to higher-level primitives like :func:`~jax.lax.scan` and :func:`~jax.lax.while_loop`, JIT used in implementations of :mod:`jax.numpy` functions, and any other case where :func:`jit` is used within an API's implementation. Note however that even under `disable_jit`, individual primitive operations will still be compiled by XLA as in normal eager op-by-op execution. Values that have a data dependence on the arguments to a jitted function are traced and abstracted. For example, an abstract value may be a :py:class:`ShapedArray` instance, representing the set of all possible arrays with a given shape and dtype, but not representing one concrete array with specific values. You might notice those if you use a benign side-effecting operation in a jitted function, like a print: >>> import jax >>> >>> @jax.jit ... def f(x): ... y = x * 2 ... print("Value of y is", y) ... return y + 3 ... >>> print(f(jax.numpy.array([1, 2, 3]))) # doctest:+ELLIPSIS Value of y is Tracedwith [5 7 9] Here ``y`` has been abstracted by :py:func:`jit` to a :py:class:`ShapedArray`, which represents an array with a fixed shape and type but an arbitrary value. The value of ``y`` is also traced. If we want to see a concrete value while debugging, and avoid the tracer too, we can use the :py:func:`disable_jit` context manager: >>> import jax >>> >>> with jax.disable_jit(): ... print(f(jax.numpy.array([1, 2, 3]))) ... Value of y is [2 4 6] [5 7 9] N)r$ disable_jit)rs rjrrs^'""   EEE                  s )--rfint | Iterable[int]axis_env%Sequence[tuple[AxisName, int]] | None tuple_argsinstantiate_const_outputs bool | None return_shapec  |td|td|tdttt  t  t dd nt jj fd tt f d} | S) aCreates a function that produces its XLA computation given example args. .. warning:: This function is deprecated as of JAX v0.4.30, and will be removed in a future JAX release. You can replace it with :ref:`ahead-of-time-lowering` APIs; for example, ``jax.xla_computation(fn)(*args)`` can be replaced with ``jax.jit(fn).lower(*args).compiler_ir('hlo')``. See the `JAX 0.4.30 Change log`_ for more examples. Args: fun: Function from which to form XLA computations. static_argnums: See the :py:func:`jax.jit` docstring. axis_env: Optional, a sequence of pairs where the first element is an axis name and the second element is a positive integer representing the size of the mapped axis with that name. This parameter is useful when lowering functions that involve parallel communication collectives, and it specifies the axis name/size environment that would be set up by applications of :py:func:`jax.pmap`. See the examples below. in_parts: Optional, how each argument to ``fun`` should be partitioned or replicated. This is used to specify partitioned XLA computations, see ``sharded_jit`` for more info. out_parts: Optional, how each output of ``fun`` should be partitioned or replicated. This is used to specify partitioned XLA computations, see ``sharded_jit`` for more info. backend: This is an experimental feature and the API is likely to change. Optional, a string representing the XLA backend: ``'cpu'``, ``'gpu'``, or ``'tpu'``. tuple_args: Optional bool, defaults to ``False``. If ``True``, the resulting XLA computation will have a single tuple argument that is unpacked into the specified function arguments. If `None`, tupling will be enabled when there are more than 100 arguments, since some platforms have limits on argument arity. instantiate_const_outputs: Deprecated argument, does nothing. return_shape: Optional boolean, defaults to ``False``. If ``True``, the wrapped function returns a pair where the first element is the XLA computation and the second element is a pytree with the same structure as the output of ``fun`` and where the leaves are objects with ``shape`` and ``dtype`` attributes representing the corresponding types of the output leaves. donate_argnums: Specify which arguments are "donated" to the computation. It is safe to donate arguments if you no longer need them once the computation has finished. In some cases XLA can make use of donated buffers to reduce the amount of memory needed to perform a computation, for example recycling one of your input buffers to store a result. You should not reuse buffers that you donate to a computation, JAX will raise an error if you try to. Returns: A wrapped version of ``fun`` that when applied to example arguments returns a built XLA Computation (see xla_client.py), from which representations of the unoptimized XLA HLO computation can be extracted using methods like ``as_hlo_text``, ``as_serialized_hlo_module_proto``, and ``as_hlo_dot_graph``. If the argument ``return_shape`` is ``True``, then the wrapped function returns a pair where the first element is the XLA Computation and the second element is a pytree representing the structure, shapes, dtypes, and named shapes of the output of ``fun``. Concrete example arguments are not always necessary. For those arguments not indicated by ``static_argnums``, any object with ``shape`` and ``dtype`` attributes is acceptable (excepting namedtuples, which are treated as Python containers). For example: >>> import jax >>> >>> def f(x): return jax.numpy.sin(jax.numpy.cos(x)) >>> c = jax.xla_computation(f)(3.) # doctest: +SKIP >>> print(c.as_hlo_text()) # doctest: +SKIP HloModule xla_computation_f.6 ENTRY xla_computation_f.6 { constant.2 = pred[] constant(false) parameter.1 = f32[] parameter(0) cosine.3 = f32[] cosine(parameter.1) sine.4 = f32[] sine(cosine.3) ROOT tuple.5 = (f32[]) tuple(sine.4) } Alternatively, the assignment to ``c`` above could be written: >>> import types >>> scalar = types.SimpleNamespace(shape=(), dtype=np.dtype(np.float32)) >>> c = jax.xla_computation(f)(scalar) # doctest: +SKIP Here's an example that involves a parallel collective and axis name: >>> def f(x): return x - jax.lax.psum(x, 'i') >>> c = jax.xla_computation(f, axis_env=[('i', 4)])(2) # doctest: +SKIP >>> print(c.as_hlo_text()) # doctest: +SKIP HloModule jaxpr_computation.9 primitive_computation.3 { parameter.4 = s32[] parameter(0) parameter.5 = s32[] parameter(1) ROOT add.6 = s32[] add(parameter.4, parameter.5) } ENTRY jaxpr_computation.9 { tuple.1 = () tuple() parameter.2 = s32[] parameter(0) all-reduce.7 = s32[] all-reduce(parameter.2), replica_groups={{0,1,2,3}}, to_apply=primitive_computation.3 ROOT subtract.8 = s32[] subtract(parameter.2, all-reduce.7) } Notice the ``replica_groups`` that were generated. Here's an example that generates more interesting ``replica_groups``: >>> from jax import lax >>> def g(x): ... rowsum = lax.psum(x, 'i') ... colsum = lax.psum(x, 'j') ... allsum = lax.psum(x, ('i', 'j')) ... return rowsum, colsum, allsum ... >>> axis_env = [('i', 4), ('j', 2)] >>> c = jax.xla_computation(g, axis_env=axis_env)(5.) # doctest: +SKIP >>> print(c.as_hlo_text()) # doctest: +SKIP HloModule jaxpr_computation__1.19 [removed uninteresting text here] ENTRY jaxpr_computation__1.19 { tuple.1 = () tuple() parameter.2 = f32[] parameter(0) all-reduce.7 = f32[] all-reduce(parameter.2), replica_groups={{0,2,4,6},{1,3,5,7}}, to_apply=primitive_computation__1.3 all-reduce.12 = f32[] all-reduce(parameter.2), replica_groups={{0,1},{2,3},{4,5},{6,7}}, to_apply=primitive_computation__1.8 all-reduce.17 = f32[] all-reduce(parameter.2), replica_groups={{0,1,2,3,4,5,6,7}}, to_apply=primitive_computation__1.13 ROOT tuple.18 = (f32[], f32[], f32[]) tuple(all-reduce.7, all-reduce.12, all-reduce.17) } .. _JAX 0.4.30 Change log: https://jax.readthedocs.io/en/latest/changelog.html#jax-0-4-30-june-18-2024 Nzinstantiate_const_outputs has been deprecated. Please use the ahead of time APIs. You can read more here: https://jax.readthedocs.io/en/latest/aot.htmlzin_parts has been deprecated. Please use the ahead of time APIs. You can read more here: https://jax.readthedocs.io/en/latest/aot.htmlzout_parts has been deprecated. Please use the ahead of time APIs. You can read more here: https://jax.readthedocs.io/en/latest/aot.html__name__unknownctj|ddS|tjdDz}t \}}tj|||S)Nrfc3 K|] \}}|V dSNrf)rhrqsizes rj z9xla_computation..make_axis_env..s&@@t@@@@@@rl)r+AxisEnvmathprodrQ)nrepsnamessizesrs rj make_axis_envz&xla_computation..make_axis_envsg  #E2r 2 22di@@x@@@@@@eH%%leU  #E5% 8 88rlc t zdt|kr&td ddt|dtj}t | |d\}}t ||f\}}rtd |}nd t|z}t||\}}tt|} t5} pgD].\} } | tj| | d/tj|| \} }}\t#j| } rtj| d D} t#j| }t+t,j| j}t3jd tj| ||gt9j|t=jtAd |ddt3j! }t3j"|j#}tHj%j&|!d}dddn #1swxYwYd|D}d|D}tO||}|D]&}tQ|tRstUd'r||fS|S)N)defaultz#jitted function has static_argnums=z, donate_argnums= but was called with only  positional arguments.F) allow_invalidrfFch|]\}}|Srfrf)rh axis_namers rj z=xla_computation..computation_maker..s;;;,)QI;;;rlxla_computation_xla_computation) ordered_effectsbackend_or_name platforms axis_context name_stack donated_args arg_shardingsresult_shardingslowering_parametersT)use_tuple_args return_tuplecBg|]}t|j|jSrfShapeDtypeStructshapedtyperhas rjrkz>xla_computation..computation_maker..0'MMMa'99MMMrlcBg|]}t|j|jSrfrrs rjrkz>xla_computation..computation_maker..1rrlzgAs we want to propagate the weak_type, we need to get a ShapedArray, otherwise this information is lost)+maxlen ValueErrorlu wrap_initr8rr:r4mapr>r enter_contextr%extend_axis_envpetrace_to_jaxpr_dynamicr&apply_outfeed_rewriterremove_named_axis_effectsjaxpr_replicaslistr'r filter_inrZlower_jaxpr_to_module ClosedJaxprr+ReplicaAxisContextr-new_name_stackrTLoweringParametersmodule_to_bytecodemodulexc_xlamlir_module_to_xla_computationr isinstancer2 RuntimeError)"ryrzfdyn_args args_flatin_treedonated_invars jaxtree_funout_treeavalsstackrrjaxpr out_avalsconsts axis_env_rlowering_resultmbuiltout_shapes_flat out_shapeout_avalrrrrxfun_namerplatformrrrs" rjcomputation_makerz*xla_computation..computation_makers >N *B7773t99DD QnQQQQ/24yyQQQ R RR SA(NDPUVVVKAx%x&899Iw1&~r7CCnn#i..0n'733K "I . .E ;%^II/)T D0D$GGHHHH%'%>{E%R%R"eY-e44e  . ;;(;;;   - 7 > >??i  ! + +EM : :<>JJ) rr@r=r;rxb get_backendr rUrO)rxrrin_parts out_partsrrrrrr r rr s``` `` `` @@@rjrrRsYb*  9 : ::  M N NN  M N NN&~66.&~66.(HH. S*i 0 0(+WW1A1A1J(99999 ::8888888888888<:8t rlargnumsint | Sequence[int]has_aux holomorphic allow_int reduce_axesSequence[AxisName]c |rtd~t||||| d}t|||t fd}t|||t fd}|r|n|S)aCreates a function that evaluates the gradient of ``fun``. Args: fun: Function to be differentiated. Its arguments at positions specified by ``argnums`` should be arrays, scalars, or standard Python containers. Argument arrays in the positions specified by ``argnums`` must be of inexact (i.e., floating-point or complex) type. It should return a scalar (which includes arrays with shape ``()`` but not arrays with shape ``(1,)`` etc.) argnums: Optional, integer or sequence of integers. Specifies which positional argument(s) to differentiate with respect to (default 0). has_aux: Optional, bool. Indicates whether ``fun`` returns a pair where the first element is considered the output of the mathematical function to be differentiated and the second element is auxiliary data. Default False. holomorphic: Optional, bool. Indicates whether ``fun`` is promised to be holomorphic. If True, inputs and outputs must be complex. Default False. allow_int: Optional, bool. Whether to allow differentiating with respect to integer valued inputs. The gradient of an integer input will have a trivial vector-space dtype (float0). Default False. Returns: A function with the same arguments as ``fun``, that evaluates the gradient of ``fun``. If ``argnums`` is an integer then the gradient has the same shape and type as the positional argument indicated by that integer. If argnums is a tuple of integers, the gradient is a tuple of values with the same shapes and types as the corresponding arguments. If ``has_aux`` is True then a pair of (gradient, auxiliary_data) is returned. For example: >>> import jax >>> >>> grad_tanh = jax.grad(jax.numpy.tanh) >>> print(grad_tanh(0.2)) 0.961043 *reduce_axes argument to grad is deprecated)rrrzGradient of {fun} with respect to positional argument(s) {argnums}. Takes the same arguments as {fun} but returns the gradient, which has the same shape as the arguments at positions {argnums}.docstrrc|i|\}}|Srrf)ryrzrgvalue_and_grad_fs rjgrad_fzgrad..grad_fts#  T ,V , ,DAq Hrlc(|i|\\}}}||fSrrf)ryrzrauxrrs rj grad_f_auxzgrad..grad_f_auxzs+#"D3F33KHQa c6Mrl)NotImplementedErrorvalue_and_gradrUrO) rxrrrrrrr r#rs @rjgradr&@sPL J K KK#C'0;.7999 #&  VW---    <.-  VW---<.- *F*rlCallable[..., tuple[Any, Any]]c|rtd~d}ttjtt |t fd}|S)a{Create a function that evaluates both ``fun`` and the gradient of ``fun``. Args: fun: Function to be differentiated. Its arguments at positions specified by ``argnums`` should be arrays, scalars, or standard Python containers. It should return a scalar (which includes arrays with shape ``()`` but not arrays with shape ``(1,)`` etc.) argnums: Optional, integer or sequence of integers. Specifies which positional argument(s) to differentiate with respect to (default 0). has_aux: Optional, bool. Indicates whether ``fun`` returns a pair where the first element is considered the output of the mathematical function to be differentiated and the second element is auxiliary data. Default False. holomorphic: Optional, bool. Indicates whether ``fun`` is promised to be holomorphic. If True, inputs and outputs must be complex. Default False. allow_int: Optional, bool. Whether to allow differentiating with respect to integer valued inputs. The gradient of an integer input will have a trivial vector-space dtype (float0). Default False. Returns: A function with the same arguments as ``fun`` that evaluates both ``fun`` and the gradient of ``fun`` and returns them as a pair (a two-element tuple). If ``argnums`` is an integer then the gradient has the same shape and type as the positional argument indicated by that integer. If argnums is a sequence of integers, the gradient is a tuple of values with the same shapes and types as the corresponding arguments. If ``has_aux`` is True then a tuple of ((value, auxiliary_data), gradient) is returned. ra'Value and gradient of {fun} with respect to positional argument(s) {argnums}. Takes the same arguments as {fun} but returns a two-element tuple where the first element is the value of {fun} and the second element is the gradient, which has the same shape as the arguments at positions {argnums}.rc t tr nt }|t|kr)t d d|dzdt|dt j |}t| |d\}}t|D]}t |st|g|R\}}nt|g|Rdd i\}}} t|ttt||tj|} t tr| d n| } s|| fS|| f| fS) Nz(differentiating with respect to argnums=z requires at least z? positional arguments to be passed by the caller, but got only rFrequire_static_args_hashablerTr)rintrr TypeErrorrrr7r_check_input_dtype_grad_vjp _check_scalarrr _check_output_dtype_grad lax_internal_one)ryrz max_argnumr f_partialrr}ansvjp_pyr"rrrrxrrs rjrz(value_and_grad..value_and_grad_fs'w44F#g,,JSYY HHH#aHH&)$iiHHH I II S&!!A)!WdGLNNNIxH%%<<k9d;;;; ..X...kc66 ....(,..c63# W-{ ; ;SAAA| %%&&A7C((/!aA  !Vm3Z]rl)r$r@r%concrete_or_errorr<rUrO)rxrrrrrrrs````` rjr%r%s>L J K KK B&   "=' : :'VW---<.-4 rlcPdj} tj|}t|tr-|jdkr t |d|jdSt |d|#t $r!}t |d||d}~wwxYw)Nz=Gradient only defined for scalar-output functions. Output {}.rfz had shape: zhad abstract value zwas )formatr%get_avalrr2rr.)xmsgavales rjr1r1sGN# 9 =  D$ $$9 r  6$*6677888   cc66677 8 88 ,,, CC q OO $ $!+,sA:: B%B  B%ctj|tj|}|rCt j|jtjst|d|jj dt j|jtj sHt j|jtj s$t j|jtj r#|st|d|jj ddSt j|jtjst|d|jj ddS)NzC with holomorphic=True requires inputs with complex dtype, but got .zb requires real- or complex-valued inputs (input dtype that is a sub-dtype of np.inexact), but got zY. If you want to use Boolean- or integer-valued inputs, use vjp or set allow_int to True.zf requires numerical-valued inputs (input dtype that is a sub-dtype of np.bool_ or np.number), but got )r& check_argr%r<r* issubdtypernpcomplexfloatingr.rqextendedintegerbool_inexact)rqrrr=r?s rj_check_input_dtype_revderivrKsr Q q  $5  TZ); < <5 44!%444 5 55  FO44 X  BJ// X  BH-- X 3 22EIZ_222 3 3333  TZ 4 4X tWWDHJOWWW X XXXXrlr&cLtj|}tj|jtjrt |d|jj|rEtj|jtj st |d|jjddStj|jtj rt |d|jjdtj|jtj st |d|jjddS)Nz with output element type zD with holomorphic=True requires outputs with complex dtype, but got rBzY requires real-valued outputs (output dtype that is a sub-dtype of np.floating), but got z. For holomorphic differentiation, pass holomorphic=True. For differentiation of non-holomorphic functions involving complex outputs, use jax.vjp directly.zN. For differentiation of functions with integer outputs, use jax.vjp directly.) r%r<r*rDrrGr.rqrErFfloating)rqrr=r?s rj_check_output_dtype_revderivrNsZ q  $ tz6?33>  <<4:?<< > >>)  TZ); < <5 44!%444 5 5555R%788 ) t55<@JO555 6 66  TZ 5 5) t((<@JO((( ) ))))rlcttd}t|fd}|S)aJacobian of ``fun`` evaluated column-by-column using forward-mode AD. Args: fun: Function whose Jacobian is to be computed. argnums: Optional, integer or sequence of integers. Specifies which positional argument(s) to differentiate with respect to (default ``0``). has_aux: Optional, bool. Indicates whether ``fun`` returns a pair where the first element is considered the output of the mathematical function to be differentiated and the second element is auxiliary data. Default False. holomorphic: Optional, bool. Indicates whether ``fun`` is promised to be holomorphic. Default False. Returns: A function with the same arguments as ``fun``, that evaluates the Jacobian of ``fun`` using forward-mode automatic differentiation. If ``has_aux`` is True then a pair of (jacobian, auxiliary_data) is returned. >>> import jax >>> import jax.numpy as jnp >>> >>> def f(x): ... return jnp.asarray( ... [x[0], 5*x[2], 4*x[1]**2 - 2*x[2], x[2] * jnp.sin(x[0])]) ... >>> print(jax.jacfwd(f)(jnp.array([1., 2., 3.]))) [[ 1. 0. 0. ] [ 0. 0. 5. ] [ 0. 16. -2. ] [ 1.6209 0. 0.84147]] Jacobian of {fun} with respect to positional argument(s) {argnums}. Takes the same arguments as {fun} but returns the jacobian of the output with respect to the arguments at positions {argnums}.rctj |}t| |d\}}tt t | sAt t ||}t|dt|\}}nCt t ||d}t|dt|\}}}tt t|t tr|dn|} tt t| ||} s| S| |fS) NFr+Nr)out_axesTr)NrNr) rrr7rr _check_input_dtype_jacfwd_jvpvmap _std_basis_check_output_dtype_jacfwdrr-_jacfwd_unravel)ryrzrr6rpushfwdyjacr" example_argsjac_treerrxrrs rjjacfunzjacfwd..jacfun,sB S&!!A)!WdGLNNNIx W. < >3GGH  o s]rl)r@r<rU)rxrrrrr`s```` rjjacfwdrasv@ ' " "' #&  VW---.-& -rlr=r Nonectj|tj|}t j|jtjrtd|jj |rCt j|jtj std|jj ddSt j|jtj std|jj ddS)Nzjacfwd with input element type zIjacfwd with holomorphic=True requires inputs with complex dtype, but got rBz]jacfwd requires real-valued inputs (input dtype that is a sub-dtype of np.floating), but got z. For holomorphic differentiation, pass holomorphic=True. For differentiation of non-holomorphic functions involving complex inputs or integer inputs, use jax.jvp directly.) r&rCr%r<r*rDrrGr.rqrErFrMrr=r?s rjrUrUBs Q q  $ tz6?33= ;$*/;; = == O  TZ); < << ;(, ;;; < <<<<  TZ 5 5O N<@JONNN O OOOOrlctj|}|rAtj|jt jstd|jjddSdS)NzJjacfwd with holomorphic=True requires outputs with complex dtype, but got rB) r%r<r*rDrrErFr.rqrds rjrYrYSst q  $5  TZ); < <5 4!%444 5 555555rlcttd}t|fd}|S)aJacobian of ``fun`` evaluated row-by-row using reverse-mode AD. Args: fun: Function whose Jacobian is to be computed. argnums: Optional, integer or sequence of integers. Specifies which positional argument(s) to differentiate with respect to (default ``0``). has_aux: Optional, bool. Indicates whether ``fun`` returns a pair where the first element is considered the output of the mathematical function to be differentiated and the second element is auxiliary data. Default False. holomorphic: Optional, bool. Indicates whether ``fun`` is promised to be holomorphic. Default False. allow_int: Optional, bool. Whether to allow differentiating with respect to integer valued inputs. The gradient of an integer input will have a trivial vector-space dtype (float0). Default False. Returns: A function with the same arguments as ``fun``, that evaluates the Jacobian of ``fun`` using reverse-mode automatic differentiation. If ``has_aux`` is True then a pair of (jacobian, auxiliary_data) is returned. >>> import jax >>> import jax.numpy as jnp >>> >>> def f(x): ... return jnp.asarray( ... [x[0], 5*x[2], 4*x[1]**2 - 2*x[2], x[2] * jnp.sin(x[0])]) ... >>> print(jax.jacrev(f)(jnp.array([1., 2., 3.]))) [[ 1. 0. 0. ] [ 0. 0. 5. ] [ 0. 16. -2. ] [ 1.6209 0. 0.84147]] rPrctj |}t| |d\}}tt t  |st |g|R\}}nt |g|Rddi\}}}tt t|t|t|}t tr|dn|}t tr|dn|} tt t|| |} tt| t|| } s| S| |fS)NFr+rTr)rrr7rr _check_input_dtype_jacrevr0_check_output_dtype_jacrevrWrXrr-_jacrev_unravelrr)ryrzrr6rr\pullbackr"r]r^r_rrrxrrs rjr`zjacrev..jacfuns_ S&!!A)!WdGLNNNIx W. Y G GRRR B.X...kaiA(AAADAAa3 W/ = =qAAA $x..A ' 'Cw,, 5#a&&#C",Wc":":H8A;;L33\3GGHn\::N1>> import jax >>> >>> g = lambda x: x[0]**3 - 2*x[0]*x[1] - x[1]**6 >>> print(jax.hessian(g)(jax.numpy.array([1., 2.]))) [[ 6. -2.] [ -2. -480.]] :py:func:`hessian` is a generalization of the usual definition of the Hessian that supports nested Python containers (i.e. pytrees) as inputs and outputs. The tree structure of ``jax.hessian(fun)(x)`` is given by forming a tree product of the structure of ``fun(x)`` with a tree product of two copies of the structure of ``x``. A tree product of two tree structures is formed by replacing each leaf of the first tree with a copy of the second. For example: >>> import jax.numpy as jnp >>> f = lambda dct: {"c": jnp.power(dct["a"], dct["b"])} >>> print(jax.hessian(f)({"a": jnp.arange(2.) + 1., "b": jnp.arange(2.) + 2.})) {'c': {'a': {'a': Array([[[ 2., 0.], [ 0., 0.]], [[ 0., 0.], [ 0., 12.]]], dtype=float32), 'b': Array([[[ 1. , 0. ], [ 0. , 0. ]], [[ 0. , 0. ], [ 0. , 12.317766]]], dtype=float32)}, 'b': {'a': Array([[[ 1. , 0. ], [ 0. , 0. ]], [[ 0. , 0. ], [ 0. , 12.317766]]], dtype=float32), 'b': Array([[[0. , 0. ], [0. , 0. ]], [[0. , 0. ], [0. , 3.843624]]], dtype=float32)}}} Thus each leaf in the tree structure of ``jax.hessian(fun)(x)`` corresponds to a leaf of ``fun(x)`` and a pair of leaves of ``x``. For each leaf in ``jax.hessian(fun)(x)``, if the corresponding array leaf of ``fun(x)`` has shape ``(out_1, out_2, ...)`` and the corresponding array leaves of ``x`` have shape ``(in_1_1, in_1_2, ...)`` and ``(in_2_1, in_2_2, ...)`` respectively, then the Hessian leaf has shape ``(out_1, out_2, ..., in_1_1, in_1_2, ..., in_2_1, in_2_2, ...)``. In other words, the Python tree structure represents the block structure of the Hessian, with blocks determined by the input and output pytrees. In particular, an array is produced (with no pytrees involved) when the function input ``x`` and output ``fun(x)`` are each a single array, as in the ``g`` example above. If ``fun(x)`` has shape ``(out1, out2, ...)`` and ``x`` has shape ``(in1, in2, ...)`` then ``jax.hessian(fun)(x)`` has shape ``(out1, out2, ..., in1, in2, ..., in1, in2, ...)``. To flatten pytrees into 1D vectors, consider using :py:func:`jax.flatten_util.flatten_pytree`. )rr)rarl)rxrrrs rjhessianrns:@ sGW+NNNk C C CCrlcddlm}t|\}}tt t j|}tj|}| ||}t|dd|S)Nr)rr*) jax.numpynumpyrsumrrErr* result_typeeye_unravel_array_into_pytree)pytreejnpleavesrndimr flat_basiss rjrXrXst6"")&! S& ! ! " "$  f %%wwt5w))* #FAtZ @ @@rlc&t|d||SrRru) input_pytreeoutput_pytree_leafarrs rjrZrZs #"(# / //rlc&t|d||SNrr|) output_pytreeinput_pytree_leafrs rjrjrjs #1' . ..rlc2tj|jtjr3tjt |tjs|j}|dnt |}|dntj|}tj |||Sr) r*rDrrErF_dtyperealis_weakly_typedr3_convert_element_type)r=exampler weak_types rj_possible_downcastrs !344  F7OOR-? @ @ A/$$vg%odd6+A'+J+J)  +Aui @ @@rlc >t|\}}jzfd|D}ttjt tj|dd}fdt|||D}t||S)aUnravel an array into a PyTree with a given structure. Args: pytree: The pytree that provides the structure. axis: The parameter axis is either -1, 0, or 1. It controls the resulting shapes. example: If specified, cast the components to the matching dtype/weak_type, or else use the pytree leaf type if example is None. arr: The array to be unraveled. c~g|]9}jdtj|zjdzdz:S)Nr*)rrE)rhlraxiss rjrkz._unravel_array_into_pytree..sE P P PACIete rx{{ *SYtAvww-? ? P P PrlNrcfg|]-\}}}ttj|||n.Sr)rrEreshape)rhr=rr}rs rjrkz._unravel_array_into_pytree.. sM888 !UDAu--wttGTT888rl) rry_splitrEcumsumrrzipr) rvrrrrxtreedefshapespartsreshaped_partss ``` rjrurus!((/&' $ P P P P P P P P& biBGVCRC[ 9 9::D A A%8888vv66888.  0 00rlct|tjrtj|||S|||Sr)rrEndarraysplitr)r=indicesrs rjrrs>2:# 8Aw % %% 88GT " ""rlin_axesint | None | Sequence[Any]rSrAxisName | None axis_size int | Nonespmd_axis_name&AxisName | tuple[AxisName, ...] | Nonectd}jr|dz }|jz } tjnt t urft trt >> import jax.numpy as jnp >>> >>> vv = lambda x, y: jnp.vdot(x, y) # ([a], [a]) -> [] >>> mv = vmap(vv, (0, None), 0) # ([b,a], [a]) -> [b] (b is the mapped axis) >>> mm = vmap(mv, (None, 1), 1) # ([b,a], [a,c]) -> [b,c] (c is the mapped axis) Here we use ``[a,b]`` to indicate an array with shape (a,b). Here are some variants: >>> mv1 = vmap(vv, (0, 0), 0) # ([b,a], [b,a]) -> [b] (b is the mapped axis) >>> mv2 = vmap(vv, (0, 1), 0) # ([b,a], [a,b]) -> [b] (b is the mapped axis) >>> mm2 = vmap(mv2, (1, 1), 0) # ([b,c,a], [a,c,b]) -> [c,b] (c is the mapped axis) Here's an example of using container types in ``in_axes`` to specify which axes of the container elements to map over: >>> A, B, C, D = 2, 3, 4, 5 >>> x = jnp.ones((A, B)) >>> y = jnp.ones((B, C)) >>> z = jnp.ones((C, D)) >>> def foo(tree_arg): ... x, (y, z) = tree_arg ... return jnp.dot(x, jnp.dot(y, z)) >>> tree = (x, (y, z)) >>> print(foo(tree)) [[12. 12. 12. 12. 12.] [12. 12. 12. 12. 12.]] >>> from jax import vmap >>> K = 6 # batch size >>> x = jnp.ones((K, A, B)) # batch axis in different locations >>> y = jnp.ones((B, K, C)) >>> z = jnp.ones((C, D, K)) >>> tree = (x, (y, z)) >>> vfoo = vmap(foo, in_axes=((0, (1, 2)),)) >>> print(vfoo(tree).shape) (6, 2, 5) Here's another example using container types in ``in_axes``, this time a dictionary, to specify the elements of the container to map over: >>> dct = {'a': 0., 'b': jnp.arange(5.)} >>> x = 1. >>> def foo(dct, x): ... return dct['a'] + dct['b'] + x >>> out = vmap(foo, in_axes=({'a': None, 'b': 0}, None))(dct, x) >>> print(out) [1. 2. 3. 4. 5.] The results of a vectorized function can be mapped or unmapped. For example, the function below returns a pair with the first element mapped and the second unmapped. Only for unmapped results we can specify ``out_axes`` to be ``None`` (to keep it unmapped). >>> print(vmap(lambda x, y: (x + y, y * 2.), in_axes=(0, None), out_axes=(0, None))(jnp.arange(2.), 4.)) (Array([4., 5.], dtype=float32), 8.0) If the ``out_axes`` is specified for an unmapped result, the result is broadcast across the mapped axis: >>> print(vmap(lambda x, y: (x + y, y * 2.), in_axes=(0, None), out_axes=0)(jnp.arange(2.), 4.)) (Array([4., 5.], dtype=float32), Array([8., 8.], dtype=float32, weak_type=True)) If the ``out_axes`` is specified for a mapped result, the result is transposed accordingly. Finally, here's an example using ``axis_name`` together with collectives: >>> xs = jnp.arange(3. * 4.).reshape(3, 4) >>> print(vmap(lambda x: lax.psum(x, 'i'), axis_name='i')(xs)) [[12. 15. 18. 21.] [12. 15. 18. 21.] [12. 15. 18. 21.]] See the :py:func:`jax.pmap` docstring for more examples involving collectives. zxVectorized version of {fun}. Takes similar arguments as {fun} but with additional array axes over which {fun} is mapped.z Original documentation: Nzvmap in_axes must be an int, None, or a tuple of entries corresponding to the positional arguments passed to the function, but got rBc3\K|]'}t|thtjvV(dSrtyper-rY spec_typesrhrs rjrzvmap..s8 R RT!WW3x23 3 R R R R R Rrlz]vmap in_axes must be an int, None, or (nested) container with those types as leaves, but got c3\K|]'}t|thtjvV(dSrrrs rjrzvmap..s8 S ST!WW3x23 3 S S S S S Srlz^vmap out_axes must be an int, None, or (nested) container with those types as leaves, but got )rc xttrOtt|kr/tdtdt|t ||ft j\}}tj}t j ||\}td|dfd}nt|||d} t j |||fd  j |}n#t j$r} td } t| } t!| d \} } | | j\}} td t%|d| jd| jdd} ~ wwxYwt|S)Nzvmap in_axes must be an int, None, or a tuple of entries corresponding to the positional arguments passed to the function, but got len(in_axes)=z , len(args)=is_leafz vmap in_axesrT)kwsrWc6tdS)N vmap out_axes)r9)rSrsrjz&vmap..vmap_f..s, HEErl)rrc |duSrrfr=s rjrz&vmap..vmap_f..s drlzat vmap out_axesz, got axis spec z but output was batched on axis )rtuplerrrrY is_vmappablerrflatten_fun_for_vmapr9_mapped_axis_sizebatch call_wrappedSpecMatchErrorrr"leaf_idxrdstsrc)ryrzrrrflat_fun in_axes_flat axis_size_out_flatr@ out_axes_flat out_axes_fullpairsrpathrrrrxrrSrs @rjvmap_fzvmap..vmap_fs('5!!Ac'llc$ii&?&? @"%g,,@@36t99@@ A AA'f~x?TUUUIw SA!6q'BBHh'14PPPL(4))#C)\6RR L Iz< E E E E E' i !hh  "LLL"?HHJJIIm$XXZZ??m' ?R?RSSSheQaj!gdA A&,,AAAA9:AA B BGKL L ((**h / //s &DF!BFF!)r@__doc__r% no_axis_namerrrrr-rYrr.allrrUrOrr_)rxrrSrrrrrs`````` rjrWrWsR I&[ //F ckF#,#4d))D$8$8$E$E$&N GnnG /T']]sE.PH._get_axis_sizesE 4[  "EEE!QYYTEh   = = = = >DEEEs A:AAc3`K|](\}}|tj||V)dSrrErrhr=drrqs rjrz$_mapped_axis_size..sNMM#'1aam .~dBHQKKCC>KmmmMMrlr*z1 must have at least one non-None value in in_axesch t|S#t$rYdSwxYw)Nr)r> str_shortr.rs rj_get_argument_typez-_mapped_axis_size.._get_argument_typesC  " " , , . ..  YYs # 11z5 got inconsistent sizes for array axes to be mapped: cRg|]#\}}dt|d|$S)ry of type rrhpr=rs rjrkz%_mapped_axis_size..sW888a555//2255888rlcRg|]#\}}dt|d|$S)rzrrrs rjrkz%_mapped_axis_size..sW<<<17VAYY7711!4477<<. s666T3/4466Q4T46!9944..q11446666rlc\g|](\}}|tj||nd)Srrrs rjrkz%_mapped_axis_size..sL,,,1a89}~~dBHQKK333$,,,rlc3K|]}||V dSrrf)rhss rjrz$_mapped_axis_size..s"#J#J!AMAMMMM#J#JrlcxtD]\}}tj||r|cSJ|fr) enumerater%definitely_equal)sziisz all_sizess rj_all_sizes_indexz+_mapped_axis_size.._all_sizes_indexsNI&&223 sB ' '11!2y/!!!rlc3@K|]\}}|VdSrrf)rhrrr key_pathss rjrz$_mapped_axis_size..s8GGur19--b112GGGGGGrlc3@K|]\}}|VdSrrf)rhrrrdimss rjrz$_mapped_axis_size..s8 = =URd##B''( = = = = = =rlz * one axis had size z: axis z of z; z * most axes (z of them) had size z , e.g. axis z * some axes ()rqrrrrr-rr)rrr%dedup_referentsrrinspect signaturebindr.r! argumentsitems collectionsCounter most_commonappendjoin)fntreevalsrrqryrzrrr>ba args_paths kwargs_paths size_countsct other_countscountsexexamplesaxaxsrrrrrs `` @@@@@rjrrs; !$--LD&   M M04 M MDJ M M   E E E E  MMMMM+.tT??MMM M M%ZZ1__ CB I   D D DC S//  H H HI#d++,$ # 2   #T 4V 4 4BB Z  BBBZ8888066888J<<<< 26 : :<<%>>(2r\F""""" HGGGGGGG-"x = = = = =f = = =("s1WWJJBBB2BB2BBBCCCCJJWWWWWWWPRWWWXXXh\::[[b"hr2 Qww jjD"DDRDDRDDDEEEE jjY2YY"YY"YYRTYYYZZZZ2773<<$%%%s3CC)(C))rrSstatic_broadcasted_argnumsdevicesrrrglobal_arg_shapesr r Sequence[xc.Device] | Noner"tuple[tuple[int, ...], ...] | Nonec | tdtjjrddlm} | ||||||||| St ||||||||| S)a*Parallel map with support for collective operations. The purpose of :py:func:`pmap` is to express single-program multiple-data (SPMD) programs. Applying :py:func:`pmap` to a function will compile the function with XLA (similarly to :py:func:`jit`), then execute it in parallel on XLA devices, such as multiple GPUs or multiple TPU cores. Semantically it is comparable to :py:func:`vmap` because both transformations map a function over array axes, but where :py:func:`vmap` vectorizes functions by pushing the mapped axis down into primitive operations, :py:func:`pmap` instead replicates the function and executes each replica on its own XLA device in parallel. The mapped axis size must be less than or equal to the number of local XLA devices available, as returned by :py:func:`jax.local_device_count()` (unless ``devices`` is specified, see below). For nested :py:func:`pmap` calls, the product of the mapped axis sizes must be less than or equal to the number of XLA devices. .. note:: :py:func:`pmap` compiles ``fun``, so while it can be combined with :py:func:`jit`, it's usually unnecessary. :py:func:`pmap` requires that all of the participating devices are identical. For example, it is not possible to use :py:func:`pmap` to parallelize a computation across two different models of GPU. It is currently an error for the same device to participate twice in the same `pmap`. **Multi-process platforms:** On multi-process platforms such as TPU pods, :py:func:`pmap` is designed to be used in SPMD Python programs, where every process is running the same Python code such that all processes run the same pmapped function in the same order. Each process should still call the pmapped function with mapped axis size equal to the number of *local* devices (unless ``devices`` is specified, see below), and an array of the same leading axis size will be returned as usual. However, any collective operations in ``fun`` will be computed over *all* participating devices, including those on other processes, via device-to-device communication. Conceptually, this can be thought of as running a pmap over a single array sharded across processes, where each process "sees" only its local shard of the input and output. The SPMD model requires that the same multi-process pmaps must be run in the same order on all devices, but they can be interspersed with arbitrary operations running in a single process. Args: fun: Function to be mapped over argument axes. Its arguments and return value should be arrays, scalars, or (nested) standard Python containers (tuple/list/dict) thereof. Positional arguments indicated by ``static_broadcasted_argnums`` can be anything at all, provided they are hashable and have an equality operation defined. axis_name: Optional, a hashable Python object used to identify the mapped axis so that parallel collectives can be applied. in_axes: A non-negative integer, None, or nested Python container thereof that specifies which axes of positional arguments to map over. Arguments passed as keywords are always mapped over their leading axis (i.e. axis index 0). See :py:func:`vmap` for details. out_axes: A non-negative integer, None, or nested Python container thereof indicating where the mapped axis should appear in the output. All outputs with a mapped axis must have a non-None ``out_axes`` specification (see :py:func:`vmap`). static_broadcasted_argnums: An int or collection of ints specifying which positional arguments to treat as static (compile-time constant). Operations that only depend on static arguments will be constant-folded. Calling the pmapped function with different values for these constants will trigger recompilation. If the pmapped function is called with fewer positional arguments than indicated by ``static_broadcasted_argnums`` then an error is raised. Each of the static arguments will be broadcasted to all devices. Arguments that are not arrays or containers thereof must be marked as static. Defaults to (). Static arguments must be hashable, meaning both ``__hash__`` and ``__eq__`` are implemented, and should be immutable. devices: This is an experimental feature and the API is likely to change. Optional, a sequence of Devices to map over. (Available devices can be retrieved via jax.devices()). Must be given identically for each process in multi-process settings (and will therefore include devices across processes). If specified, the size of the mapped axis must be equal to the number of devices in the sequence local to the given process. Nested :py:func:`pmap` s with ``devices`` specified in either the inner or outer :py:func:`pmap` are not yet supported. backend: This is an experimental feature and the API is likely to change. Optional, a string representing the XLA backend. 'cpu', 'gpu', or 'tpu'. axis_size: Optional; the size of the mapped axis. donate_argnums: Specify which positional argument buffers are "donated" to the computation. It is safe to donate argument buffers if you no longer need them once the computation has finished. In some cases XLA can make use of donated buffers to reduce the amount of memory needed to perform a computation, for example recycling one of your input buffers to store a result. You should not reuse buffers that you donate to a computation, JAX will raise an error if you try to. Note that donate_argnums only work for positional arguments, and keyword arguments will not be donated. For more details on buffer donation see the `FAQ `_. Returns: A parallelized version of ``fun`` with arguments that correspond to those of ``fun`` but with extra array axes at positions indicated by ``in_axes`` and with output that has an additional leading array axis (with the same size). For example, assuming 8 XLA devices are available, :py:func:`pmap` can be used as a map along a leading array axis: >>> import jax.numpy as jnp >>> >>> out = pmap(lambda x: x ** 2)(jnp.arange(8)) # doctest: +SKIP >>> print(out) # doctest: +SKIP [0, 1, 4, 9, 16, 25, 36, 49] When the leading dimension is smaller than the number of available devices JAX will simply run on a subset of devices: >>> x = jnp.arange(3 * 2 * 2.).reshape((3, 2, 2)) >>> y = jnp.arange(3 * 2 * 2.).reshape((3, 2, 2)) ** 2 >>> out = pmap(jnp.dot)(x, y) # doctest: +SKIP >>> print(out) # doctest: +SKIP [[[ 4. 9.] [ 12. 29.]] [[ 244. 345.] [ 348. 493.]] [[ 1412. 1737.] [ 1740. 2141.]]] If your leading dimension is larger than the number of available devices you will get an error: >>> pmap(lambda x: x ** 2)(jnp.arange(9)) # doctest: +SKIP ValueError: ... requires 9 replicas, but only 8 XLA devices are available As with :py:func:`vmap`, using ``None`` in ``in_axes`` indicates that an argument doesn't have an extra axis and should be broadcasted, rather than mapped, across the replicas: >>> x, y = jnp.arange(2.), 4. >>> out = pmap(lambda x, y: (x + y, y * 2.), in_axes=(0, None))(x, y) # doctest: +SKIP >>> print(out) # doctest: +SKIP ([4., 5.], [8., 8.]) Note that :py:func:`pmap` always returns values mapped over their leading axis, equivalent to using ``out_axes=0`` in :py:func:`vmap`. In addition to expressing pure maps, :py:func:`pmap` can also be used to express parallel single-program multiple-data (SPMD) programs that communicate via collective operations. For example: >>> f = lambda x: x / jax.lax.psum(x, axis_name='i') >>> out = pmap(f, axis_name='i')(jnp.arange(4.)) # doctest: +SKIP >>> print(out) # doctest: +SKIP [ 0. 0.16666667 0.33333334 0.5 ] >>> print(out.sum()) # doctest: +SKIP 1.0 In this example, ``axis_name`` is a string, but it can be any Python object with ``__hash__`` and ``__eq__`` defined. The argument ``axis_name`` to :py:func:`pmap` names the mapped axis so that collective operations, like :func:`jax.lax.psum`, can refer to it. Axis names are important particularly in the case of nested :py:func:`pmap` functions, where collective operations can operate over distinct axes: >>> from functools import partial >>> import jax >>> >>> @partial(pmap, axis_name='rows') ... @partial(pmap, axis_name='cols') ... def normalize(x): ... row_normed = x / jax.lax.psum(x, 'rows') ... col_normed = x / jax.lax.psum(x, 'cols') ... doubly_normed = x / jax.lax.psum(x, ('rows', 'cols')) ... return row_normed, col_normed, doubly_normed >>> >>> x = jnp.arange(8.).reshape((4, 2)) >>> row_normed, col_normed, doubly_normed = normalize(x) # doctest: +SKIP >>> print(row_normed.sum(0)) # doctest: +SKIP [ 1. 1.] >>> print(col_normed.sum(1)) # doctest: +SKIP [ 1. 1. 1. 1.] >>> print(doubly_normed.sum((0, 1))) # doctest: +SKIP 1.0 On multi-process platforms, collective operations operate over all devices, including those on other processes. For example, assuming the following code runs on two processes with 4 XLA devices each: >>> f = lambda x: x + jax.lax.psum(x, axis_name='i') >>> data = jnp.arange(4) if jax.process_index() == 0 else jnp.arange(4, 8) >>> out = pmap(f, axis_name='i')(data) # doctest: +SKIP >>> print(out) # doctest: +SKIP [28 29 30 31] # on process 0 [32 33 34 35] # on process 1 Each process passes in a different length-4 array, corresponding to its 4 local devices, and the psum operates over all 8 values. Conceptually, the two length-4 arrays can be thought of as a sharded length-8 array (in this example equivalent to jnp.arange(8)) that is mapped over, with the length-8 mapped axis given name 'i'. The pmap call on each process then returns the corresponding length-4 output shard. The ``devices`` argument can be used to specify exactly which devices are used to run the parallel computation. For example, again assuming a single process with 8 devices, the following code defines two parallel computations, one which runs on the first six devices and one on the remaining two: >>> from functools import partial >>> @partial(pmap, axis_name='i', devices=jax.devices()[:6]) ... def f1(x): ... return x / jax.lax.psum(x, axis_name='i') >>> >>> @partial(pmap, axis_name='i', devices=jax.devices()[-2:]) ... def f2(x): ... return jax.lax.psum(x ** 2, axis_name='i') >>> >>> print(f1(jnp.arange(6.))) # doctest: +SKIP [0. 0.06666667 0.13333333 0.2 0.26666667 0.33333333] >>> print(f2(jnp.array([2., 3.]))) # doctest: +SKIP [ 13. 13.] Nzglobal_arg_shapes only worked with sharded_jit which has long been removed from JAX. Please migrate to pjit and remove global_arg_shapes from pmap.r)pmaprrSr r rrr)rr$pmap_shmap_mergertjax.experimental.shard_mapr _cpp_pmap) rxrrrSr r rrrrrs rjrr&sJ"     "/////// 4Y(+E#-  / / //  !;# % % % %rlc~eZdZUded<ded<ded<ded<d ed <d ed <d ed<ded<ded<d ed<ded<dS) PmapCallInfo lu.WrappedFunrrrzCallable[[], PyTreeDef]r Sequence[Any] flat_argszSequence[bool]rzSequence[int | None]rr-local_axis_sizerout_axes_thunkrr global_axis_sizeris_explicit_global_axis_sizeNr __module__ __qualname____annotations__rfrlrjrr's####    $$$$%%%%$$$$$$rlrrr- backend_namerrctjdkr|||krtd|d|d||tj|dntj||tjdkr|}ne|rt |}nS|tjz}t fdttjDsJ|S)z/Determine global_axis_size for multi-host pmap.r*NzSpecified axis_size z" doesn't match received axis_size rBrc3K|]<}ttj|tjkV=dSr)rr local_deviceslocal_device_count)rhpirs rjrz(_get_global_axis_size..Ms\66 br7++ , ,0Eg0N0N N666666rl)r process_countrget_device_backendrrrrange)r in_devicesr$rrs @rj_get_global_axis_sizer.5sA  A"2">/))  (/ ( ($ ( ( ( ) )) 4#JqM22GGn\**G   A%%( 6Z(2+;G+D+DD 6666"*7334466666666 rlc V|"t|dkrtdt|} tj|} t d| | || d} t j|} rtt|kr;tddt|dt|dkrd nd d fd tt|D}t| ||\} }ttrtfd |D}n}n|}}t|| f\}}|r#tjjst#|d|}ndt|z} tt%|df|| fd}nY#t$rLt'|df|| f^}}|d}|j\}|dz }| r|dz }|dz }t|dwxYwt+||||d}t-| \} }t/| |\} }t1| |\}}t3|| |}|du}t5||||}t7|||||||||dnt||| S)Nrz6'devices' argument to pmap must be non-empty, or None.rrfz0pmapped function has static_broadcasted_argnums=rz positional argumentr*rrz?. All static broadcasted arguments must be passed positionally.cg|]}|v| Srfrf)rhrstatic_broadcasted_tuples rjrkz!_prepare_pmap..gs.999777777rlc3(K|] }|V dSrrf)rhrrs rjrz _prepare_pmap..ls'::'!*::::::rlrc |duSrrfrs rjrz_prepare_pmap..ys AIrlrz pmap in_axesz The 'full pytree' here is the tuple of arguments passed positionally to the pmapped function, and the value of `in_axes` must be a tree prefix of that tuple. But it was not a prefix.aG When some arguments are passed by keyword to the pmapped function, they are not included in the comparison to `in_axes`. Instead, each argument passed by keyword is mapped over its leading axis. See the description of `in_axes` in the `pmap` docstring: https://jax.readthedocs.io/en/latest/_autosummary/jax.pmap.html#jax.pmapz Check that the value of the `in_axes` argument to `pmap` is a tree prefix of the tuple of arguments passed positionally to the pmapped function.) rrrrrrrrr rr)rrrEr# fun_signaturerArrrr,r7rrrr$rsrtr:rr ryrrBrCr4rDr.r)rxrrSr1 donate_tupler-r$rryrzrrdbgr dyn_argnumsr dyn_in_axesrrrr@rrr>r res_pathsrrrrrs ` ` rj _prepare_pmapr:SsJ1 4 4 M N NNs#$S)))63 4+R 1 1# l3!* #$$D 11  J=U J J'*4yy J JIIMMSSr J J J K KK 9999eCII..999K!![$77KAx'5!!::::k:::::kkkk 'kH&122-$*&+1*$\2w??NND )N$);*:Xv ? ?EA >  B 7DC LMCZ YZc  $%C S//t##$$&c7D,OO/a,!Y#Ax00!^"1g..(H hY 7 7(!*$!6*?J+7DD x%' $%3#/&5%3&0&8ddeJ>O>O'73O Q Q Q Qs $F55AH ct||tj|n|}t|}t t||}t dt |Dstd|dt dt |Dstd|d|||fS)Nc3BK|]}t|tuVdSrrr-rs rjrz$_shared_code_pmap..s, : :T!WW^ : : : : : :rlz]pmap in_axes must be an int, None, or (nested) container with those types as leaves, but got rBc3BK|]}t|tuVdSrr=rs rjrz$_shared_code_pmap..s, ; ;T!WW^ ; ; ; ; ; ;rlz^pmap out_axes must be an int, None, or (nested) container with those types as leaves, but got )r@r% _TempAxisNamer=r;rrr.)rxrr rrrSr1r5s rj_shared_code_pmapr@s )2):d %%% )01KLL&.))+CEE, : :[%9%9 : : : : :G F;BFFF G GG ; ;[%:%: ; ; ; ; ;H G;CGGG H HH ,l ::rlc~eZdZUded<ded<ded<ded<ded<d ed <d ed <d ed<d ed<d ed<d ed<dS)_PmapFastpathDatar-versionzxc.LoadedExecutablexla_executabler in_handler out_handlerout_pytree_defSequence[xc.Device] input_deviceszSequence[sharding_specs.Index] input_indicesrinput_array_shardingsrout_array_shardings out_committedNr rfrlrjrBrBs,,,%%%%///$$$$////&&&&$$$$rlrBrc  t||\ ~~t f d} tj| dtj} t | t| } tfd} t f d| | _ | _ | S)Nc F t|| }|jD]}tj|t |j|j|j|j|j |j j |j |j  }tjt j|j g|jRi|\}}}}}d} t%|tjr$t!j|g|Ri|} || |} n+|t j||||} |j| } } | } t/| | } d}| bt%| t jrHt3jt j| }|j o |j ot;d| D}|ryt3jt j| }|j}|j}d| D}d| D}tAd|j!||| |j"|j#|j$|j%|| }nd}| |fS)N) rrrrr rrrqrrFc3JK|]}t|tjVdSr)rr ArrayImplrhr=s rjrz0_cpp_pmap..cache_miss..s. : :AJq", ' ' : : : : : :rlcg|] }|j Srf)shardingrhouts rjrkz1_cpp_pmap..cache_miss..s>>>cS\>>>rlcg|] }|j Srf) _committedrUs rjrkz1_cpp_pmap..cache_miss..s:::#s~:::rlr*) rCrDrErFrGrIrJrKrrLrM)&r:rr&rCdictrrr rrrrrrr%map_bind_with_continuationr\ xla_pmap_pr EvalTracexla_pmap_impl_lazyprocessrrExecuteReplicatedtypingrhas_unordered_effectshas_host_callbacksrrFrErBrDr'rJrr)ryrzrrparamsmap_bind_continuation top_tracefun_tracersexecuterVrrrG use_fastpathexecute_replicatedrFrErLrM fastpath_datarrrr r5rxrrSr1s rj cache_missz_cpp_pmap..cache_misssc7H.F"GWv / /A{ #+ ' Z '%&%C   F ' @)* @ @ @8> @ @<9dGV $G)T^,,E'AwAAA&AAg ! !''7"3 4 4cc ! ! / ! !)T7F C C E EcShHXZZN  2 2CLz'43IJJ!;t'=wGG  44 ;"5 5 ; : : : : ::: !;t'=wGG&2k%0j>>X>>>:::::m'+:!'"0"0 * 7)1%   mmm  rlc<tj|g|gdSr)r\ shard_args)r=rs rjrz_cpp_pmap..%s4?A3,,Q/rl)pytree_registryc8|i|Sr)lower)ryrztraces rjrqz_cpp_pmap..lower+s$ 5$ !& ! ! ' ' ) ))rlcd t || }ttt|j}t j|j |j|j |j |jj |j |j | \}}}}}tt j|j |j|j |j |jj |j |j|j||||||} t%j|j|} t%j|| |jj || S)N)rrr rqrrr) rrr rqrrrr closed_jaxprrreplicasshardspci)r:rrr>rr\get_pmap_jaxprrrrr rrrr lower_parallel_callablerrrmake_args_inforTracedr)ryrzr abstract_argsrt xc_backendrurvrwlower_callable args_inforrrr r5rxrrSr1s rjrrz_cpp_pmap..trace/s7 Wh 8,)T6 3 3A/==>>M6:6I GY#a6H Z '7773L*h $aj)#a6H Z '%&%C!    N%ai MMI =y!*2E~ 7 77rl) r@rOrIrrPdefault_registry_pmap_cache_clearsaddrUrqrr)rxrrrSr r rrrrl cpp_mapped_fpmap_frqr5r1rrs```` ``` @@@rjrrsq7H 90.'773) %|!.GGGGGGGGGGGG<GR :///0222,&&& 5::l # #&****<*777777777777<7>&,&, -rltuple[Any, ...]cjt|ttj||||S)aComputes a (forward-mode) Jacobian-vector product of ``fun``. Args: fun: Function to be differentiated. Its arguments should be arrays, scalars, or standard Python containers of arrays or scalars. It should return an array, scalar, or standard Python container of arrays or scalars. primals: The primal values at which the Jacobian of ``fun`` should be evaluated. Should be either a tuple or a list of arguments, and its length should be equal to the number of positional parameters of ``fun``. tangents: The tangent vector for which the Jacobian-vector product should be evaluated. Should be either a tuple or a list of tangents, with the same tree structure and array shapes as ``primals``. has_aux: Optional, bool. Indicates whether ``fun`` returns a pair where the first element is considered the output of the mathematical function to be differentiated and the second element is auxiliary data. Default False. Returns: If ``has_aux`` is ``False``, returns a ``(primals_out, tangents_out)`` pair, where ``primals_out`` is ``fun(*primals)``, and ``tangents_out`` is the Jacobian-vector product of ``function`` evaluated at ``primals`` with ``tangents``. The ``tangents_out`` value has the same Python tree structure and shapes as ``primals_out``. If ``has_aux`` is ``True``, returns a ``(primals_out, tangents_out, aux)`` tuple where ``aux`` is the auxiliary data returned by ``fun``. For example: >>> import jax >>> >>> primals, tangents = jax.jvp(jax.numpy.sin, (0.1,), (0.2,)) >>> print(primals) 0.09983342 >>> print(tangents) 0.19900084 rT)r@rVrr)rxprimalstangentsrs rjjvprWs4P bl3(G D D DDrlrc Ft|ttfrt|ttfs:tdt |jdt |jdt |\}}t |\}}||krtd|d|dt||D]\}} tj t|t| krRtdt|dtj t|dt| d tj |tj | kr9td tj |d tj | |sht||\} } tj| ||\} } | } t%| | t%| | fSt'||\} }tj| d \}}|||\} } |\} }t%| | t%| | t%||fS)z-Variant of jvp() that takes an lu.WrappedFun.zGprimal and tangent arguments to jax.jvp must be tuples or lists; found  and rBzgprimal and tangent arguments to jax.jvp must have the same tree structure; primals have tree structure z& whereas tangents have tree structure zprimal and tangent arguments to jax.jvp do not match; dtypes must be equal, or in case of int/bool primal dtype the tangent dtype must be float0.Got primal dtype z and so expected tangent dtype z, but got tangent dtype z instead.zEjvp called with different primal and tangent shapes;Got primal shape z and tangent shape as TrT)rrrr.rrrrr%primal_dtype_to_tangent_dtyperrErrr5rXrrrr6)rxrrrps_flattree_defts_flat tree_def_2rtrr out_primals out_tangents out_aux_treesjvp_funr"aux_trees rjrVrVs Wudm , ,V Xt} - -V U!']]3UU:>x..:QUUU V VV#7++'8$X..':  4>F44&0444 5 55'7## ] ]da )&))44q AA <+1))<<;F1IIFF << (.ayy <<< = ==  x{{bhqkk!! \+-8A;;\\NPhWXkk\\ ] ]]"  --c8<<Hh "x 0 0 = =gw O OKxzzH 8[ 1 1 8\ 2 2 444CBBHm6(D111LGS ' 4 4Wg F FK&Hh 8[ 1 1 8\ 2 2 8SSUU + + --rlrTLiteral[False]tuple[Any, Callable]cdSrrfrxrrs rj linearizer #rl Literal[True]tuple[Any, Callable, Any]cdSrrfrs rjrrrrl0tuple[Any, Callable] | tuple[Any, Callable, Any]c $t|tj|}t|\}}|rt ||\}}nt ||\}}t j|g|Rd|i^}} } } } |r|\}} n |}t||}tttj |}ttt| |||f| | }|r| \}||t| |fS| \||fS)a Produces a linear approximation to ``fun`` using :py:func:`jvp` and partial eval. Args: fun: Function to be differentiated. Its arguments should be arrays, scalars, or standard Python containers of arrays or scalars. It should return an array, scalar, or standard python container of arrays or scalars. primals: The primal values at which the Jacobian of ``fun`` should be evaluated. Should be a tuple of arrays, scalar, or standard Python container thereof. The length of the tuple is equal to the number of positional parameters of ``fun``. has_aux: Optional, bool. Indicates whether ``fun`` returns a pair where the first element is considered the output of the mathematical function to be linearized, and the second is auxiliary data. Default False. Returns: If ``has_aux`` is ``False``, returns a pair where the first element is the value of ``f(*primals)`` and the second element is a function that evaluates the (forward-mode) Jacobian-vector product of ``fun`` evaluated at ``primals`` without re-doing the linearization work. If ``has_aux`` is ``True``, returns a ``(primals_out, lin_fn, aux)`` tuple where ``aux`` is the auxiliary data returned by ``fun``. In terms of values computed, :py:func:`linearize` behaves much like a curried :py:func:`jvp`, where these two code blocks compute the same values:: y, out_tangent = jax.jvp(f, (x,), (in_tangent,)) y, f_jvp = jax.linearize(f, x) out_tangent = f_jvp(in_tangent) However, the difference is that :py:func:`linearize` uses partial evaluation so that the function ``f`` is not re-linearized on calls to ``f_jvp``. In general that means the memory usage scales with the size of the computation, much like in reverse-mode. (Indeed, :py:func:`linearize` has a similar signature to :py:func:`vjp`!) This function is mainly useful if you want to apply ``f_jvp`` multiple times, i.e. to evaluate a pushforward for many different input tangent vectors at the same linearization point. Moreover if all the input tangent vectors are known at once, it can be more efficient to vectorize using :py:func:`vmap`, as in:: pushfwd = partial(jvp, f, (x,)) y, out_tangents = vmap(pushfwd, out_axes=(None, 0))((in_tangents,)) By using :py:func:`vmap` and :py:func:`jvp` together like this we avoid the stored-linearization memory cost that scales with the depth of the computation, which is incurred by both :py:func:`linearize` and :py:func:`vjp`. Here's a more complete example of using :py:func:`linearize`: >>> import jax >>> import jax.numpy as jnp >>> >>> def f(x): return 3. * jnp.sin(x) + jnp.cos(x / 2.) ... >>> jax.jvp(f, (2.,), (3.,)) (Array(3.26819, dtype=float32, weak_type=True), Array(-5.00753, dtype=float32, weak_type=True)) >>> y, f_jvp = jax.linearize(f, 2.) >>> print(y) 3.2681944 >>> print(f_jvp(3.)) -5.007528 >>> print(f_jvp(4.)) -6.676704 r)r@rrrr6r5rXrrrrr%r<rr _lift_linearized)rxrrr primals_flatrrrr out_pvalsrr maybe_auxr out_primal_py primal_avals lifted_jvpr"s rjrrsSFl3!&w//, =1!W==K0G<<K68l73 737373*173733+y%) !HhhxzzH ;77-c$-6677,w/ '2I??@FHH* % ES *nXs&C&C CC B * $$rlc<fd}t|||S)Ncvtttj|}t |D]A\}}tj||std|d|Btg|R}t|fd D}tdJ|S)Nz_linearized function called on tangent values inconsistent with the original primal values: got z for primal aval c~g|]9}|r|nt:Srf)is_known get_knownnext)rhpval tangents_out_s rjrkz1_lift_linearized..fun..sM'''%)MMOOL   m9L9L'''rl) rrr%r<r typecompatat_least_vspacerr1iterr) r tangent_avals primal_aval tangent_aval tangents_outfull_outrrrrrs @rjrxz_lift_linearized..funsT]H5566M%(}%E%ENN! \ _[88::L I INM ,MM?JMMNN NNeV7h777L&&M''''%'''H  t $ $ , , , Orl)r?)rrio_treerrpy_argsrxs`` `` rjrrsB         !gw 7 77rlc lt|dkr(d|dt|d|d}t||\}|\}}t|\} } | |krtd| d|t | |D]\} } t | } | }tj| |sat| |sQtd| d | d |d | || }t||S) Nr*z.The function returned by `jax.vjp` applied to z was called with z arguments, but functions returned by `jax.vjp` must be called with a single argument corresponding to the single value returned by a (even if that returned value is a tuple or other container). For example, if we have: def f(x): return (x, x) _, f_vjp = jax.vjp(f, 1.0) the function `f` returns a single tuple as output, and so we call `f_vjp` with a single tuple as its argument: x_bar, = f_vjp((2.0, 2.0)) If we instead call `f_vjp(2.0, 2.0)`, with the values 'splatted out' as arguments rather than in a tuple, this error can arise.z;unexpected tree structure of argument to vjp function: got z, but expected to match zIunexpected JAX type (e.g. shape/dtype) for argument to vjp function: got z, but expected z2 because the corresponding output of the function z had JAX type ) rr.rrrr>rr%r_temporary_dtype_exceptionrr)rqout_primal_avalsrrxpy_args_r>rin_tree_expectedrryrrr?ct_avalct_aval_expectedr7s rj_vjp_pullback_wrapperr!s]]a MD M Mx== M M,0 M M MC& C.. ('&Hw''-$    P#PP=MPP Q QQt-.. ! !ic4 %%G++-- OG%5 6 6! &w0@ A A!  ""$$  5E5O5O5Q5Q  >B  ^^     ! !! T # # & &&rlct|tjr:t|tjr |j|jko|jt kSdSNF)rr%r2rrfloat0)ra_s rjrrKsK4#$$6B8H)I)I6 7bh  528v#55 rl)rrCallable[..., T]rtuple[T, Callable]cdSrrfrxrrrs rjvjprPs #rl)rCallable[..., tuple[T, U]]tuple[T, Callable, U]cdSrrfrs rjrrWs #rlc|rtd~t|ttj|g|Rd|iS)aCompute a (reverse-mode) vector-Jacobian product of ``fun``. :py:func:`grad` is implemented as a special case of :py:func:`vjp`. Args: fun: Function to be differentiated. Its arguments should be arrays, scalars, or standard Python containers of arrays or scalars. It should return an array, scalar, or standard Python container of arrays or scalars. primals: A sequence of primal values at which the Jacobian of ``fun`` should be evaluated. The number of ``primals`` should be equal to the number of positional parameters of ``fun``. Each primal value should be an array, a scalar, or a pytree (standard Python containers) thereof. has_aux: Optional, bool. Indicates whether ``fun`` returns a pair where the first element is considered the output of the mathematical function to be differentiated and the second element is auxiliary data. Default False. Returns: If ``has_aux`` is ``False``, returns a ``(primals_out, vjpfun)`` pair, where ``primals_out`` is ``fun(*primals)``. If ``has_aux`` is ``True``, returns a ``(primals_out, vjpfun, aux)`` tuple where ``aux`` is the auxiliary data returned by ``fun``. ``vjpfun`` is a function from a cotangent vector with the same shape as ``primals_out`` to a tuple of cotangent vectors with the same number and shapes as ``primals``, representing the vector-Jacobian product of ``fun`` evaluated at ``primals``. >>> import jax >>> >>> def f(x, y): ... return jax.numpy.sin(x), jax.numpy.cos(y) ... >>> primals, f_vjp = jax.vjp(f, 0.5, 1.0) >>> xbar, ybar = f_vjp((-0.7, 0.3)) >>> print(xbar) -0.61430776 >>> print(ybar) -0.2524413 z)reduce_axes argument to vjp is deprecatedr)r$r@r0rrrs rjrr\sgTK I J JJ l3 4! 4 4 4+2 4 44rlc t|\}}|D]}tj||s6t||\}}t j||\}} |}n;t ||\}} t j||d\}} } | \}} tt|} t||}ttt|j | ||f| }|s||fS||t| | fS)z-Variant of vjp() that takes an lu.WrappedFun.TrT)rr&rCr5rXrr6rr>rrr rr)rxrrrrrrrrrrr"rrrr8s rjr0r0s0&w//, 22c8-c2222 )-c7;;Hhvh 55KxzzHH3CAAHmF8\4HHHKc&Hh+[99 ;77- 70#,+h-@BBCF H H& @ &  &.3"?"? ??rlc |rtd~t|\} ttj| \}t t | t tj }t tj j }tj ||d\}}tj dgtjzd\} t!|\} t tj} t#d|| zDs2t#d|| zDst%d|d| dt& fd } t)| |S) aTranspose a function that is promised to be linear. For linear functions, this transformation is equivalent to :py:func:`vjp`, but avoids the overhead of computing the forward pass. The outputs of the transposed function will always have the exact same dtypes as ``primals``, even if some values are truncated (e.g., from complex to float, or from float64 to float32). To avoid truncation, use dtypes in ``primals`` that match the full range of desired outputs from the transposed function. Integer dtypes are not supported. Args: fun: the linear function to be transposed. *primals: a positional argument tuple of arrays, scalars, or (nested) standard Python containers (tuples, lists, dicts, namedtuples, i.e., pytrees) of those types used for evaluating the shape/dtype of ``fun(*primals)``. These arguments may be real scalars/ndarrays, but that is not required: only the ``shape`` and ``dtype`` attributes are accessed. See below for an example. (Note that the duck-typed objects cannot be namedtuples because those are treated as standard Python containers.) Returns: A callable that calculates the transpose of ``fun``. Valid input into this function must have the same shape/dtypes/structure as the result of ``fun(*primals)``. Output will be a tuple, with the same shape/dtypes/structure as ``primals``. >>> import jax >>> import types >>> >>> f = lambda x, y: 0.5 * x - 0.5 * y >>> scalar = types.SimpleNamespace(shape=(), dtype=np.dtype(np.float32)) >>> f_transpose = jax.linear_transpose(f, scalar, scalar) >>> f_transpose(1.0) (Array(0.5, dtype=float32), Array(-0.5, dtype=float32)) z/reduce_axes argument to transpose is deprecatedT) instantiatec3TK|]#}tj|tjV$dSr)r*rDrErJrhrs rjrz#linear_transpose..s1 O O1f2:.. O O O O O Orlc3TK|]#}tj|tjV$dSr)r*rDrErHrs rjrz#linear_transpose..sE22"1bj11222222rlzslinear_transpose only supports [float or complex] -> [float or complex], and integer -> integer functions, but got z -> rBct|\}} |krtd d|tttj |std d|dD}t jd|||}tt j|}t|S)Nz8cotangent tree does not match function output, expected z but got z8cotangent type does not match function output, expected c6g|]}tj|Srf)rXUndefinedPrimalrs rjrkz.transposed_fun..s#777r!!$$777rlT) rr.rrr% typecheckrX backward_passinstantiate_zerosr) const out_cotangentout_cts out_tree2dummiesin_ctsin_avalsrrrrs rjtransposed_funz(linear_transpose..transposed_funs %m44GYxzzY C"*(**CC7@CC D DD s4>9g66 7 7A @"+@@6=@@ A AA77h777G  eT5'7 C CF %v . .F '6 * **rl)r$rr5rrrr>r*rr PartialValrtrace_to_jaxpr_nounits dce_jaxprroutvarsrQrr.rOr)rxrrrr in_dtypesin_pvalsrrr out_dtypesrrrrrrs @@@@@rjlinear_transposersJQ O P PP&w//,+BL,=,=wGG(H #\ 2 2(&,))) & 1 1(5hBFHHH%E \%$#em*<*.ax_leaf..s Q$Yrl)rrYrvaluesr)rs rjax_leafz!_flat_axes_specs..ax_leafsP q$   :Jqxxzz$:$: H q% GZ3F3F%G%GIrl)r$r)rryrzrs rj_flat_axes_specsrs4 && &III /4 9 99rl.Callable[..., core.ClosedJaxpr]cdSrrfrxrrrrs rj make_jaxprr #rl+Callable[..., tuple[core.ClosedJaxpr, Any]]cdSrrfrs rjrr rrl>Callable[..., core.ClosedJaxpr | tuple[core.ClosedJaxpr, Any]]c| ttjn#t$rt YnwxYwt t fd}d|_tdrdj d|_ tdrdj d|_ |S)aZ Creates a function that produces its jaxpr given example args. Args: fun: The function whose ``jaxpr`` is to be computed. Its positional arguments and return value should be arrays, scalars, or standard Python containers (tuple/list/dict) thereof. static_argnums: See the :py:func:`jax.jit` docstring. axis_env: Optional, a sequence of pairs where the first element is an axis name and the second element is a positive integer representing the size of the mapped axis with that name. This parameter is useful when lowering functions that involve parallel communication collectives, and it specifies the axis name/size environment that would be set up by applications of :py:func:`jax.pmap`. return_shape: Optional boolean, defaults to ``False``. If ``True``, the wrapped function returns a pair where the first element is the ``ClosedJaxpr`` representation of ``fun`` and the second element is a pytree with the same structure as the output of ``fun`` and where the leaves are objects with ``shape`` and ``dtype`` attributes representing the corresponding types of the output leaves. Returns: A wrapped version of ``fun`` that when applied to example arguments returns a ``ClosedJaxpr`` representation of ``fun`` on those arguments. If the argument ``return_shape`` is ``True``, then the returned function instead returns a pair where the first element is the ``ClosedJaxpr`` representation of ``fun`` and the second element is a pytree representing the structure, shape, dtypes, and named shapes of the output of ``fun``. A ``jaxpr`` is JAX's intermediate representation for program traces. The ``jaxpr`` language is based on the simply-typed first-order lambda calculus with let-bindings. :py:func:`make_jaxpr` adapts a function to return its ``jaxpr``, which we can inspect to understand what JAX is doing internally. The ``jaxpr`` returned is a trace of ``fun`` abstracted to :py:class:`ShapedArray` level. Other levels of abstraction exist internally. We do not describe the semantics of the ``jaxpr`` language in detail here, but instead give a few examples. >>> import jax >>> >>> def f(x): return jax.numpy.sin(jax.numpy.cos(x)) >>> print(f(3.0)) -0.83602 >>> jax.make_jaxpr(f)(3.0) { lambda ; a:f32[]. let b:f32[] = cos a; c:f32[] = sin b in (c,) } >>> jax.make_jaxpr(jax.grad(f))(3.0) { lambda ; a:f32[]. let b:f32[] = cos a c:f32[] = sin a _:f32[] = sin b d:f32[] = cos b e:f32[] = mul 1.0 d f:f32[] = neg e g:f32[] = mul f c in (g,) } c .t5} pgD].\}}|tj||d/t  j|i|}dddn #1swxYwY|jrXt|j|jg\}}tj |j j |j}tj ||} n|j } r5d| j D} | tt|j| fS| S)N)rrcBg|]}t|j|jSrfr)rhos rjrkz4make_jaxpr..make_jaxpr_f..c s' I I IA agqw / / I I Irl)rrr%rrrr _num_constsrV _args_flatrconvert_invars_to_constvarsrrrrrout_info)ryrzrrrtracedrrjaxpr_rrVrrrxrrs rj make_jaxpr_fz make_jaxpr..make_jaxpr_fQ s K%^II/)T D0D$GGHHHH:s3~#2444494KCIKKfKKKKKKKKKKKKKKKV.1C0DEEifa-fl.@.4.@BBfvv..eeleI I I I I Ic N>&/#B#BCHH HH LsAA--A14A1jaxr"z make_jaxpr(rr) hashweakrefrefr.r rUrOr!rmr"r)rxrrrrrs````` rjrr s~III K  #,,CCC ::<:(", S.!!B Ac.> A A AL S*:9#,999L s#+AASharding | Nonec,||St|tjr|jSt|tjrTt j|}t|tr+t|jtjr |jjSdSr) rr(rQrTr%Tracerr<r3val)rr=r?s rj_infer_src_shardingrn s_ J5?## :!T[!! =  D$ &&:dh+P+P X  rli)maxsizec|?t|tjttt fst d|t|trrt|tjr tj }t|tstj |f|fddd| |jdSdS)Nz`jax.device_put` only accepts `None`, `jax.sharding.Sharding`, `jax.Device`, `Layout` or a pytree of these values. Received invalid value: zdevice_put argsF)allow_uneven_sharding)rrDevicerJrMrLrr% AbstractTokentoken_shaped_arrayrKr/pjit_check_aval_sharding shard_shaper)r?rs rj_check_shardingr} sm QHf6JK L L       8$*++%  $d a & &O # $0OOOOMM$* rl)rANone | xc.Device | Sharding | Layout | Any | TransferToMemoryKindrctj5t|\}}|'t|tjt tfr|gt|z}ntd||}'ttjt tfrfd|D}n4td|}ttt||}t||D]"\}}tt||#t!jj|||d} t'|| cdddS#1swxYwYdS)aTransfers ``x`` to ``device``. Args: x: An array, scalar, or (nested) standard Python container thereof. device: The (optional) :py:class:`Device`, :py:class:`Sharding`, or a (nested) :py:class:`Sharding` in standard Python container (must be a tree prefix of ``x``), representing the device(s) to which ``x`` should be transferred. If given, then the result is committed to the device(s). Returns: A copy of ``x`` that resides on ``device``. If the ``device`` parameter is ``None``, then this operation behaves like the identity function if the operand is on any device already, otherwise it transfers the data to the default device, uncommitted. For more details on data placement see the :ref:`FAQ on data placement `. This function is always asynchronous, i.e. returns immediately without blocking the calling Python thread until any transfers are completed. Nzdevice_put devicec0g|]}t|Srf)r)rhxfrs rjrkzdevice_put.. s$@@@2%c2..@@@rlzdevice_put source)r srcs)r$explicit_device_put_scoperrrrrJrLrr9rrrrrr>r& device_put_prr) r=rrx_flatr device_flatsrc_flatrrrs ` rj device_putr" s4'))--"1ooOFG6BIx1EFGG Hs6{{*kk !4gvFFk 3H.BCDD @@@@@@@hh17C@@hc-x@@AAhV[))11A(,,a0000$) 8H '8 , ,)------------------sD&EE E rvrrHct|tstdt|t |t kr0t dt |dt dfd}t j5t|g|RcdddS#1swxYwYdS)aTransfer array shards to specified devices and form Array(s). Args: shards: A sequence of arrays, scalars, or (nested) standard Python containers thereof representing the shards to be stacked together to form the output. The length of ``shards`` must equal the length of ``devices``. devices: A sequence of :py:class:`Device` instances representing the devices to which corresponding shards in ``shards`` will be transferred. This function is always asynchronous, i.e. returns immediately. Returns: A Array or (nested) Python container thereof representing the elements of ``shards`` stacked together, with each shard backed by physical device memory specified by the corresponding entry in ``devices``. Examples: Passing a list of arrays for ``shards`` results in a sharded array containing a stacked version of the inputs: >>> import jax >>> devices = jax.local_devices() >>> x = [jax.numpy.ones(5) for device in devices] >>> y = jax.device_put_sharded(x, devices) >>> np.allclose(y, jax.numpy.stack(x)) True Passing a list of nested container objects with arrays at the leaves for ``shards`` corresponds to stacking the shards at each leaf. This requires all entries in the list to have the same tree structure: >>> x = [(i, jax.numpy.arange(i, i + 4)) for i in range(len(devices))] >>> y = jax.device_put_sharded(x, devices) >>> type(y) >>> y0 = jax.device_put_sharded([a for a, b in x], devices) >>> y1 = jax.device_put_sharded([b for a, b in x], devices) >>> np.allclose(y[0], y0) True >>> np.allclose(y[1], y1) True See Also: - device_put - device_put_replicated z:device_put_sharded `shards` input must be a sequence; got zlen(shards) = z must equal len(devices) = rBc d|D}tdt|dd|ddDsPtdt|dd|ddD\}}td|d|d|d t f|d jz }tj|j}ttj |}tj |jtjr"|jj||| St$jjr]g}|D]W}t+|tjt.jfstj|}||dXn|}t7j|||t; S) NcZg|](}tjtj|)Srf)r%raise_to_shapedr<rRs rjrkzCdevice_put_sharded.._device_put_sharded.. s- @ @ @T !$-"2"2 3 3 @ @ @rlc3(K|] \}}||kVdSrrfrha1a2s rjrzBdevice_put_sharded.._device_put_sharded.. s*AAFBrRxAAAAAArlrr*c30K|]\}}||k ||fVdSrrfr(s rjrzBdevice_put_sharded.._device_put_sharded.. s;!!RRxxHxxx!!rlzVthe shards passed to device_put_sharded must have consistent shape and dtype, but got rrBr)r)rrrrupdaterrr,create_pmap_sharding_specrKrEr(r*rDrrG_rulesdevice_put_shardedr$pmap_no_rank_reductionrtrrr)Arrayasarrayrr\batched_device_putr) xsrr)r* stacked_aval sharding_specrTysr=r s rj_device_put_shardedz/device_put_sharded.._device_put_sharded s @ @R @ @ @E AAc%*eABBi&@&@AAA A AN!!3uSbSz59+E+E!!!!!fb" M>@MMGIMMM N NN8??#g,,58>)I?JJL"<\=OPPMBHW--}==H +V_==_   & 9 9"lHV] ^ ^^ $* b!!bj)/:;; jmm! !D' b  "<2tG}} M MMrlN) rrr.rrrr$rr)rvr r8s ` rjr/r/ sAb FH % %, + LL++ , ,,[[CLL  7c&kk77'*7||777 8 88NNNNN.'))22 ' 1& 1 1 1222222222222222222s B88B<?B<cttrstdfd}tj5t ||cdddS#1swxYwYdS)aTransfer array(s) to each specified device and form Array(s). Args: x: an array, scalar, or (nested) standard Python container thereof representing the array to be replicated to form the output. devices: A sequence of :py:class:`Device` instances representing the devices to which ``x`` will be transferred. This function is always asynchronous, i.e. returns immediately. Returns: An Array or (nested) Python container thereof representing the value of ``x`` broadcasted along a new leading axis of size ``len(devices)``, with each slice along that new leading axis backed by memory on the device specified by the corresponding entry in ``devices``. Examples: Passing an array: >>> import jax >>> devices = jax.local_devices() >>> x = jax.numpy.array([1., 2., 3.]) >>> y = jax.device_put_replicated(x, devices) >>> np.allclose(y, jax.numpy.stack([x for _ in devices])) True See Also: - device_put - device_put_sharded zJ`devices` argument to `device_put_replicated must be a non-empty sequence.c tjttjdtjtj|}t |tsJtj |j }tj j r`t |tjt jfrt%|dd}n3t%|dd}nt%|d}t'tj|}t+j|jt*jr"|jj|||Stt7j|dksJt;j|||gtzS)Nrr*)r% unmapped_avalrrr&r<rr2r,r-rr$r0rtrErr)r1r"rKr(r*rDrrGr.device_put_replicatedr]aval_to_xla_shapesr\r3)r=r?r6bufrTr s rj_device_put_replicatedz5device_put_replicated.._device_put_replicated4 so  c'llD,=q"24=3C3CDD F FD dK ( ((((" GX & &.g. - . ..RRRRR$'))// *A . .//////////////////sAA!$A!ct|tjr|S |j}|S#t$r|cYSwxYwr)rr%r  __array__AttributeError)r=toarrays rj _device_getrDK s[4; HkG 799    HHH s / >>ctj5t|D]'} |#t$rY$wxYwt t |cdddS#1swxYwYdS)aTransfer ``x`` to host. If ``x`` is a pytree, then the individual buffers are copied in parallel. Args: x: An array, scalar, Array or (nested) standard Python container thereof representing the array to be transferred to host. Returns: An array or (nested) Python container thereof representing the value of ``x``. Examples: Passing a Array: >>> import jax >>> x = jax.numpy.array([1., 2., 3.]) >>> jax.device_get(x) array([1., 2., 3.], dtype=float32) Passing a scalar (has no effect): >>> jax.device_get(1) 1 See Also: - device_put - device_put_sharded - device_put_replicated N)r$explicit_device_get_scopercopy_to_host_asyncrBrrD)r=r\s rj device_getrHU s>'))$$ ^^          K # # $$$$$$$$$$$$$$$$$$s1A-<A- A A-A  A--A14A1ceZdZdZgdZiZd dZedZedZ edZ dZ d Z e Z d Zd ZdS) ra=A container for the shape, dtype, and other static attributes of an array. ``ShapeDtypeStruct`` is often used in conjunction with :func:`jax.eval_shape`. Args: shape: a sequence of integers representing an array shape dtype: a dtype-like object sharding: (optional) a :class:`jax.Sharding` object )rrrT_dllNcV~t||_|tdtj|tjr|nt j||_|?t|ttfs#td|dt|dt|tr,t|j trtd|t|tr|jn||_t|tr|j nd|_dS)Nz*ShapeDtypeStruct: dtype must be specified.zcsharding should be an instance of `jax.sharding.Sharding` or `jax.experimental.layout.Layout`. Got rrBzg`DeviceLocalLayout.AUTO` cannot be used in place of a device-local layout in a `ShapeDtypeStruct`. Got )rrrr*rDrGrErrrJrMrdevice_local_layoutrNr.rTrJ)selfrr named_shaperTs rj__init__zShapeDtypeStruct.__init__ s4uDJ } C D DD +E6?CCXRWDJJx(F9K$L$L  4<  8nn    ! !! 8V$$>8/<<>  =2: = = > >>*4Hf)E)ESH%%8DM0:8V0L0LV,,RVDIIIrlc4tj|jSr)rrrrMs rjrzShapeDtypeStruct. sty44rlc*t|jSr)rrrQs rjrzShapeDtypeStruct. ss4:rlc6t|j|jSr)rMrJrTrQs rjlayoutzShapeDtypeStruct.layout s $)T] + ++rlcb |jdS#t$r}td|d}~wwxYw)Nrzlen() of unsized object)rrr.)rMr@s rj__len__zShapeDtypeStruct.__len__ sC8 Z] 888 / 0 0a78s  .).c|j d|jnd}|j d|jnd}t|jd|jd|jj||dS)Nz , sharding=rz , layout=z(shape=z, dtype=r)rTrJrTrrrrrq)rMshrs rj__repr__zShapeDtypeStruct.__repr__ s*.-*C &t} & & &B%)Y%:!DK!!!ADzz" / /4: / /Z_ /&( /*+ / / /0rlct|tsdS|j|j|j|jf|j|j|j|jfkSr)rrrrrTrT)rMothers rj__eq__zShapeDtypeStruct.__eq__ sO e- . .D U{EKFz4:t}dkBCDrlcPt|j|j|j|jfSr)rrrrTrTrQs rj__hash__zShapeDtypeStruct.__hash__ s# TZ D E EErl)NN)rr!r"r __slots__rNrOpropertyrryrTrVrY__str__r\r^rfrlrjrr} s544)+WWWW& 44 5 5$ .. / /$ ,, 8,888 000 'DDDFFFFFrlrcbt|jtj|jddS)NT)allow_extended_dtypeF)r)r2rr*canonicalize_dtyperrs rjrr s1k!'6#>> import jax >>> import jax.numpy as jnp >>> >>> f = lambda A, x: jnp.tanh(jnp.dot(A, x)) >>> A = jax.ShapeDtypeStruct((2000, 3000), jnp.float32) >>> x = jax.ShapeDtypeStruct((3000, 1000), jnp.float32) >>> out = jax.eval_shape(f, A, x) # no FLOPs performed >>> print(out.shape) (2000, 1000) >>> print(out.dtype) float32 All arguments passed via :func:`eval_shape` will be treated as dynamic; static arguments can be included via closure, for example using :func:`functools.partial`: >>> import jax >>> from jax import lax >>> from functools import partial >>> import jax.numpy as jnp >>> >>> x = jax.ShapeDtypeStruct((1, 1, 28, 28), jnp.float32) >>> kernel = jax.ShapeDtypeStruct((32, 1, 3, 3), jnp.float32) >>> >>> conv_same = partial(lax.conv_general_dilated, window_strides=(1, 1), padding="SAME") >>> out = jax.eval_shape(conv_same, x, kernel) >>> print(out.shape) (1, 32, 28, 28) >>> print(out.dtype) float32 )rr.r r eval_shape)rxryrzs rjrfrf sQD Cyyyy &&&'#,,CCC& S d -f - --s --rqrqcN||j}tj||S)aAdds a user specified name to a function when staging out JAX computations. When staging out computations for just-in-time compilation to XLA (or other backends such as TensorFlow) JAX runs your Python program but by default does not preserve any of the function names or other metadata associated with it. This can make debugging the staged out (and/or compiled) representation of your program complicated because there is limited context information for each operation being executed. `named_call` tells JAX to stage the given function out as a subcomputation with a specific name. When the staged out program is compiled with XLA these named subcomputations are preserved and show up in debugging utilities like the TensorFlow Profiler in TensorBoard. Names are also preserved when staging out JAX programs to TensorFlow using :func:`experimental.jax2tf.convert`. Args: fun: Function to be wrapped. This can be any Callable. name: Optional. The prefix to use to name all sub computations created within the name scope. Use the fun.__name__ if not specified. Returns: A version of `fun` that is wrapped in a name_scope. )rr-extend_name_stack)rxrqs rj named_callrj s,8 \ >> import jax >>> >>> @jax.jit ... def layer(w, x): ... with jax.named_scope("dot_product"): ... logits = w.dot(x) ... with jax.named_scope("activation"): ... return jax.nn.relu(logits) It can also be used as a decorator: >>> @jax.jit ... @jax.named_scope("layer") ... def layer(w, x): ... logits = w.dot(x) ... return jax.nn.relu(logits) z+named_scope name argument must be a string.N)rrr.r-rirgs rj named_scoperm, s` D#  C A B BB)$//   EEE                  sA  AAcBtjdS)z?Waits until existing functions have completed any side-effects.N)r&runtime_tokensblock_until_readyrfrlrjeffects_barrierrqa s ++-----rlc"d}g}t|D]=}t|tjr||2||>|sn9t |dkr||dnt j||S)a Tries to call a ``block_until_ready`` method on pytree leaves. Args: x: a pytree, usually with at least some JAX array instances at its leaves. Returns: A pytree with the same structure and values of the input, where the values of all JAX array leaves are ready. cP |S#t$r|cYSwxYwr)rprBrs rj try_to_blockz'block_until_ready..try_to_blockp s=  " ""  hhhs  %%r*r)rrr(rQrrrbatched_block_until_ready)r=rtarraysr}s rjrprpe s &!nnd$(( mmDl4  )  6{{aL ((( (rlctjtjtjt jtjtj tj tj tjjdS)zQ Clear all backend clients so that new backend clients can be created later. N)r_clear_backendsr' cache_clearr*r&xla_primitive_callabler/_infer_params_cached_pjit_lower_cached_create_pjit_jaxpr_cpp_pjit_cacheclearrrPjitFunctionCache clear_allrfrlrjclear_backendsr s       !--///'')))%%'''%%''''%%'''''rlcNtj|S)zkReturn all live arrays in the backend for `platform`. If platform is None, it is the default backend. )rr live_arrays)r s rjrr s  ! ! - - / //rlctjtjtjtjtj j tD]}| tjdS)zClear all compilation and staging caches. This doesn't clear the persistent cache; to disable it (e.g. for benchmarks), set the jax_enable_compilation_cache config option to False. N)rWclear_all_cachesclear_all_weakref_lru_cachesr/r~rr{ryrrrrr _cache_clearr&rz)rxs rj clear_cachesr s#%%%'')))'%%''' c !--/////rl)rxrrrrrrrrrrrrrrrrrrrrr)T)rr) rfNNNNFNFrf)rxrrrrrrrrrrrrrrrrr)rFFFrf)rxrrrrrrrrrrrrr)rxrrrrrrrrrrrrr')rFF) rxrrrrrrrrr)rrr=r rrb)rFFF) rxrrrrrrrrrrr)rrNNN)rxr_rrrSr rrrrrrrr_r)rxrrrr rr rrrrrrrrrrr )rr-r$rrr)rxrrrr rr rrrrrrrrr r)rxrrrrr)rxr)rxrrrrr)rxrrrrr)rxrrrrr)rr) rxrrr rrrrrr) rxrrr rrrrrr)rxrrr)rr)rfN.N) rxrrrrrrrrrrr) rxrrrrrrrrrrr)rfNFN) rxrrrrrrrrrrr)rr )rrrr)rvrr rH)r=r r rH)r=r )rxr)rxr_rqrrr_)rqrrrk)r __future__rrcollections.abcrrrrr functoolsr r rrr`r r r rrrrrqrE contextlibrrjax._srcrrrjax._src.tree_utilrrrrrrrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r jax._src.corer1r2r3jax._src.api_utilr4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrE jax._src.laxrFr3 jax._src.librGrHrrIjax._src.shardingrJjax._src.sharding_implsrKrLjax._src.layoutrMrNjax._src.traceback_utilrOrP jax._src.utilrQrRrSrTrUrVrWjax._src.interpretersrXrYrZr[rr\r]register_exclusion__file__rrAxisNamerr_rarbr unsafe_mapr unsafe_zipr~rrrs _add_hooksrur UNSPECIFIEDrrrr&r%r1rKr/rNr2rarUrYrljacobianrhrirnrXrZrjrrurrWrrrr.r:r@rBrWeakSetrrrVrrrrrr0rrrrrr"r/r<rDrHrpytype_aval_mappingsrfrjrmrqrprrrrfrlrjrsy#"""""MMMMMMMMMMMMMM(((((((( 00000000&&&&&&????????????????????????????????############%%%%%%######%%%%%%@@@@@@@@@@CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC -,,,,, ))))))!!!!!!&&&&&&FFFFFFFF........000000''''''''''''''''$$$$$$******&&&&&&444444&&&&&&%%%%%%"!(+++ D 1 1 1   GCx    GCLL GCLLCZCZ((( ,,, 2229?AAA9?AAA  )*/304/304! $N5N5N5N5N5b/ / / / / f;=FJ-1*.',=A).:<lllll\8949 +-@+@+@+@+@+DBC>CNPHHHHHT 9 9 9XXX$"'"=vFF)))(#7#?HH:;6;<<<<<|OOOO"555:;OT?????@ #G$?JJ$W%A8LL;<7<ACACACACACFAAA///...AAA111&###01&*!%BF CCCCCJI&I&I&\"&~%  68*. *,<@~%~%~%~%~%~%B % % % % %: % % %