
    .PhY                        d dl mZmZmZ d dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlZddlmZmZmZmZ ddlmZ dd	lmZmZ d
dlmZmZ d
dlmZmZmZ  ej        d          Z ej        d          Zd dlm Z m!Z!m"Z" d dl#mZ d dl$m%Z%m&Z&m'Z'  ej(        dd          d             Z)d Z*dJdZ+d Z,d Z-d Z.ej/        0                    dddg          ej/        0                    deeg          d                         Z1eej/        0                    dddg          ej/        0                    d d!d"g          ej/        0                    deeg          d#                                                 Z2eej/        0                    deeg          d$                         Z3d% Z4d& Z5ej/        0                    d'dd(g          d)             Z6d* Z7 G d+ d,e8          Z9dKd-Z:d. Z;d/ Z<ej/        0                    d0 e= e>d                              d1             Z?d2 Z@d3 ZAd4 ZBd5 ZCd6 ZDd7 ZEd8 ZFd9 ZGd: ZHd; ZId< ZJej/        0                    d=d>d?g          ej/        K                    ejL        d@k    o
ejL        dAk    dBC          dD                         ZMdE ZNej/        0                    dFdGdHg          dI             ZOdS )L    )absolute_importdivisionprint_functionNrandom)sleep)uuid4   )Paralleldelayedparallel_backendparallel_config)DaskDistributedBackend)AutoBatchingMixinThreadingBackend   )np
with_numpy)_recursive_backend_info_test_deadlock_with_generator8_test_parallel_unordered_generator_returns_fastest_firstdistributeddask)ClientLocalCluster
get_client)time)cleanupclusterincfunctionT)scopeautousec              #      K   ddl m} d |j        D             }d V  |                                D ]7\  }}|!t          j                            |d            (|t          j        |<   8d S )Nr   )ParallelBackendBasec                 N    i | ]"}|t           j                            |          #S  )osenvironget).0ks     U/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/joblib/test/test_dask.py
<dictcomp>z(avoid_dask_env_leaks.<locals>.<dictcomp>(   s(    XXX!BJNN1%%XXX    )joblib._parallel_backendsr%   MAX_NUM_THREADS_VARSitemsr(   r)   pop)tmp_pathr%   	old_valuer,   vs        r-   avoid_dask_env_leaksr7   !   s      
 >=====XX/B/WXXXI	EEE !!  19JNN1d####BJqMM	 r/   c                      d S Nr'   )argskwargss     r-   noopr<   3   s    Dr/   皙?c                 F    t          |           | rt          d          d S )Nzcondition evaluated to True)r   
ValueError)	conditiondurations     r-   slow_raise_value_errorrB   7   s.    	(OOO 867778 8r/   c                      |                     d           }i }|                                D ]0\  }}t           fdt          |          D                       ||<   1|S )Nc                     | j         S r9   )log)dask_workers    r-   <lambda>zcount_events.<locals>.<lambda>>   s    ;? r/   c                 ,    g | ]}|d          k    |S r   r'   )r+   event
event_names     r-   
<listcomp>z count_events.<locals>.<listcomp>B   s'    GGGuaJ0F0FU0F0F0Fr/   )runr2   lenlist)rK   clientworker_eventsevent_countsweventss   `     r-   count_eventsrU   =   sv    JJBBCCML"((** 
 
	6GGGGVGGG
 
Q r/   c           
         t                      5 \  }\  }}t          |d         |           5 }t          d          5   t                      d t	          d          D                       }|d t	          d          D             k    sJ t          j        t                    5   t                      d t	          d          D                        d d d            n# 1 swxY w Y    t                      d	 t	          d          D                       }|d
 t	          d          D             k    sJ 	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Naddressloopr   backendc              3   P   K   | ]!} t          t                    |          V  "d S r9   r   r    r+   is     r-   	<genexpr>ztest_simple.<locals>.<genexpr>K   0       D DQa D D D D D Dr/   
   c                 ,    g | ]}t          |          S r'   r    r^   s     r-   rL   ztest_simple.<locals>.<listcomp>L       999!s1vv999r/   c              3   X   K   | ]%} t          t                    |d k              V  &dS )   N)r   rB   r^   s     r-   r`   ztest_simple.<locals>.<genexpr>O   sK        DE7 677Q??     r/   c              3   P   K   | ]!} t          t                    |          V  "d S r9   r]   r^   s     r-   r`   ztest_simple.<locals>.<genexpr>S   ra   r/   c                 ,    g | ]}t          |          S r'   rd   r^   s     r-   rL   ztest_simple.<locals>.<listcomp>T   re   r/   )r   r   r   r   rangepytestraisesr?   rY   sabrP   seqs         r-   test_simplerr   G   s   	 :kq&1aAiLt,,, 	: 000 
: 
: hjj D D%)) D D DDD99uRyy9999999]:..  HJJ  INr                   
 !hjj D D%)) D D DDD99uRyy99999999
: 
: 
: 
: 
: 
: 
: 
: 
: 
: 
: 
: 
: 
: 
:	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	:: : : : : : : : : : : : : : : : : :s}   E4EA'E%/C 	E C$$E'C$(AE9EE		EE	EE4E 	 E4#E 	$E44E8;E8c                 t   t           j        t          j        u sJ t                      5 \  }\  }}t	          |d         |           5 }t          d          5  t                      5 }|j        }t          |t                     sJ |j	        |u sJ |j
        dk    sJ  |d t          t          d                    D                        |j
        dk    sJ 	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )	NrW   rX   r   rZ   r   c              3   F   K   | ]} t          d                        V  dS )c                      d S r9   r'   r'   r/   r-   rG   z?test_dask_backend_uses_autobatching.<locals>.<genexpr>.<lambda>j   s    T r/   Nr   r+   _s     r-   r`   z6test_dask_backend_uses_autobatching.<locals>.<genexpr>j   s5      NN2W\\2244NNNNNNr/        @rb   )r   compute_batch_sizer   r   r   r   r   _backend
isinstanceparallel_effective_batch_sizerj   int)rY   rn   ro   rp   rP   r}   r[   s          r-   #test_dask_backend_uses_autobatchingr   W   sO   1/	0 	0 	0 	0 
 >kq&1aAiLt,,, 	> 000 > >ZZ >8 '/G%g/EFFFFF"+x7777"8A==== HNNeCHHooNNNNNN"82=====> > > > > > > > > > > > > > >> > > > > > > > > > > > > > >	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	>> > > > > > > > > > > > > > > > > >s~   D-DC>'A3C'	C>'C++C>.C+/C>2D>DDDD	D-D	D-D	D--D14D1n_jobscontextc                     t                               dd          5   |d          5  t          d |            d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S Nr
   	n_workersthreads_per_workerr   )r   r   r   )r   r   s     r-   Atest_parallel_unordered_generator_returns_fastest_first_with_daskr   n   s    
		aA		>	> O O O O@vNNNO O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O O Os3   AAAA
	
AA
	AA"%A"	return_as	generatorgenerator_unorderedc                     t                               dd          5   | d          5  t          d ||           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S r   )r   r   r   )r   r   r   s      r-   %test_deadlock_with_generator_and_daskr   u   s    
 
		aA		>	> ? ? ? ?%dIv>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?s3   AAAA	AA	AA#&A#c                 x   t                               dd          5  t          j        t	          d          t          j                  }t          d          D ]e} | d          5  t          |          }d d d            n# 1 swxY w Y   t          |          dk    sJ t          d |D                       sJ f | d          5  t                      }d d d            n# 1 swxY w Y   t          |          dk    sJ t          d	 |D                       sJ 	 d d d            d S # 1 swxY w Y   d S )
Nr
   r   g    cAdtyper   data   c              3   (   K   | ]\  }}|d k    V  dS r   Nr'   r+   namerx   s      r-   r`   z4test_nested_parallelism_with_dask.<locals>.<genexpr>   s=        5<T100     r/   c              3   (   K   | ]\  }}|d k    V  dS r   r'   r   s      r-   r`   z4test_nested_parallelism_with_dask.<locals>.<genexpr>   s=       
 
18qD,,
 
 
 
 
 
r/   )
r   r   r   onesr   uint8rj   r   rN   all)r   r   r_   backend_types_and_levelss       r-   !test_nested_parallelism_with_daskr   ~   sg    
		aA		>	> 
 
ws3xxrx000q 	 	A N N+B+M+M+M(N N N N N N N N N N N N N N N/00A5555  @X        
 WV__ 	A 	A'>'@'@$	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A+,,1111 
 
<T
 
 
 
 
 	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sZ   A
D/'B8D/BD/B?D/C&D/&C*	*D/-C*	.3D//D36D3c                      t                      S r9   r   r'   r/   r-   random2r      s    88Or/   c           	         t                      5 \  }\  }}t          |d         |           5 }t          d          5   t                      d t	          d          D                       \  }}||k    sJ 	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NrW   rX   r   rZ   c              3   N   K   | ] } t          t                                V  !d S r9   )r   r   r^   s     r-   r`   z3test_dont_assume_function_purity.<locals>.<genexpr>   s3      !G!G"2''"2"2"4"4!G!G!G!G!G!Gr/   r
   r   r   r   r   rj   )rY   rn   ro   rp   rP   xys          r-    test_dont_assume_function_purityr      s   	 kq&1aAiLt,,, 	 000  !xzz!G!GeAhh!G!G!GGG1Avvvvv              	 	 	 	 	 	 	 	 	 	 	 	 	 	 	                 sW   B4B:B9BB		BB	BB4B 	 B4#B 	$B44B8;B8mixedFc                    ddl m} |sd t          d          D             }d}nd t          d          D             }d}t           ||                    |k    sJ t	                      5 \  }\  }}t          |d         | 	          5 }t          d
          5   t          dd          |          }	d d d            n# 1 swxY w Y   d }
|                    dd          }|	                    |
          }t          d |D                       sJ 	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   )Batchc                 H    g | ]} t          t                    |           S r'   r]   r^   s     r-   rL   z&test_dask_funcname.<locals>.<listcomp>   s&    333Qa333r/   r   batch_of_inc_4_callsc                     g | ]A}|d z  r t          t                    |          n t          t                    |          BS )r
   )r   absr    r^   s     r-   rL   z&test_dask_funcname.<locals>.<listcomp>   sA    QQQ1AE>a|ws||AQQQr/   mixed_batch_of_inc_4_callsrW   rX   r   rZ   r
   r   )
batch_sizepre_dispatchc                 *    t          | j                  S r9   )rO   transition_log)dask_schedulers    r-   fztest_dask_funcname.<locals>.f   s    N9:::r/   42c              3   *   K   | ]}d |d         v V  dS )batch_of_incr   Nr'   )r+   tups     r-   r`   z%test_dask_funcname.<locals>.<genexpr>   s+      ??C~Q/??????r/   )joblib._daskr   rj   reprr   r   r   r   replacerun_on_schedulerr   )rY   r   r   tasks
batch_reprrn   ro   rp   rP   rx   r   rE   s               r-   test_dask_funcnamer      sl   """""" 233%((333+

QQaQQQ1
e++++	 
@kq&1aAiLt,,, 		@ 000 F F>H>>>uEEF F F F F F F F F F F F F F F; ; ; $++C55J))!,,C??3?????????		@ 		@ 		@ 		@ 		@ 		@ 		@ 		@ 		@ 		@ 		@ 		@ 		@ 		@ 		@
@ 
@ 
@ 
@ 
@ 
@ 
@ 
@ 
@ 
@ 
@ 
@ 
@ 
@ 
@ 
@ 
@ 
@s[   ,D;
D#C6D#CD#	C
AD#D;#D'	'D;*D'	+D;;D?D?c                  L   d t          d          D             } t          j        d           j        t	          d                    dfd	t          dd          }t          |          }	 t          d	
          5   t                      fd| D                       }d d d            n# 1 swxY w Y   | d t          d          D             k    sJ t          d|          }t          |                                          dk    sJ t          d |D                       sJ t          d	
          5   t                      fd| D                       }d d d            n# 1 swxY w Y   t          d|          }t          |                                          dk    sJ t          d |D                       sJ 	 |                    d            |j        d           d S # |                    d            |j        d           w xY w)Nc                     g | ]}g S r'   r'   rw   s     r-   rL   z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>   s    $$$AR$$$r/   d   numpyg    .Ac                     |j                             |           |                     t                      j                   | S r9   )testingassert_array_equalappendr	   hex)list_r   Xr   s     r-   isolated_operationzCtest_no_undesired_distributed_cache_hit.<locals>.isolated_operation   s>    J))$222UWW[!!!r/   r   r
   r   r   rZ   c              3   H   K   | ]} t                    |          V  d S r9   rv   )r+   r   r   s     r-   r`   z:test_no_undesired_distributed_cache_hit.<locals>.<genexpr>   s7      SSE8W%788??SSSSSSr/   c                     g | ]}g S r'   r'   rw   s     r-   rL   z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>   s    000000r/   receive-from-scatterr   c                 4    g | ]}t          |          d k    S rI   rN   r+   rs     r-   rL   z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>   "    ---ACFFaK---r/   c              3   L   K   | ]} t                    |           V  dS )r   Nrv   )r+   r   r   r   s     r-   r`   z:test_no_undesired_distributed_cache_hit.<locals>.<genexpr>   sM        ?D+*++E:::     r/   c                 4    g | ]}t          |          d k    S rI   r   r   s     r-   rL   z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>   r   r/      timeoutr9   )rj   rk   importorskiparanger   r   r   r   r   rU   sumvaluesr   close)listsr   rP   rescountsr   r   r   s        @@@r-   'test_no_undesired_distributed_cache_hitr      s    %$s$$$E		W	%	%B	#c((A       Q1===GG__F"V,,, 	T 	T(**SSSSUSSSSSC	T 	T 	T 	T 	T 	T 	T 	T 	T 	T 	T 	T 	T 	T 	T 00U3ZZ0000000
 4f==6==??##q((((-----.....V,,, 	 	 (**     HM    C	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 4f==6==??##a''''-----......R   b!!!!!! 	R   b!!!!!sV   6G: $B6*G: 6B::G: =B:>BG: %E3'G: 3E77G: :E7;AG: :)H#c                   $    e Zd Zd Zd ZeZd ZdS )CountSerializedc                 "    || _         d| _        d S )Nr   )r   count)selfr   s     r-   __init__zCountSerialized.__init__   s    


r/   c                 4    | j         t          |d|          z   S )Nr   )r   getattr)r   others     r-   __add__zCountSerialized.__add__   s    vsE2222r/   c                 @    | xj         dz  c_         t          | j        ffS )Nr   )r   r   r   )r   s    r-   
__reduce__zCountSerialized.__reduce__   s     

a

$&++r/   N)__name__
__module____qualname__r   r   __radd__r   r'   r/   r-   r   r      sH          3 3 3 H, , , , ,r/   r   c                     | |z   |z   |z   |z   S r9   r'   )ro   rp   cdes        r-   add5r     s    q519q=1r/   c           
         d t          d          D             \  }t          t                    fdt          d          D             }| dd           d                     gz  }d |D             }t                      5 \  }}t	          |d         | 	          5 t          d
|g          5   t          d          |          }||k    sJ 	 d d d            n# 1 swxY w Y   t          j        t                    5  t          d
| d          5  	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |j
        }j
        |k    sJ j
        |k    sJ j
        }|fD ]	}	d|	_
        
t                      5 \  }}t	          |d         | 	          5 t                      |fD ])}
                    |
d          t          |
          <   *fd|D             }||k    sJ 	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |j
        }j
        |k    sJ j
        |k    sJ ||k    sJ t          d t          j                            d          D                       }|dk     rj
        |dz   k    sJ d S j
        |k    sJ d S )Nc              3   4   K   | ]}t          |          V  d S r9   )r   r^   s     r-   r`   z&test_manual_scatter.<locals>.<genexpr>  s*      77/!$$777777r/   r   c           	      0    g | ]} d d          S )r      r   r   r'   )r+   rx   r   r   r   zs     r-   rL   z'test_manual_scatter.<locals>.<listcomp>  s.    555aQQq!Q!q!!!555r/   rb   r   r   c                 &    g | ]\  }}} ||i |S r'   r'   )r+   funcr:   r;   s       r-   rL   z'test_manual_scatter.<locals>.<listcomp>  s.    FFF*<$fd%f%%FFFr/   rW   rX   r   )r[   scatterr   )r   )r[   rY   r   r   T)	broadcastc                    g | ]}\  }}} j         |gfd |D             R i t          fd|                                D                       dt          t	                                i                                ~S )c              3   ^   K   | ]'}                     t          |          |          V  (d S r9   r*   id)r+   arg	scattereds     r-   r`   z1test_manual_scatter.<locals>.<listcomp>.<genexpr>:  s7      BBcimmBsGGS11BBBBBBr/   c              3   h   K   | ],\  }}|                     t          |          |          fV  -d S r9   r  )r+   keyvaluer  s      r-   r`   z1test_manual_scatter.<locals>.<listcomp>.<genexpr>;  sT        (S% immBuIIu==>     r/   r	  )submitdictr2   strr	   result)r+   r   r:   r;   rP   r  s       r-   rL   z'test_manual_scatter.<locals>.<listcomp>7  s        )T4 BBBBTBBB       ,2LLNN      EGG   &((  r/   c              3   4   K   | ]}t          |          V  d S r9   )r   )r+   r6   s     r-   r`   z&test_manual_scatter.<locals>.<genexpr>M  s(      SS1ASSSSSSr/   .)i  r   )rj   r   r   r   r   r   r   rk   rl   	TypeErrorr   r  r   r  tupler   __version__split)rY   rS   r   expectedrn   rx   results_parallel%n_serialization_scatter_with_paralleln_serialization_with_parallelvarobjresults_nativen_serialization_scatter_nativedistributed_versionrP   r   r  r   r   r   s                 @@@@@@r-   test_manual_scatterr    sj    87eAhh777JAq!QA5555555599555E		!QQ!	!QQ!	!QQ! E
 GFFFFH	 
fq!AiLt,,, 		 !QCCC 4 4#98q#9#9#9%#@#@ '8333334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 y))  $V$JJJ                              		 		 		 		 		 		 		 		 		 		 		 		 		 		 		
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 -.G)7;;;;;7;;;;;$%G! 1a|  			 .fq!AiLt,,, 	.I!Qw I I%+^^C4^%H%H	"S''""     -2  N "X-----!	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	.. . . . . . . . . . . . . . .* &'W"74444474444404RRRRRSS0G0M0Mc0R0RSSSSSY&&
 w7!;;;;;;;w7777777s   #F>E5"D6E5DE5	D
E5&E9E	;EEEEEE5E""E5%E"&E5)F5E9	9F<E9	=FFFI18AII1I	I1!I	"I11I58I5c           	      ~  	 t          j        d          } |j        t          d          |j                  	 |j        t          d          |j                  }	gdz  |gdz  z   }t                      5 \  }\  }}t          |d         |           5 }t          d          5   t                      d	 t          |          D                        d d d            n# 1 swxY w Y   t          d
|          }||d                  ||d                  z   dk    sJ 	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t                      5 \  }\  }}t          |d         |           5 }t          d          5   t                      	fdt          d          D                        d d d            n# 1 swxY w Y   t          d
|          }||d                  dk    sJ ||d                  dk    sJ 	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   ry   r   rg   rW   rX   r   rZ   c              3   ^   K   | ](\  }} t          t                    ||||           V  )dS ))optNr   r<   )r+   r_   r   s      r-   r`   z$test_auto_scatter.<locals>.<genexpr>f  sS        4 "GDMM$aT:::     r/   r   r
   c              3   d   K   | ]*} t          t                    d d         |          V  +d S )Nrg   r"  )r+   r_   data1s     r-   r`   z$test_auto_scatter.<locals>.<genexpr>s  s=      II1=74==rrA66IIIIIIr/   r   r   )rk   r   r   r   r   r   r   r   r   	enumeraterU   rj   )
loop_in_threadr   data2data_to_processrn   ro   rp   rP   r   r$  s
            @r-   test_auto_scatterr)  [  s    		W	%	%BBGCHHBH---EBGCHHBH---Ew{w{3O	 Dkq&1aAiL~666 	D& 000   

  #,_#=#=                    ""8&AAF!I,'&9*>>!CCCCC	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	DD D D D D D D D D D D D D D D 
 -kq&1aAiL~666 	-& 000 J J

IIIIaIIIIIIJ J J J J J J J J J J J J J J ""8&AAF!I,'1,,,,!I,'1,,,,,	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	-- - - - - - - - - - - - - - - - - -s   9ED.(/C#D.#C''D.*C'+6D."E.D2	2E5D2	6EE	E	H2<H1G
>H
GHG;HH2H	H2!H	"H22H69H6retry_noc           	        	
 t          j        d          
dd}
fd		fdt                      5 \  }\  }}t          |d         |           5 }t	          d          5   
j        d	           t                      fd
t          |          D                       }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   rb   c                 $     j         |           S r9   )r   )r   r_   jr   s      r-   my_sumz#test_nested_scatter.<locals>.my_sum  s    rvayyr/   c                      t                      }t          d          5   t                       fdt                    D                       }d d d            n# 1 swxY w Y   t	          |          S )Nr   rZ   c              3   \   K   | ]&} t                    |d          |          V  'd S r9   rv   )r+   r-  arrayr_   r.  s     r-   r`   zEtest_nested_scatter.<locals>.outer_function_joblib.<locals>.<genexpr>  sP       ! !56abb	1a00! ! ! ! ! !r/   )r   r   r   rj   r   )r1  r_   rP   resultsNUM_INNER_TASKSr.  s   ``  r-   outer_function_joblibz2test_nested_scatter.<locals>.outer_function_joblib  s    V,,, 	 	 hjj ! ! ! ! ! !:?:P:P! ! !  G	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 7||s   3A!!A%(A%rW   rX   r   rZ   i'  c              3   Z   K   | ]%} t                    |d          |          V  &d S r9   rv   )r+   r_   my_arrayr4  s     r-   r`   z&test_nested_scatter.<locals>.<genexpr>  sT         3G1228ABB<CC     r/   )rk   r   r   r   r   r   r   rj   )rY   r*  NUM_OUTER_TASKSrn   ro   rp   rx   r3  r6  r.  r   r4  s          @@@@@r-   test_nested_scatterr8  {  s   		W	%	%BOO          
 kq&1aAiLt,,, 	 000  "275>>HJJ     "?33                  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	                 sZ   C#C&AB4(C4B88C;B8<C?C#C	C#C	C##C'*C'c           	      ,   d t                      5 \  }\  }}t          |d         |           5 }t          d          5   t          d          fdt	          d	          D                       }|D ]$}t          t          |                    dk    sJ %	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          |d         |           5 }t          d          5   t          d          fd
t	          d	          D                       }|D ]$}t          t          |                    dk    sJ %	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nc            	          t           t          d          d t          d          D                                 } | t           t          d          d t          d          D                                 z  } | S )Nr
   r   c              3   X   K   | ]%} t          t          j                              V  &d S r9   r   r(   getpidrw   s     r-   r`   zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>  s5      %M%Mq&8gbi&8&8&:&:%M%M%M%M%M%Mr/   c              3   X   K   | ]%} t          t          j                              V  &d S r9   r=  rw   s     r-   r`   zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>  s5      &N&N'9wry'9'9';';&N&N&N&N&N&Nr/   )setr   rj   )pidss    r-   get_nested_pidsz<test_nested_backend_context_manager.<locals>.get_nested_pids  s|    %81%%%%M%ME!HH%M%M%MMMNN&HA&&&&N&NU1XX&N&N&NNNOOOr/   rW   rX   r   rZ   r
   r;  c              3   F   K   | ]} t                                V  d S r9   rv   r+   rx   rB  s     r-   r`   z6test_nested_backend_context_manager.<locals>.<genexpr>  E       0 034,GO,,..0 0 0 0 0 0r/   rb   c              3   F   K   | ]} t                                V  d S r9   rv   rD  s     r-   r`   z6test_nested_backend_context_manager.<locals>.<genexpr>  rE  r/   )r   r   r   r   rj   rN   r@  )r&  rn   ro   rp   rP   
pid_groups	pid_grouprB  s          @r-   #test_nested_backend_context_managerrI    s'     
 
 4kq&1aAiL~666 	4& 000 4 4/XQ/// 0 0 0 08=b		0 0 0  
 ", 4 4Is9~~..!333334	4 4 4 4 4 4 4 4 4 4 4 4 4 4 4	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 AiL~666 	4& 000 4 4/XQ/// 0 0 0 08=b		0 0 0  
 ", 4 4Is9~~..!333334	4 4 4 4 4 4 4 4 4 4 4 4 4 4 4	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	44 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4s   F	C AB)C )B--C 0B-1C 4F	 C	F	C	F	"E13AEE1EE1!E"E1%F	1E5	5F	8E5	9F		FFc           
      $  	
 d 		fd
t                      5 \  }\  }}t          |d         |           5 }t          d          5  t                      5 } 	|          dk    sJ |j        dk    sJ  |
fd	t          d
          D                       }d d d            n# 1 swxY w Y   |D ]\  }}|dk    sJ |dk    sJ 	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nc                 $    | j         j        j        S r9   )r{   	__class__r   )ps    r-   _backend_typezJtest_nested_backend_context_manager_implicit_n_jobs.<locals>._backend_type  s    z#,,r/   c                  x    t                      5 }  |           | j        fcd d d            S # 1 swxY w Y   d S r9   )r   r   )rM  rN  s    r-   get_nested_implicit_n_jobszWtest_nested_backend_context_manager_implicit_n_jobs.<locals>.get_nested_implicit_n_jobs  s    ZZ 	.1 =##QX-	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	.s   /33rW   rX   r   rZ   r   r   c              3   F   K   | ]} t                                V  d S r9   rv   )r+   rx   rP  s     r-   r`   zFtest_nested_backend_context_manager_implicit_n_jobs.<locals>.<genexpr>  sF       * *BC; :;;==* * * * * *r/   r
   )r   r   r   r   r   rj   )rY   rn   ro   rp   rP   rM  all_nested_n_jobsbackend_typenested_n_jobsrN  rP  s            @@r-   3test_nested_backend_context_manager_implicit_n_jobsrU    sw   - - -. . . . . 
 /kq&1aAiLt,,, 
	/ 000 	/ 	/ZZ 1(=++/GGGGG8r>>>>() * * * *GLQxx* * * ) )%               4E / //L-'+CCCCC(B...../	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/
	/ 
	/ 
	/ 
	/ 
	/ 
	/ 
	/ 
	/ 
	/ 
	/ 
	/ 
	/ 
	/ 
	/ 
	// / / / / / / / / / / / / / / / / /s}   DC-CAB&	C&B**C-B*.C
C-CC-CC-!D-C1	1D4C1	5DD	D	c                    t          j        t                    5 }t          d          5  	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   dt	          |j                                                  v sJ d S )Nr   rZ   zcreate a dask client)rk   rl   r?   r   r  r
  lower)rY   infos     r-   test_errorsrY    s    	z	"	" dV,,, 	 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	               "S__%:%:%<%<<<<<<<s.   A9A=	A =	AAAc           	         t                      5 \  }\  }}t          |d         |           5 }t          d          5   t          d          d t	          d          D                       }t          |d	         d	         d	         t                    sJ 	 d d d            n# 1 swxY w Y   t          d          5   t          d          d
 t	          d          D                       }t          |d	         d	         d	         t                    sJ 	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NrW   rX   r   rZ   r
   r;  c              3   R   K   | ]"} t          t                    d           V  #d S )Nnested_requirer   outerrw   s     r-   r`   z.test_correct_nested_backend.<locals>.<genexpr>  sF       , ,<=NGENN$777, , , , , ,r/   r   r   c              3   R   K   | ]"} t          t                    d           V  #dS )	sharedmemr\  Nr^  rw   s     r-   r`   z.test_correct_nested_backend.<locals>.<genexpr>  sF       , ,CDNGENN+>>>, , , , , ,r/   )r   r   r   r   rj   r|   r   r   )rY   rn   ro   rp   rP   r  s         r-   test_correct_nested_backendrb    s   	 Ekq&1aAiLt,,, 	E 000 K K++++ , ,AFq, , ,   "&)A,q/3IJJJJJJ	K K K K K K K K K K K K K K K !000 E E++++ , ,HMa, , ,   "&)A,q/3CDDDDDD	E E E E E E E E E E E E E E E	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	EE E E E E E E E E E E E E E E E E Es}   ED>AB%D>%B))D>,B)-D> AD'D>'D++D>.D+/D>2E>E	EE	EEEc                 j      t          dd           fdt          d          D                       S )Nr
   threads)r   preferc              3   R   K   | ]!} t          t                              V  "d S r9   )r   middle)r+   rx   r]  s     r-   r`   zouter.<locals>.<genexpr>  sD       0 0,-''0 0 0 0 0 0r/   r   r   rj   r\  s   `r-   r_  r_    sP    /81Y/// 0 0 0 016q0 0 0   r/   c                 d     t          d|           d t          d          D                       S )Nr
   )r   requirec              3   N   K   | ] } t          t                                V  !d S r9   )r   innerrw   s     r-   r`   zmiddle.<locals>.<genexpr>  s0      .R.RA~wu~~/?/?.R.R.R.R.R.Rr/   r   rh  )rj  s    r-   rg  rg    s4    .81g....R.Rq.R.R.RRRRr/   c                  (    t                      j        S r9   )r   r{   r'   r/   r-   rl  rl    s    ::r/   c                    t          | dd          5  t          d          5   t          d          d t          d	          D                        d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )
NFT)rY   	processesset_as_defaultr   rZ   r   r;  c              3   P   K   | ]!} t          t                    |          V  "d S r9   )r   r  r^   s     r-   r`   z0test_secede_with_no_processes.<locals>.<genexpr>  s0      @@!{wr{{1~~@@@@@@r/   r
   )r   r   r   rj   rX   s    r-   test_secede_with_no_processesrr    s6   	TU4	@	@	@ A AV,,, 	A 	AHA@@uQxx@@@@@@	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	AA A A A A A A A A A A A A A A A A As4   A91A!A9!A%	%A9(A%	)A99A= A=c                 ,    ddl m}  |            j        S )Nr   )
get_worker)r   rt  rW   )rx   rt  s     r-   _worker_addressru    s#    &&&&&&:<<r/   c           	      ~   t                      5 \  }\  }}t          |d         |           5 }t          d|d                   5   t                      d t	          d          D                       }||d         gdz  k    sJ 	 d d d            n# 1 swxY w Y   t          d|d                   5   t                      d t	          d          D                       }||d         gdz  k    sJ 	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NrW   rX   r   )r[   workersc              3   P   K   | ]!} t          t                    |          V  "d S r9   r   ru  r^   s     r-   r`   z-test_dask_backend_keywords.<locals>.<genexpr>  5       P P!9!9!9!!<!< P P P P P Pr/   rb   c              3   P   K   | ]!} t          t                    |          V  "d S r9   ry  r^   s     r-   r`   z-test_dask_backend_keywords.<locals>.<genexpr>  rz  r/   r   rm   s         r-   test_dask_backend_keywordsr|    sF   	 2kq&1aAiLt,,, 	2 9FFF 2 2 hjj P PeBii P P PPPq|nr1111112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 !9FFF 2 2 hjj P PeBii P P PPPq|nr1111112 2 2 2 2 2 2 2 2 2 2 2 2 2 2	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	22 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s~   D2DABDBDBD5AD7DDD
DDD2D	D2!D	"D22D69D6c                    t          d|           5 }t          d          5   t                      d t          d          D                        d d d            n# 1 swxY w Y   t	                      }|j        j        j        r7t          d           t	                      |dz   k     sJ |j        j        j        7|j	        rJ 	 d d d            d S # 1 swxY w Y   d S )	NF)ro  rY   r   rZ   c              3   P   K   | ]!} t          t                    |          V  "d S r9   r]   r^   s     r-   r`   z/test_scheduler_tasks_cleanup.<locals>.<genexpr>  0      ::1|ws||A::::::r/   rb   g{Gz?r   )
r   r   r   rj   r   r   	schedulerr   r   futures)rY   rP   starts      r-   test_scheduler_tasks_cleanupr  
  si   	%d	+	+	+ 	"vV,,, 	; 	;HJJ::b		::::::	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; n&, 	&$KKK66EAI%%%% n&, 	& >!!!!	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"s5   C/ACA"	"C%A"	&A"CCCcluster_strategyadaptivelate_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)reasonc                    t          ddd          }t          |          }| dk    r |j        dd           n| dk    r |j        d           	 t	          d	          5   t                      d
 t          d          D                        d d d            n# 1 swxY w Y   |                                  |j                     d S # |                                  |j                     w xY w)Nr   Fr
   r   ro  r   r  )minimummaximumr  r   rZ   c              3   P   K   | ]!} t          t                    |          V  "d S r9   r]   r^   s     r-   r`   z(test_wait_for_workers.<locals>.<genexpr>+  r  r/   rb   )r   r   adaptscaler   r   rj   r   )r  r   rP   s      r-   test_wait_for_workersr    s?    Q%ANNNGG__F:%%a+++++	^	+	+
 	aV,,, 	; 	; HJJ::b		::::::	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	;
 	 	s0   C "/BC B!!C $B!%C %C2c                     t          ddd          } t          |           }	 t          dd          5  d}t          j        t
          |	          5   t                      d
 t          d          D                        d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          dd          5  d}t          j        t          |	          5   t                      d t          d          D                        d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |	                                  | j	                     d S # |	                                  | j	                     w xY w)Nr   Fr
   r  r   g?)r[   wait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)matchc              3   P   K   | ]!} t          t                    |          V  "d S r9   r]   r^   s     r-   r`   z0test_wait_for_workers_timeout.<locals>.<genexpr>:  0      >>q<73<<??>>>>>>r/   rb   z+DaskDistributedBackend has no active workerc              3   P   K   | ]!} t          t                    |          V  "d S r9   r]   r^   s     r-   r`   z0test_wait_for_workers_timeout.<locals>.<genexpr>@  r  r/   )
r   r   r   rk   rl   TimeoutErrorr   rj   RuntimeErrorr   )r   rP   msgs      r-   test_wait_for_workers_timeoutr  1  sj   Q%ANNNGG__FVcJJJ 	? 	?KC|3777 ? ?

>>E"II>>>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ?	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? VaHHH 	? 	??C|3777 ? ?

>>E"II>>>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ?	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	 	s   E  B$/BB$B	B$B	B$E  $B((E  +B(,E   D0/DD0D	D0 D	!D0$E  0D44E  7D48E   %Fr[   lokymultiprocessingc                     t          d          }t          |          }	  fd}|                    |          }|                                }t	          |          dk    sJ |d         j        }t          |t                    sJ dt          |          v sJ 	 |	                    d            |j	        d           d S # |	                    d            |j	        d           w xY w)	Nr
   )r   c                      t          j        d          5 }  t          d          d t          d          D                        d d d            n# 1 swxY w Y   | S )NT)recordr
   )r   r[   c              3   P   K   | ]!} t          t                    |          V  "d S r9   r]   r^   s     r-   r`   zftest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallel.<locals>.<genexpr>R  s0      3W3WLGCLLOO3W3W3W3W3W3Wr/   rb   )warningscatch_warningsr   rj   )r  r[   s    r-   func_using_joblib_parallelzStest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallelL  s    
 (555 X373333W3WUSUYY3W3W3WWWWX X X X X X X X X X X X X X X Ms   2AAAr   r   zdistributed.worker.daemonr   r   )
r   r   r  r  rN   messager|   UserWarningr  r   )r[   r   rP   r  futr  warnings   `      r-   /test_joblib_warning_inside_dask_daemonic_workerr  F  s   Q'''GG__F"	 	 	 	 	 mm6776{{a)#';/////*c'll:::::R   b!!!!!! 	R   b!!!!!s   A:C )C/)r=   )r   r   )P
__future__r   r   r   r(   r  r   r   r   uuidr	   rk    r   r   r   r   _daskr   r}   r   r   commonr   r   test_parallelr   r   r   r   r   r   r   r   r   distributed.metricsdistributed.utils_testr   r   r    fixturer7   r<   rB   rU   rr   r   markparametrizer   r   r   r   r   r   r   objectr   r   r  r)  rO   rj   r8  rI  rU  rY  rb  r_  rg  rl  rr  ru  r|  r  skipifr  r  r  r  r'   r/   r-   <module>r     s   @ @ @ @ @ @ @ @ @ @ 				                     C C C C C C C C C C C C * * * * * * : : : : : : : : " " " " " " " "          "f!-00v6"" 9 8 8 8 8 8 8 8 8 8 $ $ $ $ $ $ 9 8 8 8 8 8 8 8 8 8 j$///  0/"	 	 	8 8 8 8  : : : > > >. Ar7++_6F$GHHO O IH ,+O
 Ar7++{4I&JKK_6F$GHH? ? IH LK ,+ ?
 _6F$GHH
 
 IH 
*     4-00@ @ 10@24" 4" 4"n, , , , ,f , , ,   P8 P8 P8l- - -@ TT%%((^^44  5464 4 42/ / /2= = =E E E$  S S S  A A A     	2 	2 	2
" 
" 
" +j.-IJJw&N;+Bh+NL    	  KJ
*  * V->$?@@" " A@" " "r/   