.Ph1 XdZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl mZddlmZddlmZddlmZddlZddlZdd lmZmZmZdd lmZdd lmZmZmZmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%edd l&m'Z'ddl(m)Z) ddl*Z*n #e+$rdZ*YnwxYw ddl,m-Z-n #e+$rdZ-YnwxYw ddl.Z.n #e+$rdZ.YnwxYwddl/m0Z0m1Z1m2Z2m3Z3m4Z4ddl5m6Z6m7Z7m8Z8m9Z9m:Z:mZm;Z;mZ?e?@dddgeAe6jBzZCeCde6Dz ZCegZDnddgZDeDdgzZEeFedr#eCGejHddZIdZJdZKdZLGddeMZNdd ZOd!ZPdd"ZQd#ZRd$ZSd%ZTd&ZUe"d'eZ]d?Z^e"d@e6e"dAe6dBZ_dCZ`edDZadEZbe"d1gdFdGZce"d0eEdHZdedIZeeee"dJgdKe"dLddMgdNZfe"d0eEdOZgee"d0eEdPZhee"d0eie?dQhz e"dRdSdTgdUZjee"d0eDdVZkee"d0e6dWZldXZme"d0e6e"dYdgdZfd[gd\fgd]Znee"d0eEd^Zod_Zpee"d0eDd`ZqdaZrdbZsdcZtee"d0eEejVuddedfZvGdgdhe3ZwdiZxe"d0eCdjZyee"d0eEe"d1gdkdlZzdmZ{dnZ|e$edudoedpZ}dqZ~eEddZedredsDee"d0ee"d'e;er/Rs#KKK;x ,..KKKloky threading get_contextspawnc0ttjSN)r!r DEFAULT_BACKENDr+r0r.get_default_backend_instancer8^s H, --r0c Bt|dt|ddS)N_pool_workers)getattrbackends r. get_workersr?fs 7GWWj$%G%G H HHr0c ||z Sr6r+)xys r.divisionrCjs q5Lr0c |dzSNr+rAs r.squarerHns a4Kr0ceZdZdZdZdS)MyExceptionWithFinickyInitz,An exception class with non trivial __init__cdSr6r+)selfabcds r.__init__z#MyExceptionWithFinickyInit.__init__us r0N__name__ __module__ __qualname____doc__rQr+r0r.rJrJrs)66     r0rJFcH|dkr|rtddddnt|S)NrMrNrOrP)rJ ValueError)rAcustom_exceptions r.exception_raiserr[ys8Avv   &sCc : : : Hr0c8tjdt)Ng?)timer KeyboardInterruptrGs r.interrupt_raiserr_sJt r0c|dz|z|zS)zNA module-level function so that it can be spawn with multiprocessing. rFr+)rArBzs r.frbs a4!8a<r0cNttjdSNr)typer get_active_backendr+r0r._active_backend_typergs +--a0 1 11r0cdt||dtdDS)Nn_jobsr>c3PK|]!}tt|V"dSr6r$rHr,is r. z parallel_func..sC:: ::::::r0r"range) inner_n_jobsr>s r. parallel_funcrtsE 98< 9 9 9::$)!HH:::  r0c.tdksJdSrd)r#r+r0r.test_cpu_countrvs ;;??????r0c.tdksJdSrdr%r+r0r.test_effective_n_jobsrys    ! ! ! ! ! !r0contextzbackend_n_jobs, expected_n_jobs)rprprjNr)z positive-intz negative-intNone)idsc|d|5td|ksJ dddn #1swxYwYtddksJdS)Nr2r|rrx)rzbackend_n_jobsexpected_n_jobss r.test_effective_n_jobs_Noners ^ 4 4 4@@ t,,,?????@@@@@@@@@@@@@@@ 4 ( ( (A - - - - - -s 266r>rj)rrFr{verbose)rF dcdtdDt|||dtdDksJdS)Nc,g|]}t|Sr+rHr,rAs r.r/z(test_simple_parallel..s ( ( (!F1II ( ( (r0rjr>rc3PK|]!}tt|V"dSr6rlrs r.roz'test_simple_parallel..s2++Qogfooa++++++r0rrr")r>rjrs r.test_simple_parallelrsw ) (uQxx ( ( (-Hw---++%((+++-,-, , , , , , ,r0rFcd}tjd}Gddt}|||d}|dt|Dt }|jD]H}||x}r/|\} } || | z It|dksJdS) Nrz(Done\s+\d+ out of \d+ \|)ceZdZgZdZdS)/test_parallel_pretty_print..ParallelLogc:|j|dSr6)messagesappend)rLmsgs r._printz6test_parallel_pretty_print..ParallelLog._prints M  % % % % %r0N)rSrTrUrrr+r0r. ParallelLogrs( & & & & &r0ri'rcHg|]}tt| Sr+r$rbrms r.r/z.test_parallel_pretty_print..s& 4 4 4jgajjmm 4 4 4r0r) recompiler"rrsetrsearchspanaddlen) r>rjn_taskspatternrexecutorlensmessagesrMrNs r.test_parallel_pretty_printrsGj677G&&&&&h&&& {&'5IIIH H 4 4U7^^ 4 4 4555 55D$w'' '1 6688DAq HHQUOOO t99>>>>>>r0c|tjddtjd5}t d|dt d D}|gd ksJ dddn #1swxYwYd |D}|d vst|trd fd|D}t|dksJdS)Nname!some_new_name_for_the_main_thread)targetrvalueTrecordrFric3PK|]!}tt|V"dSr6rlrs r.roz6test_main_thread_renamed_no_warning..sC6 6 #$OGFOOA  6 6 6 6 6 6 r0rp)rrcrg|]4}dt|jvt|jt2|5S)zworker timeout)strr isinstanceDeprecationWarningr,ws r.r/z7test_main_thread_renamed_no_warning..sJ 3qy>> 1 119&899 2 1 1 1r0)Nr)z3multi-threaded, use of fork() may lead to deadlocksc@g|]}t|jv|Sr+)rr)r,r message_parts r.r/z7test_main_thread_renamed_no_warning..s+NNN! monkeypatchwarninforesultsrs @r.#test_main_thread_renamed_no_warningrsw ')) 1   - - -$5(!W5556 6 (-a6 6 6   )))##### $$$$$$$$$$$$$$$H+++z'00+M NNNNxNNN x==A      srscg|] }|j Sr+)rrs r.r/z*_assert_warning_nested..s,,,a ,,,r0c34K|]}d|jdvVdS)zbacked parallel loops cannotrN)args)r,eachs r.roz)_assert_warning_nested.. s?''CG.$)A,>''''''r0rF)rr simplefilterrtallrr)r>rsexpectedrwarnings_are_correctwarnings_have_the_right_lengths r._assert_warning_nestedrs<   - - -Bh'''gLAAAABBBBBBBBBBBBBBB-,8,,,H  K#&''KS'''$$ '6MH ""3x==A;M +(J,J Juts&AA A z%parent_backend,child_backend,expected))r1r)T)r1r1F)r)r)T)r)r1T)r2r)T)r2r1Tc,td|fdtdDtd|fdtdD}|dkrt|sJdSt|sJdS)NrFric3XK|]$}ttddV%dS)rFr>rsrNr$rr,_ child_backends r.roz0test_nested_parallel_warnings..(sZ//  (&''!E   //////r0rc3XK|]$}ttdV%dS)rFrNr)r,rrrs r.roz0test_nested_parallel_warnings..0sZ55  (&''!H   555555r0r2)r"rranyr)parent_backendrrress `` r.test_nested_parallel_warningsrs/HA~...////q /// 5(!^ 4 4 455555q 555  C$$3xx3xxr0)r1r)r2cdg}d}tj||f}|||dsJdS)NFc$tjd5}tddtdDdddn #1swxYwYt t |t |dk|d<dS)NTrrFr|c3PK|]!}ttdV"dS)皙?Nr$r r,rs r.rozPtest_background_thread_parallelism..background_thread..Fs2EEq~wu~~c22EEEEEEr0rr)rrr"rrprintr)is_run_parallelrs r.background_threadz=test_background_thread_parallelism..background_threadDs  $D 1 1 1 FX HA   EEE!HHEEE E E E F F F F F F F F F F F F F F F c(mm ]]a/s1AAA)rrr)r2Threadstartjoin)r>rrts r."test_background_thread_parallelismr?sfgO000  18JKKKAGGIIIFFHHH 1 r0chtd|dtdDdS)NrFric3PK|]!}ttdV"dS{Gz?Nrlrs r.roznested_loop..Qs2'P'P!(=(='P'P'P'P'P'Pr0rqr=s r. nested_looprPs:'HAw''''P'PuQxx'P'P'PPPPPPr0rrcntd|fdtdDdS)NrFric3RK|]!}ttV"dSr6)r$rrs r.roz#test_nested_loop..WsG//01 ]++//////r0rq)rrs `r.test_nested_looprTs\/HA~...////5:1XX///r0ctr6)rYr=s r.raise_exceptionr\s r0c 8tt5tdd5}|ttdtt dgdddn #1swxYwYddddS#1swxYwYdS)NrFr1ri)rrYr"r$rr)r s r.)test_nested_loop_with_exception_with_lokyr`s8   WW Q / / / W8 H*gk**6224LGO4L4LV4T4TU V V V W W W W W W W W W W W W W W WWWWWWWWWWWWWWWWWWWs5BAA7+ B7A; ;B>A; ?BBBctdtddfdtdDsJdS)z1Input is mutable when using the threading backendr)maxsizerFr2ric3RK|]!}tjdV"dSrN)r$put)r,rqs r.roz1test_mutate_input_with_threads..js5+P+P!NGAENN1,=,=+P+P+P+P+P+Pr0N)rr"rrfull)rs@r.test_mutate_input_with_threadsrgs` aA+HA{++++P+P+P+PuQxx+P+P+PPPP 6688OOOOOr0)rrFrpctd}d|Dt|d|DksJdS)z.Check the keyword argument processing of pmap. c0g|]}t|dSrrBrbrs r.r/z(test_parallel_kwargs..rs" # # #!Aa1III # # #r0r|c3TK|]#}tt|dV$dSrrNrrs r.roz'test_parallel_kwargs..rsH??  1??????r0Nr)rjlsts r.test_parallel_kwargsrnst ))C # #s # # #'>xv'>'>'>??$'???((      r0ctd}d|D}td|5}|j}||d|DksJ||d|DksJt%t |t |jusJdddn #1swxYwYtt |jJ||d|DksJtt |jJdSdS)Nrc0g|]}t|dSrrrs r.r/z4test_parallel_as_context_manager..zs"'''a!q '''r0rric3TK|]#}tt|dV$dSrrrs r.roz3test_parallel_as_context_manager..7==AZWQZZQ///======r0c3TK|]#}tt|dV$dSrrrs r.roz3test_parallel_as_context_manager..r r0c3TK|]#}tt|dV$dSrrrs r.roz3test_parallel_as_context_manager..s799A+++999999r0)rrr"_backendrr?)r>rrpmanaged_backends r. test_parallel_as_context_managerrws ))C''3'''H G , , , K*11===========11=========== >//;qz3J3JJJJJ K K K K K K K K K K K K K K K  ~1:&&... qq99S99999 9 9 9 9 ~1:&&...~..sA*B##B'*B'cGddtttd5tddfdt d Dd d d d S#1swxYwYd S) z`Check that pmap captures the errors when it is passed an object that cannot be pickled. ceZdZdZdS)1test_parallel_pickling..UnpicklableObjectc td)N123) RuntimeErrorrLs r. __reduce__z.UnpicklableObject.__reduce__su%% %r0N)rSrTrUrr+r0r.UnpicklableObjectrs# & & & & &r0rzthe task to sendmatchrFr1ric3bK|])}ttV*dSr6r$id)r,rrs r.roz)test_parallel_pickling..sN+ + 12KGBKK))++ , ,+ + + + + + r0rN)objectrrr"rr)rs@r.test_parallel_picklingr s &&&&&F&&&  %8 9 9 9  *6***+ + + + 6;Bii+ + +                      s4A,,A03A0 byteorder)<>= max_nbytes1Mcdtjdd|djj}t dd|fdtd D}|D]?\}}||ksJ||jjksJtj |@dS) Nc||jjfSr6)dtyper!rGs r.inspect_byteorderz=test_parallel_byteorder_corruption..inspect_byteorders!'###r0)rFrpi4rFr1)rjr>r%c3HK|]}tVdSr6r$)r,rr*rAs r.roz5test_parallel_byteorder_corruption..sPGG*+"!""1%%GGGGGGr0rp) rarangereshapeviewr)r!r"rrtestingassert_array_equal)r!r%initial_np_byteorderresult x_returnedbyteorder_in_workerr*rAs @@r."test_parallel_byteorder_corruptionr8s $$$ ! V$$))Y*:*:*:;;A7, FXQ: F F FGGGGG/4QxxGGGF,255' '"&:::::"j&6&@@@@@ %%a444455r0cttd|ddtdDdksJdS)NrFrjr>timeoutc3PK|]!}ttdV"dS)gMbP?Nrrs r.roz0test_parallel_timeout_success..sC<<*+u%%<<<<<<r0r)rr"rrr=s r.test_parallel_timeout_successr>st ;HAw ; ; ;<</4Ryy<<<           r0ctt5td|ddtdDddddS#1swxYwYdS)NrFrr;c3PK|]!}ttdV"dSrNrrs r.roz-test_parallel_timeout_fail..sC: : #$NGENN2  : : : : : : r0r)rrr"rrr=s r.test_parallel_timeout_failrBs     97D999: : (-b : : :                      3AAA sequential return_as generatorgenerator_unorderedc ^tt5ttd||ddt dDdddn #1swxYwYttd||ddt dDdS)NrFr)rjr>rEr<c3PK|]!}ttdV"dSrArrs r.roz.sKRR'(r""RRRRRRr0rc3PK|]!}ttdV"dSrrrs r.roz.sKM M %&NGENN4 M M M M M M r0)rrlistr"rrr>rEs r.)test_parallel_timeout_fail_with_generatorrMs%      QHAw)S Q Q QRR,1"IIRRR                     L7iLLLM M */))M M M   sAA""A&)A&c ttt5td|dt ddDdddn #1swxYwYtt 5td|ddDdddn #1swxYwYtd|5}t |jJt |j}tt5|dt ddDdddn #1swxYwYt |jJt |j|usJdtd D|d td DksJt |j}tt 5|d dDdddn #1swxYwYt |jJt |j|usJd td D|d td Dks"J|j |j |j |j f dddn #1swxYwYt |jJnOtt 5tdddDdddn #1swxYwYtt5tddt ddDdddn #1swxYwYtt5tdddtdDddddS#1swxYwYdS)NrFricPg|]#\}}tt||$Sr+r$rCr,rArBs r.r/z&test_error_capture..s1IIITQ"""1a((IIIr0)rr)rrcHg|]}tt| Sr+r$r_rs r.r/z&test_error_capture..s,>>>!*)**1-->>>r0cPg|]#\}}tt||$Sr+rPrQs r.r/z&test_error_capture..s1RRRda+'(++Aq11RRRr0c0g|]}t|dSrrrs r.r/z&test_error_capture.."111!Aa1III111r0rc3TK|]#}tt|dV$dSrrrs r.roz%test_error_capture..H>>'(  1""">>>>>>r0cHg|]}tt| Sr+rSrs r.r/z&test_error_capture.. s,GGG13'"233A66GGGr0c0g|]}t|dSrrrs r.r/z&test_error_capture..rVr0c3TK|]#}tt|dV$dSrrrs r.roz%test_error_capture..rXr0r|cHg|]}tt| Sr+rSrs r.r/z&test_error_capture.."s,MMM 9(8 9 9! < <MMMr0cPg|]#\}}tt||$Sr+rPrQs r.r/z&test_error_capture..'s1TTT1-GH--a33TTTr0r)rjrc3TK|]#}tt|dV$dS)T)rZNr$r[rms r.roz%test_error_capture..+s; T TQ &W% & &q4 @ @ @ T T T T T Tr0r:)rrZeroDivisionErrorr"zipr^r?r rr _iteratingn_completed_tasksn_dispatched_tasks _abortingrJ)r>r original_workerss r.test_error_capturergs~  ~ % & &   /HAw / / /IIS5H5HIII                   % & &   /HAw / / />>v>>>                   Q 0 0 0# Hx011===*8+<== )** T TRRc&&>Q>QRRRSSS T T T T T T T T T T T T T T T x011===x0119IIIII11uRyy111XX>>,1"II>>>66 +8+<== )** I IGGGGGHHH I I I I I I I I I I I I I I Ix011===x0119IIIII11uRyy111XX>>,1"II>>>66#*+" 9# # # # # # # # # # # # # # # N8,--5555 % & & O O HA   MMfMMM N N N O O O O O O O O O O O O O O O ! " "VVTTFF@S@STTTUUUVVVVVVVVVVVVVVV * + +  %1%%% T T%PR)) T T T                     s3AA #A ;%B,,B03B0?J$D7+ J7D; ;J>D; ?BJG9- J9G= =JG= BJJ"%J"$LL L#2M!!M%(M%2N??OOc0d d}td|5}ttd5|d|dDdddn #1swxYwYttd5|d |d Ddddn #1swxYwYttd5|d |d Ddddn #1swxYwYddddS#1swxYwYdS)Nrc3`KtdD]}||krtd|VdS)NIterator Raising Error)rrrY)raise_atrns r. my_generatorz1test_error_in_task_iterator..my_generator2sHr  AH}} !9:::GGGG  r0rFrirkrc3PK|]!}tt|V"dSr6rlrms r.roz.test_error_in_task_iterator..;2 C CQogfooa  C C C C C Cr0)rlc3PK|]!}tt|V"dSr6rlrms r.roz.test_error_in_task_iterator..@ror0rc3PK|]!}tt|V"dSr6rlrms r.roz.test_error_in_task_iterator..Ds2 D DQogfooa  D D D D D Dr0r)r"rrY)r>rmrs r.test_error_in_task_iteratorrt/s G , , , E J&> ? ? ? D D A C C,,*B*B*B C C C C C C D D D D D D D D D D D D D D D J&> ? ? ? D D A C C,,*B*B*B C C C C C C D D D D D D D D D D D D D D DJ&> ? ? ? E E A D D,,*C*C*C D D D D D D E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E Es|D  A D A D  A !D : B& D &B* *D -B* .D  C3' D 3C7 7D :C7 ;D  DDc6|d|zdS)Nz Consumed %s)r)queueitems r.consumerrxGs LL%&&&&&r0zbatch_size, expected_queue) Produced 0 Consumed 0 Produced 1 Consumed 1 Produced 2 Consumed 2 Produced 3 Consumed 3 Produced 4 Consumed 4 Produced 5 Consumed 5r) ryr{r}rrzr|r~rrrrrctfd}td||fd|D|ksJtdksJdS)z=Test that with only one job, Parallel does act as a iterator.c3hKtdD]}d|z|VdSNr+z Produced %irrrrnrvs r.producerz'test_dispatch_one_job..producerxHq  A LL* + + +GGGG  r0rrj batch_sizer>c3TK|]"}tt|V#dSr6r$rx)r,rArvs r.roz(test_dispatch_one_job..}sI??()%##??????r0 N)rKr"r)r>rexpected_queuerrvs @r.test_dispatch_one_jobrKsV FFE ?HA*g>>>????-5XZZ??? N " " " " u::      r0ctj}|fd}tddd|fd|Dt}|ddksJ|d d d }|d ksJ|d }||ksJt dksJd S)zRCheck that using pre_dispatch Parallel does indeed dispatch items lazily. c3hKtdD]}d|z|VdSrrrs r.rz/test_dispatch_multiprocessing..producerrr0rFrrp)rjr pre_dispatchr>c3TK|]"}ttdV#dS)rNr)r,rrvs r.roz0test_dispatch_multiprocessing..sQFF,-%''FFFFFFr0rryNrz Consumed anyr{rr)rManagerrKr"indexr)r>managerrqueue_contentsfirst_consumption_indexproduced_3_indexrvs @r.test_dispatch_multiprocessingrs' jllG LLNNE FHA!!WEEEFFFF19FFF%[[N !  , , , ,-RaR066~FF "R ' ' ' '%++L99 5 5 5 5 5 u::      r0ctddd5}|dtdD|jdksJ ddddS#1swxYwYdS)NrFautor2rc3PK|]!}tt|V"dSr6rrms r.roz/test_batching_auto_threading..0 . .Q+'"++a.. . . . . . .r0rr"rrr compute_batch_size)rs r.test_batching_auto_threadingrs v{ C C C4q  . .%++ . . ....z,,..!33333444444444444444444AA##A'*A'ctdd|5}|dtdD|jdksJ ddddS#1swxYwYdS)NrFrrc3PK|]!}tt|V"dSr6rrms r.roz2test_batching_auto_subprocesses..rr0rrr)r>rs r.test_batching_auto_subprocessesrs vw ? ? ?31  . .%++ . . .... z,,..22222 333333333333333333rctt5tddddtdDddddS#1swxYwYdS)zCMake sure that exception raised during dispatch are indeed capturedrFr)rjrrc3PK|]!}tt|V"dSr6r_rms r.roz*test_exception_dispatch..sG7 7 -. %G$ % %a ( (7 7 7 7 7 7 r0r:N)rrYr"rrr+r0r.test_exception_dispatchrs     6A6667 7 27))7 7 7                      rCcftddtdDdS)NrFr|c3PK|]!}tt|V"dSr6r_r,js r.roz(nested_function_inner..s6GG0w/0033GGGGGGr0r:rqrns r.nested_function_innerrs8HAGGU2YYGGGGGGGGr0cftddtdDdS)NrFr|c3PK|]!}tt|V"dSr6)r$rrs r.roz(nested_function_outer..s6LLQ5w455a88LLLLLLr0r:rqrs r.nested_function_outerrs8HALL%))LLLLLLLLr0z'https://github.com/joblib/loky/pull/255reasonctt5}td|dtdDdddn #1swxYwYt |j|j|j}d|}d|vsJd|vsJd |vsJt |jtusJdS) zEnsure errors for nested joblib cases gets propagated We rely on the Python 3 built-in __cause__ system that already report this kind of information to the user. rFric3PK|]!}tt|V"dSr6)r$rrms r.roz1test_nested_exception_dispatch..sG, , 23 *G) * *1 - -, , , , , , r0r:Nrrr[) rrYr"rrr rertbr)r>excinfo report_linesreports r.test_nested_exception_dispatchrs%    w+7+++, , 7tt5}td|dddn #1swxYwYdt |jvsJtt5}td|dddn #1swxYwYdt |jvsJtt5}td|dddn #1swxYwYdt |jvsJtt5}td|dddn #1swxYwYdt |jvsJdS)Nrriz&n_jobs == 0 in Parallel has no meaning?z2.3z$n_jobs could not be converted to int invalid_str)rrYr"_initialize_backendrr)r>rs r.test_invalid_njobsrs   Bw7+++??AAABBBBBBBBBBBBBBB 3s7=7I7I I I I I   DwW---AACCCDDDDDDDDDDDDDDD 3s7=7I7I I I I I   Fww///CCEEEFFFFFFFFFFFFFFF 1S5G5G G G G G   Nw w777KKMMMNNNNNNNNNNNNNNN 1S5G5G G G G G G GsG$AA  A <$B,,B03B0#$DDD $E::E>E>)2gffffff@rFct||}|dksJ|dtdD}tdt |DsJdS)NrirFc3PK|]!}tt|V"dSr6rlrms r.roz.test_njobs_converted_to_int..s2 2 21OGFOOA   2 2 2 2 2 2r0rc3BK|]\}}|t|kVdSr6r)r,rnrs r.roz.test_njobs_converted_to_int..s099$!QqF1II~999999r0)r"_effective_n_jobsrrr enumerate)r>rjrrs r.test_njobs_converted_to_intrs 000A   A % % % % ! 2 2b 2 2 2 2 2C 99)C..999 9 999999r0c tdtdtvsJtdtksJ td=dS#td=wxYw)N test_backend)r(rr!r+r0r.test_register_parallel_backendrse%!.2EFFF))))'+>>>>>> ^ $ $ $H^ $$$$$s 8A AcJtj}ttksJ t dt ddtt ksJ |t_n#|t_wxYwttksJdS)Nr2T) make_default)r r7rgr8r(r!r )default_backend_origs r.test_overwrite_default_backendr%s#3  ! !%A%C%C C C C C8!+x /DSWXXXX#%%)999999$8  #7 7777  ! !%A%C%C C C C C C Cs 6A22BzOnly without multiprocessingcpttd5tddtdDdddn #1swxYwYt d5tdtdDddddS#1swxYwYdS)Nz)joblib backend '.*' is not available on.*rr1r=c3PK|]!}tt|V"dSr6rlrms r.roz2test_backend_no_multiprocessing..4s2 F F!3!3 F F F F F Fr0rpc3PK|]!}tt|V"dSr6rlrms r.roz2test_backend_no_multiprocessing..8s288!?76??1%%888888r0)r UserWarningr"rrr'r+r0r.test_backend_no_multiprocessingr1sL {"M N N NGG    F FU1XX F F FFFFGGGGGGGGGGGGGGG  ( ( (99 88uQxx888888999999999999999999s#1AAA//B++B/2B/c||d5tj\}}|dksJtddksJt}|jdksJ|dkr6t |t usJt |jt usJn|dkr6t |tusJt |jtusJn|dkr6t |tusJt |jtusJnJ| dr5t |tusJt |jtusJddddS#1swxYwYdS)Nrpr|r)r1r2test_) r rfr%r"rjrerr rr startswithr)rz backend_nameactive_backend active_n_jobsrs r.check_backend_context_managerr;s a ( ( (;;(0(C(E(E% !!!!""a'''' JJx1}}}} , , ,''+AAAAA ##'====== V # #'';6666 ##{22222 [ ( (''+;;;;; ##'777777  $ $W - - ;''+>>>>> ##':::::#;;;;;;;;;;;;;;;;;;sE E((E,/E,cg|]}d|zS)ztest_backend_%dr+rms r.r/r/Qs(Q(Q(Q1):Q)>(Q(Q(Qr0rpc|tvr!|t|ttt ksJt |tt ksJt ddfdtDtt ksJdS)NrFr2ric3XK|]$}|tt|V%dSr6)r$r)r,rNrzs r.roz/test_backend_context_manager..csR,, ,.-..w::,,,,,,r0)r!setitemrrgr8rr" all_backends_for_context_manager)rr>rzs `r.test_backend_context_managerrTshHg/BCCC  ! !%A%C%C C C C C!'7333 ! !%A%C%C C C C C,HA{+++,,,,1,,, ! !%A%C%C C C C C C Cr0ceZdZdZddZdS)ParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.Nc6|td||_dS)Nzparam should not be None)rYparam)rLrs r.rQz%ParameterizedParallelBackend.__init__ps! =788 8 r0r6rRr+r0r.rrms.CCr0rch|tdttt ksJ|ddd5t j\}}t|tusJ|jdksJ|dksJt}|j dksJ|j |usJ|dtdD}dddn #1swxYwY|dtdDksJtt ksJdS)N param_backend*rp)rrjc3PK|]!}tt|V"dSr6r$rrms r.roz=test_parameterized_backend_context_manager..277MGDMM!$$777777r0rc,g|]}t|Sr+rrms r.r/z>test_parameterized_backend_context_manager..1111tAww111r0) rr!rrgr8r rfrerr"rjr rr)rrzrrrrs r.*test_parameterized_backend_context_managerrvs/3OPPP  ! !%A%C%C C C C C 1 5 5 588(0(C(E(E% N##'CCCCC#r))))!!!! JJx1}}}}z^++++!77eAhh77777888888888888888 11a111 1 1 1 1 ! !%A%C%C C C C C C CsB C((C,/C,c@ttksJ|tdd5tj\}}t |tusJ|jdksJ|dksJt}|jdksJ|j |usJ|dtdD}dddn #1swxYwY|dtdDksJttksJdS)N+)rrr|c3PK|]!}tt|V"dSr6rrms r.rozFtest_directly_parameterized_backend_context_manager..rr0c,g|]}t|Sr+rrms r.r/zGtest_directly_parameterized_backend_context_manager..rr0) rgr8rr rfrerr"rjr rr)rzrrrrs r.3test_directly_parameterized_backend_context_managerrs  ! !%A%C%C C C C C -B777 B B B88(0(C(E(E% N##'CCCCC#r))))!!!! JJx1}}}}z^++++!77eAhh77777888888888888888 11a111 1 1 1 1 ! !%A%C%C C C C C C CsB CCCcFtdtjS)Nr)r osgetpidr+r0r.sleep_and_return_pidrs #JJJ 9;;r0cttksJtdksJtddt dDS)NrrFr|c3NK|] }ttV!dSr6)r$rrs r.roz"get_nested_pids..s4PP!;g&:;;==PPPPPPr0)rgr r"rrrr+r0r.get_nested_pidsr so  ! !%5 5 5 5 5 :: ' ' ) )Q . . . . 81   PPuQxxPPP P PPr0c"eZdZdZfdZxZS) MyBackendz:Backend to test backward compatibility with older backendsc^tt|dSrd)superr get_nested_backend)rL __class__s r.rzMyBackend.get_nested_backends&Y%%88::1==r0)rSrTrUrVr __classcell__)rs@r.r r s>DD>>>>>>>>>r0r back_compat_backend)r2r1r)rc||5tddtdD}|D]$}tt|dksJ% ddddS#1swxYwYdS)NrFr|c3NK|] }ttV!dSr6)r$r rs r.roz6test_nested_backend_context_manager..s3'V'Vq(@(@(@(B(B'V'V'V'V'V'Vr0rr)r"rrrr)rzr> pid_groups pid_groups r.#test_nested_backend_context_managerrs   ,,'XQ''''V'VERTII'V'V'VVV # , ,Is9~~&&!+++++ ,,,,,,,,,,,,,,,,,,,sAA22A69A6)rFr{Nc*dtdfdtdD|5tdfdtdDddddS#1swxYwYdS)Nctt|ksJt|}t|ksJdSr6)rgr!r%r"r)expected_backend_typeexpected_n_jobs r.check_nested_backendz?test_nested_backend_in_sequential..check_nested_backendsW$%%2G)HHHHH*.99zz++--??????r0rr|c3^K|]'}ttjdV(dSr)r$r r7)r,rrs r.roz4test_nested_backend_in_sequential..sMGH%$%%h&>BBr0rc3JK|]}tVdSr6r.)r,rr>rrjs r.roz4test_nested_backend_in_sequential..sJ  ?@ )G( ) )'6 : :      r0rq)r>rjrzrs`` @r.!test_nested_backend_in_sequentialrs5@@@HALQRTII  ( ( (        DI"II                        s5BB B c||5}|tur|d}|tur|d}|j|ksJ ddddS#1swxYwYdS)Nr>r)r'r& nesting_level)rz inner_backendexpected_levelctxr>s r.check_nesting_levelr$s   73 o % %)nG & & &!fG$66666 777777777777777777s0A  AA outer_backendr!cBt|dtd|fdtdDd5tfdtdDddddS#1swxYwYdS)NrrFric3VK|]#}ttdV$dSrr$r$r,rrzr!s r.roz-test_backend_nesting_level..sL..DE$#$$WmQ??......r0rr|c3VK|]#}ttdV$dSrr(r)s r.roz-test_backend_nesting_level..sL  HI (G' ( (- C C      r0)r$r"rr)rzr%r!s` `r.test_backend_nesting_levelr+s/ 222-HA}---.....INr... q ) ) )        MRSUYY                        s2BBBwith_retrieve_callbackTcddlGfddt}td|d|d5}tdfdt d D|t ur|d jd ksJ|tur|djd ksJddddS#1swxYwYdS) Nrc8eZdZdZZjdZdS))test_retrieval_context..MyBackendrc32K|xjdz c_dVdSr}rrs r.retrieval_contextz;test_retrieval_context..MyBackend.retrieval_context s" FFaKFF EEEEEr0N)rSrTrUrnsupports_retrieve_callbackcontextmanagerr1) contextlibr,sr.r r/s= %;"  "   # "   r0r  retrievalcbtddt|DS)NrFr|c3PK|]!}tt|V"dSr6rrms r.roz>test_retrieval_context..nested_call..s0!C!CQ+'"++a..!C!C!C!C!C!Cr0rq)ns r. nested_callz+test_retrieval_context..nested_calls2!xq!!!!C!C%((!C!C!CCCCr0rFr|c3HK|]}t|VdSr6r.)r,rnr9s r.roz)test_retrieval_context..s6EEq/7;//22EEEEEEr0rr>r)r4r r(r"rrr'rnr&)rzr,r r#r4r9s ` @@r.test_retrieval_contextr;s\$k9555DDD   !EEEEE!HHEEEEEE o % %y>#q(((( & & &q68q==== !!!!!!!!!!!!!!!!!!sA+B44B8;B8r)rr{gQ?ctt5t|ddddS#1swxYwYdS)Nr)rrYr"r=s r.test_invalid_batch_sizer>s   ((J''''((((((((((((((((((s 377z)n_tasks, n_jobs, pre_dispatch, batch_size) )rFrFrr)rFrFrjrrrFrjr)irFrjrr?)rrrjr)rrjr)r 2 * n_jobsr)rrr)rCrrBrX)r@rrBrc|||d}dt|D}tdi|dt|D}||ksJdS)N)rjrrc,g|]}t|Sr+rrms r.r/z0test_dispatch_race_condition..:s222aq 222r0c3PK|]!}tt|V"dSr6rlrms r.roz/test_dispatch_race_condition..;s2 L L!3!3 L L L L L Lr0r+r)rrjrrparamsrrs r.test_dispatch_race_conditionrH%su( J W WF225>>222H h    L LU7^^ L L LLLG h      r0ctj}tdd}|jd}|}||ksJdS)NrFr)rirz)rget_start_methodr"_backend_kwargsget)mp_start_methodrrz start_methods r.test_default_mp_contextrO?sa)++O#4555A##I..G++--L ? * * * * * *r0c@|dkrtjd}tjd}|ddtjjtd|fdtdDdS)Nr)r4rirFric3hK|],}ttjjV-dSr6)r$rdotT)r,rnrMs r.rozAtest_no_blas_crash_or_freeze_with_subprocesses..^s9'R'RA13(?(?'R'R'R'R'R'Rr0) rr3rrandom RandomStaterandnrRrSr"rr)r>rngrMs @r..test_no_blas_crash_or_freeze_with_subprocessesrXHs###.)) )   # #C $AF1acNNN(HAw''''R'R'R'Rq'R'R'RRRRRRr0afrom joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) c|dkr+tjdkrtjdt|}t tjd|gdddS)Nr)forkzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr\[0, 1, 4, 9, 16\]r< stdout_regex) rrJpytestskip,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINformatrsys executable)r>codes r.2test_parallel_with_interactively_defined_functionsreqs ###(;(=(=(G(G  .    8 > >w G GD t$b?Tr0aimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) z!def square(x): return x ** 2 zcdef gen_square(): def square(x): return x ** 2 return square square = gen_square() zsquare = lambda x: x ** 2 define_funccallable_position)r$rkwargsc |dvr/|tkstjdkrtjdt |||tj tj tj }| d}| |ttj|jgdddS) N)r)r4win32zNot picklable with pickle)rfr>rgjoblib_root_folderzunpicklable_func_script.pyrr[r\) SQUARE_MAINrbplatformr^r_)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINrarpathdirnamejoblib__file__rwriterrcstrpath)r>rfrgtmpdirrd code_files r.0test_parallel_with_unpicklable_functions_in_argsrws...{""clg&=&= /000 4 ; ;+7??27??6?+K+KLL <  D  899I OOD *+*r0aimport sys import faulthandler # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # To make debugging easier faulthandler.dump_traceback_later(30, exit=True) # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(backend="loky", n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) )rkc|d}|tttj|jgdddS)Nz(joblib_interactively_defined_function.pyr[)r]r<)rrs5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrrbrcrtruscripts r.7test_parallel_with_interactively_defined_functions_lokyr|s\ [[C D DF LLFGGG (*r0aimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(backend="loky", n_jobs=2)( delayed(l.append)(i) for i in range(3) )) c|d}|tttj|jgddddS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\] LokyProcess)r] stderr_regexr<)rrs9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrrbrcrtrzs r.:test_parallel_with_interactively_defined_bound_method_lokyrs] [[D E EF LLJKKK (,# r0cbtg}td|gksJdS)NrFr|)iterr")exhausted_iterators r.%test_parallel_with_exhausted_iteratorr)s;b 81   0 1 1R 7 7 7 7 7 7r0ct|tjstdt ||S)Nz#Expected np.memmap instance, got %r)rrmemmap TypeErrorrecopy)rMs r. check_memmapr.s; a # #H=tAwwGGG 6688Or0cd}tdd|d|dD}t||t|D]%\}}tj||&tdd|d|dD}t||t|D]%\}}tj||&dS) Nc3xKt|D]'}tjdtj|zV(dS)Nrr))rrronesfloat32)r8rns r.generate_arrayszBtest_auto_memmap_on_arrays_from_generator..generate_arrays<sJq 4 4A'"BJ///!3 3 3 3 3 4 4r0rFrrjr%r>c3PK|]!}tt|V"dSr6r$rr,rMs r.roz.BN@@%& a  @@@@@@r0rrc3PK|]!}tt|V"dSr6rrs r.roz.Krr0)r"rarrr2r3)r>rrr5rs r.)test_auto_memmap_on_arrays_from_generatorr4sM444 @haAw???@@*9/#*>*>@@@G W)F)FGG88 %%h7777 @haAw???@@*9/#*>*>@@@G W)F)FGG88 %%h777788r0c|Sr6r+)args r.identityrRs Jr0c|dj}tj}t j|dt j|dg}t||t|dtdfdd D\}td tj sJd j |ksJtj ||dS) Nz test.mmapuint8rr) mmap_moderFr|c3RK|]!}ttV"dSr6)r$r)r,rrs r.roz.test_memmap_with_big_offset..^s6"J"J#478#4#4V#<#<"J"J"J"J"J"Jr0rsr)rrtmmapALLOCATIONGRANULARITYrzerosrr r r"rroffsetr2r3)rufnamesizeobjr5rs @r.test_memmap_with_big_offsetrVs KK $ $ ,E  %D 8D ( ( ("'$g*F*F*F GCe %3 ' ' 'F"""""J"J"J"Jc"J"J"JJJIV fQi + ++++ !9 d " " " "J!!#v.....r0chtjd5}tdddtdDdddn #1swxYwYt |dksJ|d}t |jtsJt|jdksJdS) NTrr)rjr<c3PK|]!}tt|V"dSr6rlrms r.rozFtest_warning_about_timeout_not_supported_by_backend..fs2%L%LQogfooa&8&8%L%L%L%L%L%Lr02rzThe backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.) rrr"rrrrrrr)rrs r.3test_warning_about_timeout_not_supported_by_backendrds   - - -M%1%%%%L%L%))%L%L%LLLLMMMMMMMMMMMMMMM x==A     A ai - ---- qy>>       s2AAAc|||<|Sr6r+) input_listrrs r.set_list_valuerqsJu Lr0)rrFrcdgdzt|ddfdtdD}t|}tdt |DsJdS)NrrrFr2rjrEr>c3VK|]#}tt||V$dSr6)r$r)r,rnrs r.rozPtest_parallel_return_order_with_return_as_generator_parameter..|sSQQ67 Aq11QQQQQQr0c3(K|] \}}||kVdSr6r+r,vrs r.rozPtest_parallel_return_order_with_return_as_generator_parameter..s*::$!QqAv::::::r0)r"rrrKrra)rjr5rs @r.=test_parallel_return_order_with_return_as_generator_parameterrvs qJ PXV{K P P PQQQQ;@88QQQF &\\F ::#j&"9"9::: : ::::::r0cB|rtdt|S)Nr:)r r)edelays r._sqrt_with_delayrs   b 77Nr0cRt|d|dtdDtfdtdD}dgttddz}t d t ||DsJdS) NrGrc3`K|])}tt|dz|dkV*dS)rFrN)r$rrms r.rozK_test_parallel_unordered_generator_returns_fastest_first..sYWW67! !!!Q$a11WWWWWWr0rc36K|]}tVdSr6)next)r,rr5s r.rozK_test_parallel_unordered_generator_returns_fastest_first..s)==qd6ll======r0 rrFc3(K|] \}}||kVdSr6r+rs r.rozK_test_parallel_unordered_generator_returns_fastest_first..s*SS$!QqAvSSSSSSr0)r"rrsortedrKrra)r>rjquickly_returnedexpected_quickly_returnedr5s @r.8_test_parallel_unordered_generator_returns_fastest_firstrsWXV/Dg V V VWW;@99WWWF====E!HH=====!"d5B<<&8&8 8 SS#&?AQ"R"RSSS S SSSSr0c&t||dSr6)rr>rjs r.7test_parallel_unordered_generator_returns_fastest_firstrs=WfMMMMMr0)rrFrr{cdgdgdzz}tt5tj}t||d|Ddddn #1swxYwYtj|z }|dksJdS)NrMrrric3ZK|]&}ttj|V'dSr6r$r]r rms r.roz%test_abort_backend..s70X0XA1D1D1DQ1G1G0X0X0X0X0X0Xr0rj)rrr]r")rjr>delayst_startdts r.test_abort_backendrsUbTCZ F   YY)++00000X0XQW0X0X0XXXXYYYYYYYYYYYYYYY w B 777777s8A##A'*A'c`tjtdt}d|d<|S)NgArFr)rrintbool)rr5s r.get_large_objectrs* WS\\ . . .FF1I Mr0ct|||5}|dtdD}t|t|~ddddS#1swxYwYdS)N)rjr>rEc3PK|]!}tt|V"dSr6)r$rrms r.roz0_test_deadlock_with_generator..s6JJ13'"233A66JJJJJJr0r)r"rrr)r>rErjr r5s r._test_deadlock_with_generatorrs I F F F(JJb JJJJJ V  V  sAA""A&)A&c(t|||dSr6)r)r>rErjs r.test_deadlock_with_generatorrs "'9f=====r0c~ttd5t|||}|dtdD}t j}|dtdD}dddn #1swxYwYt j|z dks Jd ~dS) N)This Parallel instance is already runningrrLc3PK|]!}ttdV"dSrrrs r.roz/test_multiple_generator_call..s2::1^WU^^A&&::::::r0rc3PK|]!}tt|V"dSr6rrms r.roz/test_multiple_generator_call..s0;;1  A;;;;;;r0rrF[The error should be raised immediately when submitting a new task but it took more than 2s.)rrr"rrr])r>rErjr grgen2s r.test_multiple_generator_callrs  $O P P P<<FGyIII H::b ::: : :)++x;;c ;;;;; <<<<<<<<<<<<<<< 9;; 1 $ $ $ $ % $ $ sA*B  BBct|||5}|dtdD}tj}ttd5|dtdD}dddn #1swxYwYtj|z dks Jd  dddn #1swxYwY~dS) NrLc3PK|]!}ttdV"dSrArrs r.roz7test_multiple_generator_call_managed..s2;;A^WU^^B'';;;;;;r0rrrc3PK|]!}tt|V"dSr6rrms r.roz7test_multiple_generator_call_managed..s0==Q+'"++a..======r0rrFr)r"rrr]rr)r>rErjr rrg2s r.$test_multiple_generator_call_managedrs^ &'Y ? ? ?  8 H;;r;;; ; ;)++ L(S T T T > >==%**=====B > > > > > > > > > > > > > > >y{{W$q((( ()(((                 s6A C #B C B C B 'C  CC return_as_1 return_as_2c t|||dtdD}t|||dtddD}|dkrt|}|dkrt|}tdt |tdDsJtdt |tddDsJdS) NrLc3VK|]$}tt|dzV%dSrFNrrms r.roz9test_multiple_generator_call_separated..sOAA !  adAAAAAAr0rc3VK|]$}tt|dzV%dSrrrms r.roz9test_multiple_generator_call_separated..sOBB !  adBBBBBBr0rjrGc3(K|] \}}||kVdSr6r+r,rrns r.roz9test_multiple_generator_call_separated..s*88FCsax888888r0c3(K|] \}}||kVdSr6r+rs r.roz9test_multiple_generator_call_separated..s*==FCsax======r0)r"rrrrra)r>rrrjrrs r.&test_multiple_generator_call_separatedrs6 AK@@@AA%*2YYAAA  A B&'[ A A ABB%*2r]]BBB  B+++ 1II+++ BZZ 88c!U2YY&7&7888 8 8888 ==c"eBmm&<&<=== = ======r0zbackend, error))r1T)r2F)rDFc |dkrttjdtd||}|dt dD}t j|d}|rttd n t}|5tj }td||d t dd D}|d krt|}td t|t dd DsJ dddn #1swxYwYtj |z dksJd} |jr,| dkr&| dz } tjd|jr| dk&tj |z dksJ|jdkr |jsJdSdS)Nr1zRequires multiprocessingrFrLc3PK|]!}ttdV"dSrArrms r.roz.&s277##777777r0rc tdS)NzGenerator collected)rr+r0r.z;test_multiple_generator_call_separated_gc..'su-B'C'Cr0z The executor underlying Parallelrc3VK|]$}tt|dzV%dSrrrms r.roz.3sO@ @ $%MGDMM!Q$  @ @ @ @ @ @ r0rjrGc3(K|] \}}||kVdSr6r+rs r.roz.:s*@@Q3!8@@@@@@r0rrrprr)rr^r_r"rrweakreffinalizerrrr]rrraaliver rre) r>rrerrorr rg_wrr#rretrys r.)test_multiple_generator_call_separated_gcrsh 6  .///7kBBBH77U2YY77777A  ACC D DD  |#EFFFF ]] A A )++ ?HQ; ? ? ?@ @ ).r2@ @ @    / / /q A@@#ar2*?*?@@@@@@@@@ A A A A A A A A A A A A A A A 9;; 1 $ $ $ $ E *   3 * 9;; 1 $ $ $ $!x!##q((!!!!)( "!sBD44D8;D8c|j}tdd||5}|dtjdgdzDt t j|dksJ dddn #1swxYwYtdD]'}t j|sntd(td tdd| }|d tjdgdzDtdD](}t j|sdStd)td ) NrFr)rjr%r> temp_folderc3PK|]!}tt|V"dSr6rrs r.roz(test_memmapping_leaks..Vs5 G Gq ',   " " G G G G G Gr0rrrrz/temporary directory of Parallel was not removedrc3PK|]!}tt|V"dSr6rrs r.roz(test_memmapping_leaks..fs5CC1glACCCCCCr0) rtr"rrTrrlistdirrrr AssertionError)r>rurrs r.test_memmapping_leaksrKs ^F q'v N N N+RS  G GRY-=-=b-A-A,BQ,F G G GGGG2:f%%&&***** +++++++++++++++3ZZPPz&!!  E c NOOO a999AACC)9)9")=)=(>(BCCCCCC 3ZZPPz&!!  EE c NOOOsABB  B )Nr1r2ctd|dtdD}|dtdDksJdS)NrFric3HK|]}td|VdS)c |dzSrEr+rGs r.rz2test_lambda_expression...vs !Q$r0Nr.rms r.roz)test_lambda_expression..usH22'(""222222r0rcg|]}|dzSrFr+rms r.r/z*test_lambda_expression..xs///q!t///r0rq)r>rs r.test_lambda_expressionrpsr 2ha11122,1"II222G //U2YY/// / / / / / /r0cd}d}d|z td||}|fdt|D|jj|jjksJ|jj|jjksJ|fdt|D|jj|jjksJ|jj|jjksJdS) zCTest that a parallel backend correctly resets its batch statistics.rFig@r)rrjr>c3\K|]&}ttjV'dSr6rr,rn task_times r.roz6test_backend_batch_statistics_reset..8>>gdj)$$>>>>>>r0c3\K|]&}ttjV'dSr6rr s r.roz6test_backend_batch_statistics_reset..r r0N)r"rrr _effective_batch_size_DEFAULT_EFFECTIVE_BATCH_SIZE_smoothed_batch_duration _DEFAULT_SMOOTHED_BATCH_DURATION)r>rjn_inputsrr s @r.#test_backend_batch_statistics_resetr{s FHhIFG<<>>>eHoo>>>>>> : +qz/W W W W W + : 6 7 7 7 7A>>>>eHoo>>>>>> : +qz/W W W W W + : 6 7 7 7 7 7 7r0cdD]}tt|jtusJt|d}t|jtusJt|d}t|jt usJt|d}t|jtusJtdd d }t|jt usJ|d d5td }t|jt usJ|jdksJ dddn #1swxYwY|d d5td d}t|jt usJ|jd ksJ dddn #1swxYwY|d d5td }t|jtusJ|jdksJ dddn #1swxYwY|d d5td d}t|jtusJ|jd ksJ ddddS#1swxYwYdS)N)rrFr{r|threadsrjprefer processes sharedmemrjrequirerFr1)rjr>rrrprr)rer"r r8r rrj)rzrjrs r.$test_backend_hinting_and_constraintsrs 4 4HF+++4559U9W9WWWWW F9 5 5 5AJ#33333 F; 7 7 7AJ;.... FK 8 8 8AJ#333333 6)<<;EE  E .s0001000r0rrc3PK|]!}tt|V"dSr6rlrms r.roz6test_backend_hinting_always_running..sC55 555555r0)rrjc3PK|]!}tt|V"dSr6rlrms r.roz6test_backend_hinting_always_running..s2CCA_WV__Q//CCCCCCr0)rrr"r')rjrexpected_resultsrs r.#test_backend_hinting_always_runningr$s+ 10eBii0004hfV44455$)"II555G & & & & & v 6 6 6DD(**CCrCCCCCDDDDDDDDDDDDDDD & & & & & & &s$/BB#&B#cGddt}||5tdd}t|j|usJtdd}t|j|usJ dddn #1swxYwYGdd t}||5tdd}t|j|usJ|\}}|d ksJ|d ksJtddd }t|jt usJ|\}}d }||ksJ|d ksJ dddn #1swxYwYtt5t|dddddS#1swxYwYdS)Nc"eZdZdZdZdZdZdS)[test_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackendTcdSr6r+rs r. apply_asynczgtest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.apply_async Dr0c|Sr6r+rs r.r%zltest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.effective_n_jobsMr0NrSrTrUsupports_sharedmem use_threadsr)r%r+r0r.MyCustomThreadingBackendr's=!         r0r0rFrrrrc"eZdZdZdZdZdZdS)\test_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackendFcdSr6r+rs r.r)zhtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.apply_asyncr*r0c|Sr6r+rs r.r%zmtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.effective_n_jobsr,r0Nr-r+r0r.MyCustomProcessingBackendr2s="         r0r5rr)rjrrzUsing ThreadingBackend as joblib backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.r>r) rr"rer  readouterrr striprrY)capsysrzr0rr5outerrrs r.9test_backend_hinting_and_constraints_with_custom_backendsr<s#6 ))++ , ,<< Ak 2 2 2AJ#;;;;; A{ 3 3 3AJ#;;;;;; <<<<<<<<<<<<<<<$7 **,, - - Ak 2 2 2AJ#<<<<<$$&&Sbyyyybyyyy A{B ? ? ?AJ#33333$$&&S 8  yy{{h&&&&byyyyy%(   KK2244kJJJJKKKKKKKKKKKKKKKKKKs7ABB B 8B;FFFGG  G cxtt5tddddn #1swxYwYtt5tddddn #1swxYwYtt5tdddddn #1swxYwYt}tt5tdddddn #1swxYwYtt5td dddddS#1swxYwYdSdS) Ninvalidrrrr)rrr1r6r))rrYr"rr+r0r.,test_invalid_backend_hinting_and_constraintsr? s   ## """"###############   $$####$$$$$$$$$$$$$$$   ::  [9999:::::::::::::::  ~J   : : V[ 9 9 9 9 : : : : : : : : : : : : : : : J   E E . D D D D E E E E E E E E E E E E E E E E E E ~sT266A..A25A2 B++B/2B/C//C36C3D--D14D1c .td5}t|jj|jjfg}dkr|cdddS|fdt dD}||dzcdddS#1swxYwYdS)zCPerform nested parallel calls and introspect the backend on the wayrFr|rNc3VK|]#}ttdddz iV$dS)limitrNr+)r$_recursive_backend_info)r,rnrhrBs r.roz*_recursive_backend_info..'sY   -G+ , , G G519 G G G      r0r)r"rer rSr rr)rBrhr this_levelrs`` r.rCrC s    'qAJ''0!*2JKL A::''''''''!     1XX     GAJ&''''''''''''''''''s.B /B  BBc||d5t}dddn #1swxYwY|dz}|dfdddg}||ksJdS)NrFr|Backendr)r r)rrF)rCtitle)rzr>backend_types_and_levelstop_level_backend_typeexpected_types_and_levelss r.test_nested_parallelism_limitrK.s  # # #==#:#<#< ===============%]]__y8 #  ! $'@ @ @ @ @ @ @s )--c^tdtdDS)z6A horrible function that does recursive parallel callsc3NK|] }ttV!dSr6)r$_recursive_parallelrms r.roz&_recursive_parallel..As4GG2g12244GGGGGGr0rFrq) nesting_limits r.rNrN?s+ 8::GGeAhhGGG G GGr0cn||d5tt5}tdddn #1swxYwYdddn #1swxYwY|j}|dkr1ddlm}t ||tfrtj dt |tsJdS)NrFr|r1r)TerminatedWorkerErrorz1Loky worker crash when serializing RecursionError) r BaseExceptionrNr&joblib.externals.loky.process_executorrQrrr^xfailRecursionError)rzr>rexcrQs r.test_thread_bomb_mitigationrWDsK  # # #"" M " " "g  ! ! ! " " " " " " " " " " " " " " """""""""""""""" -C& QPPPPP c1=A B B N LL M M M c> * ******s1A> AA AA AAAcvi}dD]$}tj|||<%|tdfS)N)OMP_NUM_THREADSOPENBLAS_NUM_THREADSMKL_NUM_THREADSVECLIB_MAXIMUM_THREADSNUMEXPR_NUM_THREADSNUMBA_NUM_THREADS ENABLE_IPCr)renvironrLr)env_varsvars r._run_parallel_sumrc`sFH , , s++ \#&& &&r0zNeed OpenMP helper compiledcptd|dtdD}ttdzd}|D]b\}}||ksJ|D]@\}}|dr|t |ksJ0|dksJ|dksJAcdS)NrFric3NK|] }ttV!dSr6)r$rcrs r.roz-test_parallel_thread_limit..rsE22)*"!""$$222222r0r_THREADSr_1)r"rrrr#itemsendswithr)r>rexpected_num_threadsworker_env_varsomp_num_threadsrrs r.test_parallel_thread_limitrmos2ha11122.3Ahh222Gy{{a/33,3$$("66666*0022 $ $KD%}}Z(( $$8 9 999999|++++|||||  $$$r0zThis test requires daskczttd5|dddddS#1swxYwYdS)NzPlease install daskrdask)rrY)rzs r.)test_dask_backend_when_dask_not_installedrps  "7 8 8 8s 044cNGddt}d}||5tjt|5t ddt dDdddn #1swxYwYddddS#1swxYwYdS)Nc"eZdZdZddZdZdS)3test_zero_worker_backend..ZeroWorkerBackendcdSrdr+)rLrrhs r.rz=test_zero_worker_backend..ZeroWorkerBackend.configure1r0Nc td)NzNo worker availabler)rLfunccallbacks r.r)z?test_zero_worker_backend..ZeroWorkerBackend.apply_asyncs455 5r0cdSrdr+rs r.r%zDtest_zero_worker_backend..ZeroWorkerBackend.effective_n_jobsrur0r6)rSrTrUrr)r%r+r0r.ZeroWorkerBackendrssF    6 6 6 6     r0rzz&ZeroWorkerBackend has no active workerrrFr|c3PK|]!}tt|V"dSr6rrms r.roz+test_zero_worker_backend..s0@@!{wr{{1~~@@@@@@r0)r r^rrr"rr)rzrz expected_msgs r.test_zero_worker_backendr}sq,.check_globalss!!r0rFr|c3FK|]}tVdSr6r.r,rnrs r.roz.E11%&   111111r0z changed valuec3FK|]}tVdSr6r.rs r.roz.rr0)rr"rrr)workers_global_variablers @r.)test_globals_update_at_each_parallel_callrs* *""" =??. . . . .0ha0001111*/((111 & ' ',<+= = = = =) =??o - - - -0ha0001111*/((111 & ' 'O+< < < < < < >|LL  r0cv|D]5}|D]0}t||}t|||h}|d|vsJ16dS)Nr)rmin) workers_inforrchild_threadpool_inforparent_max_num_threadsrs r.check_child_num_threadsrsw".;;1 ; ;L%@k&& "K)?@@+NH .(:::::  ;;;r0)rFrrr{c|t}t|dkrtjdt d|dt dD}t |}|dkr|dd }ntt|zd}t|||dS) Nr&Need a version of numpy linked to BLASrr1rc3NK|] }ttV!dSr6r$rrms r.roz;test_threadpool_limitation_in_child_loky..sMGG67/.//11GGGGGGr0rFrr rrr^r_r"rrr%rr#r)rjrworkers_threadpool_infosexpected_child_num_threadss r.(test_threadpool_limitation_in_child_lokyrs122K ;1 CDDDDFxvFFFGG;@88GGG  f % %F {{%0^M%B""%()>%B%B" +/Ir0inner_max_num_threads)rrFrNct}t|dkrtjd|d|5t |dt dD}dddn #1swxYwYt |}|d kr|dd }n$| tt|zd }n|}t|||dS) Nrrrr1rr|c3NK|] }ttV!dSr6rrms r.roz>test_threadpool_limitation_in_child_context.. sE; ; :; 3G2 3 3 5 5; ; ; ; ; ; r0rFrrr)rzrjrrrrs r.+test_threadpool_limitation_in_child_contextrsV122K ;1 CDDDD /D E E E  #:86#:#:#:; ; ?DQxx; ; ; $ $                 f % %F {{%0^M%B""  &%()>%B%B""%:" +/Is1BBBvar_name)rZr[rYct|dkrtjdtddt j} dt j<t|fdtd D}|ddgksJ|d d 5t|fd td D}dddn #1swxYwY|d d gksJ |t j=dS|t j<dS#|t j=n|t j<wxYw)NrzSkip test when n_jobs == 1Treusec@tj|Sr6)rr`rL)rs r._get_envz>test_threadpool_limitation_in_child_override.._get_env,sz~~h'''r04r|c3HK|]}tVdSr6r.r,rnrrs r.roz?test_threadpool_limitation_in_child_override..3s6)X)X!*;'(*;*;H*E*E)X)X)X)X)X)Xr0rFr1rc3HK|]}tVdSr6r.rs r.roz?test_threadpool_limitation_in_child_override..7sG..01!!!(++......r0rg) r%r^r_rshutdownrr`rLr"rr)rzrjroriginal_var_valuerrs ` @r.,test_threadpool_limitation_in_child_overriders1$$ 0111%%%..000(((116" 8)(&))))X)X)X)X)XuUVxx)X)X)XXX3*$$$$ WV1 5 5 5  -hf---.....5:1XX...G               3*$$$$$  % 8$$$#5BJx   % 8$$#5BJx 5 5 5 5s1/AE4D< ED  ED  E!E!)rFrr{cd}||td}tdD]%}||td}||ksJ&dS)Nctdt|fdtdDdS)Nrr|c3RK|]!}ttV"dSr6)r$sum)r,rnrAs r.rozAtest_loky_reuse_workers..parallel_call..Ks1CCA  QCCCCCCr0r)rjrAs @r. parallel_callz.test_loky_reuse_workers..parallel_callIsJ "IICCCCrCCCCCCCCr0Trr)rrr)rjrfirst_executorrrs r.test_loky_reuse_workersrCs DDD M&*666N2YY** f(t444>)))))**r0c2d|tj<dS)N initialized)rr)statuss r._set_initializedrYs'F29;;r0cLtj}||d}|dvs Jd||sdSd||<tjdz}t d|D}tj|krb||kr\tjdt d|D}tj|kr||k\tj|krtd|S) N)rstartedz2worker should have been in initialized state, got rr:c$g|] \}}|dk |Srr+r,pidrs r.r/z!_check_status..is!HHHVS!iSr0rc$g|] \}}|dk |Srr+rs r.r/z!_check_status..ls!LLLaQ)^^^^^r0z-Waited more than 30s to start all the workers)rrrLr]rrhr r)rrj wait_workersrstatedeadline n_starteds r. _check_statusr]s! )++C JJsD ! !E . . . .DUDD / . . F3Ky{{RHHHv||~~HHHIII )++ Y%7%7 3LL6<<>>LLLMM  )++ Y%7%7 y{{hJKKK Jr0ctj}|||tf5t fdt dDddddS#1swxYwYdS)Nr>rj initializerinitargsc3TK|]"}ttV#dSr6r$rr,rnrjrs r.roz+test_initializer_context..s8NNa)7=))&&99NNNNNNr0r)rrdictrr"rr)rjr>rzrrs` @r.test_initializer_contextrts jllG \\^^F $    OO  NNNNN5::NNNNNNOOOOOOOOOOOOOOOOOOs2A>>BBctj}|t|tffdt dDdS)Nrc3TK|]"}ttV#dSr6rrs r.roz,test_initializer_parallel..s8AAgmVV,,AAAAAAr0r)rrrr"rrr)rjr>rrs` @r.test_initializer_parallelrsjllG \\^^FH$  BAAAAeCjjAAA BBBBBr0cd}tj}|t}t |D]`}t dt ffdt D}|t|}at|ks JddS)Nrpr1rc3XK|]$}ttdV%dST)rNrrs r.roz*test_initializer_reused..P   #GM " "66 E E E      r0z=The workers should be reused when the initializer is the same) rrrrrrr"runionrrj n_repetitionsrpidsrnrrs` @r.test_initializer_reusedrs MjllG \\^^F 55D = ! ! ( ( ((Y         6]]     zz#g,,'' t99   G     r0cd}tj}t}t|D]t}|t dt ffdtD}|t|}ut||zks JddS)Nrpr1rc3XK|]$}ttdV%dSrrrs r.roz.test_initializer_not_reused..rr0zFThe workers should not be reused when the initializer arguments change) rrrrrrr"rrrrs` @r.test_initializer_not_reusedrs MjllG 55D = ! ! ( ( ((Y         6]]     zz#g,,'' t99 . . . .P / . . . .r0)F)rr)rpr6)rVrrrrbr2r]rrr4rmathrr)rpicklerr tracebackr r^rqr r r joblib._multiprocessing_helpersrjoblib.test.commonrrrrjoblib.testingrrrrrjoblib.externals.lokyrrvrposix ImportError _openmp_test_helper.parallel_sumr distributedjoblib._parallel_backendsrrrrr joblib.parallelr!r"r#r$r%r&r'r(rRETURN_GENERATOR_BACKENDSpoprkeysALL_VALID_BACKENDSPROCESS_BACKENDSPARALLEL_BACKENDShasattrrr3r8r?rCrH ExceptionrJr[r_rbrgrtrvrymarkrrrrrrrrrrrrrrr r8r>rBrrMrgrtrxrrrrrrrrTrrrrrrrrrrextendrrrrrrrr _parallel_backendsr rrr$r+r;r>rHrOrXr`rernrl SQUARE_LOCAL SQUARE_LAMBDArwrarorprrryr|rrrrrrrrrrrrrrrrrrrrrrrrrr$r<r?rCrKrNno_coverrWrcrmrpr}rrrrrrrrrrrrrrr+r0r.rs[  """"""(((((( &&&&&& ''''''''''...... UTTTTTTTTTTTTT>;;;;;;LLLL EEE>>>>>>>LLLKKK                      *HMOO/666Vff]X]__555KK(KKKK:)62$ }4 72}7nbnW55666...III             222""" Y*:;<<% b"""--- . :000 .. =<. Y*++ X~~~&& Y %%,,&%'&,+,  Y*++ X1v,+( Y*++'',+'T0 +    2 Y@@@AA  BA QQQ _h''  x(()((' WWW  Xyyy!!"! Y)**//+*/:       [///** \At9%%55&%+* 5$ Y)**  +*  Y)**  +*  Y566,GHH [;(=>??@?IH$ Y())C C *)C L Y!!EE"!E,''' Y!!       $     %$''P  Q''"!R   Y)**+*<444 Y())33*)3   HHHMMM Y)**CDD--ED+*-,     +       Y*++HH,+H$ Y)** X}}}%%::&%+*:%%% D D D$=>>>99?>9;;;*$5QQQ#7 ''(Q(Qa(Q(Q(QRRR Y899 Y)?;<<DD=<:9D,#4 Y*:;<<DD=<D& Y*:;<<DD=<D( QQQ>>>>>)5>>>/;;; YWWWXX Y*:;<<,,=<YX, X}}}%% Y)** Y*:;<<  =<+*&% 0777 _/00 _/00 Y*:;<<   =<1010   Y*:;<<  %e}55!!65=<!: \===))((*)(  /    ! +++  Y())SS*) S, 0,  Y())  *) -)@    Y("*BB7)LMM ][, FGG  "?"?"?@@A@HGNM,(9P Fbgoobgoofo.N.NOOFPPQ6V   =, Fbgoobgoofo.N.NOOFPP-:2888    Y())88*) 86  / /  /    999-- ; ;.- ;"Aq6** Y566,GHHNNIH +* N Y*++ X~~~&&'&,+  Y122 [;(=>?? X~~~&&>>'&@?32 > Y122 [;(=>?? X~~~&&  '&@?32 & Y122 [;(=>?? X~~~&&  '&@?32 & Y122 ][*?@AA ][*?@AA X~~~&&>>'&BABA32>&  ][*?@AA ][*?@AA)")"BABA)"X  Y())PP*) PD  rz{##7R7R7R000 Y())*), Y*:;<<++=<+\ X1v X55566 ' '76 ' Y*:;<<3K3K=<3KlEEE* ' ' ' ' Y-.. Y*:;<< A A=</. AHHHH  Y*:;<< Y"*+6;:OQQ++RQ=<+2 ' ' ' Y2>$vGG %BCCC $ $DCHG $ Y*:;<< 4(ABBBCB=<  Y*:;<<AA=<A&===H + + + ; ; ;  X~~~&&'& 0   $ooo66 X2w Y*:;<<=< 76  6 X2w ZWWWXX Y*:;<< 6 6=<YX  6F Xzzz""**#"*((((. X1v Y()) Y*:;<< O O=<*) O X1v Y()) B B*) BAq6**+*0Aq6**+*s6B BBB""B,+B,0B55B?>B?