
    .PhӪ                     	   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlZd dlmZ d dlmZmZmZmZmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d dl m!Z!m"Z"m#Z#m$Z$m%Z% d d	l&m'Z'm(Z(m)Z) d
 Z*d Z+d Z,d Z-d Z.e%e$d                         Z/e%e$ e)ej        dk    d          d                                     Z0e%e$d                         Z1e%d             Z2e%e$ e'deej3        gddg          d                                     Z4e%e$ e'deej3        gddg          d                                     Z5e%e$ e'dddg          d                                     Z6e%e$ e'dddg          d                                     Z7e%e$ e'dddg          d                                     Z8e%e$ e'dddg          d                                      Z9e%e$d!                         Z:e%e$d"                         Z;e%e$ e'dddg          d#                                     Z<e%e$ e'dddg          d$                                     Z=e%e$ e'dddg          d%                                     Z>e%e$ e'deej3        gddg          d&                                     Z?e%e$ e'd ej@        dejA        B                    d'          (          dg          d)                                     ZCe%e$ e'deej3        gddg          d*                                     ZDe%e$e# e'deej3        gddg          d+                                                 ZEe%e$e# e'deej3        gddg          d,                                                 ZFe%e$ e'deej3        gddg          d-                                     ZGd. ZHe%e$ e'deej3        gddg          d/                                     ZId0 ZJe%e$ e'd1 eK ejL        eej3        g eMd2                              d3  ejL        ddg eNeO eMd2                              D                       d4                                     ZPd5 ZQd6 ZRe% e)ej        dk    d7           e'd8d9d:g          d;                                     ZSe%d<             ZTd= ZUe%e$d>                         ZVe%e$d?                         ZWdS )@    N)sleep)ArrayMemmapForwardReducer_get_backing_memmap_get_temp_dir_strided_from_memmap_WeakArrayKeyMaphas_shareable_memory)make_memmap)_TestingMemmappingExecutor)Paralleldelayed)MemmappingPool)IS_GIL_DISABLEDnpwith_dev_shmwith_multiprocessing
with_numpy)parametrizeraisesskipifc                  2    t          j        dd           d S )Ni,  T)timeoutexit)faulthandlerdump_traceback_later     [/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/joblib/test/test_memmapping.pysetup_moduler   &   s    %c======r   c                  ,    t          j                     d S N)r   cancel_dump_traceback_laterr   r   r   teardown_moduler#   *   s    ,.....r   c                 (    t          |           J | S r!   )r   )arrays    r   check_memmap_and_send_backr&   .   s    u%%111Lr   c                 ^    | \  }}}t           j                            ||         |           dS )zDummy helper function to be executed in subprocesses

    Check that the provided array has the expected values in the provided
    range.

    Nr   testingassert_array_equalargsdatapositionexpecteds       r   check_arrayr0   3   s2      $D(HJ!!$x.(;;;;;r   c                     | \  }}}||         |k    sJ ||xx         dz  cc<   t           j                            ||         d|z             dS )zDummy helper function to be executed in subprocesses


    Check that the input array has the right values in the provided range
    and perform an inplace modification to double the values in the range by
    two.

       Nr(   r+   s       r   inplace_doubler3   >   sa      $D(H>X%%%%NNNaNNNJ!!$x.!h,?????r   c                 P   t           j        j        }|                     d          j        }t          j        |t           j        dd          }dt          j        |j        d         |j	                  z  |dd<   |
                                 t          j        |t           j        d	d
dd          }t          j        d                              |j                  |dd<   |ddddddf         }|                    dd          }t          j        |          }|j        }t          d| j        dd          fd}	 |	|          }
t          |
          sJ t!          |
t           j                  sJ  ||
|            |	|          }t          |          sJ  |||            |	|          }t          |          sJ  |||            |	|          }t!          |t           j                  rJ t          |          sJ  |||            |	|          }t!          |t           j                  rJ t          |          sJ  |||           |dz  }t          |          rJ  |	|          }t          |          rJ t!          |t           j                  rJ  |||dz             t          j        |          }t          |          rJ  |	|          }t!          |t           j                  sJ t          |          rJ  |||           dS )z9Check that it is possible to reduce a memmap backed array	test.mmapi  w+dtypeshapemodeg      r   r8   N)         r+Fr>   )r8   r9   r:   orderoffset<      r2   
   2   cTc                 *     |           \  }} || S r!   r   xconsr,   reducers      r   reconstruct_array_or_memmapzEtest_memmap_based_array_reducing.<locals>.reconstruct_array_or_memmaps        WQZZ
dtT{r   r<   )r   r)   r*   joinstrpathmemmapfloat64aranger9   r8   flushreshapeasarrayTr   r	   
isinstancendarray)tmpdirr*   filenamebufferabb2rH   drN   a_reconstructedb_reconstructedb2_reconstructedc_reconstructedd_reconstructeda3a3_reconstructedb3b3_reconstructedrM   s                      @r    test_memmap_based_array_reducingrk   M   s    6{{;''/H Yxrz4HHHF ryaEEEEF111I
LLNNN 		
)$cRS	 	 	A 9R==  ))AaaaD
 	
!B$"ac/A 
B		B 	
1A	A (fnc4HHG    
 21!44O00000ory11111*** 21!44O00000*** 32266 011111',,, 21!44O/295555500000***11!44O/295555500000*** 
QB#B'''''22266#$455555*BI66666'Q/// 
BB#B'''''22266&
33333#$455555',,,,,r   win32z2PermissionError only easily triggerable on Windows)reasonc                    |                      d          j        }d                    |          }t          j        t
          j        d|gt          j        t          j                  }|                                 |	                                \  }}|j
        dk    sJ |                                            |dk    sJ d                    |          }||                                v sJ d S )	Nr5   a  if 1:
    import os
    import numpy as np
    import time
    from joblib.externals.loky.backend import resource_tracker
    resource_tracker.VERBOSE = 1

    # Start the resource tracker
    resource_tracker.ensure_running()
    time.sleep(1)

    # Create a file containing numpy data
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    memmap[:] = np.arange(10).astype(np.int8).data
    memmap.flush()
    assert os.path.exists(r"{filename}")
    del memmap

    # Create a np.memmap backed by this file
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    resource_tracker.register(r"{filename}", "file")

    # Ask the resource_tracker to delete the file backing the np.memmap , this
    # should raise PermissionError that the resource_tracker will log.
    resource_tracker.maybe_unlink(r"{filename}", "file")

    # Wait for the resource_tracker to process the maybe_unlink before cleaning
    # up the memmap
    time.sleep(2)
    )r\   -cstderrstdoutr   r   z'tried to unlink {}, got PermissionError)rP   rQ   format
subprocessPopensys
executablePIPEwaitcommunicate
returncodedecode)r[   r\   cmdpouterrmsgs          r   2test_resource_tracker_retries_when_permissionerrorr      s     {{;''/H: 	!!; < 		s#JOJO	 	 	A FFHHH}}HC<1cjjll#::::
3
:
:8
D
DC#**,,r   c                 n   t           j        j        }|                     d          j        }t          j        |t           j        dd          }t          j        d                              |j	                  |d d <   |dd         }|d d ddf         }|d d d d d d df         }|d	d
d         }t          d | j        dd          fd} ||          }	t          |	          sJ t          |	t           j                  sJ  ||	|            ||          }
t          |
          sJ  ||
|            ||          }t          |          sJ  |||            ||          }t          |          sJ  |||            ||          }t          |          sJ  |||           d S )Nr5   )d      r   r<   r6   r7   i r   rF   r=   rD   r<   r>   rH   Tc                 *     |           \  }} || S r!   r   rJ   s      r   rN   zNtest_high_dimension_memmap_array_reducing.<locals>.reconstruct_array_or_memmap   rO   r   )r   r)   r*   rP   rQ   rR   rS   rT   rV   r9   r   r	   rY   )r[   r*   r\   r^   r_   rH   ra   erN   rb   rc   re   rf   e_reconstructedrM   s                 @r   )test_high_dimension_memmap_array_reducingr      s    6{{;''/H 		("*4D4PPPA9&''//88AaaaD 	
!B$A	!!!QrT'
A	!!!QQQ1*A	!Aa%A (fnc4HHG     21!44O00000ory11111***11!44O00000***11!44O00000***11!44O00000***11!44O00000*****r   c                    |                      d          j        }dt          j        z  }t          j        dz   }t	          j        |d||z             }t          |dd|d|d d d	
	  	        }t          |t          j                  sJ |j        |k    sJ t          |dd|d|dz  fd|d	
	  	        }t          |          j        |k    sJ d S )Nr5   r=   rD   r6   )r:   r9   uint8rCF)r8   r:   rB   rA   r9   stridestotal_buffer_lenunlink_on_gc_collectr2   )r2   )
rP   rQ   mmapALLOCATIONGRANULARITYr   rR   r   rY   rB   r   )r[   fnamesizerB   
memmap_objmemmap_backed_objs         r   test__strided_from_memmapr     s   KK$$,Et))D'!+F5t4&=AAAJ%"
 
 
J j"),,,,,&&&&,qyl"
 
 
 0118FBBBBBBr   factorymultiprocessingloky)idsc           	         t           j        j        }|                    d          j        } | dd|          }	 |                    d          j        }t          j        |t           j        dd                              d	           |	                    t          fd
t          j        d                   D                         |dt          j        j                  z             t          j        |t           j        dd          |	                    t          fdt          j        d                   D                        t          j        |          g k    sJ  |dt          j        j                  z              |dt          j        j                  z             t          j        |t           j        ddd          t!          t"                    5  |	                    t$          fdt          j        d                   D                        ddd           n# 1 swxY w Y   t!          t&          t(          f          5  |	                    t          fdt          j        d                   D                        ddd           n# 1 swxY w Y   |                                 ~dS # |                                 ~w xY w)z@Check that subprocess can access and update shared memory memmappoolrF   r2   
max_nbytestemp_folderr5   r<   r=   r6   r7         ?c                 X    g | ]&}t          j        d                    D ]	}||fdf
'S rD   r   ranger9   ).0ijr^   s      r   
<listcomp>z)test_pool_with_memmap.<locals>.<listcomp>=  @    TTT!%PQ
BSBSTTQa!QTTTTr   r   )r=   r<   rH   c                 X    g | ]&}t          j        d                    D ]	}||fdf
'S )rD          @r   )r   r   r   r_   s      r   r   z)test_pool_with_memmap.<locals>.<listcomp>G  r   r   rF   r      )r8   r9   r:   rB   c                     g | ]}|d fS )g      @r   r   r   rH   s     r   r   z)test_pool_with_memmap.<locals>.<listcomp>V  s    GGGAsGGGr   Nc                     g | ]}|d fS )r   r   r   s     r   r   z)test_pool_with_memmap.<locals>.<listcomp>[  s    "J"J"J1Aq#;"J"J"Jr   )r   r)   r*   mkdirrQ   rP   rR   float32fillmapr3   r   r9   onesoslistdirr   AssertionErrorr0   RuntimeError
ValueError	terminate)	r   r[   r*   pool_temp_folderr~   r\   r^   r_   rH   s	         @@@r   test_pool_with_memmapr   (  sU    6 ||F++3q.>???A);;{++3IhbjTJJJ	s	TTTTuQWQZ'8'8TTT	
 	
 	

 	1a"'!'"2"22333 IhbjSIII	TTTTuQWQZ'8'8TTT	
 	
 	
 z*++r1111 	1a"'!'"2"223331a"'!'"2"22333 IhbjCPUVVVN## 	I 	IEE+GGGGU171:5F5FGGGHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I
 \:.// 	L 	LEE."J"J"J"Jagaj8I8I"J"J"JKKK	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	
AA 	
sV   F4K 2A H>2K >IK IK $A J0$K 0J44K 7J48K K)c                 N   t           j        j        }|                    d          j        } | dd|          }	 |                    d          j        }t          j        |t           j        dd                              d	           t          j	                  t          t           j                  rJ t                    sJ |                    t          fd
t          j        d                   D                         |dt          j        j                  z              |dt          j        j                  z             t#          j        |          g k    sJ 	 |                                 ~dS # |                                 ~w xY w)z?Check that subprocess can access and update shared memory arrayr   rF   r2   r   r5   r   r6   r7   r   c                 X    g | ]&}t          j        d                    D ]	}||fdf
'S r   r   )r   r   r   r^   a_views      r   r   z4test_pool_with_memmap_array_view.<locals>.<listcomp>|  sY       qwqz**   !Q%   r   r   N)r   r)   r*   r   rQ   rP   rR   r   r   rW   rY   r	   r   r3   r   r9   r   r   r   r   )r   r[   r*   r   r~   r\   r^   r   s         @@r    test_pool_with_memmap_array_viewr   b  s    6 ||F++3q.>???A;;{++3IhbjTJJJ	s Afbi00000#F+++++	    qwqz**  	
 	
 	
 	1a"'!'"2"2233361rwqw'7'7#7888 z*++r11111 	
AA 	
s   D8F F$backendc                 v   d                     |           }t          j        t          j        d|gt          j        t          j                  }|                                 |                                \  }}|j        dk    s4J |	                                dz   |	                                z               d S )N  if 1:
        import numpy as np
        from joblib import Parallel, delayed


        data = np.random.rand(int(2e6)).reshape((int(1e6), 2))

        # Build a complex cyclic reference that is likely to delay garbage
        # collection of the memmapped array in the worker processes.
        first_list = current_list = [data]
        for i in range(10):
            current_list = [current_list]
        first_list.append(current_list)

        if __name__ == "__main__":
            results = Parallel(n_jobs=2, backend="{b}")(
                delayed(len)(current_list) for i in range(10))
            assert results == [1] * 10
    r_   ro   rp   r   

)
rs   rt   ru   rv   rw   rx   ry   rz   r{   r|   r   r}   r~   r   r   s        r   -test_permission_error_windows_reference_cycler     s    $ 	% & 		s#JOJO	 	 	A FFHHH}}HC<1cjjllV3cjjllBr   c                    d                     |           }t          d          D ]}t          j                                        }t          j                            t                    |d<   t          j	        t          j        d|gt          j        t          j        |          }|                                 |                                \  }}|j        dk    s
J |            |dk    sJ d	|vsJ d S )
Nak  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(int(2e6))

        if __name__ == '__main__':
            # warm-up call to launch the workers and start the resource_tracker
            _ = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(id)(i) for i in range(20))

            time.sleep(0.5)

            slice_of_data = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(return_slice_of_data)(data, 0, 20) for _ in range(10))
    r   r<   
PYTHONPATHro   rq   rr   envr   r      resource_tracker)rs   r   r   environcopypathdirname__file__rt   ru   rv   rw   rx   ry   rz   r{   )r   r}   _r   r~   r   r   s          r   3test_permission_error_windows_memmap_sent_to_parentr     s    ( 	) , 1XX . .jooGOOH55L^T3'??	
 
 
 	
==??S|q   #   czzzz"#-----. .r   c                    t          j        t          d                     t          d| d          fdt	          d          D                       \  } t          d| d          fdt	          d          D                       \  }t
          j                            |          t
          j                            |          k    sJ d S )N      Y@r2   rF   n_jobsr   r   c              3   T   K   | ]"} t          t                    d           V  #dS r\   Nr   getattrr   r   r%   s     r   	<genexpr>z6test_parallel_isolated_temp_folders.<locals>.<genexpr>  Q       F F01
++F F F F F Fr   rD   c              3   T   K   | ]"} t          t                    d           V  #dS r   r   r   s     r   r   z6test_parallel_isolated_temp_folders.<locals>.<genexpr>  r   r   r   rT   intr   r   r   r   r   )r   
filename_1
filename_2r%   s      @r   #test_parallel_isolated_temp_foldersr     s     Ic#hhEE81g"EEE F F F F5:1XXF F F  LZ F81g"EEE F F F F5:1XXF F F  LZ 7??:&&"'//**E*EEEEEEEr   c                    t          j        t          d                    t          d| d          5 } |fdt	          d          D                       \  } |fdt	          d          D                       \  }d d d            n# 1 swxY w Y   t
          j                            |          t
          j                            |          k    sJ d S )Nr   r2   rF   r   c              3   T   K   | ]"} t          t                    d           V  #dS r   r   r   s     r   r   z9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>  8      OO)))%<<OOOOOOr   rD   c              3   T   K   | ]"} t          t                    d           V  #dS r   r   r   s     r   r   z9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>  r   r   r   )r   r~   r   r   r%   s       @r   &test_managed_backend_reuse_temp_folderr     s   
 Ic#hhE	G	;	;	; PqqOOOOeAhhOOOOOqOOOOeAhhOOOOOP P P P P P P P P P P P P P P 7??:&&"'//**E*EEEEEEEs   ABBBc                     t          j        t          d                    } t                      }t                      }d }t	          j        || |f          }t	          j        || |f          }|                                 |                                 |                                 |                                 t          |          dk    sJ t          |          dk    sJ ||k    sJ d S )Nr   c                 4    t          ddd          5 }t          d          D ]Z} | fdt          d          D                       \  }|                    t          j                            |                     [	 d d d            d S # 1 swxY w Y   d S )Nr   r2   rF   )r   r   r   c              3   T   K   | ]"} t          t                    d           V  #dS r   r   r   s     r   r   z]test_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename.<locals>.<genexpr>  s8      UUq/ww//zBBUUUUUUr   rD   )r   r   addr   r   r   )r%   	temp_dirsr~   r   r\   s   `    r   concurrent_get_filenamezJtest_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename  s    fQ2>>> 	9!2YY 9 9QUUUUERSHHUUUUU
bgooh7788889	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9s   A+BBB)targetr,   rD   )	r   rT   r   set	threadingThreadstartrP   len)r%   temp_dirs_thread_1temp_dirs_thread_2r   t1t2s         r   )test_memmapping_temp_folder_thread_safetyr      s    Ic#hhE9 9 9 
	&e5G-H
 
 
B 
	&e5G-H
 
 
B HHJJJHHJJJGGIIIGGIII!""a''''!""a''''!3333333r   c            	         d} g d}|D ]\  }}}t          j        t          j        d|                     ||          gt           j        t           j                  }|                                 |                                \  }}|j        |k    sJ |	                                            d|vsJ |	                                            d S )Na>  if 1:
        import os
        import numpy as np
        from joblib import Parallel, delayed
        from joblib.externals.loky.backend import resource_tracker
        from concurrent.futures import ThreadPoolExecutor, wait

        resource_tracker.VERBOSE = 0

        array = np.arange(int(1e2))

        temp_dirs_thread_1 = set()
        temp_dirs_thread_2 = set()


        def raise_error(array):
            raise ValueError


        def parallel_get_filename(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(getattr)(array, "filename") for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        def parallel_raise(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(raise_error)(array) for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        executor = ThreadPoolExecutor(max_workers=2)

        # both function calls will use the same loky executor, but with a
        # different Parallel object.
        future_1 = executor.submit({f1}, array, temp_dirs_thread_1)
        future_2 = executor.submit({f2}, array, temp_dirs_thread_2)

        # Wait for both threads to terminate their backend
        wait([future_1, future_2])

        future_1.result()
        future_2.result()
    ))parallel_get_filenamer   r   )r   parallel_raiserD   )r   r   rD   ro   )f1f2rp   r   )
rt   ru   rv   rw   rs   rx   ry   rz   r{   r|   )r}   functions_and_returncodesr  r  r{   r~   r   r   s           r   ?test_multithreaded_parallel_termination_resource_tracker_silentr  &  s    1Cd! ! ! 8 	< 	<B
^T3:::#;#;<??
 
 

 	
3|z)))3::<<)))"#---szz||----	< 	<r   c                 "   d                     |           }t          j                                        }t          j                            t                    |d<   t          j        t          j
        d|gt          j        t          j        |          }|                                 |                                \  }}|j        dk    sJ |                                            |dk    sJ |                                            d|vsJ d S )	Na  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(100)

        if __name__ == '__main__':
            for i in range(5):
                slice_of_data = Parallel(
                    n_jobs=2, max_nbytes=1, backend='{b}')(
                        delayed(return_slice_of_data)(data, 0, 20)
                        for _ in range(10)
                    )
    r   r   ro   r   r   r   r   )rs   r   r   r   r   r   r   rt   ru   rv   rw   rx   ry   rz   r{   r|   )r   r}   r   r~   r   r   s         r   'test_many_parallel_calls_on_same_objectr  q  s    $ 	% & *//

C11C	s#		 	 	A FFHHH}}HC<1cjjll#:::szz||:::c))))))r   c                     t          j        t          d                     t          d| d          fdt	          d          D                       \  }t          |          J d S )Ng     @@r2   r   r   c              3   R   K   | ]!} t          t                              V  "d S r!   )r   r&   )r   r   r-   s     r   r   z8test_memmap_returned_as_regular_array.<locals>.<genexpr>  sP       C C67+*++D11C C C C C Cr   rD   )r   r   r   r   r   r   )r   resultr-   s     @r   %test_memmap_returned_as_regular_arrayr
    s     73s88DBxq'cBBB C C C C;@88C C C  HV v&&.....r   c                    | dk    r3t           j                            d          rt          j        d           d                    |           }t          j        t           j        d|gt          j	        t          j	                  }|
                                 |                                \  }}|                                }|                                }|j        dk    sJ |d	z   |z               d
|vs
J |            d S )Nr   winzVThe temporary folder cannot be deleted on Windows in the presence of a reference cycler   r   ro   rp   r   r   resource_tracker)rv   platform
startswithpytestxfailrs   rt   ru   rw   rx   ry   rz   r|   r{   r   s        r   2test_resource_tracker_silent_when_reference_cyclesr    s   " &S\44U;; 	,	
 	
 	

$ 	% & 		s#JOJO	 	 	A FFHHH}}HC
**,,C
**,,C<1cFlS0S(((#(((((r   c                 *   t          j        |j                  g k    sJ  | dd|j        d          }	 t          j        |j                  g k    sJ t           j                            |j                  rJ t          j        dt          j                  j	        dk    sJ |
                    t          fdt          j        d	                   D                        t          j        |j                  g k    sJ t          j        d
t          j                  j	        dk    sJ |
                    t          fdt          j        d	                   D                        t           j                            |j                  sJ t          j        |j                  }t!          |          dk    sJ t          j        dgd
z  d          }|
                    t$          |g          }|d	         rJ 	 |                                 t          d          D ]7}t)          d           t           j                            |j                  s n(8t+          d                    |j                            ~dS # |                                 t          d          D ]7}t)          d           t           j                            |j                  s n(8t+          d                    |j                            ~w xY w)z0Check that large arrays are not copied in memoryr<   (   r2   )r   r   verboser=   r;   r   c                     g | ]}|d fS r   r   )r   r   smalls     r   r   z9test_memmapping_pool_for_large_arrays.<locals>.<listcomp>      KKKUAsOKKKr   r   r      c                     g | ]}|d fS r  r   r   r   larges     r   r   z9test_memmapping_pool_for_large_arrays.<locals>.<listcomp>  r  r   rD   abcobjectrF   皙?z#temporary folder {} was not deletedN)r   r   rQ   r   exists_temp_folderr   r   r   nbytesr   r0   r   r9   rS   isdirr   r%   r	   r   r   r   rs   )	r   r[   r~   dumped_filenamesobjectsresultsr   r  r  s	          @@r   %test_memmapping_pool_for_large_arraysr(    s    :fn%%++++ 	bfnaHHHA(z&.))R////7>>!.11111,,,|r!!!!	kKKKKU5;q>5J5JKKKLLL z&.))R//// 2:...|s""""	kKKKKU5;q>5J5JKKKLLL w}}Q^,,,,,:an55#$$)))) (E7S=999%%,wi881: 	
r 	 	A#JJJ7>>!.11  !5<<Q^LL   AA 	
r 	 	A#JJJ7>>!.11  !5<<Q^LL   s   GJ BLz,https://github.com/joblib/joblib/issues/1086)marksc                    d                     |           }t          j                                        }t          j                            t                    |d<   t          j        t          j
        d|gt          j        t          j        |          }|                                 |                                \  }}|                                |                                }}|                    d          d         }|j        dk    sJ |p|            |dk    sJ t          j                            |          rJ d S )	Na  if 1:
        import os
        from pathlib import Path
        from time import sleep

        import numpy as np
        from joblib import Parallel, delayed
        from testutils import print_filename_and_raise

        data = np.random.rand(1000)

        def get_temp_folder(parallel_obj, backend):
            if "{b}" == "loky":
                return Path(parallel_obj._backend._workers._temp_folder)
            else:
                return Path(parallel_obj._backend._pool._temp_folder)


        if __name__ == "__main__":
            try:
                with Parallel(n_jobs=2, backend="{b}", max_nbytes=100) as p:
                    temp_folder = get_temp_folder(p, "{b}")
                    p(delayed(print_filename_and_raise)(data)
                              for i in range(1))
            except ValueError as e:
                # the temporary folder should be deleted by the end of this
                # call but apparently on some file systems, this takes
                # some time to be visible.
                #
                # We attempt to write into the temporary folder to test for
                # its existence and we wait for a maximum of 10 seconds.
                for i in range(100):
                    try:
                        with open(temp_folder / "some_file.txt", "w") as f:
                            f.write("some content")
                    except FileNotFoundError:
                        # temp_folder has been deleted, all is fine
                        break

                    # ... else, wait a bit and try again
                    sleep(.1)
                else:
                    raise AssertionError(
                        str(temp_folder) + " was not deleted"
                    ) from e
    r   r   ro   r   
r    )rs   r   r   r   r   r   r   rt   ru   rv   rw   rx   ry   rz   r|   splitr{   r!  )r   r}   r   r~   r   r   r\   s          r   &test_child_raises_parent_exits_cleanlyr.    s   :-Z 	[ \ *//

C11C	s#		 	 	A FFHHH}}HCzz||SZZ\\Cyyq!H<1cjS"9999w~~h'''''''r   c                     | dd|j                   }	 t          j        |j                   g k    sJ t          j        dt          j                  j        dk    sJ |                    t          fdt          j
        d                   D                        t          j        |j                   g k    sJ 	 |                                 ~dS # |                                 ~w xY w)	z2Check that large arrays memmapping can be disabledr<   Nr   r   r;   r  c                     g | ]}|d fS r  r   r  s     r   r   zBtest_memmapping_pool_for_large_arrays_disabled.<locals>.<listcomp>  r  r   r   )rQ   r   r   r   r   rS   r#  r   r0   r   r9   r   )r   r[   r~   r  s      @r   .test_memmapping_pool_for_large_arrays_disabledr1  w  s     	d???Az&.))R//// 2:...|s""""	kKKKKU5;q>5J5JKKKLLL z&.))R///// 	
AA 	
s   B*C C/c                    t           j        }	 t          d          t           _         | dd          }	 |j        }d}|                    |          sJ t
          j                            |          sJ t          j	        dt          j
                  }|j        dk    sJ |                    t          |gdz             t          t          j        |                    d	k    sJ t          j	        dt          j
                  d
z  }|j        dk    sJ |                    t          |gdz             t          t          j        |                    d
k    sJ 	 |                                 ~n# |                                 ~w xY wt#          d          D ]2}t
          j                            |          s nt%          d           3t'          d          	 |t           _        dS # |t           _        w xY w)z1Check that memmapping uses /dev/shm when possibleg    ~Ar<   rF   r   z"/dev/shm/joblib_memmapping_folder_r   r;   r  rD   r2   r   z(temporary folder of pool was not deletedN)jmrSYSTEM_SHARED_MEM_FS_MIN_SIZEr   r"  r  r   r   r!  r   r   rS   r#  r   idr   r   r   r   r   r   )r   	orig_sizer~   r   folder_prefixr^   r_   r   s           r   'test_memmapping_on_large_enough_dev_shmr9    s    1I+6 -0II)GA"%%%	  !~@M#..}=====7>>"233333 2:...A8s????EE"qcBh rz"233449999
 2:...2A8s????EE"qcBhrz"2334499999 KKMMM KKMMMEEEEs 	M 	MA7>>"233 #JJJJ !KLLL 
 -6)))I)5555s$   &G0 D*E6  G0 6FAG0 0G>c                    t           j        }	 t          d          t           _         | dd          }	 |j        }|                    d          rJ 	 |                                 ~n# |                                 ~w xY wt          j                            |          rJ 	 |t           _        d S # |t           _        w xY w)Ng@6Dr<   rF   r3  z/dev/shm)	r4  r5  r   r"  r  r   r   r   r!  )r   r7  r~   r   s       r   $test_memmapping_on_too_small_dev_shmr;    s     1I6 -0JJ)GA"%%%	  !~'22:>>>>>> KKMMM KKMMMEEEE 7>>"2333333,5)))I)5555s"   &B4 A* B4 *B$B4 4Cc                 v   t           j        j        } | dd|j                  }	 |                    t           j        d          }|                                }t          |          rJ  ||t          j        d                     |                                 ~dS # |                                 ~w xY w)z:Check that large arrays are not copied in memory in returnr<   rF   r   )  r,   r=  N)	r   r)   r*   rQ   apply_asyncr   getr	   r   )r   r[   r*   r~   resr  s         r   /test_memmapping_pool_for_large_arrays_in_returnrB    s     6 	bfn===AmmBG'm22		'.....5"'$--000	AA 	
s   A$B! !B8c                 .    t          |           sJ | |z  S )z4Multiplication function to be executed by subprocess)r	   )r^   n_timess     r   _worker_multiplyrE  
  s     """""w;r   c                    t           j        j        } | dd|j                  }	 t          j        t          j        d                              d          d          dddd	ddf         }|                    t          |df
          	                                }t          |          rJ  ||d|z             |                                 ~dS # |                                 ~w xY w)zCheck that memmaps with a bad buffer are returned as regular arrays

    Unary operations and ufuncs on memmap instances return a new memmap
    instance with an in-memory buffer (probably a numpy bug).
    r<   rF   r   ip  )r=  r2   r<   r@   )rA   NrD   r>  )r   r)   r*   rQ   rW   rT   rV   r?  rE  r@  r	   r   )r   r[   r*   r~   r^   r_   s         r   6test_workaround_against_bad_memmap_with_copied_buffersrG    s     6bfn===A Jry..|<<CHHHBQBPQPQPQR MM*!QM88<<>>'*****1a!e$$$	AA 	
s   BC C.c                     | S r!   r   )args    r   identityrJ  /  s    Jr   zfactory,retry_nor<   c                 @    g | ]\  }}d                      ||          S )z{}, {})rs   )r   rK   ys      r   r   r   <  s:     	 	 	Aq 	1	 	 	r   c                    |                     d          j        }dt          j        z  }t          j        dz   }t	          |d|d|          } | d|j                  }|                    t          |f	                                          }t          |t          j
                  sJ |j        |k    sJ t          j                            ||           |                                 d S )
Nr5   r=   rD   r6   r   )r:   r9   r8   rB   r2   r   r>  )rP   rQ   r   r   r
   r?  rJ  r@  rY   r   rR   rB   r)   r*   r   )	r   retry_nor[   r   r   rB   objr~   r	  s	            r    test_pool_memmap_with_big_offsetrQ  3  s    & KK$$,Et))D'!+F
e$d'&
Q
Q
QCv~...A]]83&]115577Ffbi(((((=F""""J!!#v...KKMMMMMr   c                 &   d}t          || j                  \  }}|du sJ ||                     d          j        k    sJ t          |d           \  }}t          j                            d          r|du sJ |                    |          sJ d S )Ntest.tmpdirFrN  r  )r   rQ   rP   rv   r  r  endswith)r[   pool_folder_namepool_folder
shared_mems       r   test_pool_get_temp_dirrX  S  s    $+,<fnMMK&++m44<<<<<+,<$OOOK
|u%% #U"""" 01111111r   c                 &   d}ddl }t          |j        j        d          r |                    |j        j        d           t          |d          \  }}t          j                            d          r|du sJ |	                    |          sJ dS )z`Check that _get_temp_dir works when os.statvfs is not defined

    Regression test for #902
    rS  r   NstatvfsrN  r  F)
joblib._memmapping_reducerhasattr_memmapping_reducerr   delattrr   rv   r  r  rT  )r[   monkeypatchrU  joblibrV  rW  s         r   !test_pool_get_temp_dir_no_statvfsra  _  s    
 %%%%%v),i88 FF699EEE+,<$OOOK
|u%% #U"""" 01111111r   z1This test fails with a PermissionError on Windows	mmap_moder?   r6   c                    d d t          d          D             } t          | dd          fdt          |          D                       }t          |          D ]%\  }}t          j                            ||           &d S )Nc                     || d d <   | S r!   r   )arrvalues     r   funcz4test_numpy_arrays_use_different_memory.<locals>.funcw  s    AAA
r   c                 :    g | ]}t          j        d d          S ))rF   rF   rS   r;   )r   zeros)r   r   s     r   r   z:test_numpy_arrays_use_different_memory.<locals>.<listcomp>{  s'    EEEabhxy111EEEr   rF   r   r2   )rb  r   r   c              3   P   K   | ] \  }} t                    ||          V  !d S r!   r   )r   r   re  rg  s      r   r   z9test_numpy_arrays_use_different_memory.<locals>.<genexpr>}  sR       D D"(!Sc1D D D D D Dr   )r   r   	enumerater   r)   r*   )rb  arraysr'  r   re  rg  s        @r   &test_numpy_arrays_use_different_memoryrn  q  s       FE599EEEFChqCCC D D D D,5f,=,=D D D  G G$$ . .3

%%c1----. .r   c                  X   dd} t          j        d          }t                                          |d                               |          dk    sJ |}                    |          dk    sJ                     |d                               |          dk    sJ ~t          j                     t          j                  dk    sJ                     |          dk    sJ ~ | j                   t          j        d          }                    |d           t          j                  dk    sJ                     |          dk    sJ t          t                    5                      t          j        d                     d d d            n# 1 swxY w Y   ~ | j                   d t          fd	t          d
          D                       }t          j                    dk    r t          rdnd}t          |          |k     sJ d S d S )Nr   c                     t          |          D ]:}t          |           dk    r d S t          j                     t	          d           ;t          |           dk    sJ d S )Nr   r   )r   r   gccollectr   )	containerretriesr   s      r   assert_empty_after_gc_collectz>test_weak_array_key_map.<locals>.assert_empty_after_gc_collect  sd    w 	 	A9~~""JLLL#JJJJ9~~""""""r   *   r^   r_   rD   rH   c                 ,   t          j        d          }t          t                    5  |                     |           d d d            n# 1 swxY w Y   |                     ||           |                     |          |k    sJ t          |          S )Nrv  )r   r   r   KeyErrorr@  r   r6  )mr   r^   s      r   get_set_get_collectz4test_weak_array_key_map.<locals>.get_set_get_collect  s    GBKKH 	 	EE!HHH	 	 	 	 	 	 	 	 	 	 	 	 	 	 		auuQxx1}}}}!uus   AAAc                 (    g | ]} |          S r   r   )r   r   rz  ry  s     r   r   z+test_weak_array_key_map.<locals>.<listcomp>  s'    EEEA))!Q//EEEr   r=  CPythoni  )r   )r   r   r   r   r@  rq  rr  r   _datar   rx  r   r  python_implementationr   )ru  r^   r_   rH   
unique_idsmax_len_unique_idsrz  ry  s         @@r   test_weak_array_key_mapr    s^   # # # # 	AAEE!SMMM5588s????	A5588s????EE!SMMM5588s????	JLLLqw<<15588s????	!!!'***
AEE!SMMMqw<<15588s????			  	bgbkk               	
!!!'***   EEEEEtEEEFFJ%''944 %4<SS:!33333 54 43s   5(F))F-0F-c                      t                      } t          t          j                  5  t          j        |            d d d            d S # 1 swxY w Y   d S r!   )r   r   picklePicklingErrordumps)ry  s    r   #test_weak_array_key_map_no_picklingr    s    A	$	%	%  Q                 s   A

AAc                 R   t          |                     d                    t          j        dd          }|                               fdd              } t          d          fd	|fD                       } t          d          fd
|fD                       }t          j                            ||            t          d          fdt          d          D                       }t          j                            |d         |           d S )Nzarr.datrF   r   r;   c                      t                    5 } t          j        |                                 dt          j        d          }d d d            n# 1 swxY w Y   t	          j        dt          j        |d          S )Nr   )accessrB   r   )r8   r]   rB   )openr   filenoACCESS_READr   rZ   r   )fdmmtestfiles     r   _read_arrayz%test_direct_mmap.<locals>._read_array  s    (^^ 	Nr299;;$2B1MMMB	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	Nz%rx1EEEEs   5AAAc                     | dz  S Nr2   r   rK   s    r   rg  ztest_direct_mmap.<locals>.func      !tr   r2   r   c              3   H   K   | ]} t                    |          V  d S r!   rk  r   rK   rg  s     r   r   z#test_direct_mmap.<locals>.<genexpr>  3      ;;!]WT]]1--;;;;;;r   c              3   H   K   | ]} t                    |          V  d S r!   rk  r  s     r   r   z#test_direct_mmap.<locals>.<genexpr>  3       A Aaq!1!1 A A A A A Ar   c              3   F   K   | ]} t                                V  d S r!   rk  r   r   r  s     r   r   z#test_direct_mmap.<locals>.<genexpr>  4       J JA!5!5!5!7!7 J J J J J Jr   rD   r   )	strrP   r   rT   tofiler   r)   r*   r   r[   r^   re  refr'  r  rg  r  s        @@@r   test_direct_mmapr    sW    6;;y))**H
	"G$$$AHHXF F F F F
   +--C (!


;;;;s;;;
;
;C !ha    A A A AC5 A A AAAGJ!!'3/// !ha    J J J Jq J J JJJGJ!!'!*c22222r   c                    t          |                     d                    t          j        dd                              dd          }|                               fdd              } t          d	          fd
|fD                       } t          d	          fd|fD                       }|d         j        d         rJ t          j        	                    ||            t          d	          fdt          d          D                       }t          j        	                    |d         |           d S )Nzarr2.datrF   r   r;   r=   r2   c                  X    t          j                  } |                     dd          S )Nr=   r2   )r   rR   rV   )r  r  s    r   r  z=test_parallel_memmap2d_as_memmap_1d_base.<locals>._read_array  s&    Yx  zz!Qr   c                     | dz  S r  r   r  s    r   rg  z6test_parallel_memmap2d_as_memmap_1d_base.<locals>.func  r  r   r  c              3   H   K   | ]} t                    |          V  d S r!   rk  r  s     r   r   z;test_parallel_memmap2d_as_memmap_1d_base.<locals>.<genexpr>  r  r   c              3   H   K   | ]} t                    |          V  d S r!   rk  r  s     r   r   z;test_parallel_memmap2d_as_memmap_1d_base.<locals>.<genexpr>  r  r   r   F_CONTIGUOUSc              3   F   K   | ]} t                                V  d S r!   rk  r  s     r   r   z;test_parallel_memmap2d_as_memmap_1d_base.<locals>.<genexpr>  r  r   rD   )r  rP   r   rT   rV   r  r   flagsr)   r*   r   r  s        @@@r   (test_parallel_memmap2d_as_memmap_1d_baser    sz   
 6;;z**++H
	"G$$$,,Q22AHHX            +--C (!


;;;;s;;;
;
;C !ha    A A A AC5 A A AAAGqz////J!!'3/// !ha    J J J Jq J J JJJGJ!!'!*a00000r   )Xr   rq  	itertoolsr   r   r  r  rt   rv   r   timer   r  r[  r]  r4  r   r   r   r   r   r	   joblib.backportsr
   joblib.executorr   TestExecutorjoblib.parallelr   r   joblib.poolr   joblib.test.commonr   r   r   r   r   joblib.testingr   r   r   r   r#   r&   r0   r3   rk   r   r   r   get_memmapping_executorr   r   r   r   r   r   r   r  r  r
  r  r(  parammarkr  r.  r1  r9  r;  rB  rE  rG  rJ  listproductr   r   r  rQ  rX  ra  rn  r  r  r  r  r   r   r   <module>r     s2
       				      				       



            ( ( ( ( ( (                ) ( ( ( ( ( F F F F F F - - - - - - - - & & & & & &              7 6 6 6 6 6 6 6 6 6> > >/ / /  
< < <@ @ @ S- S-  S-l LG$X  * *   
*Z )+ )+  )+X C C CD \9:	F#  
0 0   0f \9:	F#  
$ $   $N Y*F344C C 54  CB Y*F344*. *. 54  *.Z Y*F344
F 
F 54  
F Y*F344F F 54  F !4 !4  !4H F< F<  F<R Y*F344%* %* 54  %*P Y*F344/ / 54  / Y*F3442. 2. 54  2.j \9:	F#  
1 1   1h +##E $  	
 	
 	
 	 K( K(   K(\ \9:	F#  
    , \9:	F#  
.6 .6    .6b \9:	F#  
6 6    60 \9:	F#  
    ,   \9:	F#  
    0   D	\ABEE!HH	
 	
 
	 	%I%'8&&A33sEERSHHCUCUVV	 	 	      $	2 	2 	22 2 2$ LG$W   [4,''. . ('  
. 84 84 84v   3 3  36 1 1  1 1 1r   