.Pho`dZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z ddl mZmZmZmZmZddlmZ en#e$redZYnwxYw ddlZddlmZn #e$rdZYnwxYwdd lmZdd lm Z dd l!m"Z"dd l#m$Z$m%Z%m&Z&d Z'e(dZ)ej*ej+zej,zZ-ej*ej+zZ.e/Z0dZ1dZ2dZ3e3e"j4d<GddZ5dZ6d$dZ7dZ8dZ9dZ:dZ;Gdde<Z= d%d!Z>Gd"d#e<Z?dS)&z/ Reducer using memory mapping for numpy arrays N)mmap)util)HIGHEST_PROTOCOL PicklingErrordumpsloads whichmodule)uuid4) as_strided) make_memmap) delete_folder)resource_tracker)dumploadload_temporary_memmapz/dev/shmgeAcddlm}tjdt j|t j| |ddS)Nr )_resource_trackerzj[FINALIZER CALL] object mapping to {} about to be deleted, decrementing the refcount of the file (pid: {})file) 'externals.loky.backend.resource_trackerrrdebugformatospathbasenamegetpid maybe_unlink)filenamers Z/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/joblib/_memmapping_reducer.py_log_and_unlinkr ;suJJJJJJJ ;;A6 G  X & & < <  ""8V44444c 4tjdt|t |t j|jt j tj |t|jdS)NzD[FINALIZER ADD] adding finalizer to {} (id {}, filename {}, pid {})) rrrtypeidrrrrrweakreffinalizer )memmaps radd_maybe_unlink_finalizerr(GsuJNUU LL"V**bg&6&6v&G&G    V_fo>>>>>r!cd}td|dzD]v} tj|dS#t$rEt jd|||krtjdYht$rYswxYwdS)a=Wrapper around os.unlink with a retry mechanism. The retry mechanism has been implemented primarily to overcome a race condition happening during the finalizer of a np.memmap: when a process holding the last reference to a mmap-backed np.memmap/np.array is about to delete this array (and close the reference), it sends a maybe_unlink request to the resource_tracker. This request can be processed faster than it takes for the last reference of the memmap to be closed, yielding (on Windows) a PermissionError in the resource_tracker loop. r z9[ResourceTracker] tried to unlink {}, got PermissionErrorg?N) rangerunlinkPermissionErrorrrrtimesleepFileNotFoundError)r NUM_RETRIESretry_nos r unlink_filer3PsK![1_--  Ih    EE JKRR    ;&& 3     D s0A B > B  B rc*eZdZdZdZdZdZdZdS)_WeakArrayKeyMapaA variant of weakref.WeakKeyDictionary for unhashable numpy arrays. This datastructure will be used with numpy arrays as obj keys, therefore we do not use the __get__ / __set__ methods to avoid any conflict with the numpy fancy indexing syntax. ci|_dSN_dataselfs r__init__z_WeakArrayKeyMap.__init__{s  r!cv|jt|\}}||urt||Sr7)r9r$KeyError)r;objrefvals rgetz_WeakArrayKeyMap.get~s<:bgg&S 355  3--  r!ct| j\}}||urt|n+#t$rfd}tj||}YnwxYw||fj<dS)Ncj=dSr7r8)_keyr;s r on_destroyz(_WeakArrayKeyMap.set..on_destroysJsOOOr!)r$r9r>r%r@)r;r?valuer@rErGrFs` @rsetz_WeakArrayKeyMap.setsgg /Z_FCsuuCsmm#  / / / $ $ $ $ $ $+c:..CCC /u* 3s+?%A'&A'c td)Nz"_WeakArrayKeyMap is not pickleable)rr:s r __getstate__z_WeakArrayKeyMap.__getstate__s@AAAr!N)__name__ __module__ __qualname____doc__r<rBrIrKr!rr5r5ss_%%%$BBBBBr!r5cxt|dd}|dSt|tr|St|S)z@Recursively look up the original np.memmap instance base if any.baseN)getattr isinstancer_get_backing_memmap)abs rrUrUsF64  Ayt At  &#1%%%r!cd}| tjdd}|tjt rt tdr tjt }|j|j z}|tkr\t }tj ||}tj|stj |d}n#ttf$rd}YnwxYw|tj}tjtj|}tj ||}||fS)aGet the full path to a subfolder inside the temporary folder. Parameters ---------- pool_folder_name : str Sub-folder name used for the serialization of a pool instance. temp_folder: str, optional Folder to be used by the pool for memmapping large arrays for sharing memory with worker processes. If None, this will try in order: - a folder pointed by the JOBLIB_TEMP_FOLDER environment variable, - /dev/shm if the folder exists and is writable: this is a RAMdisk filesystem available by default on modern Linux distributions, - the default system temporary folder that can be overridden with TMP, TMPDIR or TEMP environment variables, typically /tmp under Unix operating systems. Returns ------- pool_folder : str full path to the temporary folder use_shared_mem : bool whether the temporary folder is written to the system shared memory folder or some other temporary folder. FNJOBLIB_TEMP_FOLDERstatvfsT)renvironrBrexistsSYSTEM_SHARED_MEM_FShasattrrZf_bsizef_bavailSYSTEM_SHARED_MEM_FS_MIN_SIZEjoinmakedirsIOErrorOSErrortempfile gettempdirabspath expanduser)pool_folder_name temp_folderuse_shared_mem shm_statsavailable_nbytes pool_folders r _get_temp_dirrpsS<Njnn%94@@  7>>. / / #GB 4J4J # #J';<< #,#4y7I#I #&CCC#7K"$',,{>+661 K000%)NW% # # ##  #)++ '//"'"4"4["A"ABBK',,{,<==K  &&s!BC11DDc$t|duS)z?Return True if a is backed by some mmap buffer directly or not.N)rU)rVs rhas_shareable_memoryrrs q ! ! --r!c |dkrd}|t|||||||St|||||||} t| ||S)z2Reconstruct an array view on a memory mapped file.zw+zr+N)dtypeshapemodeoffsetorderunlink_on_gc_collect)rtrurwrvrxry)rustrides)r r ) rrtrvrwrxrurztotal_buffer_lenryrRs r_strided_from_memmapr|s t|| !5     "!5   $eW====r!c tjd|jt j ddlm}n#ttf$r ddl m}YnwxYw||\}}||d}||z }||j z }|j dkr|j drd}nd}|j ds |j drd }d } n|j}||z |jz} t |j|j|j|||j|| d f fS) aPickling reduction for memmap backed arrays. a is expected to be an instance of np.ndarray (or np.memmap) m is expected to be an instance of np.memmap on the top of the ``base`` attribute ancestry of a. ``m.base`` should be the real python mmap object. zC[MEMMAP REDUCE] reducing a memmap-backed array (shape, {}, pid: {})r) byte_boundsr F_CONTIGUOUSFC C_CONTIGUOUSNF)rrrrurrnumpy.lib.array_utilsr~ModuleNotFoundError ImportErrornumpyrwndimflagsrzitemsizer|rrtrv) rVmr~a_starta_endm_startrwrxrzr{s r_reduce_memmap_backedrsi JMTT GRY[[   &5555555  -&&&%%%%%%%%&![^^NGUk!nnQG w F ahF vzzagn-zw~;!'."9;)!GO :  J G F   G      sAA! A!ct|}t|tjr|jt vrt ||Stttj |tffS)z5reduce a np.array or a np.memmap from a child processprotocol) rUrTnpr'r JOBLIB_MMAPSrrrasarrayr)rVrs rreduce_array_memmap_backwardrWshAA!RY KAJl$B$B%Q***bjmm6FGGGIJJr!c@eZdZdZ d dZedZdZdZdS) ArrayMemmapForwardReducera#Reducer callable to dump large arrays to memmap files. Parameters ---------- max_nbytes: int Threshold to trigger memmapping of large arrays to files created a folder. temp_folder_resolver: callable An callable in charge of resolving a temporary folder name where files for backing memmapped arrays are created. mmap_mode: 'r', 'r+' or 'c' Mode for the created memmap datastructure. See the documentation of numpy.memmap for more details. Note: 'w+' is coerced to 'r+' automatically to avoid zeroing the data on unpickling. verbose: int, optional, 0 by default If verbose > 0, memmap creations are logged. If verbose > 1, both memmap creations, reuse and array pickling are logged. prewarm: bool, optional, False by default. Force a read on newly memmapped array to make sure that OS pre-cache it memory. This can be useful to avoid concurrent disk access when the same data array is passed to different worker processes. rTc&||_||_||_t||_|dkr&|jt |_n||_||_t|_ t|_ ||_ dS)Nauto) _max_nbytes_temp_folder_resolver _mmap_modeintverbose _temp_folder startswithr]_prewarmr5_memmaped_arraysrI_temporary_memmaped_filenames_unlink_on_gc_collect)r; max_nbytestemp_folder_resolver mmap_moderyrprewarms rr<z"ArrayMemmapForwardReducer.__init__s&%9"#7|| f   $ 1 < <=Q R RRDMM#DM 0 2 2-0UU*%9"""r!c*|Sr7)rr:s rrz&ArrayMemmapForwardReducer._temp_folders))+++r!c^|jd|j|jf}|j|jd}t ||fS)N)rr)rrrrrr)r;argskwargss r __reduce__z$ArrayMemmapForwardReducer.__reduce__s? $9ST|}  )$66r!c t|}|*t|tjrt ||S|jjs|j|j|jkr tj |j tj |j tn.#t$r!}|jtjkr|Yd}~nd}~wwxYw |j|}n#t&$rudtjt-t/jt3j}|j||YnwxYwtj|j |}||jv}|j||j rtCj"|d|rtCj"|dtj#|stIj%d|j&|j|tO||D]}tj |tP|j)r(tU||j+,nPtIj%d|j&|jtj-|t\||j+|j ffStIj%d|j&|jt^ta|tbffS)Nz {}-{}-{}.pklrzP[ARRAY DUMP] Pickling new array (shape={}, dtype={}) creating a new memmap at {})rzN[ARRAY DUMP] Pickling known array (shape={}, dtype={}) reusing memmap file: {}zB[ARRAY DUMP] Pickling array (NO MEMMAPPING) (shape={}, dtype={}).r)2rUrTrr'rrt hasobjectrnbytesrrcrchmodFOLDER_PERMISSIONSreerrnoEEXISTrrBr>rrr$ threadingcurrent_threadr hexrIrrbraddrrregisterr\rrrurFILE_PERMISSIONSrrrmaxrrrrr)r;rVrerr is_new_memmapdumped_filenames r__call__z"ArrayMemmapForwardReducer.__call__s;  " " =Z2955=(A.. .!^ C ,4+++  D-...*,>????   7el**G+****  7044Q77 7 7 7 *00IKKI$<$>$>!?!?%))!X66666 7w||D$5x@@H %D,NNM  . 2 28 < < <) <!)(F;;; < !)(F;;;7>>(++  228&!'82T2T(,Ax'8'8@@OH_.>????=D T_===AACCC ..4f"'*:*:8*D*D//&4?D,FG  J$fQWag66   E!.>???AB Bs+#8B C&CC C&&A$&<# )E"*%'C")$ . ..r!c<eZdZdZd dZdZdZdZdZ d d Z dS) TemporaryResourcesManagera9Stateful object able to manage temporary folder and pickles It exposes: - a per-context folder name resolving API that memmap-based reducers will rely on to know where to pickle the temporary memmaps - a temporary file/folder management API that internally uses the resource_tracker. Ncd|_||_d|_t|_t j|_i|_|t j}| |dSr7) _current_temp_folder_temp_folder_root_use_shared_memr_cached_temp_foldersr r_id _finalizersset_current_context)r;temp_folder_root context_ids rr<z"TemporaryResourcesManager.__init__Lsh$(!!1#$(FF!77;   J   ,,,,,r!c>||_||dSr7)_current_context_idregister_new_context)r;rs rrz-TemporaryResourcesManager.set_current_context[s$#-  !!*-----r!c||jvrdSdtj|j|}t ||j\}}|||||j|<dS)Nz!joblib_memmapping_folder_{}_{}_{})rrrrrrprregister_folder_finalizer)r;rnew_folder_namenew_folder_pathrEs rrz.TemporaryResourcesManager.register_new_context_s 2 2 2 FBHH TXzO"/@V!W!W OQ  * *?J G G G4CD %j 1 1 1r!c&|j|jS)z@Return a folder name specific to the currently activated context)rrr:s rresolve_temp_folder_namez2TemporaryResourcesManager.resolve_temp_folder_namevs()ABBr!cttdtjdfd}t j||j|<dS)Nrfolderctdgj} |dtjddS#t$r+t jdYdSwxYw)Nr)fromlistTallow_non_emptyrz%Failed to delete temporary folder: {}) __import__rr unregisterrewarningswarnr)rpool_module_namepool_subfolders r_cleanupzETemporaryResourcesManager.register_folder_finalizer.._cleanups' O+<   ndCCCC +NHEEEEE    ;BB>RR s">1A32A3)r rrratexitr)r;rrrrs ` @rrz3TemporaryResourcesManager.register_folder_finalizer|sf '}oFF!.(;;;      ((.x'@'@$$$r!Fc|1t|jD]}||||dS|j|}|r-tj|rt j|D]k}|r4tj tj ||d8tj tj ||dl||z} t|||j |dtj |d|j |d}|tj |dSdS#t $rYdSwxYwdSdS)z9Clean temporary resources created by a process-based poolN)forcerrrr)listr_clean_temporary_resourcesrBrrr\listdirrrrbrrpoprrre)r;rrrrkr finalizers rrz4TemporaryResourcesManager._clean_temporary_resourcess  #4#<==   //e_0   377 CCK# rw~~k::#  " ; 7 7  H )3GLLh??)5GLLh??  5(!+OOOO-11*dCCC$/ XFFF!% 0 4 4Z F FI ,))44444-,DD ?# # # # s7A2E-- E;:E;)NN)NFF) rLrMrNrOr<rrrrrrPr!rrrBs - - - -...DDD.CCC AAA@=B000000r!rr7)NNNrrrFT)@rOrrrstatrfrr.rr%rmultiprocessingrpicklerrrrr uuidr WindowsError NameErrorr#rrnumpy.lib.stride_tricksr r backportsr diskrexternals.loky.backendr numpy_picklerrrr]rraS_IRUSRS_IWUSRS_IXUSRrrrIrr r(r3_CLEANUP_FUNCSr5rUrprrr|rrobjectrrrrPr!rrse   MMMMMMMMMMMMMMLL4::LLL2222222 BBB#"""""444444;;;;;;;;;;"!$C \DL04<?<$,. suu  5 5 5???@+6'&B&B&B&B&B&B&B&BZ&&&"8'8'8'8'v... '>'>'>T999xKKK"hChChChChChChChCX ,/,/,/,/^IIIIIIIIIIs$A AA" A--A76A7