
    0PhF                       d dl Z d dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZ d dlmZm Z  d d	l!m"Z" d d
l#m$Z$ d dl%m&Z& d dl'm(Z(m)Z)m*Z*m+Z+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2m3Z3m4Z4 eeegZ5ddZ6d Z7d Z8d Z9d Z:ej;        <                    dddg          d             Z=d Z>d Z?d Z@d ZAd ZBd ZCd ZDd  ZEd! ZFd" ZGd# ZHej;        <                    d$ ejI        g d%          g d&f ejI        g d%          dfg d'g d(fg d'dfg          d)             ZJej;        <                    d$ ejI        g d*          g d&f ejI        g d+          g d,fg          d-             ZKej;        <                    d$ ejI        g d.          dfg d/dfg d.g d&fg d/g d(fg          d0             ZLej;        <                    d1g d2          d3             ZMd4 ZNej;        <                    d5d6 ejI        g d.          g d7fd6 ejI        g d/          g d8fd9 ejI        g d*          dfd: ejI        g d/          g d;fd< ejI        g d.          d d=gfd< ejI        g d/          d>d?gfd@ ejI        g d.          g dAfd@ ejI        g d/          g dBfdC ejI        g dD          g d(fdC ejI        g dB          g d(fdC ejI        g dA          g d&fg          ej;        <                    dEdFdGg          dH                         ZOej;        <                    dIdJdKdFdLfdMdKdGdLfdNdFg dOfdPdFdQdRfdSi fg          dT             ZPdU ZQej;        <                    dVe5          dW             ZRej;        <                    dVe5          dX             ZSej;        T                    dY          ej;        <                    dVeeg          ej;        <                    dZd[d\g          d]                                     ZUej;        <                    dVe5          d^             ZVej;        <                    dddg          d_             ZWd` ZXej;        <                    dddg          da             ZYdb ZZdc Z[dd Z\de Z]df Z^dg Z_ej;        <                    dhg dig djd gd gfg dig dkd gd gfg dig dld gd gfg dig dmd gd gfg dng djdQgd gfg dng dkdQgd gfg dng dldQgd gfg dng dmdQgd gfg dog djdpgd gfg dog dkd gd gfg dog dld gd gfg dog dmd gd gfg dqg djg drg dofg dqg dkg drg dofg dqg dlg drg dofg dqg dmg drg dofg dsg djg dtg dufg dsg dkg dtg dufg dsg dlg dtg dufg dsg dmg dtg dufg          dv             Z`ej;        <                    dhd=d gdQdQgd=gd gfd d=gdQdQgd=gd gfg dig dwdQgd gfg dng dwdQgd gfg dog dwd gd gfg dqg dwd=gd gfg dsg dwd=gd gfg dtg dwd=gd gfg          dx             Zady Zbej;        <                    dzg d{          d|             Zcej;        <                    d}g d~g dg dg dg dg          d             Zdej;        <                    dd d=gg djdfg dod dQgdfg dg djdfg dg djdfg dg ddfg          d             Zed Zfd Zgd Zhd Zid Zjd Zkd Zl	 ddZmej;        <                    degekejehf          ej;        <                    deelf          d                         Znd Zoej;        <                    dd          ej;        <                    dd          ej;        <                    d epd=                    d                                     Zqd Zrd Zsd Ztej;        <                    dg dsg dwfg dsg dwgfg dsgg dwfg          d             Zud Zvej;        <                    de0          d             Zwd Zxd Zyd Zzd Z{d Z|d Z}d Z~d Zej;        <                    dddg          d             Zd Zd Zd Zd Zej;        <                    dg dAd=dfg dAddQfg dAddfg          d             Zej;        <                    d ejI        g d          d=d=f ejI        g d          d=dQf ejI        g d          dd=f ejI        g d          d=d=f ejI        g d          d=dQf ejI        g d          dd=fg          d             Zej;        <                    d ejI        g dĢ          dg dAf ejI        g dŢ          dQg dAf ejI        g dƢ          dQg dAf ejI        g dǢ          dg dȢfg          ej;        <                    dddg          dʄ                         Zd˄ Zej;        <                    dg dAd=dfg dAddQfg dAdd=fg          d̄             Zej;        <                    dg dAdfg dAdfg          dτ             Zej;        <                    dg dѢg dҢg dӢg dԢg dբgddfg dAg dҢg dӢg dԢg dբgddfg dآg dҢg dӢg dԢg dբgg d/dfg dآg dҢg dӢg dԢg dբgg d;dfg dۢg dҢg dӢg dԢg dբgg dAdfg dۢg dҢg dӢg dԢg dբgg dݢdfd d=gg dߢg dgddfg          d             Zej;        <                    de0          d             Zej;        <                    deeeeg          ej;        <                    dg d          d                         Zd Zd ZdS )    N)stats)datasetssvm)make_multilabel_classification)UndefinedMetricWarning)LogisticRegression)accuracy_scoreaucaverage_precision_scorecoverage_error	dcg_score	det_curve%label_ranking_average_precision_scorelabel_ranking_loss
ndcg_scoreprecision_recall_curveroc_auc_score	roc_curvetop_k_accuracy_score)_dcg_sample_scores_ndcg_sample_scores)train_test_split)label_binarize)_sparse_random_matrix)_convert_containerassert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equal)softmax)CSR_CONTAINERS)check_arraycheck_consistent_lengthcheck_random_stateFc                    | t          j                    } | j        }| j        }|r||dk              ||dk              }}|j        \  }}t          j        |          }t          d          }|                    |           ||         ||         }}t          |dz            }t
          j
                            d          }t
          j        ||                    |d|z            f         }t          j        ddd          }	|	                    |d|         |d|                                       ||d                   }
|r|
ddd	f         }
|	                    ||d                   }||d         }|||
fS )
zMake some classification predictions on a toy dataset using a SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    N   %   r      linearT)kernelprobabilityrandom_state   )r   	load_irisdatatargetshapenparanger$   shuffleintrandomRandomStatec_randnr   SVCfitpredict_probapredict)datasetbinaryXy	n_samples
n_featuresprnghalfclfy_scorey_predy_trues                b/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/sklearn/metrics/tests/test_ranking.pymake_predictionrL   8   sy    $&&AA "Qx1q51GIz
	)A
R
 
 CKKNNNQ41qAy1}D )


"
"C
a9cJ&67778A 't!
D
D
DCggah%4%))77$%%AAG   !!!Q$-[[455""FtuuXF67""    c                 N   t          j        |           d         }|| |k             }|| |k             }|                    dd          |                    dd          z
  }t          j        |dk              }|t	          t          |          t          |          z            z  S )zKAlternative implementation to check for correctness of
    `roc_auc_score`.r-   r   )r2   uniquereshapesumfloatlen)rJ   rH   	pos_labelposnegdiff_matrix	n_corrects          rK   _aucrZ   h   s     	&!!!$I &I%
&C
&I%
&C++a$$s{{2q'9'99K{Q''IuSXXC01111rM   c                    t          j        |           d         }t          j        | |k              }t          j        |          ddd         }||         }| |         } d}t	          t          |                    D ]D}| |         |k    r6d}t	          d|dz             D ]}| |         |k    r|dz  }||dz   z  }||z  }E||z  S )a>  Alternative implementation to check for correctness of
    `average_precision_score`.

    Note that this implementation fails on some edge cases.
    For example, for constant predictions e.g. [0.5, 0.5, 0.5],
    y_true = [1, 0, 0] returns an average precision of 0.33...
    but y_true = [0, 0, 1] returns 1.0.
    r-   NrO   r         ?)r2   rP   rR   argsortrangerT   )	rJ   rH   rU   n_posorderscoreiprecjs	            rK   _average_precisionre   w   s     	&!!!$IF6Y&''EJw"%EenGE]FE3w<<   	 	!9	!! D1a!e__    !9	))CKDAGODTME5=rM   c                 "   t          | |          \  }}}t          t          |                    }t          t          |                    }d}t          dt	          |                    D ]"}|||         ||         ||dz
           z
  z  z  }#|S )ao  A second alternative implementation of average precision that closely
    follows the Wikipedia article's definition (see References). This should
    give identical results as `average_precision_score` for all inputs.

    References
    ----------
    .. [1] `Wikipedia entry for the Average precision
       <https://en.wikipedia.org/wiki/Average_precision>`_
    r   r-   )r   listreversedr^   rT   )rJ   rH   	precisionrecall	thresholdaverage_precisionrb   s          rK   _average_precision_slowrm      s     $:&'#J#J IvyXi(())I(6""##F1c)nn%% H HYq\VAYA-FGGrM   c                     d } || ||          \  }}t          ||          }d}|}d||z
  z  ||z   z  }	||z
  }
dd||	z
  |
|	z
  z  z   z  S )zcAlternative implementation to check for correctness of `roc_auc_score`
    with `max_fpr` set.
    c                 X   t          | |          \  }}}|||k             }t          j        ||          }|||k             }t          j        ||k              }|dz
  }	||	         ||         g}
||	         ||         g}t          j        |t          j        ||
|                    }||fS )Nr-   )r   r2   appendargmaxinterp)rJ   	y_predictmax_fprfprtpr_new_fprnew_tpridx_outidx_inx_interpy_interps               rK   _partial_rocz,_partial_roc_auc_score.<locals>._partial_roc   s    	22S!cWn%)GW--cWn%)C'M**1KW.KW.)GRYw(%K%KLL!!rM   r         ?r-   )r
   )rJ   rs   rt   r~   rx   ry   partial_aucfpr1fpr2min_areamax_areas              rK   _partial_roc_auc_scorer      s    

" 
" 
" $|FIw??GWgw''K DDdTk"dTk2Hd{H!{X-(X2EFFGGrM   dropTc                 B   t          d          \  }}}t          ||          }t          |||           \  }}}t          ||          }t	          ||d           t          |t          ||                     |j        |j        k    sJ |j        |j        k    sJ d S )NTr?   drop_intermediater&   decimal)rL   rZ   r   r
   r   r   r   r1   )	r   rJ   rw   rH   expected_aucru   rv   
thresholdsroc_aucs	            rK   test_roc_curver      s     )555FAw((L$VWMMMCj#smmGg|Q????vw!?!?@@@9	!!!!9
(((((((rM   c                  \   t           j                            d          } t          j        dgdz  dgdz  z             }|                     dd          }t          ||d          \  }}}|d         dk    sJ |d	         dk    sJ |j        |j        k    sJ |j        |j        k    sJ d S )
Nr   2   r-      d   sizeTr   rO   )r2   r6   r7   arrayrandintr   r1   )rE   rJ   rI   ru   rv   thrs         rK   test_roc_curve_end_pointsr      s     )


"
"CXqcBh!r)**F[[[%%FffEEEMCcq6Q;;;;r7a<<<<9	!!!!9	!!!!!!rM   c                  d   t          d          \  } }}t          | |          \  }}}g }|D ]L}t          j        ||k    | z            }t          j        |           }	|                    d|z  |	z             Mt          ||d           |j        |j        k    sJ |j        |j        k    sJ d S )NTr   r\   r&   r   )rL   r   r2   rR   rp   r   r1   )
rJ   rw   rH   ru   rv   r   tpr_correctttprD   s
             rK   test_roc_returns_consistencyr      s     )555FAw$VW55Cj K ) )VW\V+,,F6NN38a<(((( c;::::9	!!!!9
(((((((rM   c                      t          d          \  } }}t          j        t                    5  t	          | |           d d d            d S # 1 swxY w Y   d S )NFr   )rL   pytestraises
ValueErrorr   )rJ   rw   rH   s      rK   test_roc_curve_multir      s    (666FAw	z	"	" # #&'"""# # # # # # # # # # # # # # # # # #s   AAAc                      t          d          \  } }}t          | |dz
            \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ d S )NTr   r   ?r&   r   )rL   r   r
   r   r1   )rJ   rw   rH   ru   rv   r   r   s          rK   test_roc_curve_confidencer      s    (555FAw$VWs];;Cj#smmGgtQ77779	!!!!9
(((((((rM   c                     t          d          \  } }}t          j        | j                  }t	          | |          \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ t          j        | j                  }t	          | |          \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ t	          | |          \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ d S )NTr   r   r&   r   g(\?)rL   r2   onesr1   r   r
   r   zeros)rJ   predrH   trivial_predru   rv   r   r   s           rK   test_roc_curve_hardr     sp   +4888FD' 76<((L$V\::Cj#smmGgtQ77779	!!!!9
((((( 8FL))L$V\::Cj#smmGgtQ77779	!!!!9
((((( %VT22Cj#smmGgtQ77779	!!!!9
(((((((rM   c                     g d} g d}d}t          j        t          |          5  t          | |          \  }}}d d d            n# 1 swxY w Y   t	          |t          j        t          |          t
          j                             |j	        |j	        k    sJ |j	        |j	        k    sJ d}t          j        t          |          5  t          d | D             |          \  }}}d d d            n# 1 swxY w Y   t	          |t          j        t          |          t
          j                             |j	        |j	        k    sJ |j	        |j	        k    sJ d S )N)
r-   r-   r-   r-   r-   r-   r-   r-   r-   r-   )
r   r-   r   r-   r   r-   r   r-   r   r-   INo negative samples in y_true, false positive value should be meaninglessmatchHNo positive samples in y_true, true positive value should be meaninglessc                     g | ]}d |z
  S )r-    ).0xs     rK   
<listcomp>z,test_roc_curve_one_label.<locals>.<listcomp>5  s    )@)@)@A!a%)@)@)@rM   )
r   warnsr   r   r   r2   fullrT   nanr1   )rJ   rI   expected_messageru   rv   r   s         rK   test_roc_curve_one_labelr   !  s   +++F+++F 	T  
,4D	E	E	E 9 9(88S*9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 sBGC
OORV<<===9	!!!!9
((((( 	S  
,4D	E	E	E J J()@)@)@)@)@&IIS*J J J J J J J J J J J J J J J sBGC
OORV<<===9	!!!!9
(((((((s#   AAAC88C<?C<c                  J   ddg} ddg}t          | |          \  }}}t          | |          }t          |g d           t          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |g d           t          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |ddg           t          |ddg           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |g d           t          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |ddg           t          |ddg           t          |d           ddg} dd	g}d
}t	          j        t          |          5  t          | |          \  }}}d d d            n# 1 swxY w Y   t          |g d           t          |t          j        t          j        t          j        g           d}t	          j        t          |          5  t          | |          }d d d            n# 1 swxY w Y   t          j
        |          sJ ddg} dd	g}d}t	          j        t          |          5  t          | |          \  }}}d d d            n# 1 swxY w Y   t          |t          j        t          j        t          j        g           t          |g d           d}t	          j        t          |          5  t          | |          }d d d            n# 1 swxY w Y   t          j
        |          sJ t          j        ddgddgg          } t          j        ddgddgg          }t	          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   t	          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   t          t          | |d          d           t          t          | |d          d           t          j        ddgddgg          } t          j        ddgddgg          }t	          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   t	          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   t          t          | |d          d           t          t          | |d          d           t          j        ddgddgg          } t          j        ddgddgg          }t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          j        ddgddgg          } t          j        ddgddgg          }t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           d S )Nr   r-   r   r   r-   r   r-   r-   r\           r         ?      ?r   r   r   r   r\   OOnly one class is present in y_true. ROC AUC score is not defined in that case.r   macroaverageweightedsamplesmicro)r   r   r   r   r   r   r   r2   r   mathisnanr   )rJ   rH   rv   ru   rw   r   r   r
   s           rK   test_roc_curve_toydatar   <  sd	   VF!fGFG,,KCaFG,,Gc999---c999---%%%VF!fGFG,,KCaFG,,Gc999---c999---%%%VF!fGFG,,KCaFG,,GcAq6***cAq6***%%%VF!fGFG,,KCaFG,,Gc999---c999---%%%VFCjGFG,,KCaFG,,GcAq6***cAq6***%%% VFTlG 	S  
,4D	E	E	E 1 100S!1 1 1 1 1 1 1 1 1 1 1 1 1 1 1c???333cBFBFBF#;<<<	5  
,4D	E	E	E - -FG,,- - - - - - - - - - - - - - -:c?? VFTlG 	T  
,4D	E	E	E 1 100S!1 1 1 1 1 1 1 1 1 1 1 1 1 1 1cBFBFBF#;<<<c???333	5  
,4D	E	E	E - -FG,,- - - - - - - - - - - - - - -:c?? X1v1v&''FhAA'((G	,4D	E	E	E 8 8fgw77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	,4D	E	E	E ; ;fgz::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;fgyIII3OOOfgwGGGMMMX1v1v&''FhAA'((G	,4D	E	E	E 8 8fgw77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	,4D	E	E	E ; ;fgz::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;fgyIII3OOOfgwGGGMMMX1v1v&''FhAA'((GfgwGGGKKKfgzJJJANNNfgyIII1MMMfgwGGGKKKX1v1v&''Fhc
S#J/00GfgwGGGMMMfgzJJJCPPPfgyIII3OOOfgwGGGMMMMMs   H''H+.H+J//J36J35LLLNN"%N"P--P14P1Q22Q69Q6T++T/2T/U00U47U4c            	          g d} g d}t          | |d          \  }}}t          |t          j        dddg           g d} g d	}t          | |d          \  }}}t          |t          j        dd
dddg           d S )Nr   r   r   r   r-   r-   r   皙?r   333333?ffffff?r\   Tr   r\   r   r   r   r   r   r   r   r   r   r-   r-   r-   r-   r-   r-   r   皙?r   r   r   皙?r   r   r   r   r   r   r\   r   r   )r   r   r2   inf)rJ   rH   rv   ru   r   s        rK    test_roc_curve_drop_intermediater     s    F,,,G$VWMMMCjj263S*ABBB 544FOOOG$VWMMMCjj263S#s*KLLLLLrM   c                  ,   g d} g d}t          j        dd          }t          | ||          \  }}}t          j        |          dk                                     dk    sJ t          j        |          dk                                     dk    sJ d S )Nr   r   r-   r-   r-   )r   r   333333?皙?r   r      sample_weightr   )r2   repeatr   diffrR   )rJ   rH   r   ru   rv   rw   s         rK   !test_roc_curve_fpr_tpr_increasingr     s     __F'''GIc1%%MFG=IIIKCaGCLL1!!##q((((GCLL1!!##q((((((rM   c                     ddg} ddg}t          t          | |          d           ddg} ddg}t          t          | |          d           g d} g d}t          t          | |          d           ddg} ddg}t          t          | |          d           g d} g d}t          t          | |          d           d S )Nr   r-   r   r-   r   r   r   r   r   r-   )r   r
   )r   rA   s     rK   test_aucr     s    	
AA	
AAc!Qii---	
AA	
AAc!Qii---		A		Ac!Qii---	
AA	
AAc!Qii+++AAc!Qii-----rM   c                      t          j        t                    5  t          g dddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          dgdg           d d d            n# 1 swxY w Y   g d} g d}d                    t          j        |                     }t          j        t          t          j        |                    5  t          | |           d d d            d S # 1 swxY w Y   d S )	Nr   r   r   r   )r&   r-   r      )r            z+x is neither increasing nor decreasing : {}r   )	r   r   r   r
   formatr2   r   reescape)r   rA   error_messages      rK   test_auc_errorsr     s   	z	"	" ) )OOOc3Z((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) 
z	"	"  SEC5               	AAAHHRSUUM	z=)A)A	B	B	B  Aq			                 s1   ;??A>>BB%DD
Dzy_true, labels)r   r-   r   r&   r   r-   r&   )abr   c)r   r   r   c           	         t          j        g dg dg dg dg          }t          g dg d          }t          g dg d          }||z   d	z  }t          g d
g d          }t          g dg d          }||z   d	z  }t          ddgddg          }	t          ddgddg          }
|	|
z   d	z  }||z   |z   dz  }t          t          | ||d          |           |||g}g d}t          j        ||          }t          t          | ||dd          |           d}t          j        t          |          5  t          | ||dd            d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   r   r   ffffff?r   333333?r   r   r   r-   r   r-   )r   r   r   r   r-   r   )r   r   r   r&   r-   r-   r   )r   r   r   r   )r   r   r   r-   r   r   r   r   r   r   ovolabelsmulti_class)r   r   r   )weightsr   r  r  r   z6average=None is not implemented for multi_class='ovo'.r   )r2   r   r   r   r   r   r   NotImplementedError)rJ   r  y_scoresscore_01score_10average_score_01score_02score_20average_score_02score_12score_21average_score_12ovo_unweighted_scorepair_scores
prevalenceovo_weighted_scorer   s                    rK   #test_multiclass_ovo_roc_auc_toydatar    sQ    x	///+<+<+<mmmL H YYY(8(8(899HYYY88H 8+q0 YYY77HYYY(8(8(899H 8+q0 aVc3Z00HaVc3Z00H 8+q0 -/??BRRVWWfhv5III   $%57GHK###JKDDDHV
	
 	
 	
 		   MM	*-	@	@	@ X Xfhv5RVWWWWX X X X X X X X X X X X X X X X X Xs   ;EE!$E!)r   r&   r   r&   )r   dr   r  )r   r   r  c           	      ,   t          j        g dg dg dg dg          }t          g dg d          }t          g dg d          }||z   d	z  }t          t          | ||d
          |           t          t          | ||d
d          |           d S )N)r   r   r   )r   r   r   )皙?r   ?)r   r   r   )r-   r   r-   r   )r   r   r  r   )r   r-   r   r-   )r   r   r  r   r&   r   r  r   r  r2   r   r   r   )rJ   r  r  r  r	  	ovo_scores         rK   *test_multiclass_ovo_roc_auc_toydata_binaryr  '  s     x	///+<+<+<oooN H \\\+@+@+@AAH\\\+@+@+@AAHH$)Ifhv5III9  
 HV
	
 	
 	
 		    rM   )r   r-   r&   r&   )r   r   r   r   c           	         t          j        g dg dg dg dg          }t          g d|d d df                   }t          g d|d d df                   }t          g d	|d d d
f                   }t          t          | |d|d           |||g           ||z   |z   dz  }t          t          | |d|          |           |dz  |dz  z   |dz  z   }t          t          | |d|d          |           d S )Nr\   r   r   )r   r   r   )r   r   r   )r   r   r   r-   r   r   r   r   )r   r-   r   r   r-   r   r   r-   r-   r&   ovr)r  r  r   g      @)r  r  r   r   r   r  )rJ   r  r  out_0out_1out_2result_unweightedresult_weighteds           rK   #test_multiclass_ovr_roc_auc_toydatar'  K  s]    x	///???OOOL H
 ,,,A77E,,,A77E,,,A77EfhE&RVWWW	u   .#5fhE&III   dlUT\1ECK?OH%
	
 	
 	
 		    rM   zmulti_class, average))r!  r   )r!  r   )r   r   c                 b   t          j        g d          }g dg dg dg dg}t          t          ||| |          d           g dg dg dg dg}t          ||| |          dk     sJ dt          j        d	          z  }t          ||| |          t          j        d
          k    sJ d S )N)r   r-   r&   r   )r   r   r   r\   )r   r\   r   r   )r   r   r\   r   )r   皙?r)  r   r  r   r\   r   )r   r   r   )r2   r   r   r   r   r   approx)r  r   rJ   	y_perfecty_imperfecty_chances         rK   0test_perfect_imperfect_chance_multiclass_roc_aucr/  v  s#    Xlll##F 	   	I fi['RRR   		K 	fk{GTTT
	 	 	 	
 bgfoo%Hk7  	s		     rM   c                   	 | 	t           j                            g dd	          }t          j        	fd|D                       }t          |g d          }t          |                                |                                          \  }}}t          ||          }t          ||dd	          }|t          j        |          k    sJ d S )
N)       @r\   r     )r   r,   c                 x    g | ]6}t           j                            d |                                          7S )r-   )nrD   r,   )r   multinomialrvsrq   )r   y_pred_iseeds     rK   r   z3test_micro_averaged_ovr_roc_auc.<locals>.<listcomp>  sM     	
 	
 	
 !!A!EELLNN	
 	
 	
rM   r   )classesr!  r   r*  )r   	dirichletr6  r2   asarrayr   r   ravelr
   r   r   r+  )
global_random_seedrI   rJ   y_onehotru   rv   rw   roc_auc_by_handroc_auc_autor8  s
            @rK   test_micro_averaged_ovr_roc_aucrA    s    D
 _  t$ OOFZ	
 	
 	
 	
"	
 	
 	
 F fiii888HHNN,,fllnn==KCa#smmO UGTTTLfmL99999999rM   zmsg, y_true, labelsz!Parameter 'labels' must be uniquer   r&   r   )r   r   r   zKNumber of classes in y_true not equal to the number of columns in 'y_score'z"Parameter 'labels' must be ordered)r   r   r   zMNumber of given labels, 2, not equal to the number of columns in 'y_score', 3r-   r   r   zMNumber of given labels, 4, not equal to the number of columns in 'y_score', 3)r   r-   r&   r   )r   r   r   r  z2'y_true' contains labels not in parameter 'labels')r   r   r   er  r   r!  c                     t          j        g dg dg dg dg          }t          j        t          |           5  t          ||||           d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   r   r  )r2   r   r   r   r   r   )msgrJ   r  r  r  s        rK   *test_roc_auc_score_multiclass_labels_errorrF    s    R x	///+<+<+<mmmL H 
z	-	-	- P Pfhv;OOOOP P P P P P P P P P P P P P P P P Ps   AA!$A!zmsg, kwargszLaverage must be one of \('macro', 'weighted', None\) for multiclass problemsr   )r   r  zUaverage must be one of \('micro', 'macro', 'weighted', None\) for multiclass problemszksample_weight is not supported for multiclass one-vs-one ROC AUC, 'sample_weight' must be None in this case)r  r   z|Partial AUC computation not available in multiclass setting, 'max_fpr' must be set to `None`, received `max_fpr=0.5` insteadr   )r  rt   z'multi_class must be in \('ovo', 'ovr'\)c                 "   t          d          }|                    dd          }t          |          }|                    ddd          }t	          j        t          |           5  t          ||fi | d d d            d S # 1 swxY w Y   d S )N     r   r   r   r   )r$   randr    r   r   r   r   r   )rE  kwargsrE   rH   y_probrJ   s         rK   #test_roc_auc_score_multiclass_errorrM    s    N S
!
!Chhr1ooGWF[[AB[''F	z	-	-	- 0 0ff/////0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   (BBBc                  l   t          d          } |                     d          }t          j        dd          }d}t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j        dd          }t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j	        ddd          }t	          j        t          |          5  t          ||           d d d            d S # 1 swxY w Y   d S )NrH  
   r5   dtyper   r   rO   )
r$   rJ  r2   r   r   r   r   r   r   r   )rE   rI   rJ   warn_messages       rK   test_auc_score_non_binary_classrS  4  s    S
!
!CXXb\\FXb&&&F	5  
,L	A	A	A & &ff%%%& & & & & & & & & & & & & & &WRu%%%F	,L	A	A	A & &ff%%%& & & & & & & & & & & & & & &WR5)))F	,L	A	A	A & &ff%%%& & & & & & & & & & & & & & & & & &s6   A55A9<A91CCCD))D-0D-
curve_funcc                    t          d          }|                    ddd          }|                    d          }d}t          j        t
          |          5   | ||           d d d            d S # 1 swxY w Y   d S )NrH  r   r   rO  r   z"multiclass format is not supportedr   )r$   r   rJ  r   r   r   )rT  rE   rJ   rI   rE  s        rK   &test_binary_clf_curve_multiclass_errorrV  I  s    
S
!
!C[[AB[''FXXb\\F
.C	z	-	-	- # #
66"""# # # # # # # # # # # # # # # # # #s   A44A8;A8c                    d}t          j        t          |          5   | t          j        ddgd          ddg           d d d            n# 1 swxY w Y   t          j        t          |          5   | t          j        ddgt
                    ddg           d d d            n# 1 swxY w Y   g d	} | g d
|          } | g d|          }t          ||          D ]%\  }}t          j                            ||           &d S )Nzy_true takes value in {'a', 'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.r   r   r   z<U1rP  r   r\   )r   r\   r   gzG?)r   r-   r-   r   )r   r\   r\   r   )	r   r   r   r2   r   objectziptestingr   )rT  rE  rI   	int_curvefloat_curveint_curve_partfloat_curve_parts          rK   (test_binary_clf_curve_implicit_pos_labelr_  S  s   
	  
z	-	-	- B B
28S#Je444sCjAAAB B B B B B B B B B B B B B B 
z	-	-	- C C
28S#Jf555SzBBBC C C C C C C C C C C C C C C
 #""F
<<<00I*1116::K,/	;,G,G E E((

"">3CDDDDE Es#   %AAA5*B++B/2B/z.ignore:Support for labels represented as byteslabels_typerg   r   c                     t          ddg|          }d}t          j        t          |          5   | |ddg           d d d            d S # 1 swxY w Y   d S )N   a   bzy_true takes value in {b'a', b'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.r   r   r\   )r   r   r   r   )rT  r`  r  rE  s       rK   .test_binary_clf_curve_implicit_bytes_pos_labelrd  m  s      tk::F	0 
 
z	-	-	- ' '
6C:&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's   AAAc                     g d}g d}g d} | |||          } | |d d         |d d         |d d                   }t          ||          D ]\  }}t          ||           d S )Nr   )r   r   r   r   r   )r-   r-   r-   r   r   r   rO   )rY  r   )rT  rJ   rH   r   result_1result_2arr_1arr_2s           rK   (test_binary_clf_curve_zero_sample_weightrj  }  s    __F'''G%%%Mz&'GGGHz&"+wss|=QTRTQTCUVVVHHh// & &uu%%%%& &rM   c                    t          d          \  }}}t          |||            t          |dd          |dd          |           \  }}}|d         dk    sJ |d         |dd                                          k    sJ d|t	          j        |dk              <   |                                }t          |||            t          ||           g d}g d	}	t          ||	|           \  }}}| r7t          |g d
           t          |g d           t          |g d           n6t          |g d           t          |g d           t          |g d	           |j	        |j	        k    sJ |j	        |j	        dz   k    sJ d S )NTr   r-   r   r   r\   rO   r-   r   r   r-   )r-   r&   r   r   )r   QUU?r\   r\   )r\   r   r   r   )r-   r&   r   )r   rm  r   r\   r\   )r\   r   r   r   r   )
rL   _test_precision_recall_curver   meanr2   wherecopyr   r   r   )
r   rJ   rw   rH   rD   rr   y_true_copyr  predict_probass
             rK   test_precision_recall_curveru    s   (555FAw $777 %VABBZPTUUUGAq!Q43;;;;Q46!"":??$$$$$$ %'F28FaK  !++--K $777{F+++\\F!\\N$V^tTTTGAq! )666777///000999%%%%;;;<<<444555<<<(((6QV6QVaZrM   c                    t          | ||          \  }}}t          | |          }t          |dd           t          |t          | |                     t	          t          | |          |d           |j        |j        k    sJ |j        |j        dz   k    sJ t          | t          j        |          |          \  }}}|j        |j        k    sJ |j        |j        dz   k    sJ d S )Nr   grh|?r   r&   r   r-   )	r   rm   r   r   r   re   r   r2   
zeros_like)rJ   rH   r   rD   rr  r   precision_recall_aucs          rK   rn  rn    s"   -fgQUVVVAq*267CC2E1===5fgFF   67++-A1    6QV6Z_q(((((-g&&$  Aq* 6QV6Z_q(((((((rM   c           	      (   t          j        d          5  ddg}ddg}t          |||           \  }}}t          ||          }t	          |g d           t	          |g d           t          |d           ddg}ddg}t          |||           \  }}}t          ||          }t	          |g d	           t	          |g d
           t          |d           ddg}ddg}t          |||           \  }}}t          ||          }t	          |ddg           t	          |ddg           t          |d           ddg}ddg}t          |||           \  }}}t          ||          }t	          |g d           t	          |g d           t          |d           ddg}ddg}t          |||           \  }}}t          ||          }t	          |ddg           t	          |ddg           t          |d           ddg}ddg}t          j        t          d          5  t          |||           \  }}}d d d            n# 1 swxY w Y   t          j        t          d          5  t          ||          }d d d            n# 1 swxY w Y   t          |g d           t          |g d           t          |d           ddg}ddg}t          |||           \  }}}t          t          ||          d           t	          |g d           t	          |g d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          t          ||d          d           t          t          ||d          d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          t          ||d          d           t          t          ||d          d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          t          ||d          d           t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          j
        ddgddgg          }t          j
        ddgddgg          }t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           d d d            n# 1 swxY w Y   t          j        d          5  t          j
        ddgddgg          }t          j
        ddgddgg          }t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nraise)allr   r-   r   )r   r-   r-   r   r\   )r   r   r\   r  r   r   r   r   z!No positive class found in y_truer   r   )r\   r\   r\   )r-   r   r   r   r   r   r   r   ignore)r2   errstater   r   r   r   r   r   UserWarningr   r   )r   rJ   rH   rD   rr  rw   auc_prcs          rK   #test_precision_recall_curve_toydatar    s   		!	!	! V
 V
Qa&(DQQQ1a)&'::!![[[111!!YYY///GS)))Qa&(DQQQ1a)&'::!!___555!!___555 	GS)))Qa&(DQQQ1a)&'::!!c1X...!!c1X...GS)))Qa&(DQQQ1a)&'::!![[[111!!YYY///GS)))Q*(DQQQ1a)&'::!!c1X...!!aX...GS)))Q,\+-PQQQ 	V 	V,VWPTUUUGAq!	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V\+-PQQQ 	? 	?-fg>>G	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	?999%%%999%%%###Q,(DQQQ1a3FGDDcJJJ!!___555!!]]]333 Aq6Aq6*++(QFQF+,,\+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 \+-PQQQ 	 	'LLLc  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	#FGYGGG	
 	
 	
 	/QQQSVWWWAq6Aq6*++(QFQF+,,\+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 \+-PQQQ 	 	'LLLc  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	#FGYGGG	
 	
 	
 	/QQQSVWWWAq6Aq6*++(QFQF+,,#FGWEEEs	
 	
 	
 	#FGZHHH#	
 	
 	
 	#FGYGGG	
 	
 	
 	#FGWEEEs	
 	
 	
 Aq6Aq6*++(QFQF+,,\+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	#FGZHHH#	
 	
 	
 \+-PQQQ 	 	'KKKS  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 \+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	
 Aq6Aq6*++(QFQF+,,/QQQSVWWW#FGZHHH#	
 	
 	
 	#FGYGGG	
 	
 	
 	/QQQSVWWWAq6Aq6*++(S#Jc
344#FGWEEEs	
 	
 	
 	#FGZHHH#	
 	
 	
 	#FGYGGG	
 	
 	
 	#FGWEEEs	
 	
 	
iV
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
p 
	"	"	"  Aq6Aq6*++(QFQF+,,\+-PQQQ 	 	'LLLa  	 	 	 	 	 	 	 	 	 	 	 	 	 	 		                 s  Hc$I;cI	cI	c-J
>c
J	cJ	C&c8!N%c%N)	)c,N)	-c!O8,c8O<	<c?O<	 Bc!R?3c?S	cS	c%!TcT	cT	Ec !ZcZ	cZ	>c!\ 4c \	c\	c&!]c]	c]	E+ccc2Af!e/#f/e3	3f6e3	7fffc                     g d} g d}t          | |d          \  }}}t          |g d           g d} g d}t          | |d          \  }}}t          |g d           g d	} g d
}t          | |d          \  }}}t          |ddg           g d} g d
}t          | |d          \  }}}t          |g d
           dS )z9Check the behaviour of the `drop_intermediate` parameter.r   r   Tr   )r   r   r\   r   r   )r   r   r   r   r   r\   r   r   r   r   )r   r   r   r   r   r   r-   r-   r-   r-   N)r   r   )rJ   rH   ri   rj   r   s        rK   -test_precision_recall_curve_drop_intermediater  d  sH   F,,,G$:4% % %!Ivz J000 544FOOOG$:4% % %!Ivz J > > >??? \\F"""G$:4% % %!Ivz Jc
+++ \\F"""G$:4% % %!Ivz J 4 4 455555rM   c                      t          j        dt                    } d| d d d<   t          j        d          }t	          | |          dk    sJ d S )Nr   rP  r-   r   r   )r2   r   r5   r   r   rJ   rH   s     rK   &test_average_precision_constant_valuesr    sW    
 Xc%%%FF33Q3KgcllG #6733t;;;;;;rM   c                      t          j        ddg          } t          j        ddg          }d}t          j        t          |          5  t          | |d           d d d            d S # 1 swxY w Y   d S )Nr   r-   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r   r&   rU   r2   r   r   r   r   r   rJ   rI   err_msgs      rK   4test_average_precision_score_binary_pos_label_errorsr    s    Xq!fFXq!fFOG	z	1	1	1 = =!<<<<= = = = = = = = = = = = = = = = = =s   
A**A.1A.c                     t          j        ddgddgddgddgg          } t          j        ddgddgddgddgg          }d}t          j        t          |          5  t          | |d	           d d d            d S # 1 swxY w Y   d S )
Nr-   r   r   r   r   r   znParameter pos_label is fixed to 1 for multilabel-indicator y_true. Do not set pos_label or set pos_label to 1.r   r  r  r  s      rK   8test_average_precision_score_multilabel_pos_label_errorsr    s     X1v1v1v1v677FXSzC:SzC:FGGF	6  
z	1	1	1 = =!<<<<= = = = = = = = = = = = = = = = = =s   A>>BBc            	         t          j        g d          } t          j        g dg dg dg dg dg dg          }d}t          j        t          |          5  t          | |d	           d d d            d S # 1 swxY w Y   d S )
N)r   r-   r&   r   r-   r&   )r   r   r   )r   r   r   )r   r   r   )r   r   r   zdParameter pos_label is fixed to 1 for multiclass y_true. Do not set pos_label or set pos_label to 1.r   r   r  r  r  s      rK   8test_average_precision_score_multiclass_pos_label_errorsr    s    X((())FXOOOOOOOOOOOO	
	 	F	6  
z	1	1	1 = =!<<<<= = = = = = = = = = = = = = = = = =s   A::A>A>c                     t          d          \  } }}t          | |          }t          | d|z            }t          | d|z            }t          | |dz
            }||k    sJ ||k    sJ ||k    sJ t          | |          }t          | d|z            }t          | d|z            }	t          | |dz
            }
||k    sJ ||	k    sJ ||
k    sJ d S )NTr   r   gư>rO  )rL   r   r   )rJ   rw   rH   r   roc_auc_scaled_uproc_auc_scaled_downroc_auc_shiftedpr_aucpr_auc_scaled_uppr_auc_scaled_downpr_auc_shifteds              rK   test_score_scale_invariancer    s    )555FAwFG,,G%fcGm<<'w??#FGbL99O''''')))))o%%%%$VW55F.vsW}EE0HH,VWr\BBN%%%%%'''''^######rM   z(y_true,y_score,expected_fpr,expected_fnrr   r   )r   r   r   )r   r   r-   r   r   r   r   r   r   r   )r-   r-   r   r   r   )r   r   r   c                 n    t          | |          \  }}}t          ||           t          ||           d S Nr   r   rJ   rH   expected_fprexpected_fnrru   fnrrw   s          rK   test_det_curve_toydatar    s?    6 FG,,KCaC&&&C&&&&&rM   r   r   r   c                 n    t          | |          \  }}}t          ||           t          ||           d S r  r  r  s          rK   test_det_curve_tie_handlingr    s?     FG,,KCaC&&&C&&&&&rM   c                  n    t          t          g dg d          t          g dg d                     d S )Nr   r   r   )r   r   r   r   r-   r-   )r   r   r   rM   rK   test_det_curve_sanity_checkr    sN    )))[[[))$$$&<&<&<==    rM   rH   )r   r   r   r   r-   c                     t          g dt          j        d|                     \  }}}t          |dg           t          |dg           t          || g           d S )N)r   r-   r   r-   r   r-   r   r  r-   r   )r   r2   r   r   )rH   ru   r  rk   s       rK   test_det_curve_constant_scoresr    sr    #!!!271g+>+>  Ci C!C!Iy)))))rM   rJ   )r   r   r   r   r   r-   r   )r   r   r   r-   r-   r-   )r   r   r-   r-   r-   r-   )r   r-   r-   r-   r-   r-   c                 t    t          | |           \  }}}t          |dg           t          |dg           d S )Nr  r   r  )rJ   ru   r  rw   s       rK   test_det_curve_perfect_scoresr     sF     66:::KCaC!C!rM   zy_true, y_pred, err_msgzinconsistent numbers of samplesr   r   r   z#Only one class is present in y_truer-   r-   r-   )cancerr  
not cancer)r   r   r   zpos_label is not specifiedc                     t          j        t          |          5  t          | |           d d d            d S # 1 swxY w Y   d S )Nr   )r   r   r   r   r  s      rK   test_det_curve_bad_inputr  1  s      
z	1	1	1 " "&&!!!" " " " " " " " " " " " " " " " " "   :>>c                     dgdz  dgdz  z   } t          j        g d          }d|z
  }t          | |d          \  }}}t          | |d          \  }}}|d         t          j        d	          k    sJ |d         t          j        d
          k    sJ t          ||d d d                    t          ||d d d                    d S )Nr  r   r  r   )
r   r   r   r   r   r   r   r   r   r   r-   r  r   r   r   rO   )r2   r   r   r   r+  r   )	rJ   y_pred_pos_not_cancery_pred_pos_cancerfpr_pos_cancerfnr_pos_cancerth_pos_cancerfpr_pos_not_cancerfnr_pos_not_cancerth_pos_not_cancers	            rK   test_det_curve_pos_labelr  E  s   Z!^|nq00FH%W%W%WXX114=5 5 51NNM
 AJA A A=*,= v}S111111Q6=#5#55555 N$6ttt$<===N$6ttt$<=====rM   c                    t           | ddggddgg          d           t           | ddggddgg          d           t           | ddggddgg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | ddggddgg          d           t           | ddggddgg          d           t           | ddggddgg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           d S )Nr   r-   r   r   r   r   r  r   r   r   gUUUUUU?r   g?r   g?r   r   r   r  r   r   r   r  )r   r   r   UUUUUU?)r-   r-   r-   r   )r   r   r   r   )r   
lrap_scores    rK   check_lrap_toyr  _  s/   

QF8tTl^<<a@@@

QF8tTl^<<eDDD

QF8tTl^<<a@@@

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDeLLL

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDeLLL
III;!2!2!2 3446I   
III;!2!2!2 3446I   

III;1B1B1B0CDDeLLL

III;1B1B1B0CDDeLLL
III;!2!2!2 3446I   

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDoVVV

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDeLLL

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDoVVV

III;1B1B1B0CDDeLLL
III;!2!2!2 3446I   

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDaHHH 

QF8sCj\::C@@@

QF8sCj\::C@@@

QF8sCj\::A>>>

III;1A1A1A0BCCSIII

III;1A1A1A0BCCSIII

III;1A1A1A0BCCQGGG

III;1A1A1A0BCCUKKK
III;!1!1!1 2335H   
III;!1!1!1 2335H   

III;1A1A1A0BCCQGGG

III;0ABBEJJJ

LLL>4H4H4H3IJJERRRRRrM   c           	         t          d          }t          dd          D ]}|                    d|f          }t          j        |          }t          j        d|f          } | ||          dk    sJ  | ||          dk    sJ t          j        d|f          } | ||          dk    sJ  | ||          dk    sJ t           | dgdgdgdggdgdgdgdgg          d           d S )Nr   r&   r   r-   r   r\   r   )r$   r^   uniformr2   rw  r   r   r   )r  r,   n_labelsrH   y_score_tiesrJ   s         rK   !check_zero_or_all_relevant_labelsr    sA   %a((L!QKK 7 7&&QM&::}W-- 1h-((z&'**c1111z&,//36666 !X''z&'**c1111z&,//366666 
QC!qcA3'3%#u)EFF    rM   c                    t          j        t                    5   | g dg d           d d d            n# 1 swxY w Y   t          j        t                    5   | g dg dg dg dg           d d d            n# 1 swxY w Y   t          j        t                    5   | g dg dg dg dg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggddg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggddgg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggdgdgg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5   | dgdggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggdgdgg           d d d            d S # 1 swxY w Y   d S )	Nr   )r   r   r   r   )r   r   r   )r   r   r   )r   r   r   r   r-   )r   r   r   r  s    rK   check_lrap_error_raisedr    s   	z	"	" 0 0
999...///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0	z	"	" U U
999000///???STTTU U U U U U U U U U U U U U U	z	"	" 
 

OO//////R	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
z	"	" - -
QFQF#aV,,,- - - - - - - - - - - - - - -	z	"	" / /
QFQF#q!fX.../ / / / / / / / / / / / / / /	z	"	" 1 1
QFQF#qcA3Z0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1	z	"	" / /
QF8q!fq!f-.../ / / / / / / / / / / / / / /	z	"	" 1 1
QC!:AA/0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1	z	"	" 1 1
QFQF#qcA3Z0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s   7;;A??BB#CCC+DDD0EEE6FF F=GG#&G#H&&H*-H*
I..I25I2c           	      &   t          dd          D ]}t          j        d|f          }t          d|          D ]V}t          ||z
            D ]A}t          j        d|f          }d|d|||z   f<   t	           | ||          ||z             BWd S )Nr&   rO  r-   r   )r^   r2   r   r   r   )r  r  rH   
n_relevantrV   rJ   s         rK   check_lrap_only_tiesr    s     !RLL 	X 	X'1h-((  8,, 	X 	XJX
233 X X1h-0045q#j 0001#JJvw$?$?hAVWWWWX	X		X 	XrM   c                 .   t          dd          D ] }|t          j        |                              d|f          dz   z
  }t          j        d|f          }d|d<   d|d<   t           | ||          d|z  dz   dz             t          d|          D ]zt          |z
            D ]et          j        d|f          }d|dz   f<   t           | ||          t          fdt                    D                                  f{d S )Nr&   rO  r-   )r   r   )r   rO   r   c              3   :   K   | ]}|d z   |z   d z   z  z  V  dS )r-   Nr   )r   rr  r  rV   s     rK   	<genexpr>z>check_lrap_without_tie_and_increasing_score.<locals>.<genexpr>  sM         QC!GaK:#=>     rM   )r^   r2   r3   rQ   r   r   rR   )r  r  rH   rJ   r  rV   s       @@rK   +check_lrap_without_tie_and_increasing_scorer    sv    !RLL  bi11991h-HH1LM 1h-((tuJJvw77!h,:Ja9OPPP  8,, 	 	JX
233 	 	1h-0045q#j 0001#Jvw//     !&z!2!2       		 rM   c                    t          | |           t          |           } t          |          }| j        \  }}t          j        |f          }t          |          D ]}t          j        ||         d          \  }}|j        }||z
  t          j        |dz             	                                }	|	         | |         
                                d         }
|
j        dk    s|
j        |k    rd||<   d||<   |
D ]7t          fd|
D                       }||xx         |         z  z  cc<   8||xx         |
j        z  cc<   |                                S )z8Simple implementation of label ranking average precisionT)return_inverser-   )	minlengthr   r   c              3   <   K   | ]}|                  k    V  d S r  r   )r   rr  labelranks     rK   r  z_my_lrap.<locals>.<genexpr>  s0       J JAaDK!7 J J J J J JrM   )r#   r"   r1   r2   emptyr^   rP   r   bincountcumsumnonzerorR   ro  )rJ   rH   rB   r  ra   rb   unique_rankinv_rankn_ranks	corr_rankrelevantn_ranked_abover  r  s               @@rK   _my_lrapr    s   FG,,,  F'""G ,IxHi\""E9 " " !#	'!*T J J JX"! K!<<<CCEE	!9$$&&q)=A(!:!:E!Ha 	5 	5E ! J J J J J J J JJJN !HHHe44HHHHaHM!::<<rM   r   rI  c                    t          dd|||          \  }}t          |j        d         |j        d         |          }t          |d          r|                                }t          ||          }t          ||          }t          ||           t          |          }|	                    ||f          }t          ||          }t          ||          }t          ||           d S )Nr-   F)rC   allow_unlabeledr,   	n_classesrB   r   )n_componentsrC   r,   toarrayr   )
r   r   r1   hasattrr  r   r  r   r$   r  )	r  r  rB   r,   rw   rJ   rH   
score_lrapscore_my_lraps	            rK   %check_alternative_lrap_implementationr    s    /!  IAv $\!_<?!  G w	"" $//##6vwGGJVW--M
M222 &l33L""I(>"??G6vwGGJVW--M
M22222rM   checkfuncc                      | |           d S r  r   )r  r  s     rK   test_label_ranking_avpr  ;  s     
E$KKKKKrM   c                  .    t          t                     d S r  )r  r   r   rM   rK   test_lrap_error_raisedr  I  s    ABBBBBrM   rB   )r-   r&   r   rI  r  )r&   r   rO  r,   c                 4    t          t          || |           d S r  )r  r   )rB   r  r,   s      rK   $test_alternative_lrap_implementationr  M  s)     *-y)\    rM   c                  t   t          j        g dg dg dgt                    } t          j        g dg dg dg          }t          j        g d          }t          j        g d	          }t          t	          | ||
          t          j        ||z            t          j        |          z             d S )Nr  rl  r  rP  )r   r   r   r   )r   r   r   r   r   r   r   r   )r   r   r\   )r\   r\   r   r   )r2   r   boolr   r   rR   )rJ   rH   samplewise_lrapsr   s       rK   &test_lrap_sample_weighting_zero_labelsr  V  s     X|||\\\<<<@MMMFh			3335I5I5IJ G x 0 0 011H___--M-G=	
 	
 	
 	}//0026-3H3HH	    rM   c            	         t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	g dgg dg dg          d           t          t          g d	g dg d
gg dg dg dg          d           t          t          g d	g dg d
gg dg dg dg          d           d S )Nr   r-   r   r   r&   r  r  r   r   r   r   r   r   r   r  r  r  r   g      $@r   r-   r   r1  r   rO  r  rB  g@r   r-   r   r   r   r   rM   rK   test_coverage_errorr  k  st   Ax4,@@!DDDAx4,@@!DDDAx4,@@!DDDAx4,@@!DDD			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL 			999-/KLL  
 YY			999-yyy)))/T	
 	
 		   YY			999-yyy)))/T	
 	
 		    rM   c                  f   t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           d S )Nr   r   r-   r&   r  r  r   r   r   r   r   r   r   r  r   r   rM   rK   test_coverage_tie_handlingr    s   Ax3*>>BBBAx3*>>BBBAx3*>>BBBAx3*>>BBB			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKKKKrM   zy_true, y_scorec                     t          j        t          d          5  t          | |           d d d            d S # 1 swxY w Y   d S )Nz'Expected 2D array, got 1D array insteadr   )r   r   r   r   r  s     rK   test_coverage_1d_error_messager    s     
z)S	T	T	T ( (vw'''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (r  c            	         t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dg dgg dg dg          d           t          t          g dg dg d	gg dg dg dg          d           t          t          g dg dg d	gg dg dg dg          d           d S )Nr   r-   r   r   r   r  r   r   r   r   r\   r   r   r  r  r  r  r  r  r  rB  r  r   r   r   rM   rK   test_label_ranking_lossr    s   *QF8tTl^DDaHHH*QF8tTl^DDaHHH*III;9J9J9J8KLLaPPP*III;9J9J9J8KLLeTTT*III;9J9J9J8KLLaPPP*III;9J9J9J8KLLeTTT*III;9J9J9J8KLLeTTT*III;9J9J9J8KLLeTTT *QF8tTl^DDaHHH*QF8tTl^DDaHHH*QF8sCj\BBAFFF*QF8sCj\BBAFFF*III;9J9J9J8KLLaPPP*III;9J9J9J8KLLaPPP*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOO IIIyyy1OOOYYY3OPP  
 YY			999-yyy)))/T	
 	
 	"	   YY			999-yyy)))/T	
 	
 	"	    rM   csr_containerc                     t          t           | t          j        g dg dg                    g dg dg          d           d S )Nr   r   r  r  r   )r   r   r2   r   )r	  s    rK   test_label_ranking_loss_sparser    sc    M"(IIIyyy#9::;;mmmYYY=W	
 	
 		    rM   c                     t          j        t                    5  t          ddgddggddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddgddggddgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddgddggdgdgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          dgdggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddgddggdgdgg           d d d            d S # 1 swxY w Y   d S )Nr   r-   )r   r   r   r   r   rM   rK   $test_ranking_appropriate_input_shaper    sE   	z	"	" 5 5QFQF+aV4445 5 5 5 5 5 5 5 5 5 5 5 5 5 5	z	"	" 7 7QFQF+q!fX6667 7 7 7 7 7 7 7 7 7 7 7 7 7 7	z	"	" 9 9QFQF+qcA3Z8889 9 9 9 9 9 9 9 9 9 9 9 9 9 9	z	"	" 7 7QF8q!fq!f%56667 7 7 7 7 7 7 7 7 7 7 7 7 7 7	z	"	" 9 9QC!:AA'78889 9 9 9 9 9 9 9 9 9 9 9 9 9 9	z	"	" 9 9QFQF+qcA3Z8889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9si   ?AA#B		BB-CCC8DD"%D"E))E-0E-F55F9<F9c                  F   t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           d S )Nr-   r   r   r   r  r   r   r   r   r   r  r   rM   rK   test_ranking_loss_ties_handlingr    sX   *QF8sCj\BBAFFF*QF8sCj\BBAFFF*III;9I9I9I8JKKUSSS*III;9I9I9I8JKKUSSS*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOOOOrM   c                      t          dd          \  } }| dz   }t          ||           t          j                            d                              d          \  }}t          ||           d S Nr   rO  )r,   r  r-   )r&   r   rO  )r   _test_dcg_score_forr2   r6   r7   random_samplerw   rJ   rH   s      rK   test_dcg_scorer    sp    .ALLLIAvgkG(((i++A..<<\JJOFG(((((rM   c                 :   t          j        t          j        | j        d                   dz             }t	          | |           }t	          | |          }||k                                    sJ t	          | | d          |k                                    sJ |j        | j        d         fk    sJ |j        | j        d         fk    sJ |t          j        t          j        |           d d d d df         |z  	                    d                    k    sJ d S )Nr-   r&   r   kr   rO   axis)
r2   log2r3   r1   r   r{  r   r+  sortrR   )rJ   rH   discountidealra   s        rK   r  r    s   wrya11A566Hvv..Evw//EUN!!!!!vv333u<AACCCCC;6<?,,,,,;6<?,,,,,FM276??111ddd7#;h#F"K"KQR"K"S"STTTTTTTTrM   c            	         t          j        t          j        d          g          } t          j        | j                  }t          | |          }t          | |d          }dt          j        t          j        dd                    z  }|t          j        |	                                | 
                                z  g          k    sJ |t          j        || d d d d df         z  	                                g          k    sJ d|dd	d f<   t          | |          }t          | |d          }|t          j        || d d d d df         z  	                                g          k    sJ |t          j        |d d         	                                | dd	d f         
                                z  |dd          	                                | dd d	f         
                                z  z   g          k    sJ d S )
Nr   Tignore_tiesr-   r&   r   rO   r   r   )r2   r;  r3   r   r1   r   r  r   r+  rR   ro  )rJ   rH   dcgdcg_ignore_ties	discountss        rK   test_dcg_tiesr%  %  s   Z1''Fhv|$$G
VW
-
-C(dKKKOBGBIaOO,,,I&-6;;==!@ ABBBBBBfmi&DDbD/.I-N-N-P-P,QRRRRRRGAqrrEN
VW
-
-C(dKKKOfmi&DDbD/.I-N-N-P-P,QRRRRRR&-bqbM&ABB-"4"4"6"66m!!F1bqb5M$6$6$8$889	
       rM   c            	          t          j        d                              d          } t          | | dd          t	          j        t          | | dd                    k    sJ d S )N   )r&   r   r   T)r  r!  )r2   r3   rQ   r   r   r+  )r   s    rK   test_ndcg_ignore_ties_with_kr(  9  sq    
	"f%%AaaT222fm1a1$///7 7      rM   c                      t          j        g dg          } t          j        g dg          }d}t          j        t          |          5  t          | |           ddd           dS # 1 swxY w Y   dS )zDCheck `ndcg_score` exception when `y_true` contains negative values.)g{Gzg(\gGz޿g(\?gQ?)gQ?gףp=
?r   gQ?gHzG?z7ndcg_score should not be used on negative y_true valuesr   N)r2   r   r   r   r   r   )rJ   rH   r   s      rK    test_ndcg_negative_ndarray_errorr*  @  s    X8889::Fh666788GP	z)9	:	:	: $ $67###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $s   A**A.1A.c                     t          j        d                              dd          } | t           j                            d                              dd| j                  z   }t          | |          }t          | |d	          }|t          j	        |          k    sJ |t          j	        d
          k    sJ |dz  }t          | |          t          j	        d
          k    sJ d S )NF   r   rO  r   皙ɿr   r   Tr   r\   r2  )
r2   r3   rQ   r6   r7   r  r1   r   r   r+  )rJ   rH   ndcgndcg_no_tiess       rK   test_ndcg_invariantr0  I  s    Yr]]""1b))Fry,,Q//77c7UUUGfg&&Dfg4@@@L6=......6=%%%%%%tOGfg&&&-*<*<<<<<<<rM   r!  c           
      v   dt          j        d          d d         z  }t          j        t          j        ddd          d          }|t           j                            d                              dd	|j        
          z   }t          |||           t          j
        dt          j        t          j        dd                    z            k    sJ t          |||           t          j
        dt          j        t          j        dd                    z            k    sJ t          |||           t          j
        dt          j        t          j        dd                    z            k    sJ t          ||d|           t          j
        dt          j        t          j        dd                    z            k    sJ t          |||           t          j
        dt          j        t          j        dd                    z                                            k    sJ t!          |||           t          j
        dt          j        t          j        dd                    z                                            k    sJ dt          j        d          z  }dt          j        t          j        dd                    z                                  }t          |||           t          j
        |t          j        d          z            k    sJ t          |||           t          j
        t          j        d                    k    sJ t!          |||           t          j
        |          k    sJ t          |||           t          j
        d          k    sJ d S )Nr   r   r   r   rO   )r   r-   r   r-  r   r   r   r&   r-   rO  )log_baser!  )r   r   	   r\   )r2   eyetiler3   r6   r7   r  r1   r   r   r+  r  r   log10r   ro  r   r   rR   )r!  rJ   rH   y_score_noisyexpected_dcg_scores        rK   test_ndcg_toy_examplesr9  T  s   2A2Fgbi2r**F33Gbi33A66>>c ?   M [  	q2729Q??333	4	45 5 5 5 ;  	q2729Q??333	4	45 5 5 5 [  	q2729Q??333	4	45 5 5 5 "+  	q28BIaOO444	5	56 6 6 6 fg;???6=	
RWRYq!__%%	%++--D D     VW+>>>&-	
RWRYq!__%%	%++--C C      Fbgbi1oo666;;==[  	)BGAJJ6	7	78 8 8 8 [  	rwqzz	"	"# # # # VW+>>>&-C C     fg;???6=QTCUCUUUUUUUrM   c                      d} t          j        t          |           5  t          dggdgg           ddd           dS # 1 swxY w Y   dS )zhCheck that we raise an informative error message when trying to
    compute NDCG with a single document.zTComputing NDCG is only meaningful when there is more than 1 document. Got 1 instead.r   r-   N)r   r   r   r   )r  s    rK   test_ndcg_error_single_documentr;  {  s    	  
z	1	1	1 ! !QC5A3%   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s   A  AAc                      t          dd          \  } }| dz   }t          ||           t          j                            d                              d          \  }}t          ||           d S r  )r   _test_ndcg_score_forr2   r6   r7   r  r  s      rK   test_ndcg_scorer>    sp    .ALLLIAvgkG)))i++A..<<\JJOFG)))))rM   c                 J   t          | |           }t          | |          }||k                                    sJ | dk                        d          }||          t          j        t	          j        |                                                     k    sJ ||         t          j        t	          j        |                                                    k    sJ ||          t          j        t          | |          |          t          | |           |          z            k    sJ ||         t          j        t	          j        |                                                    k    sJ |j	        | j	        d         fk    sJ |j	        | j	        d         fk    sJ d S )Nr   r-   r  )
r   r{  r   r+  r2   r   rR   r   r   r1   )rJ   rH   r  ra   all_zeros        rK   r=  r=    s   //E00EUN!!!!!!  a ((H(v}RWxi__5F5F-G-GHHHHHH?fmBHX\\^^,D,DEEEEEE(v}67++XI6
VV
,
,hY
7	8        ?fmBHX\\^^,D,DEEEEEE;6<?,,,,,;6<?,,,,,,,rM   c            	         t          j        g d          } t          | | d          dk    sJ t          | | d          dk    sJ t          j        t
                    5  t          | | d          sJ 	 d d d            n# 1 swxY w Y   t          j        t
                    5  t          | | d          sJ 	 d d d            n# 1 swxY w Y   t          j        t
                    5  t          | | d          sJ 	 d d d            n# 1 swxY w Y   t          j        g d          }t          | |d          }t          | |          }||k    sJ t          | |d	          d
k    sJ t          d          \  } }}t          j        ddd          D ]1}t          t          | ||          t          | ||                     2d S )Nr   r-   )rt   gMbP?gg?r   )r   r   r   g{Gz?r   r   Tr   g-C6?r   )
r2   r   r   r   r   r   rL   linspacer   r   )rJ   r  roc_auc_with_max_fpr_oneunconstrained_roc_aucrI   rw   rt   s          rK   test_partial_roc_auc_scorerE    s   Xlll##F333q88887771<<<<	z	"	" ; ;VVT:::::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;	z	"	" : :VVS9999999: : : : : : : : : : : : : : :	z	"	" 8 8VVQ77777778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 x+++,,H,VXqIII)&(;;#'<<<<<37773>>>>'t444FFA;tQ** 
 
&&':::"667;;	
 	
 	
 	

 
s6    BB	B&CCC,DDDzy_true, k, true_scorer   r&   r   r   c                     t          j        g dg dg dg dg          }t          | ||          }|t          j        |          k    sJ d S )Nr  r   r   r   r   r   r   r   r   r   r   r   r   r  r2   r   r   r   r+  )rJ   r  
true_scorerH   ra   s        rK   test_top_k_accuracy_scorerL    su     h            		
 G !A666EFM*--------rM   zy_score, k, true_score)rO   rO   r-   r-   )rO   r-   rO   r-   )r   r   r   r   )r   r   r   r   c                 V   g d}|                                  dk    r|                                 dk    rdnd}|dk    r#| |k                        t          j                  n|}t          || |          }t          ||          }||cxk    rt          j        |          k    sn J d S )Nr   r   r-   r   r  )	minmaxastyper2   int64r   r	   r   r+  )rH   r  rK  rJ   rk   rI   ra   	score_accs           rK    test_top_k_accuracy_score_binaryrS    s     \\F{{}}))gkkmmq.@.@aI78Avvg	!))"(3336F A666Evv..II::::z!:!:::::::::rM   zy_true, true_score, labels)r   r-   r-   r&   )r   r-   r-   r-   r  )r   rC  rC  r   )r   r   r  rC  labels_as_ndarrayc                     |rt          j        |          }t          j        g dg dg dg dg          }t          | |d|          }|t	          j        |          k    sJ dS )z,Test when labels and y_score are multiclass.r  rG  rH  rI  r&   r  r  N)r2   r;  r   r   r   r+  )rJ   rK  r  rT  rH   ra   s         rK   0test_top_k_accuracy_score_multiclass_with_labelsrW    s      $F##h            		
 G !AfEEEEFM*--------rM   c                     t          j        dddd          \  t          d          \  } }}}t          d                              | |           t          | |f||f          D ]O\  fdt          dd          D             }t          j        t          j	        |          dk              sJ Pd S )NrO  r2  r   )r  rB   n_informativer,   )r,   c                 Z    g | ]'}t                                        |           (S )r  )r   r<   )r   r  r@   rG   rA   s     rK   r   z8test_top_k_accuracy_score_increasing.<locals>.<listcomp>  sD     
 
 
CD C$5$5a$8$8A>>>
 
 
rM   r&   )
r   make_classificationr   r   r;   rY  r^   r2   r{  r   )X_trainX_testy_trainy_testscoresr@   rG   rA   s        @@@rK   $test_top_k_accuracy_score_increasingra    s    'BQ  DAq (811'M'M'M$GVWf
!
,
,
,CGGGWWf%'899 + +1
 
 
 
 
 
HMaQS
 
 
 vbgfoo)******+ +rM   c                     t          j        g dg dg dg dg          }t          | ||          t          j        |          k    sJ d S )N)r   r   r   r   )r-   r   r   r   )r   r   r   r   r  r  rJ  )rJ   r  rK  rH   s       rK   test_top_k_accuracy_score_tiesrc    sh     hLLLLLLLL		
 G  1555z9R9RRRRRRRrM   z	y_true, kr   c                     t          j        g dg dg dg dg          }d}t          j        t          |          5  t          | ||          }d d d            n# 1 swxY w Y   |dk    sJ d S )	Nr  )r   r   r   r   )r   r   r   r   )r   r   r   r   zu'k' \(\d+\) greater than or equal to 'n_classes' \(\d+\) will result in a perfect score and is therefore meaningless.r   r  r-   )r2   r   r   r   r   r   )rJ   r  rH   r   ra   s        rK   !test_top_k_accuracy_score_warningre  )  s     h            		
 G	6  
,4D	E	E	E ; ;$VW:::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;A::::::s   AA!$A!zy_true, y_score, labels, msg)r   g=
ףp=?r-   r&   )r   r   r   )r   r   r   r   )r   r   r   z9y type must be 'binary' or 'multiclass', got 'continuous'zZNumber of classes in 'y_true' \(4\) not equal to the number of classes in 'y_score' \(3\).)r   r   r   r   z"Parameter 'labels' must be unique.z#Parameter 'labels' must be ordered.)r   r   r-   r&   zSNumber of given labels \(4\) not equal to the number of classes in 'y_score' \(3\).r  z3'y_true' contains labels not in parameter 'labels'.)r   r   r   )r   r   r   z}`y_true` is binary while y_score is 2d with 3 classes. If `y_true` does not contain all the labels, `labels` must be providedc                     t          j        t          |          5  t          | |d|           d d d            d S # 1 swxY w Y   d S )Nr   r&   rV  )r   r   r   r   )rJ   rH   r  rE  s       rK   test_top_k_accuracy_score_errorrg  B  s    f 
z	-	-	- B BVW&AAAAB B B B B B B B B B B B B B B B B Bs   =AAc                      | g dg dg          }t          j        g dg dg          }t          ||          }|t          j        d          k    sJ d S )Nr   r   )r   r   r   r  )r2   r   r   r   r+  )r	  rJ   rH   results       rK   Otest_label_ranking_avg_precision_score_should_allow_csr_matrix_for_y_true_inputrj    sl     ]IIIyyy122Fh344G267CCFV]5))))))))rM   metricr9  ))FT)r   r-   )r   r\   )zeroonec                 $   t           j                            d          }d|d         }}|                    ||d          }|                    |          } | |||          }t          |t                    rt          j        |          rJ dS |\  }}	}
t          j        |                                          rJ t          j        |	                                          rJ t          j        |
                                          rJ dS )zCheck that the metric works with different types of `pos_label`.

    We can expect `pos_label` to be a bool, an integer, a float, a string.
    No error should be raised for those types.
    *   rO  rO   T)r   replacer  N)	r2   r6   r7   choicerJ  
isinstancerS   r   any)rk  r9  rE   rB   rU   rJ   y_probari  metric_1metric_2r   s              rK   #test_ranking_metric_pos_label_typesrw    s
    )


#
#Cwr{yIZZiZ>>Fhhy!!GVFGy999F&%   .8F#######)/&(J8H%%))+++++8H%%))+++++8J''++-------rM   c                     t           j                            |           }|                    ddd          }|                    d          }t          ||          \  }}}t          j        |d                   sJ dS )zCheck that thresholds do not exceed 1.0 when `y_score` is a probability
    estimate.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/26193
    r   r&   rO  r   N)r2   r6   r7   r   rJ  r   isinf)r=  rE   rJ   rH   rw   r   s         rK   (test_roc_curve_with_probablity_estimatesrz    sv     )

 2
3
3C[[AB[''FhhrllG 11Aq*8JqM"""""""rM   c                  J   t          d          \  } }}d}t          j        t          |          5  t	          | |           ddd           n# 1 swxY w Y   d}t          j        t          |          5  t	          | ||           ddd           dS # 1 swxY w Y   dS )	z)Check the message for future deprecation.Tr   z)probas_pred was deprecated in version 1.5r   )probas_predNz4`probas_pred` and `y_score` cannot be both specified)r|  rH   )rL   r   r   FutureWarningr   r   r   )rJ   rw   rH   warn_msg	error_msgs        rK   /test_precision_recall_curve_deprecation_warningr    sH    )555FAw:H	m8	4	4	4 
 
	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 GI	z	3	3	3 
 
	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s#   AAA8BBB)NF)r   rI  r   )r   r   numpyr2   r   scipyr   sklearnr   r   sklearn.datasetsr   sklearn.exceptionsr   sklearn.linear_modelr   sklearn.metricsr	   r
   r   r   r   r   r   r   r   r   r   r   r   sklearn.metrics._rankingr   r   sklearn.model_selectionr   sklearn.preprocessingr   sklearn.random_projectionr   sklearn.utils._testingr   r   r   r   r   sklearn.utils.extmathr    sklearn.utils.fixesr!   sklearn.utils.validationr"   r#   r$   CURVE_FUNCSrL   rZ   re   rm   r   markparametrizer   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r'  r/  rA  rF  rM  rS  rV  r_  filterwarningsrd  rj  ru  rn  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r^   r  r  r  r  r  r  r  r  r  r  r  r%  r(  r*  r0  r9  r;  r>  r=  rE  rL  rS  rW  ra  rc  re  rg  rj  rw  rz  r  r   rM   rK   <module>r     s    				            ! ! ! ! ! ! ! ! ; ; ; ; ; ; 5 5 5 5 5 5 3 3 3 3 3 3                              M L L L L L L L 4 4 4 4 4 4 0 0 0 0 0 0 ; ; ; ; ; ;              * ) ) ) ) ) . . . . . .          )# )# )# )#`2 2 2  <  &H H H8 $//
) 
) 0/
)
" 
" 
") ) )&# # #) ) )) ) )8) ) )6oN oN oNdM M M	) 	) 	). . .&  " 	,,,		+	,,,		&			/			t$	 /X /X /Xd 	,,,		+	&&&	'	'9   : 	,,,		&			t$	yyy!			/	   D      D: : :( 	,hbh|||.D.DiiiP/BH)))**OO	
 BH\\\""	
 1BH)))**OO	
 BH\\\""F	
 BH)))**#J	
 BH\\\""LL	
 BH)))**   	
 ABH)))**OO	
 ABH)))**OO	
 ABH\\\""II	
}CF FN 77P P 87OF FPP ' "%88	
' "%88	
F "B77	
 "c22	
 
4R8=" "F	0 	0G" "F	0& & &* {33# # 43# {33E E 43E2 LMM(>	'JKK(9::
' 
' ;: LK NM
' {33	& 	& 43	& $//    0/ >) ) ), $//` ` 0/`F6 6 6D< < <= = =
= 
= 
== = =*$ $ $2 .	KKK!qc*	NNNQC!-	NNNQC!-	%%%sQC0	KKK#,	NNNSEA3/	NNNSEA3/	%%%uqc2	KKK#,	NNNQC!-	NNNQC!-	%%%sQC0	KKKiii8	NNNKKK;	NNNKKK;	%%%{{{III>	KKKMMM:	NNNIII}}}=	NNNIII}}}=	%%%yyy---@) 2' '3 2' .
Q#saS1#&
Q#saS1#&	$$$seaS1	$$$seaS1	$$$qcA3/	$$$qcA3/	$$$qcA3/	$$$qcA3/	 ' ' '   $E$E$EFF* * GF* 																 	 	 	 
Q?@	QH?@	KKK!FG	KKK!FG...OO(	

 " " "> > >48S 8S 8Sv  .1 1 14X X X  6# # #N 9:3 3 3 3@ 3)	  "G!RSS  TS C C C m44j11q22  32 21 54  *4 4 4nL L L  	$$$%	%%%&'
))&&&' ( ( (( ( (V .99  :99 9 9 	P 	P 	P) ) )U U U  (  $ $ $= = = u66#V #V 76#VL! ! !* * *- - - 
 
 
4 	q$	q#	q$ 
. 
. 
. 	...	!	!1a(	...	!	!1c*	...	!	!1a(	&&&	'	'A.	&&&	'	'C0	&&&	'	'A.
 
	; 	;
 
	;  	,,,		|||4	,,,		lll3	,,,		lll3	&&&	'	'/C/C/CD	  ,tUm<<. . =< .&+ + +& 	q$	q#	q! 
S 
S 
S 	q	q   $ " OO	 G
	
 LL	 +	
 !  	 !  0
	
 !  	 OO1
	
 LL	 LL 	
 LL	 IIA
	
 F__ooo.W	
KNQ QdB BeQ QdB
 .99* * :9* &	3I9U  CCC . .  .(# # #
 
 
 
 
rM   