
    0Ph(                       d dl Z d dlZd dlmZ d dlmZmZmZ d dlZ	d dl
Z
d dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. d dl/m0Z0m1Z1 d dl2m3Z3 d dl4m5Z5m6Z6 d dl7m8Z8 d dl9m:Z: d dl;m<Z<m=Z=m>Z>m?Z?m@Z@ d dlAmBZB d dlCmDZDmEZE d dlFmGZG ddZHd ZId ZJe
jK        L                    ddd de	jM        g          d             ZNe
jK        L                    dd gdfd dgdfg ddfg          d             ZOd  ZPd! ZQe
jK        R                    d"          d#             ZSe
jK        R                    d"          d$             ZTe
jK        R                    d"          d%             ZUd& ZVe
jK        L                    d'g d( e	jW        g d)g d*g d+g d,g          fg d-g d.fg          d/             ZXe
jK        L                    d'g d0 e	jW        g d,g d,g d1g d1g d2g          fg d3g d4fg          d5             ZYd6 ZZd7 Z[d8 Z\d9 Z]e
jK        L                    d:eD          e
jK        L                    d;eE          d<                         Z^d= Z_e
jK        L                    d>g d?          d@             Z`dA ZadB Zbe
jK        L                    dC e	jW        g dD           e	jW        g dD          dEdFf e	jW        g dG           e	jW        g dG          dEdHf e	jW        g dG           e	jW        g dD          dEdIf e	jW        g dG           e	jW        g dJ          dEdKf e	jW        g dD           e	jW        g dG          dEdLfg          dM             Zce
jK        L                    dN e	jW        g dO           e	jW        g dP          dEdQfg          dR             ZddS ZedT Zfe
jK        L                    dd de	jM        g          e
jK        L                    dUd gd gfg          e
jK        L                    dVe" ee#dW          e,e-g          dX                                     Zge
jK        L                    dUd gd gfg          e
jK        L                    dVe" ee#dW          e,e-g          dY                         ZhdZ Zid[ Zjd\ Zkd] Zle
jK        L                    d^d_d`g          da             Zmdb Zne
jK        L                    dcg dd          de             Zodf Zpdg Zqdh Zre
jK        L                    dig djfdkdlgdmfgdndogp          dq             Zse
jK        L                    drdd dgg dfg dsp          dt             Ztdu Zue
jK        L                    dvg dw          dx             Zvdy Zwdz Zxd{ Zyd| Zzd} Z{d~ Z|d Z}d Z~d Ze
jK        R                    d"          d             Zd Zd Zd Zd Zd Zd Zd Ze
jK        L                    dddg          d             Ze
jK        R                    d"          d             Ze
jK        R                    d"          d             Ze
jK        R                    d"          e
jK        L                    dddde	jM        e	jM        fg          d                         Ze
jK        L                    ddg          e
jK        L                    dcg d          e
jK        L                    dd de	jM        g          d                                     Ze
jK        L                    dcg d          d             Ze
jK        L                    dd de	jM        g          d             Zd Zd Ze
jK        L                    dd de	jM        g          d             Ze
jK        L                    ddd de	jM        g          d             Ze
jK        L                    ddd de	jM        g          d             Ze
jK        L                    ddd de	jM        g          d             Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Ze
jK        L                    dve	j        e	j        e	j        g          d             Ze
jK        L                    dve	j        e	j        e	j        g          d             Ze
jK        L                    dUg dg dfg ddd gd dgdd ggfg dg dg dg dgfg          d             Zd Zd Zd Ze
jK        L                    dg dg dfg dg dfg dg dfg          d             Ze
jK        L                    dVe&e" ee#dW          e+e,e-eg          e
jK        L                    dg d          d                         Ze
jK        L                    d e	jW        d dg           e	jW        dd g          df e	jW        d dg           e	jW        d dg          df e	jW        d dg           e	jW        d d g          df e	jW        d d g           e	jW        d d g          dfg          d             Ze
jK        L                    d e(e"e	jM                   e(e#de	jM        æ           e(e,e	jM                   e(e-e	jM                  g          dĄ             Zdń ZdƄ ZdǄ ZdS )    N)partial)chainpermutationsproduct)linalg)hamming)	bernoulli)datasetssvm)make_multilabel_classification)UndefinedMetricWarning)accuracy_scoreaverage_precision_scorebalanced_accuracy_scorebrier_score_lossclass_likelihood_ratiosclassification_reportcohen_kappa_scoreconfusion_matrixf1_scorefbeta_scorehamming_loss
hinge_lossjaccard_scorelog_lossmake_scorermatthews_corrcoefmultilabel_confusion_matrixprecision_recall_fscore_supportprecision_scorerecall_scorezero_one_loss)_check_targetsd2_log_loss_score)cross_val_score)LabelBinarizerlabel_binarize)DecisionTreeClassifier)MockDataFrame)assert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equalignore_warnings)_nanaverage)CSC_CONTAINERSCSR_CONTAINERS)check_random_stateFc                    | t          j                    } | j        }| j        }|r||dk              ||dk              }}|j        \  }}t          j        |          }t          d          }|                    |           ||         ||         }}t          |dz            }t
          j
                            d          }t
          j        ||                    |d|z            f         }t          j        ddd          }	|	                    |d|         |d|                                       ||d                   }
|r|
ddd	f         }
|	                    ||d                   }||d         }|||
fS )
zMake some classification predictions on a toy dataset using a SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    N   %   r      linearT)kernelprobabilityrandom_state   )r
   	load_irisdatatargetshapenparanger2   shuffleintrandomRandomStatec_randnr   SVCfitpredict_probapredict)datasetbinaryXy	n_samples
n_featuresprnghalfclfy_pred_probay_predy_trues                i/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/sklearn/metrics/tests/test_classification.pymake_predictionrZ   :   sz    $&&AA "Qx1q51GIz
	)A
R
 
 CKKNNNQ41qAy1}D )


"
"C
a9cJ&67778A 't!
D
D
DC771UdU8QuuX..<<QtuuXFFL * $AAAqD)[[455""FtuuXF6<''    c            
         t          j                    } t          | d          \  }}}ddddddd	d
ddddddddddddddddddd}t          ||t	          j        t          | j                            | j        d          }|                                |                                k    sJ |D ]}|dk    r2t          ||         t                    sJ ||         ||         k    sJ :||                                         ||                                         k    sJ ||         D ]*}t          ||         |         ||         |                    +t          |d         d         t                    sJ t          |d         d         t                    sJ t          |d         d         t                    sJ t          |d         d         t                    sJ d S ) NFrL   rM   g7Mo?gUUUUUU?ghQ?   )	precisionrecallf1-scoresupportUUUUUU?gc1Ƹ?g433333?   g)k??$I$I?   gCFQ?gc?gƢ?K   ra   r_   r`   rb   g?gD~WG?g]3p?)setosa
versicolor	virginica	macro avgaccuracyweighted avgT)labelstarget_namesoutput_dictrn   rj   r_   rm   rb   )r
   r<   rZ   r   r@   rA   lenrq   keys
isinstancefloatr+   rC   )irisrX   rW   _expected_reportreportkeymetrics           rY   ,test_classification_report_dictionary_outputr}   j   sd   D'UCCCFFA
 -)*	
 
 -*+	
 
 -)+	
 
 ++'	
 
 '++(	
 
5   OD #yT.//00&  F ;;==O00222222 W W*fSk511111#;/#"666666#;##%%)=)B)B)D)DDDDD)#. W W#OC$8$@&+fBUVVVVW oh/<eDDDDDok2;?GGGGGoh/	:C@@@@@ok29=sCCCCCCCr[   c                     t          g g d          } dt          j        t          j        t          j        ddt          j        t          j        t          j        ddd}t          | t                    sJ |                                 |                                k    sJ |D ]}|dk    r2t          | |         t                    sJ | |         ||         k    sJ :| |                                         ||                                         k    sJ ||         D ]*}t          ||         |         | |         |                    +d S )NT)rX   rW   rr           r   ri   )rn   rm   ro   rn   )r   r@   nanru   dictrt   rv   r+   )rz   ry   r{   r|   s       rY   2test_classification_report_output_dict_empty_inputr      so   ""RTJJJFf	
 
 f	
 
 O fd#####;;==O00222222 W W*fSk511111#;/#"666666#;##%%)=)B)B)D)DDDDD)#. W W#OC$8$@&+fBUVVVVWW Wr[   zero_divisionwarnr;   c                    g dg d}}t          j        d          5 }t          ||| d           | dk    r5t          |          dk    sJ |D ]}d}|t	          |j                  v sJ n|rJ d d d            d S # 1 swxY w Y   d S )	Nabc)r   r   dTrecord)r   rr   r   r;   z7Use `zero_division` parameter to control this behavior.)warningscatch_warningsr   rs   strmessage)r   rX   rW   r   itemmsgs         rY   0test_classification_report_zero_division_warningr      s   $__oooFF		 	-	-	- 
F-T	
 	
 	
 	
 F""v;;???? 0 0Oc$,///////0 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   AA>>BBzlabels, show_micro_avgTr   r;   r4   c                 t    ddgddg}}t          ||| d          }|rd|v sJ d|vsJ dS d|v sJ d|vsJ dS )a3  Check the behaviour of passing `labels` as a superset or subset of the labels.
    WHen a superset, we expect to show the "accuracy" in the report while it should be
    the micro-averaging if this is a subset.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/27927
    r   r;   T)rp   rr   z	micro avgrn   Nr   )rp   show_micro_avgrX   rW   rz   s        rY   1test_classification_report_labels_subset_supersetr      s     VaVFF"66&dSSSF )f$$$$''''''V####&((((((r[   c                  F   t          j        g dg dg          } t          j        g dg dg          }t          | |          dk    sJ t          | |           dk    sJ t          ||          dk    sJ t          |t          j        |                    dk    sJ t          | t          j        |                     dk    sJ t          | t          j        | j                            dk    sJ t          |t          j        | j                            dk    sJ d S )Nr   r;   r;   r;   r   r;   r   r   r;         ?r;   r   )r@   arrayr   logical_notzerosr?   y1y2s     rY   .test_multilabel_accuracy_score_subset_accuracyr      s)   	999iii(	)	)B	999iii(	)	)B"b!!S(((("b!!Q&&&&"b!!Q&&&&"bnR0011Q6666"bnR0011Q6666"bhrx0011Q6666"bhrx0011Q666666r[   c            	         t          d          \  } }}t          | |d           \  }}}}t          |ddgd           t          |ddgd           t          |d	d
gd           t          |ddg           i ddifD ]}t	          j                    5  t	          j        d           t          | |fi |}t          |dd           t          | |fi |}	t          |	dd           t          | |fi |}
t          |
d
d           t          t          | |fddi|d|z  |	z  d|z  |	z   z  d           d d d            n# 1 swxY w Y   d S )NTrM   averageg\(\?g333333?r4   g)\(?g(\?皙?gRQ?   r   rM   errorbeta      )rZ   r   r,   r-   r   r   simplefilterr    r!   r   r+   r   )rX   rW   rx   rR   rfskwargspsrsfss              rY   %test_precision_recall_f1_score_binaryr      s   't444FFA 1NNNJAq!Qa$q111a$q111a$q111q2r(###
 	8,-  $&& 	 	!'*** ::6::B%b$222ff7777B%b$222&&33F33B%b$222FF===f==R"$r	B7  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 s   B D>>E	E	z1ignore::sklearn.exceptions.UndefinedMetricWarningc                  f   dt          ddgddg          k    sJ dt          ddgddg          k    sJ dt          ddgddg          k    sJ dt          ddgddgd          k    sJ dt          ddgddg          k    sJ dt          ddgddg          k    sJ dt          ddgddg          k    sJ dt          ddgddgt	          d                    k    sJ t          ddgddgt	          d                    t          j        t          ddgddgd                    k    sJ d S )	N      ?r;   r   r   r   infg     j@)r    r!   r   r   rv   pytestapprox r[   rY   +test_precision_recall_f_binary_single_classr      s   
 /1a&1a&111111,1v1v......(Aq6Aq6******+q!fq!f15555555/2r(RH555555,Bx"b222222(B8b"X......+r2hRuU||DDDDDDDBx"be===RHr2hS111B B      r[   c                     g d} g d}t          | t          j        d                    }t          |t          j        d                    }| |f||fg}t          |          D ]\  }\  } }t	          | |g dd           }t          g d|           t	          | |g dd          }t          t          j        g d          |           d	D ]C}|d
k    r|dk    rt          t	          | |g d|          t	          | |d |                     DdD ]}t          j	        t                    5  t	          ||t          j        d          |           d d d            n# 1 swxY w Y   t          j	        t                    5  t	          ||t          j        dd          |           d d d            n# 1 swxY w Y   t          j        g dg dg          } t          j        g dg dg          }t          | |d
ddg          \  }}	}
}t          t          j        ||	|
g          t          j        g d                     d S )N)r;      r   r4   )r;   r;   r   r4   r   classes)r   r;   r4   r   r   rp   r   )r   r   r   r   r   macro)microweightedsamplesr   r   )Nr   r   r      r   r   r   r;   r   r   r;   r;   r;   r   r;   r   rp   )      ?r;   竪?)r'   r@   rA   	enumerater!   r,   meanr+   r   raises
ValueErrorr   r   )rX   rW   
y_true_bin
y_pred_binr=   iactualr   rR   r   r   rx   s               rY   $test_precision_recall_f_extra_labelsr   3  s.    \\F\\F	!===J	!===JVz:67D(  FFff___dSSS!";";";VDDD ff___gVVV!"'*C*C*C"D"DfMMM 8 	 	G)##QVVOOOWUUUVVD'JJJ   	 7  ]:&& 	W 	WZ	!gVVVV	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W]:&& 	 	JryQ/?/?   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 Xyyy))),--FXyyy))),--F0	1a&  JAq!Q !Q++RX6G6G6G-H-HIIIIIs$   2&E$$E(	+E(	'F;;F?	F?	c                  \   g d} g d}t          | t          j        d                    }t          |t          j        d                    }| |f||fg}t          |          D ]\  }\  } }t	          t
          | |ddg          }t	          t
          | |d           }t          dd	g |d 
                     t          d |d
                     t          d |d
                     t          d |d
                     dD ]} ||
           ||
          k    sJ d S )N)r;   r;   r4   r   )r;   r   r   r   r   r   r;   r   rp   r   r   r   r   r   UUUUUU?r   r   )r   r   r   )r'   r@   rA   r   r   r!   r,   r+   )	rX   rW   r   r   r=   r   	recall_13
recall_allr   s	            rY   &test_precision_recall_f_ignored_labelsr   a  s{    \\F\\F	!===J	!===JVz:67D( M MFFL&&!QHHH	\66$GGG
!3*ii.E.E.EFFFOYYw-G-G-GHHH3YYz5R5R5RSSSGYYw%?%?%?@@@ 6 	M 	MG9W---G1L1L1LLLLLL	MM Mr[   c            	      .   t          j        g dg dg dg dg dg dg          } t          j        g dg dg dg d	g d
g dg          }d}t          j        t          |          5  t          | |d           ddd           dS # 1 swxY w Y   dS )z:Test multiclass-multiouptut for `average_precision_score`.)r4   r4   r;   r;   r4   r   r   r;   r4   r;   )r4   r   r;   ffffff?皙?皙?皙?333333?r   r   r   r   r   r   r   )r   r   r   )r   r   r   z.multiclass-multioutput format is not supportedmatchr4   	pos_labelN)r@   r   r   r   r   r   )rX   y_scoreerr_msgs      rY   -test_average_precision_score_non_binary_classr   x  s   XIIIIIIIIIIII	
	 	F hOOOOOOOOOOOO	
	 	G ?G	z	1	1	1 > >1====> > > > > > > > > > > > > > > > > >s   *B

BBzy_true, y_scorer   r   r;   r4   r   r   r   r   )r   r   r   r   r;   r;   r;   r;   r;   r;   r;   )r   r   r   r   r   333333?r   re   re   r;   r;   c                 2    t          | |          dk    sJ dS )a(  
    Duplicate values with precision-recall require a different
    processing than when computing the AUC of a ROC, because the
    precision-recall curve is a decreasing curve
    The following situation corresponds to a perfect
    test statistic, the average_precision_score should be 1.
    r;   Nr   rX   r   s     rY   -test_average_precision_score_duplicate_valuesr     s%    8 #6733q888888r[   )r4   r4   r;   r;   r   )r   r   r   )r   r   r   r   )r   r   r   c                 2    t          | |          dk    sJ d S )Nr   r   r   s     rY   (test_average_precision_score_tied_valuesr     s%    : #6733s::::::r[   c                      d} t          j        t          |           5  t          g dg ddd           d d d            d S # 1 swxY w Y   d S )NzNote that pos_label \(set to 2\) is ignored when average != 'binary' \(got 'macro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   r;   r4   r4   r4   r   r   r   )r   warnsUserWarningr   r   s    rY   (test_precision_recall_f_unused_pos_labelr     s    
	  
k	-	-	- 
 
'IIyyyAw	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   AA
Ac                      t          d          \  } }}d } || |            |d | D             d |D                        d S )NTr   c                 n   t          | |          }t          |ddgddgg           |                                \  }}}}||z  ||z  z
  }t          j        ||z   ||z   z  ||z   z  ||z   z            }|dk    rdn||z  }	t          | |          }
t          |
|	d           t          |
dd           d S )	N   r         r   r4   decimal=
ףp=?)r   r-   flattenr@   sqrtr   r,   )rX   rW   cmtpfpfntnnumdentrue_mccmccs              rY   testz*test_confusion_matrix_binary.<locals>.test  s    ff--2Q!R1222BB2gRgrBw27+rBw727CDDq11cCi//!#x;;;;!#tQ777777r[   c                 ,    g | ]}t          |          S r   r   .0rO   s     rY   
<listcomp>z0test_confusion_matrix_binary.<locals>.<listcomp>      	!	!	!Q#a&&	!	!	!r[   c                 ,    g | ]}t          |          S r   r  r  s     rY   r  z0test_confusion_matrix_binary.<locals>.<listcomp>      #;#;#;qCFF#;#;#;r[   rZ   rX   rW   rx   r  s       rY   test_confusion_matrix_binaryr    sr    't444FFA8 8 8 	DD	!	!&	!	!	!#;#;F#;#;#;<<<<<r[   c                      t          d          \  } }}d } || |            |d | D             d |D                        d S )NTr   c                 b    t          | |          }t          |ddgddggddgddggg           d S )Nr  r  r   r  r   r-   )rX   rW   r  s      rY   r  z5test_multilabel_confusion_matrix_binary.<locals>.test  sF    (882"a1b' 2b!Wq"g4FGHHHHHr[   c                 ,    g | ]}t          |          S r   r  r  s     rY   r  z;test_multilabel_confusion_matrix_binary.<locals>.<listcomp>   r  r[   c                 ,    g | ]}t          |          S r   r  r  s     rY   r  z;test_multilabel_confusion_matrix_binary.<locals>.<listcomp>   r  r[   r  r  s       rY   'test_multilabel_confusion_matrix_binaryr%    su    't444FFAI I I 	DD	!	!&	!	!	!#;#;F#;#;#;<<<<<r[   c                      t          d          \  } }}dd} || |            |d | D             d |D             d           d S )	NFr   c           	         t          | |          }t          |ddgddggddgddggd	d
gddggg           |rg dng d}t          | ||          }t          |ddgddggd	d
gddggddgddggg           |rg dng d}t          | ||          }t          |ddgddggd	d
gddggddgddggddgddggg           d S )N/   r   r      &   r      r      r   r4      )021r   r4   r;   r   )r.  r/  r0  3)r   r4   r;   r   rh   r   r"  )rX   rW   string_typer  rp   s        rY   r  z9test_multilabel_confusion_matrix_multiclass.<locals>.test  s[   (882q'Ar7#r1gAw%72r(QG9LM	
 	
 	

 %0>YYY(GGG2q'Ar7#r2hB%8B7RG:LM	
 	
 	

 *5F%%%%,,,(GGGa1b'"bAr7#a2q'"a1a&!		
 	
 	
 	
 	
r[   c                 ,    g | ]}t          |          S r   r  r  s     rY   r  z?test_multilabel_confusion_matrix_multiclass.<locals>.<listcomp>#  r  r[   c                 ,    g | ]}t          |          S r   r  r  s     rY   r  z?test_multilabel_confusion_matrix_multiclass.<locals>.<listcomp>#  r  r[   T)r3  )Fr  r  s       rY   +test_multilabel_confusion_matrix_multiclassr6    s|    'u555FFA
 
 
 
6 	DD	!	!&	!	!	!#;#;F#;#;#;NNNNNNr[   csc_containercsr_containerc                 J   t          j        g dg dg dg          }t          j        g dg dg dg          } ||          } ||          } | |          } | |          }t          j        g d          }dd	gddggdd	gddggd	d
gdd	ggg}	|||g}
|||g}|
D ]'}|D ]"}t          ||          }t          ||	           #(t          ||d          }t          |dd	gddggddgd	dggd	dgd
d	ggg           t          ||d
d	g          }t          |d	d
gdd	ggdd	gddggg           t          ||d
d	gd          }t          |d	d	gddggddgd	d	ggd	dgdd	ggg           t          |||d          }t          |d
d	gd
d
ggddgd	dggd	dgdd	ggg           d S )Nr   r   r;   r   r;   r;   r   r   r   r   )r4   r;   r   r;   r   r4   T
samplewiser   )rp   r=  )sample_weightr=  r   r   )r@   r   r   r-   )r7  r8  rX   rW   
y_true_csr
y_pred_csr
y_true_csc
y_pred_cscr>  real_cmtruespreds
y_true_tmp
y_pred_tmpr  s                  rY   +test_multilabel_confusion_matrix_multilabelrH  &  s   
 Xyyy)))YYY788FXyyy)))YYY788Fv&&Jv&&Jv&&Jv&&J HYYY''MAA1a&1a&!1QFQF3CDGZ,EZ,E , ,
 	, 	,J,ZDDBr7++++	,
 
%VV	E	E	EBraVaV,1v1v.>!Q!Q@PQRRR 
%VVQF	C	C	CBraVaV,1v1v.>?@@@ 
%VVQFt	T	T	TBraVaV,1v1v.>!Q!Q@PQRRR 
%m
 
 
B raVaV,1v1v.>!Q!Q@PQRRRRRr[   c            	         t          j        g dg dg dg          } t          j        g dg dg dg          }t          j        t          d          5  t          | |d	d
g           d d d            n# 1 swxY w Y   t          j        t          d          5  t          | |g dg dg dg           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          | |dg           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          | |dg           d d d            n# 1 swxY w Y   t          j        t          d          5  t          g dg dd           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          g dg dgg dg dg           d d d            d S # 1 swxY w Y   d S )Nr   r:  r;  r   r   r   inconsistent numbers of samplesr   r;   r4   r>  zshould be a 1d arrayr;   r4   r   )r4   r   r   )r   r   r   z%All labels must be in \[0, n labels\)r   r   r   zSamplewise metricsr   r   Tr<  z'multiclass-multioutput is not supported)r4   r;   r   r;   r   r4   )r@   r   r   r   r   r   )rX   rW   r   s      rY   'test_multilabel_confusion_matrix_errorsrN  P  s   Xyyy)))YYY788FXyyy)))YYY788F 
z)J	K	K	K J J#FF1a&IIIIJ J J J J J J J J J J J J J J	z)?	@	@	@ 
 
#F999iii*K	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 7G	z	1	1	1 A A#FFB4@@@@A A A A A A A A A A A A A A A6G	z	1	1	1 @ @#FFA3????@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ 
z)=	>	>	> K K#IIIyyyTJJJJK K K K K K K K K K K K K K K 8G	z	1	1	1 T T#YYY			$:YYY			<RSSST T T T T T T T T T T T T T T T T Tsl   A77A;>A;CC	C	-DDD5EEE;FF"%F"G00G47G4z%normalize, cm_dtype, expected_results))truer   TUU?)predr   rP  )allr   geq?)Nr   r4   c                     g ddz  }t          t          t          g d                     }t          |||           }t	          ||           |j        j        |k    sJ d S )Nr   r   	normalize)listr   r   r   r*   dtypekind)rU  cm_dtypeexpected_resultsy_testrW   r  s         rY   test_confusion_matrix_normalizer\  n  so     YY]F%iii00122F	&&I	>	>	>BB()))8=H$$$$$$r[   c                  H   g d} g d}t          | |d          }|                                t          j        d          k    sJ t	          j                    5  t	          j        dt                     t          | |d          }d d d            n# 1 swxY w Y   |                                t          j        d          k    sJ t	          j                    5  t	          j        dt                     t          || d           d d d            d S # 1 swxY w Y   d S )	N)r   r   r   r   r;   r;   r;   r;   )r   r   r   r   r   r   r   r   rO  rT         @r   rQ  r   )r   sumr   r   r   r   r   RuntimeWarning)r[  rW   cm_truecm_preds       rY   ,test_confusion_matrix_normalize_single_classrc    s   %%%F%%%Fvv@@@G;;==FM#...... 
	 	"	" E Eg~666"66VDDDE E E E E E E E E E E E E E E ;;==FM#......		 	"	" ; ;g~6666::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;s$   -BBB-DDDc                      g d} g d}t          j        t          d          5  t          ||            ddd           dS # 1 swxY w Y   dS )z8Test `confusion_matrix` warns when only one label found.r   r   r   r   zA single label was found inr   N)r   r   r   r   )r[  rW   s     rY   "test_confusion_matrix_single_labelrf    s    \\F\\F	k)F	G	G	G ) )((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) )s   AA	Azparams, warn_msg)r   r   r   r   r   r   rX   rW   z2samples of only one class were seen during testing)r;   r;   r;   r   r   r   z:positive_likelihood_ratio ill-defined and being set to nanz+no samples predicted for the positive classr   r   r   r;   r;   r;   z:negative_likelihood_ratio ill-defined and being set to nanz@no samples of the positive class were present in the testing setc                     t          j        t          |          5  t          di |  d d d            d S # 1 swxY w Y   d S Nr   r   )r   r   r   r   )paramswarn_msgs     rY   test_likelihood_ratios_warningsrm    s    b 
k	2	2	2 * *))&)))* * * * * * * * * * * * * * * * * *   6::zparams, err_msg)r   r;   r   r;   r   r;   r;   r   r   r4   zeclass_likelihood_ratios only supports binary classification problems, got targets of type: multiclassc                     t          j        t          |          5  t          di |  d d d            d S # 1 swxY w Y   d S rj  )r   r   r   r   )rk  r   s     rY   test_likelihood_ratios_errorsrq    s    $ 
z	1	1	1 * *))&)))* * * * * * * * * * * * * * * * * *rn  c                  F   t          j        dgdz  dgdz  z             } t          j        dgdz  dgdz  z   dgdz  z             }t          | |          \  }}t          |d           t          |d	           t          | |           \  }}t	          |t           j        dz             t          |t          j        d          d
           t          j        dgdz  dgdz  z             }t          | ||          \  }}t          |d           t          |d           d S )Nr;   r   r   r  r4   
   r  g?g_B{	%?g-q=)rtolr      r   r   rK  gUUUUUU@gqq?)r@   r   r   r*   r-   r   r   )rX   rW   posnegr>  s        rY   test_likelihood_ratiosrx    s*    XqcAgb())FXqcAgb(A37233F&vv66HCC!!!C!!! 'vv66HCsBFQJ'''C!51111
 HcURZ3%!)344M&vv]SSSHCC   C!!!!!r[   c                     t          j        dgdz  dgdz  z             } t          j        dgdz  dgdz  z   dgdz  z   dgdz  z             }t          | |          }t          |dd	
           |t          ||           k    sJ t          j        | dgdz            } t          j        |dgdz            }t          | |ddg          |k    sJ t          t          | |           d           t          j        dgdz  dgdz  z   dgdz  z             } t          j        dgdz  dgdz  z   dgdz  z             }t          t          | |          dd
           t          j        dgdz  dgdz  z   dgdz  z             } t          j        dgdz  dgdz  z   dgdz  z             }t          t          | |          dd
           t          t          | |d          dd
           t          t          | |d          dd
           d S )Nr   (   r;   <   rg   rs  2   gʡE?r   r  r4   r   r   r   .   ,   4          g??g+?r7   weightsg_vO?	quadraticg#?)r@   r   r   r+   append)r   r   kappas      rY   test_cohen_kappar    sV    
1#(aS2X%	&	&B	1#(aS2X%b0A38;	<	<Bb"%%Eua0000%b"------ 
2sQw		B	2sQw		BRQF333u<<<<)"b113777 
1#(aS2X%b0	1	1B	1#(aS2X%b0	1	1B)"b1161EEEE 
1#(aS2X%b0	1	1B	1#(aS2X%b0	1	1B)"b1161EEEE)"b(CCCVUVWWWW"b+666     r[   zy_true, y_predr|   r   c                    t          j                    5  t          j        d            | |||          }ddd           n# 1 swxY w Y   t          j        |          rt          j        |          sJ dS ||k    sJ dS )zmCheck the behaviour of `zero_division` when setting to 0, 1 or np.nan.
    No warnings should be raised.
    r   r   N)r   r   r   r@   isnan)r|   rX   rW   r   results        rY   !test_zero_division_nan_no_warningr    s     
	 	"	" E Eg&&&mDDDE E E E E E E E E E E E E E E 
x 'x&&&&&&s   #AA
Ac                     t          j        t                    5   | ||d          }ddd           n# 1 swxY w Y   |dk    sJ dS )ztCheck the behaviour of `zero_division` when setting to "warn".
    A `UndefinedMetricWarning` should be raised.
    r   r  Nr   )r   r   r   )r|   rX   rW   r  s       rY   test_zero_division_nan_warningr  7  s     
,	-	- > >f===> > > > > > > > > > > > > > >S======s   599c                     t           j                            d          } |                     ddd          }|                     ddd          }t	          t          ||          t          j        ||          d         d           d S )Nr   r4   rg   sizer   r;   rs  )r@   rD   rE   randintr+   r   corrcoef)rS   rX   rW   s      rY   -test_matthews_corrcoef_against_numpy_corrcoefr  J  s    
)


"
"C[[AB[''F[[AB[''F&&))2;vv+F+Ft+Lb    r[   c                    	
 t           j                            d          } |                     ddd          }|                     ddd          }|                     d          }t          |||          	t          	          
t          	
fdt          
          D                       }t          	
fdt          
          D                       }t          j        	
fdt          
          D                       }|t          j	        ||z            z  }t          |||          }t          ||d	           d S )
Nr   r4   rg   r  rK  c           	          g | ]Q}t                    D ]?}t                    D ]-}||f         ||f         z  ||f         ||f         z  z
  .@RS r   range)r  kmlCNs       rY   r  z9test_matthews_corrcoef_against_jurman.<locals>.<listcomp>`  s     	
 	
 	
1XX	
 	
 1XX		
 	
  adGa1g!Q$!AqD' 11	
 	
 	
 	
 	
r[   c           	          g | ]Pd d f                                          t          j         fdt                    D                       z  QS )Nc                 T    g | ]$}t                    D ]}|k    ||f         %S r   r  r  r   gr  r  r  s      rY   r  zDtest_matthews_corrcoef_against_jurman.<locals>.<listcomp>.<listcomp>j  8    LLL!qLLAQ!VVa1gVVVVr[   r_  r@   r  r  r  r  r  s    @rY   r  z9test_matthews_corrcoef_against_jurman.<locals>.<listcomp>h  st     	
 	
 	
  aaadGKKMMfLLLLLLuQxxLLLMMN	
 	
 	
r[   c           	          g | ]Pd d f                                          t          j         fdt                    D                       z  QS )Nc                 T    g | ]$}t                    D ]}|k    ||f         %S r   r  r  s      rY   r  zDtest_matthews_corrcoef_against_jurman.<locals>.<listcomp>.<listcomp>q  r  r[   r  r  s    @rY   r  z9test_matthews_corrcoef_against_jurman.<locals>.<listcomp>o  st     	
 	
 	
  adGKKMMfLLLLLLuQxxLLLMMN	
 	
 	
r[   rs  )r@   rD   rE   r  randr   rs   r_  r  r
  r   r+   )rS   rX   rW   r>  cov_ytypcov_ytytcov_ypyp
mcc_jurmanmcc_oursr  r  s            @@rY   %test_matthews_corrcoef_against_jurmanr  T  s    )


"
"C[[AB[''F[[AB[''FHHRLLM}EEEAAA	
 	
 	
 	
 	
1XX	
 	
 	
 H 	
 	
 	
 	
 	
 1XX	
 	
 	
 H v	
 	
 	
 	
 	
 1XX	
 	
 	
 H BGHx$7888J }MMMH*b11111r[   c            	      R   t           j                            d          } d |                     ddd          D             }t	          t          ||          d           d |D             }t	          t          ||          d           t          |d	d
g          }t          j        |d	d
          }t	          t          ||          d           t	          t          g dg d          d           t	          t          |d	gt          |          z            d           g d}g d}t	          t          ||          d           dgdz  dgdz  z   }t          j
        t                    5  t	          t          |||          d           d d d            d S # 1 swxY w Y   d S )Nr   c                 "    g | ]}|d k    rdndS )r   r   r   r   r  r   s     rY   r  z*test_matthews_corrcoef.<locals>.<listcomp>}  s$    IIIQ!VVccIIIr[   r4   rg   r  r   c                 "    g | ]}|d k    rdnd S )r   r   r   r  s     rY   r  z*test_matthews_corrcoef.<locals>.<listcomp>  s$    ;;;qc##s;;;r[   r   r   r   r   re  r   )r;   r   r;   r;   r   r;   r;   r;   r   r;   r;   r;   r;   r;   r;   r;   r   r;   r;   r;   )r;   r;   r;   r   r   r;   r;   r;   r;   r   r;   r;   r;   r   r;   r;   r;   r   r;   r;   r;   rs  rK  )r@   rD   rE   r  r+   r   r'   wherers   r   r   AssertionError)rS   rX   
y_true_invy_true_inv2y_1y_2masks          rY   test_matthews_corrcoefr  {  s   
)


"
"CIIckk!QRk.H.HIIIF )&&993??? <;F;;;J)&*==rBBB #s<<<K(;S11K)&+>>CCC ),,,EEsKKK )&3%#f++2EFFLLL G
F
FC
F
F
FC)#s33S999 38qcBhD 
~	&	& R R-c3dKKKSQQQR R R R R R R R R R R R R R R R R Rs   .!FF #F c                  B  	 t           j                            d          } t          d          	d}	fd|                     d|d          D             }t          t          ||          d           g d}g d	}t          t          ||          d
           g d}g d}t          t          ||          dt          j        d          z             g d}g d}t          t          ||          d           g d}g d}t          t          ||          d           g d}g d}t          t          ||          d           g d}g d}g d}t          t          |||          d           g d}g d}g d}t          t          |||          d           d S )Nr   r   r   c                 4    g | ]}t          |z             S r   )chr)r  r   ord_as     rY   r  z5test_matthews_corrcoef_multiclass.<locals>.<listcomp>  s#    IIIc%!)nnIIIr[   rg   r  r   )r   r   r;   r;   r4   r4   )r4   r4   r   r   r;   r;   g      )r;   r;   r   r   r   r   ii  r   )r   r   r   r   	r   r;   r4   r   r;   r4   r   r;   r4   )	r;   r;   r;   r4   r4   r4   r   r   r   )r   r   r;   r;   r4   ro  r;   r;   r;   r;   r   rK  r   r   r;   r;   r   r   )r@   rD   rE   ordr  r+   r   r
  )
rS   	n_classesrX   
y_pred_bad
y_pred_minrW   r  r  r>  r  s
            @rY   !test_matthews_corrcoef_multiclassr    s   
)


"
"CHHEIIIIIckk!YRk&H&HIIIF )&&993???  F###J)&*==tDDD  F###J)&*==sRWWEUEU?UVVV YYFYYF)&&993??? YYFYYF)&&993??? &
%
%C
%
%
%C)#s33S999 __F__F#OOM&&FFF   \\F\\F LLM&&FFF    r[   n_pointsd   i'  c                    t           j                            d          d }fd}t          j        ddg|           }t	          t          ||          d           t          j        g d|           }t	          t          ||          d            ||           \  }}t	          t          ||          d           t	          t          ||           |||                     d S )Ni3c                     t          | |          }|d         }|d         }|d         }t          |           }||z   |z  }||z   |z  }||z  ||z  z
  }	||z  d|z
  z  d|z
  z  }
|	t          j        |
          z  S )Nr;   r;   )r;   r   r  r;   )r   rs   r@   r
  )rX   rW   conf_matrixtrue_pos	false_pos	false_negr  pos_rateactivitymcc_numeratormcc_denominators              rY   mcc_safez1test_matthews_corrcoef_overflow.<locals>.mcc_safe  s    &vv66t$%	%	v;;y(H4y(H4 8+h.AA"X-X>!h,Orw7777r[   c                                          |           }|d                     |           dz
  z  z   }|dk    }|dk    }||fS )Nr   r   )random_sample)r  x_truex_predrX   rW   rS   s        rY   	random_ysz2test_matthews_corrcoef_overflow.<locals>.random_ys  sT    ""8,,#!2!28!<!<s!BCC##v~r[   r   r   )r   r   r^  )r@   rD   rE   repeatr+   r   )r  r  r  arrrX   rW   rS   s         @rY   test_matthews_corrcoef_overflowr    s     )


)
)C
8 
8 
8     )S#J
)
)C)#s33S999
)OOOX
.
.C)#s33S999Yx((NFF)&&993???)&&9988FF;S;STTTTTr[   c                  N   t          d          \  } }}t          | |d           \  }}}}t          |g dd           t          |g dd           t          |g dd           t          |g d           t	          | |d	d
          }t          |dd           t          | |d
          }t          |dd           t          | |d
          }	t          |	dd           t	          | |d          }t          |dd           t          | |d          }t          |dd           t          | |d          }	t          |	dd           t	          | |d          }t          |dd           t          | |d          }t          |dd           t          | |d          }	t          |	dd           t          j        t                    5  t	          | |d           d d d            n# 1 swxY w Y   t          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t          j        t                    5  t          | |dd           d d d            n# 1 swxY w Y   t          | |g dd           \  }}}}t          |g dd           t          |g dd           t          |g dd           t          |g d           d S )NFr   r   )(\?Q?gzG?r4   )HzG?g
ףp=
?re   )Q?333333?r  )r^   rd   rg   r;   r   r   g(\?r   r   gRQ?r   gGz?r   r   r   r   r1  r   )r  g=
ףp=?r  )r  re   r   )r  r  r  )r^   rg   rd   )rZ   r   r,   r-   r    r!   r   r   r   r   r   )
rX   rW   rx   rR   r   r   r   r   r   r   s
             rY   )test_precision_recall_f1_score_multiclassr    s}   'u555FFA 1NNNJAq!Qa!3!3!3Q777a!3!3!3Q777a!3!3!3Q777q,,,''' 
1g	F	F	FBb$***	ffg	6	6	6Bb$***	&&'	2	2	2Bb$***		9	9	9Bb$***	ffg	6	6	6Bb$***	&&'	2	2	2Bb$***		<	<	<Bb$***	ffj	9	9	9Bb$***	&&*	5	5	5Bb$***	z	"	" ; ;	::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;	z	"	" 8 8VVY77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	z	"	" 4 433334 4 4 4 4 4 4 4 4 4 4 4 4 4 4	z	"	" A AFFIC@@@@A A A A A A A A A A A A A A A 1yyy$  JAq!Q a!3!3!3Q777a!3!3!3Q777a!3!3!3Q777q,,,'''''sH   G++G/2G/H..H25H2I11I58I5J55J9<J9r   )r   r   r   r   Nc                 $   t          j        g dg          }t          j        g dg          }t          ||g dg |           \  }}}}t          |d           t          |d           t          |d           | t          |g d           d S d S )Nr  r   r   r;   r;   )r   r   r;   r4   )rp   warn_forr   r   r   r;   r;   r   )r@   r   r   r-   )r   rX   rW   rR   r   r   r   s          rY   ;test_precision_refcall_f1_score_multilabel_unordered_labelsr  2  s     X|||n%%FX|||n%%F0|||b'  JAq!Q q!q!q!1lll+++++ r[   c                  T   t          j        g d          } t          j        g d          }t          | |d           \  }}}}t          | |d          \  }}}}|t          j        |          k    sJ |t          j        |          k    sJ |t          j        |          k    sJ t          | |d          \  }}}}t          j        |           }	|t          j        ||	          k    sJ |t          j        ||	          k    sJ |t          j        ||	          k    sJ d S )N)r   r;   r   r   r;   r;   r   r;   r   r   r;   r   r;   r   r;   )r;   r;   r   r;   r   r;   r;   r;   r;   r   r;   r   r;   r   r;   r   r   r   r  )r@   r   r   r   bincountr   )
rX   rW   r   r   r   rx   rR   r   r   rb   s
             rY   .test_precision_recall_f1_score_binary_averagedr  A  sD   XCCCDDFXCCCDDF 4FFDQQQMBB0QQQJAq!Q0TTTJAq!Qk&!!G
2w///////
2w///////
2w/////////r[   c                     t          j        d          } 	 t          j        g d          }t          j        g d          }t          t	          ||d          dd           t          t          ||d          dd           t          t          ||d          dd           t          j        d	i |  d S # t          j        d	i |  w xY w)
Nraise)rR  )r   r;   r4   r   r;   r4   )r4   r   r;   r;   r4   r   r   r   r   r4   r   )r@   seterrr   r+   r    r!   r   )old_error_settingsrX   rW   s      rY   test_zero_precision_recallr  R  s     w///	(,,,--,,,--OFFGLLLcSTUUULIII3PQRRRHVVWEEEsANNN 		''&'''''	''&''''s   BB9 9Cc                  X   t          d          \  } }}t          | |ddg          }t          |ddgddgg           t          | |d	dg          }t          |d
d	gddgg           t          j        |           dz   }t          | |d	|g          }t          |d
dgddgg           d S )NFr   r   r;   r   r)  r   r   r4   r-  r^   )rZ   r   r-   r@   max)rX   rW   rx   r  extra_labels        rY   .test_confusion_matrix_multiclass_subset_labelsr  c  s    'u555FFA 
&&!Q	8	8	8BrRGaV,--- 
&&!Q	8	8	8BrRGb!W-... &..1$K	&&![1A	B	B	BBrRGaV,-----r[   zlabels, err_msgz,'labels' should contains at least one label.r   r   z.At least one label specified must be in y_truez
empty listzunknown labels)idsc                     t          d          \  }}}t          j        t          |          5  t	          |||            d d d            d S # 1 swxY w Y   d S )NFr   r   r   )rZ   r   r   r   r   )rp   r   rX   rW   rx   s        rY   test_confusion_matrix_errorr  v  s     (u555FFA	z	1	1	1 8 877778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8s   AAArp   )NonerM   
multiclassc                     | rt          |           nd}t          j        ||ft                    }t	          g g |           }t          ||           d S )Nr   rW  r   )rs   r@   r   rC   r   r-   )rp   expected_n_classesexpectedr  s       rY   *test_confusion_matrix_on_zero_length_inputr    s^     )/5VAx+-?@LLLH	"b	0	0	0Br8$$$$$r[   c            	         g d} t          j        t          |                     }t          | |           }|j        t           j        k    sJ t           j        t           j        t           j        fD ]@}t          | | |	                    |d                    }|j        t           j        k    sJ At           j
        t           j        d t          fD ]@}t          | | |	                    |d                    }|j        t           j        k    sJ At          j        t          |           dt           j                  }t          | | |          }|d         dk    sJ |d         d	k    sJ t          j        t          |           d
t           j                  }t          | | |          }|d         d
k    sJ |d         dk    sJ d S )Nr   F)copyrK  l    r  r   r   r  l    l    )r@   onesrs   r   rW  int64bool_int32uint64astypefloat32float64objectfulluint32)rO   weightr  rW  s       rY   test_confusion_matrix_dtyper    s   		AWSVV__F	!Q		B8rx(BHbi0 $ $a&--E-2R2RSSSx28#####*bj$7 & &a&--E-2R2RSSSx2:%%%%% WSVVZry999F	!Qf	5	5	5Bd8z!!!!d8z!!!! WSVV0AAAF	!Qf	5	5	5Bd8*****d8r>>>>>>r[   rW  )Int64Float64booleanc                    t          j        d          }t          j        g d          }|                    ||           }|                    g dd          }t          ||          }t          ||          }t          ||           dS )zkChecks that confusion_matrix works with pandas nullable dtypes.

    Non-regression test for gh-25635.
    pandas)	r;   r   r   r;   r   r;   r;   r   r;   r  )	r   r   r;   r;   r   r;   r;   r;   r;   r  N)r   importorskipr@   r   Seriesr   r-   )rW  pd	y_ndarrayrX   y_predictedoutputexpected_outputs          rY   %test_confusion_matrix_pandas_nullabler    s     
	X	&	&B44455IYYyY..F))777w)GGKfk22F&y+>>Ov/////r[   c            	          t          j                    } t          | d          \  }}}d}t          ||t	          j        t          | j                            | j                  }||k    sJ d S )NFr]   a|                precision    recall  f1-score   support

      setosa       0.83      0.79      0.81        24
  versicolor       0.33      0.10      0.15        31
   virginica       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
rp   rq   r
   r<   rZ   r   r@   rA   rs   rq   rw   rX   rW   rx   ry   rz   s         rY   %test_classification_report_multiclassr"    s    D'UCCCFFA
O #yT.//00&	  F _$$$$$$r[   c                  J    g dg d}} d}t          | |          }||k    sJ d S )N)	r   r   r   r;   r;   r;   r4   r4   r4   r  a|                precision    recall  f1-score   support

           0       0.33      0.33      0.33         3
           1       0.33      0.33      0.33         3
           2       0.33      0.33      0.33         3

    accuracy                           0.33         9
   macro avg       0.33      0.33      0.33         9
weighted avg       0.33      0.33      0.33         9
r   )rX   rW   ry   rz   s       rY   .test_classification_report_multiclass_balancedr$    sG    0002M2M2MFF
O #6622F_$$$$$$r[   c                      t          j                    } t          | d          \  }}}d}t          ||          }||k    sJ d S )NFr]   a|                precision    recall  f1-score   support

           0       0.83      0.79      0.81        24
           1       0.33      0.10      0.15        31
           2       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
)r
   r<   rZ   r   r!  s         rY   :test_classification_report_multiclass_with_label_detectionr&    sX    D'UCCCFFA
O #6622F_$$$$$$r[   c            	          t          j                    } t          | d          \  }}}d}t          ||t	          j        t          | j                            | j        d          }||k    sJ d S )NFr]   a|                precision    recall  f1-score   support

      setosa    0.82609   0.79167   0.80851        24
  versicolor    0.33333   0.09677   0.15000        31
   virginica    0.41860   0.90000   0.57143        20

    accuracy                        0.53333        75
   macro avg    0.52601   0.59615   0.50998        75
weighted avg    0.51375   0.53333   0.47310        75
r   )rp   rq   digitsr   r!  s         rY   1test_classification_report_multiclass_with_digitsr)    s    D'UCCCFFA
O #yT.//00&  F _$$$$$$r[   c                     t          d          \  } }}t          j        g d          |          } t          j        g d          |         }d}t          | |          }||k    sJ d}t          | |g d          }||k    sJ d S )NFr   )bluegreenreda|                precision    recall  f1-score   support

        blue       0.83      0.79      0.81        24
       green       0.33      0.10      0.15        31
         red       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
a|                precision    recall  f1-score   support

           a       0.83      0.79      0.81        24
           b       0.33      0.10      0.15        31
           c       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r   rq   rZ   r@   r   r   )rX   rW   rx   ry   rz   s        rY   7test_classification_report_multiclass_with_string_labelr0    s    'u555FFAX...//7FX...//7F
O #6622F_$$$$
O #66PPPF_$$$$$$r[   c                      t          d          \  } }}t          j        g d          }||          } ||         }d}t          | |          }||k    sJ d S )NFr   )u   blue¢u   green¢u   red¢u                precision    recall  f1-score   support

       blue¢       0.83      0.79      0.81        24
      green¢       0.33      0.10      0.15        31
        red¢       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r/  rX   rW   rx   rp   ry   rz   s         rY   8test_classification_report_multiclass_with_unicode_labelr3  9  sp    'u555FFAX:::;;FF^FF^F
O #6622F_$$$$$$r[   c                      t          d          \  } }}t          j        g d          }||          } ||         }d}t          | |          }||k    sJ d S )NFr   )r+  greengreengreengreengreenr-  a                             precision    recall  f1-score   support

                     blue       0.83      0.79      0.81        24
greengreengreengreengreen       0.33      0.10      0.15        31
                      red       0.42      0.90      0.57        20

                 accuracy                           0.53        75
                macro avg       0.53      0.60      0.51        75
             weighted avg       0.51      0.53      0.47        75
r/  r2  s         rY   <test_classification_report_multiclass_with_long_string_labelr6  O  sp    'u555FFAX22233FF^FF^F
O #6622F_$$$$$$r[   c                      g d} g d}g d}d}t          j        t          |          5  t          | |ddg|           d d d            d S # 1 swxY w Y   d S )	Nr   r   r4   r   r   r   r4   r4   r   r   zclass 0zclass 1zclass 2z6labels size, 2, does not match size of target_names, 3r   r   r4   r  )r   r   r   r   )rX   rW   rq   r   s       rY   =test_classification_report_labels_target_names_unequal_lengthr;  f  s    __F__F444L
BC	k	-	-	- X XffaV,WWWWX X X X X X X X X X X X X X X X X Xs   AAAc                      g d} g d}g d}d}t          j        t          |          5  t          | ||           d d d            d S # 1 swxY w Y   d S )Nr8  r9  r:  zaNumber of classes, 2, does not match size of target_names, 3. Try specifying the labels parameterr   r.  )r   r   r   r   )rX   rW   rq   r   s       rY   @test_classification_report_no_labels_target_names_unequal_lengthr=  p  s    __F__F444L	. 
 
z	1	1	1 I Iff<HHHHI I I I I I I I I I I I I I I I I Is   A

AAc                      d} d}t          d|| d          \  }}t          d|| d          \  }}d}t          ||          }||k    sJ d S )Nr   r|  r;   r   )rQ   rP   r  r:   a                precision    recall  f1-score   support

           0       0.50      0.67      0.57        24
           1       0.51      0.74      0.61        27
           2       0.29      0.08      0.12        26
           3       0.52      0.56      0.54        27

   micro avg       0.50      0.51      0.50       104
   macro avg       0.45      0.51      0.46       104
weighted avg       0.45      0.51      0.46       104
 samples avg       0.46      0.42      0.40       104
)r   r   )r  rP   rx   rX   rW   ry   rz   s          rY   %test_multilabel_classification_reportr?  ~  s    II.	YQ  IAv /	YQ  IAvO #6622F_$$$$$$r[   c                  F   t          j        g dg dg          } t          j        g dg dg          }t          | |          dk    sJ t          | |           dk    sJ t          ||          dk    sJ t          |t          j        |                    dk    sJ t          | t          j        |                     dk    sJ t          | t          j        | j                            dk    sJ t          |t          j        | j                            dk    sJ d S )Nr   r   r   r   r   r;   )r@   r   r"   r   r   r?   r   s     rY   $test_multilabel_zero_one_loss_subsetrA    s)   	999iii(	)	)B	999iii(	)	)BR  C''''R  A%%%%R  A%%%%R^B//00A5555R^B//00A5555RXbh//00A5555RXbh//00A555555r[   c                  h   t          j        g dg dg          } t          j        g dg dg          }t          j        ddg          }t          | |          dk    sJ t          | |           dk    sJ t          ||          dk    sJ t          |d|z
            dk    sJ t          | d| z
            dk    sJ t          | t          j        | j                            dk    sJ t          |t          j        | j                            d	k    sJ t          | ||
          dk    sJ t          | d|z
  |
          dk    sJ t          | t          j        |           |
          dk    sJ t          | d         |d                   t          | d         |d                   k    sJ d S )Nr   r   r   r;   r   UUUUUU?r   r   r   rK  gUUUUUU?gUUUUUU?)r@   r   r   r   r?   
zeros_like
sp_hamming)r   r   ws      rY   test_multilabel_hamming_lossrG    s   	999iii(	)	)B	999iii(	)	)B
!QAB5((((B1$$$$B1$$$$AF##q((((AF##q((((BHRX..//58888BHRX..//36666Ba000H<<<<AF!444	AAAABM"--Q???7JJJJ1r!u%%BqE2a5)A)AAAAAAAr[   c                     t          j        g d          } t          j        g d          }d}t          j        t          |          5  t          | |dd           d d d            n# 1 swxY w Y   t          j        g dg dg          } t          j        g d	g d
g          }d}t          j        t          |          5  t          | |dd           d d d            n# 1 swxY w Y   t          j        g d          } t          j        g d          }d}t          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          | |d           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          | |dd           d d d            d S # 1 swxY w Y   d S )N)r   r;   r   r;   r;   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r   rM   r4   r   r   r   r   r   r   Target is multilabel-indicator but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted', 'samples'\].r   )r   r;   r;   r   r4   r  Target is multiclass but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted'\].r   zJSamplewise metrics are not available outside of multilabel classification.r   zNote that pos_label \(set to 3\) is ignored when average != 'binary' \(got 'micro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   )r@   r   r   r   r   r   r   r   )rX   rW   r   msg1msg2msg3r   s          rY   test_jaccard_score_validationrO    s~   Xooo&&FXooo&&FOG	z	1	1	1 E Effh!DDDDE E E E E E E E E E E E E E E Xyyy))),--FXyyy))),--F	6 	
 
z	.	.	. F Fffh"EEEEF F F F F F F F F F F F F F F Xooo&&FXooo&&F	 	
 
z	.	.	. 8 8ffh77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8WD	z	.	.	. 9 9ffi88889 9 9 9 9 9 9 9 9 9 9 9 9 9 9	  
k	-	-	- D DffgCCCCD D D D D D D D D D D D D D D D D DsZ   
A**A.1A.C&&C*-C*:EE EF  F$'F$G))G-0G-c           	         t          j        g dg dg          }t          j        g dg dg          }t          ||d          dk    sJ t          ||d          dk    sJ t          ||d          dk    sJ t          |t          j        |          d          dk    sJ t          |t          j        |          d          dk    sJ t          |t          j        |j                  d          dk    sJ t          |t          j        |j                  d          dk    sJ t          j        g dg d	g          }t          j        g d
g dg          }t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          t          ||dddg          d           t          t          ||dddg          d           t          t          ||d           t          j        g d                     t          j        g dg dg          }t          j        g d
g dg          }t          t          ||d          d           t          t          ||d          d           d}t          j	        t          |          5  t          ||dgd           d d d            n# 1 swxY w Y   d}t          j	        t          |          5  t          ||dgd           d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          t          j        ddgg          t          j        ddgg          d          dk    sJ 	 d d d            n# 1 swxY w Y   d}t          j        t          |          5  t          t          j        ddgddgg          t          j        ddgddgg          d          dk    sJ 	 d d d            n# 1 swxY w Y   t          |           rJ d S )Nr   r   r   r   r   r   r;   r   r   r   r   r   r   r   g?r4   r   r   )r   r   r   r   r   g      ?z	Got 4 > 2r   r   r   z
Got -1 < 0r   zXJaccard is ill-defined and being set to 0.0 in labels with no true or predicted samples.zXJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels.)r@   r   r   r   r   r?   r+   r-   r   r   r   r   r   rV  )recwarnr   r   rX   rW   rM  rN  r   s           rY   test_multilabel_jaccard_scorerR    s   	999iii(	)	)B	999iii(	)	)B
 R333t;;;;R333q8888R333q8888R^B//CCCqHHHHR^B//CCCqHHHHRXbh//CCCqHHHHRXbh//CCCqHHHHXyyy))),--FXyyy))),--FffgFFFPPPffgFFFPPPffiHHH(SSSffiAGGG   ffiAGGG   ffd333RX>U>U>U5V5V   Xyyy))),--FXyyy))),--FffgFFFPPPffjIII7SSSD	z	.	.	. C CffaS'BBBBC C C C C C C C C C C C C C CD	z	.	.	. D DffbT7CCCCD D D D D D D D D D D D D D D	- 
 
,C	8	8	8 
 
"(QF8,,bhAx.@.@'RRR    
 
 
 
 
 
 
 
 
 
 
 
 
 
 
	, 
 
,C	8	8	8 
 
1a&1a&)**1a&1a&)**!  
     
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 G}}sJ   K//K36K3L88L<?L< AN00N47N4A	P..P25P2c           	         g d}g d}g d}t                      }|                    |           |                    |          }|                    |          }t          t          ||          }t          t          ||          }ddgddgddgdgdgdgd g}	ddgdd	gd	dgdgdgd	gd g}
d
D ]>}t          |	|
          D ]+\  }}t           |||           |||                     ,?t          j        ddgddgddgg          }t          j        ddgddgddgg          }t                      5  t	          ||d          dk    sJ 	 d d d            n# 1 swxY w Y   t          |           rJ d S )N)antrT  catrU  rT  rU  birdrV  )rU  rT  rU  rU  rT  rV  rV  rU  )rT  rV  rU  rT  rV  rU  r   r;   r4   )r   r   r   Nr   r   r   )r&   rI   	transformr   r   zipr+   r@   r   r.   rV  )rQ  rX   rW   rp   lbr   r   multi_jaccard_scorebin_jaccard_scoremulti_labels_listbin_labels_listr   m_labelb_labels                 rY   test_multiclass_jaccard_scorer`  4  sA   GGGFGGGF###F			BFF6NNNf%%Jf%%J!-@@z:FF						 1v1v1vsQC!dCO 8   #$5 G G 	 	GW##GGDDD!!''BBB   	 X1v1v1v.//FX1v1v1v.//F			 F FVVZ@@@AEEEEEF F F F F F F F F F F F F F F G}}s   3EE Ec                    t          dgdgd          dk    sJ d}t          j        t          |          5  t          ddgddgd          dk    sJ 	 d d d            n# 1 swxY w Y   t          dgdgdd          d	k    sJ t	          j        g d
          }t	          j        g d          }t          t          ||d          d           t          t          ||dd          d           t          |           rJ d S )Nr;   r   rM   r   r   zOJaccard is ill-defined and being set to 0.0 due to no true or predicted samplesr   r   r   )r;   r   r;   r;   r   )r;   r   r;   r;   r;   r   rI  r   )r   r   r   r   r@   r   r+   rV  )rQ  r   rX   rW   s       rY   !test_average_binary_jaccard_scorerb  Y  s   !qc8444;;;;	'  
,C	8	8	8 F FaVaVX>>>#EEEEEF F F F F F F F F F F F F F F !qcQAAASHHHHXooo&&FXooo&&FffhGGGQQQffh!DDDg   G}}s   A""A&)A&c                  6   t          j        g dg dg          } t          j        g dg dg          }d}t          j        t          |          5  t          | |dd          }|t          j        d          k    sJ 	 d d d            d S # 1 swxY w Y   d S )	Nr   r   r   r   zJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels. Use `zero_division` parameter to control this behavior.r   r   r   r   r   r   )r@   r   r   r   r   r   r   )rX   rW   r   scores       rY   (test_jaccard_score_zero_division_warningrg  p  s     Xyyy))),--FXyyy))),--F	C 
 
,C	8	8	8 + +ffivVVVc*******+ + + + + + + + + + + + + + + + + +s   .BBBzzero_division, expected_scorer  )r;   r   c                 R   t          j        g dg dg          }t          j        g dg dg          }t          j                    5  t          j        dt
                     t          ||d|           }d d d            n# 1 swxY w Y   |t          j        |          k    sJ d S )Nr   rd  r   r   re  )	r@   r   r   r   r   r   r   r   r   )r   expected_scorerX   rW   rf  s        rY   *test_jaccard_score_zero_division_set_valuerj    s     Xyyy))),--FXyyy))),--F		 	"	" 
 
g'=>>>FI]
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 

 FM.11111111s   .BB	Bc                  Z   t          j        g dg dg dg          } t          j        g dg dg dg          }t          | |d           \  }}}}t          |g dd           t          |g dd           t          |g d	d           t          |g d
d           t	          | |dd           }|}t          |g dd           t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          t          j        |                     t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          d|z  |z  d|z  |z   z             t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          t          j        ||                     t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          d           d S )Nr;   r   r   r   r   r;   r   r   r  )r;   r   r;   r   r   )r   r   r   r   r4   )r   r   r   r   )r   r   r;   r   )r;   r;   r;   r;   r   r   )r   r  r;   r   r   g      ?r   g?r   r   r   r   r  r   r@   r   r   r,   r   r+   r   r   rX   rW   rR   r   r   r   f2rb   s           rY   +test_precision_recall_f1_score_multilabel_1rr    s$   
 X|||\\\<<<@AAFX|||\\\<<<@AAF0NNNJAq!Q a!5!5!5q999a!5!5!5q999a!7!7!7;;;aq111	VV!T	:	:	:BGb///1555 1QQQJAq!Q7###3+,,,999FFG<<<bgbkk  
 1QQQJAq!Q333999FFG<<<	!a1q519%   1TTTJAq!Q7###3+,,,999FFJ???

2w'''   1SSSJAq!Q333999FFINNNPSTTTTTr[   c                  \   t          j        g dg dg dg          } t          j        g dg dg dg          }t          | |d           \  }}}}t          |g dd           t          |g d	d           t          |g d
d           t          |g dd           t	          | |dd           }|}t          |g dd           t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          d|z  |z  d|z  |z   z             t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          t          j        |                     t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          t          j        ||                     t          | |d          \  }}}}t          |d           t          |d           t          |d           |J t          t	          | |dd          dd           d S )Nrl  rm  r  r   r   r   r;   r  r   )r   r   r   r   r4   )r   r   r   r   )r   gQ?r   r   r;   r4   r;   r   rn  )r   皙?r   r   r         ?r   r   r   g      ?rC  r   r   rc   r  r   g&S?ro  rp  s           rY   +test_precision_recall_f1_score_multilabel_2rx    s/    X|||\\\<<<@AAFX|||\\\<<<@AAF 1NNNJAq!Qa!5!5!5q999a!5!5!5q999a!6!6!6:::aq111	VV!T	:	:	:BGb///15550QQQJAq!Q4   4   0111999FFG<<<	!a1q519%  
 1QQQJAq!Q4   5!!!6"""999FFG<<<bgbkk   1TTTJAq!Q5!!!5!!!=)))999FFJ???

2w'''  
 1SSSJAq!Q 5!!!5!!!=)))999FFI>>>    r[   z%zero_division, zero_division_expected)r   r   r  c           	         t          j        g dg dg dg          }t          j        g dg dg dg          }t          ||d |           \  }}}}t          ||dddgd	           t          |dd
d|gd	           d}t          ||dd|gd	           t          |g dd	           t	          ||d	d |           }	|}
t          |	|dd|gd	           t          ||d|           \  }}}}t          j        |          rdn|}dt          j        |           z   }t          |d	|z   |z             t          |d|z   |z             d}t          ||           |J t          t	          ||d	d|           t          |	d                      t          ||d|           \  }}}}t          |d           t          |d
           t          |d           |J t          t	          ||d	d|           d|z  |z  d|z  |z   z             t          ||d|           \  }}}}t          ||dk    rdnd           t          |d
           d}t          |d|z             |J t          t	          ||d	d|           t          |	|
                     t          ||d          \  }}}}t          |d           t          |d           t          |d           |J d }t          t	          ||d	d|           |d	           d S )!Nrm  rl  r  re  rt  re  r   r   r4   r   r   r   r;   ru  r   r   r   rv  r   r         ?g?r  r   rf   r   r   r   r   g@r   r   rc   gZd;O?)r@   r   r   r,   r   r  r+   r/   )r   zero_division_expectedrX   rW   rR   r   r   r   
expected_frq  rb   value_to_sumvalues_to_averageexpected_results                 rY   7test_precision_recall_f1_score_with_an_empty_predictionr    s    X|||\\\<<<@AAFX|||\\\<<<@AAF 1M  JAq!Q a"8#sC!H!LLLa#sC1G!H!LLLJa*gq*!EqIIIaq111	VV!T	W	W	WBGb:tQ
"CQGGG0}  JAq!Q !788T11>TL*@!A!AABA,0AABBBC,.2CCDDD J:&&&999'	
 	
 	
 	B%%%	 	 	 1}  JAq!Q 5!!!30111999FG=	
 	
 	
 
!a1q519%	   1
-  JAq!Q $:a$?$?55SIII3M->>???999FJm	
 	
 	
 	B(((	   1SSSJAq!Q 5!!!5!!!5!!!999OFI]	
 	
 	
 		    r[   r   )r   r   r   r   c                 X   t          j        d          }t          j        |          }t          j                    5  t          j        d           t          |||| |          \  }}}}t          ||| ||          }	d d d            n# 1 swxY w Y   |J t          j        |          r!||||	fD ]}
t          j        |
          sJ d S t          |          }t          ||           t          ||           t          ||           t          |	t          |                     d S )Nrg   r   r   r   r   r   rz  )r@   r   rD  r   r   r   r   r   r  rv   r+   )r   r   r   rX   rW   rR   r   r   r   fbetar|   s              rY   "test_precision_recall_f1_no_labelsr  s  s    XgF]6""F		 	"	" 
 
g&&&4'
 
 

1a '
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
" 999 
x !Q& 	$ 	$F8F######-((M=)))=)))=)))u]3344444   AB

BBc                    t          j        d          }t          j        |          }t          }t	          j        t                    5   |||| d          \  }}}}d d d            n# 1 swxY w Y   t          |d           t          |d           t          |d           |J t	          j        t                    5  t          ||| d          }d d d            n# 1 swxY w Y   t          |d           d S )Nr  r   r  r   )	r@   r   rD  r   r   r   r   r+   r   )	r   rX   rW   funcrR   r   r   r   r  s	            rY   1test_precision_recall_f1_no_labels_check_warningsr    s   XgF]6""F*D	,	-	- E ET&&'DDD
1aE E E E E E E E E E E E E E E 111999	,	-	- G GFFG#FFFG G G G G G G G G G G G G G G q!!!!!s$   	A**A.1A.C""C&)C&c                    t          j        d          }t          j        |          }t          j                    5  t          j        d           t          ||d d|           \  }}}}t          ||dd |           }d d d            n# 1 swxY w Y   t          j        |           } t          || | | gd           t          || | | gd           t          || | | gd           t          |g dd           t          || | | gd           d S )Nr  r   r   r  rz  r4   rd  )
r@   r   rD  r   r   r   r   r   r  r,   )r   rX   rW   rR   r   r   r   r  s           rY   /test_precision_recall_f1_no_labels_average_noner    sx   XgF]6""F 
	 	"	" 
 
g&&&4'
 
 

1a Fd-
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 J}--Ma-!NPQRRRa-!NPQRRRa-!NPQRRRaA...em]M%RTUVVVVVr  c                  .   t          j        d          } t          j        |           }t          j        t
                    5  t          | |d d          \  }}}}d d d            n# 1 swxY w Y   t          |g dd           t          |g dd           t          |g dd           t          |g dd           t          j        t
                    5  t          | |dd           }d d d            n# 1 swxY w Y   t          |g dd           d S )Nr  r;   r  rd  r4   rn  )	r@   r   rD  r   r   r   r   r,   r   )rX   rW   rR   r   r   r   r  s          rY   4test_precision_recall_f1_no_labels_average_none_warnr    s   XgF]6""F 
,	-	- 
 
4FDq
 
 

1a
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 aA...aA...aA...aA...	,	-	- B BFFDAAAB B B B B B B B B B B B B B B eYYY22222s$   A''A+.A+C77C;>C;c            	      r   t           t          }} dD ]}d}t          j        ||          5   | g dg d|           d d d            n# 1 swxY w Y   d}t          j        ||          5   | g dg d|           d d d            n# 1 swxY w Y   d}t          j        ||          5   | t	          j        d	d
gd	d
gg          t	          j        d	d
gd
d
gg          d           d d d            n# 1 swxY w Y   d}t          j        ||          5   | t	          j        d	d
gd
d
gg          t	          j        d	d
gd	d
gg          d           d d d            n# 1 swxY w Y   d}t          j        ||          5   | t	          j        d	d	gd	d	gg          t	          j        d
d
gd
d
gg          d           d d d            n# 1 swxY w Y   d}t          j        ||          5   | t	          j        d
d
gd
d
gg          t	          j        d	d	gd	d	gg          d           d d d            n# 1 swxY w Y   d}t          j        ||          5   | d	d	gddgd           d d d            n# 1 swxY w Y   d}t          j        ||          5   | ddgd	d	gd           d d d            n# 1 swxY w Y   t          j        d          5 }t          j        d           t          d
d
gd
d
gd           d}t          |
                                j                  |k    sJ d}t          |
                                j                  |k    sJ d}t          |
                                j                  |k    sJ 	 d d d            d S # 1 swxY w Y   d S )NNr   r   zPrecision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.r   r   r;   r;   r4   r   zRecall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.zPrecision is ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.r;   r   r   zRecall is ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.r   zRecall is ill-defined and being set to 0.0 due to no true samples. Use `zero_division` parameter to control this behavior.r   rM   Tr   alwaysF-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.)r   r   r   r   r@   r   r   r   r   r   popr   )r   rF  r   r   r   s        rY   test_prf_warningsr    s   *,BqA. 5 5 	 \!3''' 	5 	5AiiiG4444	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	 \!3''' 	5 	5AiiiG4444	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5
	  
as	#	#	# U U	"(QFQF#
$
$bhAA/?&@&@)TTTTU U U U U U U U U U U U U U U	  
as	#	#	# U U	"(QFQF#
$
$bhAA/?&@&@)TTTTU U U U U U U U U U U U U U U
	  
as	#	#	# S S	"(QFQF#
$
$bhAA/?&@&@'RRRRS S S S S S S S S S S S S S S	  
as	#	#	# S S	"(QFQF#
$
$bhAA/?&@&@'RRRRS S S S S S S S S S S S S S S
	  
as	#	#	# . .	1a&2r(H----. . . . . . . . . . . . . . .	  
as	#	#	# . .	2r(QFH----. . . . . . . . . . . . . . . 
	 	-	-	- 0h''''AAIIII 	
 6::<<'((C//// 	 6::<<'((C//// 	 6::<<'((C/////-0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   A

A	A	-BB	B	0?C;;C?C??E))E-0E-?GGG:?II	I	(JJJ*K		KK)B5N,,N03N0c           	      <   t          j                    5  t          j        d           dD ]0}t          g dg d||            t          g dg d||            1t          t	          j        ddgddgg          t	          j        ddgddgg          d|            t          t	          j        ddgddgg          t	          j        ddgddgg          d|            t          t	          j        ddgddgg          t	          j        ddgddgg          d	|            t          t	          j        ddgddgg          t	          j        ddgddgg          d	|            t          ddgd
d
gd|            t          d
d
gddgd|            d d d            n# 1 swxY w Y   t          j        d          5 }t          j        d           t          ddgddgd|            t          |          dk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr   r  r   r  re  r;   r   r   r   r   rM   Tr   r  )r   r   r   r   r@   r   rs   )r   r   r   s      rY   )test_prf_no_warnings_if_zero_division_setr  W  sM   		 	"	" 2
 2
g&&& 3 	 	G+		999g]    ,		999g]    
 	(Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 	(Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 	(Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 	(Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 	(FRHhm	
 	
 	
 	
 	(Hq!fhm	
 	
 	
 	
a2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
 2
h 
	 	-	-	-  h''''FQFHM	
 	
 	
 	
 6{{a                                   s%   FF""F&)F&AHHHc           	      0   t          j                    5  t          j        d           t          t	          j        ddgddgg          t	          j        ddgddgg          d|            d d d            n# 1 swxY w Y   t          j        d          5 }t          j        d           t          t	          j        ddgddgg          t	          j        ddgddgg          d|            | d	k    r-t          |                                j                  d
k    sJ nt          |          dk    sJ t          ddgddg           | d	k    r,t          |                                j                  d
k    sJ d d d            d S # 1 swxY w Y   d S )Nr   r;   r   r   re  Tr   r  r   r  )
r   r   r   r!   r@   r   r   r  r   rs   r   r   s     rY   test_recall_warningsr    sM   		 	"	" 
 
g&&&Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
	 	-	-	- h'''Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 F""FJJLL()) ." " " " " " v;;!####aVaV$$$F""FJJLL()) ." " " " "+                 s%   AA88A<?A<C&FFFc           	      0   t          j        d          5 }t          j        d           t          t	          j        ddgddgg          t	          j        ddgddgg          d|            | dk    r-t          |                                j                  d	k    sJ nt          |          dk    sJ t          ddgddg           | dk    r,t          |                                j                  d	k    sJ d d d            n# 1 swxY w Y   t          j                    5  t          j        d
           t          t	          j        ddgddgg          t	          j        ddgddgg          d|            d d d            d S # 1 swxY w Y   d S )NTr   r  r;   r   r   re  r   r  r   )
r   r   r   r    r@   r   r   r  r   rs   r  s     rY   test_precision_warningsr    sM   		 	-	-	- h'''Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	
 F""FJJLL()) ." " " " " " v;;!####AA'''F""FJJLL()) ." " " " "+              6 
	 	"	" 
 
g&&&Hq!fq!f%&&Hq!fq!f%&&'		
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s%   C&DDD&AFFFc           
      0   t          j        d          5 }t          j        d           t          t	          t
          d          fD ]3} |t          j        ddgddgg          t          j        ddgddgg          d| 	           t          |          dk    sJ  |t          j        ddgddgg          t          j        ddgddgg          d| 	           t          |          dk    sJ  |t          j        ddgddgg          t          j        ddgddgg          d| 	           | d
k    r.t          |
                                j                  dk    sJ t          |          dk    sJ 5	 d d d            d S # 1 swxY w Y   d S )NTr   r  r4   r   r;   r   r   re  r   r  )r   r   r   r   r   r   r@   r   rs   r   r  r   )r   r   rf  s      rY   test_fscore_warningsr    s6   		 	-	-	- "(h'''! < < <= 	( 	(EE1a&1a&)**1a&1a&)**+	    v;;!####E1a&1a&)**1a&1a&)**+	    v;;!####E1a&1a&)**1a&1a&)**+	    &&

,-- 2- - - - - - 6{{a'''''?	("( "( "( "( "( "( "( "( "( "( "( "( "( "( "( "( "( "(s   E'FFFc                     g d} g d}d}t          j        g dg dg dg          }t          j        g dg dg dg          }d}| ||f|||ffD ]q\  }}}t          t          t          t          t          d	
          fD ]A}	t          j        t          |          5   |	||           d d d            n# 1 swxY w Y   Brd S )N)r;   r4   r   r   )r;   r4   r   r;   rK  r   r   r   r:  rJ  r4   r   r   )
r@   r   r    r!   r   r   r   r   r   r   )
	y_true_mc	y_pred_mcmsg_mc
y_true_ind
y_pred_indmsg_indrX   rW   r   r|   s
             rY   'test_prf_average_binary_data_non_binaryr  	  sV   II	1 
 999iii;<<J999iii;<<J	<  
Iv&	Z)  ' '
 Ka(((	
 	' 	'F z555 ' 'vv&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' '	'	' 's   B66B:=B:c                     d} d}d}d}d}d}| t          j        g dg dg d	g          f| t          j        d
dgdd
gddgg          f|g df|g df|g df|t          j        dgdgdgg          f|t          j        d
gdgdgg          f|t          j        dgdgdgg          f|t          j        d
dgddgddgg          f|t          j        ddgddgddgg          fg
}i | | f| ||f|||f||| fd || fd ||f|||fd ||fd ||fd | |fd ||fd ||fd ||fd ||fd | |fd ||fd ||fd ||fd | |fd ||fd ||fd i}t          |d          D ]\  \  }}	\  }
}	 |||
f         }n# t          $ r ||
|f         }Y nw xY w|t	          j        t                    5  t          |	|           d d d            n# 1 swxY w Y   ||
k    rZd                    ||
          }t	          j        t          |          5  t          |	|           d d d            n# 1 swxY w Y   |||| fvrXd                    |          }t	          j        t          |          5  t          |	|           d d d            n# 1 swxY w Y   7t          |	|          \  }}}||k    sJ |	                    d          r|j        dk    sJ |j        dk    sJ nDt          |t          j        |	                     t          |t          j        |                     t	          j        t                    5  t          |	d d         |           d d d            n# 1 swxY w Y   ddg}	d d!g}d"}t	          j        t          |          5  t          |	|           d d d            d S # 1 swxY w Y   d S )#Nzmultilabel-indicatorr  rM   
continuouszmulticlass-multioutputzcontinuous-multioutputr   r   r   r   r;   )r4   r   r;   )r   r{  r   r4   r   r   r{  r   r   r^  g?g      @)r  z@Classification metrics can't handle a mix of {0} and {1} targetsr   z{0} is not supported
multilabelcsrr   )r;   r4   )r   r4   r   )r4   )r   r4   zYou appear to be using a legacy multi-label data representation. Sequence of sequences are no longer supported; use a binary array or sparse matrix instead - the MultiLabelBinarizer transformer can convert to this format.)r@   r   r   KeyErrorr   r   r   r#   format
startswithr-   squeeze)INDMCBINCNTMMCMCNEXAMPLESEXPECTEDtype1r   type2r   r  r   merged_typey1outy2outr   s                     rY   test__check_targetsr  +	  sj    !C	B
C
C
"C
"C 
bh			999iii899:	bhAAA/001	YYY	iii	ooo	RXsQC!o&&'	bhaS1#''(	bhuse,--.	bhAAA/001	bhc
S#Jc
;<<=H	c
C	R" 
c
C 
S	4	
 
c
D 
b	2 
c
D 
c
D 
c
D 
c
D 
S	4 
c
D 
c
D 
c
D  
c
D!" 
S	4#$ 
c
D%& 
c
D	c
D	S	4	c
D- H2 %,HQ$?$?$? !, !, [eR	.u-HH 	. 	. 	.u-HHH	.z** ' 'r2&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ~~--3VE5-A-A  ]:W=== + +"2r***+ + + + + + + + + + + + + + + b#..4;;EBBGzAAA / /&r2.../ / / / / / / / / / / / / / / )7r2(>(>%K(****%%l33 :|u,,,,|u,,,,,"5"*R..999"5"*R..999z** , ,r#2#w+++, , , , , , , , , , , , , , , )	B
B	3  
z	-	-	-  r2                 sl   .
E99FF/GG	G	H++H/	2H/	.JJ	J	M&&M*	-M*	N66N:=N:c                  N    ddg} ddg}t          | |          d         dk    sJ d S )Nr   r;   r   r  )r#   rg  s     rY   Atest__check_targets_multiclass_with_both_y_true_and_y_pred_binaryr  	  s:    VFWF&&))!,<<<<<<r[   c                     t          j        g d          } t          j        g d          }t          | |          dk    sJ t          j        g d          } t          j        g d          }t          | |          dk    sJ d S )N)r   r;   r;   r   )g      !r   r{  g333333ӿr   )r   r4   r4   r   )r@   r   r   rX   pred_decisions     rY   test_hinge_loss_binaryr  	  s    Xnnn%%FH33344Mfm,,7777Xlll##FH33344Mfm,,777777r[   c            
         t          j        g dg dg dg dg dg dg          } t          j        g d          }t          j        d| d         d         z
  | d         d         z   d| d         d         z
  | d         d	         z   d| d	         d	         z
  | d	         d
         z   d| d
         d         z
  | d
         d	         z   d| d         d
         z
  | d         d	         z   d| d         d	         z
  | d         d
         z   g          }t          j        |dd |           t          j        |          }t	          ||           |k    sJ d S )N
ףp=
?(\ſ(\gGz)HzGgGz׿Q޿r  333333r  RQؿr  )r  r  r  r  gzGgHzGgHzGѿgQ?)r   r;   r4   r;   r   r4   r;   r   r4   r   r   r   outr@   r   clipr   r   )r  rX   dummy_lossesdummy_hinge_losss       rY   test_hinge_loss_multiclassr  	  s   H((((((((((((((((((	
	 	M X((())F8a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	 	L GL!T|4444w|,,fm,,0@@@@@@@r[   c                      t          j        g d          } t          j        g dg dg dg dg          }d}t          j        t          |          5  t          | |           d d d            d S # 1 swxY w Y   d S )N)r   r;   r4   r4   )gRQ?g rh?g(\gffffffr  r  zDPlease include all labels in y_true or pass labels as third argumentr   )r@   r   r   r   r   r   )rX   r  error_messages      rY   :test_hinge_loss_multiclass_missing_labels_with_labels_noner  	  s    Xlll##FH((((((((((((		
 M 	O  
z	7	7	7 * *6=)))* * * * * * * * * * * * * * * * * *   A22A69A6c            
         t          j        g d          } t          j        g d          }d}t          j        t          t          j        |                    5  t          | |           d d d            n# 1 swxY w Y   t          j        ddgddgddgddgddgddgddgg          }g d	}d
}t          j        t          t          j        |                    5  t          | ||           d d d            d S # 1 swxY w Y   d S )N)r4   r;   r   r;   r   r;   r;   )r   r;   r4   r;   r   r4   r;   zThe shape of pred_decision cannot be 1d arraywith a multiclass target. pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7,)r   r  r   r;   r4   r   zThe shape of pred_decision is not consistent with the number of classes. With a multiclass target, pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7, 2))rX   r  rp   )r@   r   r   r   r   reescaper   )rX   r  r  rp   s       rY   <test_hinge_loss_multiclass_no_consistent_pred_decision_shaper  	  s    X+++,,FH22233M	  
z=)A)A	B	B	B ? ?&>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ? Hq!fq!fq!fq!fq!fq!fqRSfUVVMYYF	  
z=)A)A	B	B	B N N&fMMMMN N N N N N N N N N N N N N N N N Ns$   A::A>A>!DDDc            	         t          j        g dg dg dg dg dg          } t          j        g d          }t          j        g d          }t          j        d| d         d         z
  | d         d         z   d| d         d         z
  | d         d         z   d| d         d         z
  | d         d	         z   d| d	         d         z
  | d	         d         z   d| d
         d         z
  | d
         d	         z   g          }t          j        |dd |           t          j        |          }t	          || |          |k    sJ d S )Nr  皙r  r  r  r  )r   r;   r4   r;   r4   )r   r;   r4   r   r;   r   r4   r   r   r  r   r  r  rX   rp   r  r  s        rY   .test_hinge_loss_multiclass_with_missing_labelsr  	  sz   H(((((((((((((((	
 M Xooo&&FXlll##F8a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
 L GL!T|4444w|,,fmF;;;?OOOOOOOr[   c            	         t          j        g dg dg dg dg dg          } t          j        g d          }t          j        g d          }t          j        d| d         d         z
  | d         d         z   d| d         d	         z
  | d         d         z   d| d	         d	         z
  | d	         d         z   d| d
         d         z
  | d
         d	         z   d| d         d	         z
  | d         d         z   g          }t          j        |dd |           t          j        |          }t	          t          || |          |           d S )N)r  r  r  )g333333ÿr  r  )r  r  r  )r  g(\gzGڿ)r   r4   r4   r   r4   r   r;   r   r4   r   r   r  r   )r@   r   r  r   r+   r   r  s        rY   @test_hinge_loss_multiclass_missing_labels_only_two_unq_in_y_truer   
  s   
 H!!!!!!!!!!!!!!!	
 M Xooo&&FXiii  F8a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
 L GL!T|4444w|,,6=888:J    r[   c            
      V   g d} g dg dg dg dg dg dg}t          j        d|d         d         z
  |d         d         z   d|d         d         z
  |d         d         z   d|d         d         z
  |d         d	         z   d|d	         d         z
  |d	         d         z   d|d
         d	         z
  |d
         d         z   d|d         d         z
  |d         d	         z   g          }t          j        |dd |           t          j        |          }t	          | |          |k    sJ d S )N)r+  r,  r-  r,  whiter-  r  r  r  r  r;   r   r4   r   r   r   r  r  )rX   r  r  r  s       rY   +test_hinge_loss_multiclass_invariance_listsr   
  sw    ?>>F$$$$$$$$$$$$$$$$$$M 8a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	 	L GL!T|4444w|,,fm,,0@@@@@@@r[   c            	         g d} t          j        ddgddgddgddgddgd	d
gg          }t          | |          }t          j        t	          j        t          j        |           dk    |d d df                              }t          ||           g d} g dg dg dg}t          | |d          }t          |d           | dz  } |dz  }t          | |d          }t          |d           g d} ddgddgddgg}t          j        t                    5  t          | |           d d d            n# 1 swxY w Y   g d} ddgddgddgddgg}t          | |          }t          |d           ddg} ddgddgg}t          j        ddgddgg          }d}t          j        t          |           5  t          | |           d d d            n# 1 swxY w Y   ddgddgddgg}d!}t          j        t          |           5  t          | |           d d d            n# 1 swxY w Y   t          j        t          j
        |d d df                              }t          | |ddg"          }t          ||           g d#} g d$g dg d%g}t          | |g d&"          }t          |t          j
        d                      d S )'Nnor  r  yesr  r  r   r   re   {Gz?Gz?r   rw  gMbP?g+?r  r;   rM  r   r   r   )r   r   r   )r   r   r   TrT  g躕ʀ?r4   Fg.L`@r   r   r   r   hamspamr  r  CT?r   r   zly_true contains only one label \(2\). Please provide the true labels explicitly through the labels argument.r   zDFound input variables with inconsistent numbers of samples: \[3, 2\]r   r   )r   r   r   r   r   r   rL  )r@   r   r   r   r	   logpmfr*   r   r   r   log)	rX   rW   loss	loss_truer   	error_strtrue_log_losscalculated_log_lossy_score2s	            rY   test_log_lossr  <
  s   444FX
sc3Z$SzD$<%QVX F FF##D)"(6*:*:e*CVAAAqD\RRSSSID)$$$ YYFoo@FFFd333DD)$$$ aKF
aKFFFe444DD-((( YYFCj3*sCj1F	z	"	" ! !   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ,++FCj3*sCj3*=FFF##DD)$$$ VFCj3*%Fhc
S#J/00G	C  
z	3	3	3 ! !   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! Cj3*sCj1FWI	z	3	3	3 ! !   ! ! ! ! ! ! ! ! ! ! ! ! ! ! !
 WRVGAAAqDM22333M"67Aq6BBB'777 YYF///BHFHYYY777DD26#;;,'''''s6   D55D9<D95GGGH""H&)H&c                     t          j        ddg|           }t          j        ddg|           }t          ||          }t          j        |          sJ dS )zCheck the behaviour internal eps that changes depending on the input dtype.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/24315
    r   r;   r  N)r@   r   r   isfinite)rW  rX   rW   r  s       rY   test_log_loss_epsr   }
  s_     Xq!fE***FXq!fE***FFF##D;tr[   c                     t          j        g d          }t          j        ddgddgddgddgg|           }t          j        t          d	
          5  t          ||           ddd           dS # 1 swxY w Y   dS )zGCheck that log_loss raises a warning when y_pred values don't sum to 1.r  r   r   r   r   r   r   r  z$The y_pred values do not sum to one.r   N)r@   r   r   r   r   r   )rW  rX   rW   s      rY   'test_log_loss_not_probabilities_warningr  
  s     Xlll##FXSzC:SzC:FeTTTF	k)O	P	P	P ! !   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !r  r:  r   r   c                 V    t          | |          t          j        d          k    sJ dS )z6Check that log_loss returns 0 for perfect predictions.r   N)r   r   r   rg  s     rY   !test_log_loss_perfect_predictionsr  
  s/     FF##v}Q'7'7777777r[   c                  n   t          j        g d          } t          j        ddgddgddgddgg          }t          t          fg}	 ddlm}m} |                    ||f           n# t          $ r Y nw xY w|D ];\  }} ||            ||          }}t          ||          }	t          |	d           <d S )	Nr  r   r   r   r   r   )	DataFramer  r  )
r@   r   r)   r  r  r  r  ImportErrorr   r*   )
y_try_prtypesr  r  TrueInputTypePredInputTyperX   rW   r  s
             rY   test_log_loss_pandas_inputr  
  s   822233D8c3Z#sc3Z#sDEED]+,E,,,,,,,,fi())))   (- ) )$}&t,,mmD.A.A''i((((	) )s   A' '
A43A4c                     t          j        g d          } t          j        g d          }t          j        | |z
            dz  t	          |           z  }t          t          | |           d           t          t          | |          |           t          t          d| z   |          |           t          t          d| z  dz
  |          |           t          j        t                    5  t          | |dd                     d d d            n# 1 swxY w Y   t          j        t                    5  t          | |dz              d d d            n# 1 swxY w Y   t          j        t                    5  t          | |dz
             d d d            n# 1 swxY w Y   t          j        g d          } t          j        g d          }d	}t          j        t          |
          5  t          | |           d d d            n# 1 swxY w Y   t          t          dgdg          d           t          t          dgdg          d           t          t          dgdg          d           t          t          dgdgd          d           t          t          dgdgd          d           d S )Nr   r;   r;   r   r;   r;   r   r   re   r   r   gffffff?r4   r   r   r;   )r   r;   r4   r   )r   r   r   r   zMOnly binary classification is supported. The type of the target is multiclassr   r   r   g{Gz?r   r  foobarr   )
r@   r   r   normrs   r+   r   r   r   r   )rX   rW   
true_scorer  s       rY   test_brier_score_lossr  
  s   X((())FX55566FVf_--2S[[@J(88#>>>(88*EEE(vv>>
KKK(Va@@*MMM	z	"	" - -,,,- - - - - - - - - - - - - - -	z	"	" / /#.../ / / / / / / / / / / / / / /	z	"	" / /#.../ / / / / / / / / / / / / / / Xlll##FX***++FW  
z	7	7	7 ) )((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) ("u55t<<<(!se44d;;;(!se44d;;;(%3%5III4PPP(%3%5III4PPPPPsH   1DDD:EE!E>FF"%F"2HHHc                      d} t          j        t          |           5  t          g dg d           d d d            d S # 1 swxY w Y   d S )Nz%y_pred contains classes not in y_truer   rd  r   )r   r   r   r   r   s    rY   #test_balanced_accuracy_score_unseenr  
  s    
1C	k	-	-	- 6 6			9995556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s   A  AAzy_true,y_pred)r   r   r   r   )r   r   r   r   )r   r   r   r   c                    t          | |dt          j        |                     }t                      5  t	          | |          }d d d            n# 1 swxY w Y   |t          j        |          k    sJ t	          | |d          }t	          | t          j        | | d                             }|||z
  d|z
  z  k    sJ d S )Nr   r   T)adjustedr   r;   )r!   r@   uniquer.   r   r   r   	full_like)rX   rW   macro_recallbalancedr  chances         rY   test_balanced_accuracy_scorer  
  s     	&0A0A  L 
		 ; ;*66::; ; ; ; ; ; ; ; ; ; ; ; ; ; ; v}\222222&vvEEEH$VR\&&)-L-LMMF6)a&j9999999s   AAAr   r   ))FTr  )r   r   )zeroonec                 \   t           j                            d          }d|d         }}|                    ||d          }| t          u r|                    |          }n|                                } | |||          }t          j        t          j        |                    rJ dS )	zCheck that the metric works with different types of `pos_label`.

    We can expect `pos_label` to be a bool, an integer, a float, a string.
    No error should be raised for those types.
    *   rs  r   T)r  replacer  r   N)	r@   rD   rE   choicer   uniformr  anyr  )r|   r   rS   rP   r   rX   rW   r  s           rY   *test_classification_metric_pos_label_typesr(  
  s    * )


#
#Cwr{yIZZiZ>>F!!!),,VFFi888Fvbhv&&'''''''r[   zy_true, y_pred, expected_scorer   r   c                 Z    t          | |d          t          j        |          k    sJ dS )zCheck the behaviour of `zero_division` for f1-score.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/26965
    r   r  N)r   r   r   )rX   rW   ri  s      rY   2test_f1_for_small_binary_inputs_with_zero_divisionr*    s4     FF#666&-:W:WWWWWWWr[   scoringr  r4   )r   r   c                     t          j        d          \  }}t          dd                              ||          }t	          |||| dd           dS )	aZ  Check that we validate `np.nan` properly for classification metrics.

    With `n_jobs=2` in cross-validation, the `np.nan` used for the singleton will be
    different in the sub-process and we should not use the `is` operator but
    `math.isnan`.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/27563
    r   )r:   r   )	max_depthr:   r4   r  )r+  n_jobserror_scoreN)r
   make_classificationr(   rI   r%   )r+  rN   rO   
classifiers       rY   9test_classification_metric_division_by_zero_nan_validatonr2  &  s^    & 'Q777DAq'!!DDDHHANNJJ1gaWUUUUUUr[   c                  z   t          j        g d          } t          j        g d          }d}t          j        t          |          5  t          | |           ddd           n# 1 swxY w Y   d}t          j        t          |          5  t          | ||           ddd           dS # 1 swxY w Y   dS )	z)Check the message for future deprecation.r  r  z$y_prob was deprecated in version 1.5r   )y_probNz/`y_prob` and `y_proba` cannot be both specified)r4  y_proba)r@   r   r   r   FutureWarningr   r   r   )rX   rW   rl  	error_msgs       rY   )test_brier_score_loss_deprecation_warningr8  ?  s]    X((())FX55566F5H	m8	4	4	4 
 
	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 BI	z	3	3	3 
 
	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s$   
A((A,/A,B00B47B4c            	         g d} g d}t          j        ddgddgddgddgdd	gd
dgg          }t          j        ddgddgddgddgddgddgg          }t          | |          }t          | |d          }t          | |d          }d||z  z
  }|t	          j        |          k    sJ t          j        g d          }|d d                                         |                                z  |d d df<   |dd                                          |                                z  |d d df<   t          | ||          }t          | ||d          }t          | ||d          }d||z  z
  }|t	          j        |          k    sJ t          j        ddgddgddgddgddgddgg          }t          | |          }d|cxk     rdk     sn J t          ||          }	|	t	          j        |          k    sJ t          j        ddgddgddgddgddgddgg          }t          | |          }|dk     sJ t          ||          }	|	t	          j        |          k    sJ g d} t          j        ddgddgddgddgddgddgg          }t          | |          }|dk    sJ t          ||          }	|	dk    sJ g d} g d}t          j        ddgddgddgddgg          }t          | |          }|dk    sJ t          ||          }	|	dk    sJ g d}t          | ||          }
|
dk    sJ g d} g d}t          j        g d g d g d!g d"g          }t          | |          }d|cxk     rdk     sn J t          | ||          }d|cxk     rdk     sn J t          j        g d#g d$g d"g d%g          }t          | |          }|dk     sJ t          | ||          }|dk     sJ d S )&Nrh  r  r   re   r   r   r   gffffff?g?r  r  rg  F)rX   rW   rU  r;   )r4   r;   r   r   r   r;   r   r   )rX   rW   r>  )rX   rW   r>  rU  r   r   r   r   rw  )r   r;   r;   r;   )r  r  r  r  )r4   r4   r4   r4   rK  )highr:  lowneutral)gffffff?r   r   r   )r   r   r   r   )r   r   r   )r   r   r   r  r  )r@   r   r$   r   r   r   r_  )rX   y_true_stringrW   y_pred_nulld2_scorelog_likelihoodlog_likelihood_nulld2_score_truer>  d2_score_stringd2_score_with_sample_weights              rY   test_d2_log_loss_scorerE  U  s   F;;;MX#J#J#J#J4L4L	
	 	F (#J#J#J#J#J#J	
	 	K !v>>>HVFeLLLN"&PUVVV)<<<Mv}]333333 H///00M%bqb)--//-2C2C2E2EEK1%abb)--//-2C2C2E2EEK1 fM  H #	  N ##	   )<<<Mv}]333333 X#J#J#J#J#J#J	
	 	F !00HC'v>>OfmH555555 X#J#J#J#J4L#J	
	 	F !00Ha<<<<'v>>OfmH555555  FX#J#J#J#J#J#J	
	 	F !00Hq===='v>>Oa \\F///MXd|dD\D$<$NOOF 00Hq===='v>>Oa LLM"3m# # # '!++++ 0//F(((MXOOOOOOOO		
 F !00HC }MMMHCXOOOOOOOO		
 F !00Ha<<<< }MMMHa<<<<<<r[   c                     g d} ddgddgddgg}d}t          j        t          |          5  t          | |           d	d	d	           n# 1 swxY w Y   g d
} ddgddgddgg}g d}d}t          j        t          |          5  t          | ||           d	d	d	           n# 1 swxY w Y   g d} g dg dg}d}t          j        t          |          5  t          | |           d	d	d	           n# 1 swxY w Y   dg} ddgg}d}t          j        t
          |          5  t          | |           d	d	d	           n# 1 swxY w Y   g d} ddgddgddgg}d}t          j        t          |          5  t          | |           d	d	d	           n# 1 swxY w Y   g d} dg}ddgddgddgg}d}t          j        t          |          5  t          | ||           d	d	d	           d	S # 1 swxY w Y   d	S )zPTest that d2_log_loss_score raises the appropriate errors on
    invalid inputs.r   r   r   r   r   r   z#contain different number of classesr   Nr   z(number of classes in labels is differentr   )r   r   r   )r   r   r   rJ  r;   zscore is not well-definedr   r   zy_true contains only one labelz.The labels array needs to contain at least two)r   r   r   r$   r   r   )rX   rW   errrp   s       rY   test_d2_log_loss_score_raisesrH    s    YYFCj3*sCj1F
/C	z	-	-	- * *&&)))* * * * * * * * * * * * * * *
 __FCj3*sCj1FYYF
4C	z	-	-	- 9 9&&88889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 YYFoo/F
+C	z	-	-	- * *&&)))* * * * * * * * * * * * * * * SFCj\F
%C	,C	8	8	8 * *&&)))* * * * * * * * * * * * * * * YYFCj3*sAh/F
*C	z	-	-	- * *&&)))* * * * * * * * * * * * * * *
 YYFSFCj3*sAh/F
:C	z	-	-	- 9 9&&88889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9sk   A

AAB$$B(+B(C55C9<C9%EE	E9FFFG00G47G4)NF)r  r   	functoolsr   	itertoolsr   r   r   numpyr@   r   scipyr   scipy.spatial.distancer   rE  scipy.statsr	   sklearnr
   r   sklearn.datasetsr   sklearn.exceptionsr   sklearn.metricsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   sklearn.metrics._classificationr#   r$   sklearn.model_selectionr%   sklearn.preprocessingr&   r'   sklearn.treer(   sklearn.utils._mockingr)   sklearn.utils._testingr*   r+   r,   r-   r.   sklearn.utils.extmathr/   sklearn.utils.fixesr0   r1   sklearn.utils.validationr2   rZ   r}   r   markparametrizer   r   r   r   r   filterwarningsr   r   r   r   r   r   r   r   r  r%  r6  rH  rN  r\  rc  rf  rm  rq  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r"  r$  r&  r)  r0  r3  r6  r;  r=  r?  rA  rG  rO  rR  r`  rb  rg  rj  rr  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  float16r   r  r  r  r  r  r  r(  r*  r2  r8  rE  rH  r   r[   rY   <module>r`     s   				        2 2 2 2 2 2 2 2 2 2            8 8 8 8 8 8 ! ! ! ! ! ! ! ! ! ! ! ! ! ! ; ; ; ; ; ; 5 5 5 5 5 5                                             . N M M M M M M M 3 3 3 3 3 3 @ @ @ @ @ @ @ @ / / / / / / 0 0 0 0 0 0              . - - - - - > > > > > > > > 7 7 7 7 7 7)( )( )( )(`>D >D >DBW W W< 61a*@AA  BA T{aVUOiii=OP ) ) )(7 7 7  D PQQ  RQ$ PQQ*J *J RQ*JZ PQQM M RQM,> > >6  LLBH#OO#OO#OO#OO	 
	
 .--===	
 (9 9) (9  OOBH#OO#OO#OO#OO#OO 	
 IIOO	
 *; ;+ *;
 
 
 = = =*	= 	= 	= O  O  OF .99.99%S %S :9 :9%SPT T T< +   % % %; ; ;&) ) ) 
 #"(#5#5#566""(#5#5#566  A	
 #"(#5#5#566""(#5#5#566  I	
 #"(#5#5#566""(#5#5#566  :	
 #"(#5#5#566""(#5#5#566  I	
 #"(#5#5#566""(#5#5#566  O	
E), ,Z* *[, ,Z*  #"(???33""(???33 
<		
 * * *" " "2  > 1a.99)aS1#J<88!$$$	 ' '  98 :9' )aS1#J<88!$$$	    98  $2 $2 $2N R  R  RF5 5 5p c5\22U U 32UB7( 7( 7(t $S$S$STT, , UT,0 0 0"( ( (". . .& 	;<
QAB 
'(   8 8 8 taVYYY'-M-M-M   % % %  6 "A"A"ABB0 0 CB0"% % %4% % %$% % %(% % %6 %  %  %F% % %,% % %.X X XI I I PQQ% % RQ%<6 6 6B B B(%D %D %DPJ J JZ" " "J  .+ + + 868:LMM	2 	2 NM	2 PQQ?U ?U RQ?UD PQQ= = RQ=@ PQQ+&&2626"23 [ [	  RQ
[| !%%$M$M$MNN1a.99"5 "5 :9 ON &%"5J $M$M$MNN" " ON"& 1a.99 W  W :9 WF3 3 38e0 e0 e0P 1a.99:  :  :9: z 61a*@AA$ $ BA$N 61a*@AA$
 $
 BA$
N 61a*@AA#( #( BA#(L' ' '>_ _ _D= = =8 8 8A A A6* * *"N N N8P P P4  @A A A8>( >( >(B 2:rz2:"FGG
 
 HG
 2:rz2:"FGG! ! HG! 	III	aVaVaV,-	YYY			99956 8 8 8) ) )$ Q  Q  QF6 6 6 			3334			3334			3334 
: 
: 
: #&&&'  CCC ( (  ($ $	1a&		828QF++S1	1a&		828QF++S1	1a&		828QF++S1	1a&		828QF++S1	 X X X HBF333Karv>>>O26:::L777	 V V V 
 
 
,S S Sl.9 .9 .9 .9 .9r[   