hMhOr dZddlmZmZmZddlZddlZddlZ ddl Z ddl m Z ddl mZddlmZddlmZmZdd lmZdd lmZdd lmZdd lmZmZd edededefdZd edededeeeffdZ d edededefdZ!d edededefdZ"d edededeeeffdZ#d edededefdZ$ d?dej%de&deej%ej%ffdZ'd?dede&defdZ(dedefdZ)d@dededefdZ*d ej%d!ej%d"ej%deej%ej%ffd#Z+ d?d$ej%de&deej%ej%ffd%Z, dAd'e j-d(e.de&d)ed*e/d+e/d,e/de j-fd-Z0 dBd0e j-d1ee1de&d)ed2e/d*e/d+e/d,e/d3e/d4e/fd5Z2 dCd'e j-d7ee1e3ffd8Z4 dDd0e j-d1ee1de&d)ed2e/d7ee1e3fd*e/d+e/d,e/d3e/d4e/fd9Z5 dEdeej%e1e j6fd:eej%e1e j6fd;e1dZ8dS)GaProject: PhiK - correlation analyzer library Created: 2018/09/05 Description: Functions for calculating the statistical significance of outliers in a contingency table. Authors: KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands Redistribution and use in source and binary forms, with or without modification, are permitted according to the terms listed in the file LICENSE. )TupleUnionOptionalN)stats)betainc) definitions)bin_datahist2d_from_rebinned_df)log_incompbeta) z_from_logp)dq_check_nunique_values)array_like_to_dataframeguess_interval_colsnobsnexpnexperrreturnc|dkrdS|dkr4|dkr|n|}|||zz }||zdz}dd|zz }t|||}n#tj|dz |}|S)a Calculate p-value for nobs observations given the expected value and its uncertainty using the Linnemann method. If the uncertainty on the expected value is known the Linnemann method is used. Otherwise the Poisson distribution is used to estimate the p-value. Measures of Significance in HEP and Astrophysics Authors: J. T. Linnemann http://arxiv.org/abs/physics/0312059 Code inspired by: https://root.cern.ch/doc/master/NumberCountingUtils_8cxx_source.html#l00086 Three fixes are added for: * nobs = 0, when - by construction - p should be 1. * uncertainty of zero, for which Linnemann's function does not work, but one can simply revert to regular Poisson. * when nexp=0, betainc always returns 1. Here we set nexp = nexperr. :param int nobs: observed count :param float nexp: expected number :param float nexperr: uncertainty on the expected number :returns: p-value :rtype: float rr )rrpoissonsf)rrrnexpalttaubxps M/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/phik/outliers.py poisson_obs_pr!s8 qyyq{{(($$7*+ cMA  SM D!Q   M  TAXt , , Hcb|dkrdtj fS|dkr4|dkr|n|}|||zz }||zdz}dd|zz }t|||}n_tj|dz |}tj|dz |} |tjd| z f}|S)a Calculate logarithm of p-value for nobs observations given the expected value and its uncertainty using the Linnemann method. If the uncertainty on the expected value is known the Linnemann method is used. Otherwise the Poisson distribution is used to estimate the p-value. Measures of Significance in HEP and Astrophysics Authors: J. T. Linnemann http://arxiv.org/abs/physics/0312059 Code inspired by: https://root.cern.ch/doc/master/NumberCountingUtils_8cxx_source.html#l00086 Three fixes are added for: * nobs = 0, when - by construction - p should be 1. * uncertainty of zero, for which Linnemann's function does not work, but one can simply revert to regular Poisson. * when nexp=0, betainc always returns 1. Here we set nexp = nexperr. :param int nobs: observed count :param float nexp: expected number :param float nexperr: uncertainty on the expected number :returns: tuple containing pvalue and 1 - pvalue :rtype: tuple rr )npinfr rrlogsfrlog) rrrrrrrtlogplogprs rlog_poisson_obs_pr'Ls8 qyy26'z{{(($$7*+ cMA  SMtQ**}""4!8T22 M  TAXt , ,rva!e}}% Lrct|||}|dks|dkrIt|||}|dkr|d}t|}n:|d}t|d}n tj| }|S)a2 Calculate the Z-value for measuring nobs observations given the expected value. The Z-value express the number of sigmas the observed value deviates from the expected value, and is based on the p-value calculation. If the uncertainty on the expected value is known the Linnemann method is used. Otherwise the Poisson distribution is used to estimate the p-value. :param int nobs: observed count :param float nexp: expected number :param float nexperr: uncertainty on the expected number :returns: Z-value :rtype: float rr T flip_sign)rr'r rnormppfrrrp_valuer%r&z_valuelog1mps r poisson_obs_zr1zsD$00G!||w!||!$g66 a<<8D!$''GG1XF!&D999GG:>>'*** Nrcjt|||}t|dz||}d||z z}||z}|S)a. Calculate the p-value for measuring nobs observations given the expected value. The Lancaster mid-P correction is applied to take into account the effects of discrete statistics. If the uncertainty on the expected value is known the Linnemann method is used for the p-value calculation. Otherwise the Poisson distribution is used to estimate the p-value. :param int nobs: observed count :param float nexp: expected number :param float nexperr: uncertainty on the expected number :returns: mid p-value :rtype: float r ?)r)rrrrpplus1mid_ps rpoisson_obs_mid_pr6sF dD'**A 4!8T7 3 3F 1v: EJA Hrct|||}t|dz||}|d}|d}tjd|ztjdtj||z zz}|d}|d} tjd| ztjdtj|| z zz} || fS)aP Calculate the logarithm of the p-value for measuring nobs observations given the expected value. The Lancaster mid-P correction is applied to take into account the effects of discrete statistics. If the uncertainty on the expected value is known the Linnemann method is used for the p-value calculation. Otherwise the Poisson distribution is used to estimate the p-value. :param int nobs: observed count :param float nexp: expected number :param float nexperr: uncertainty on the expected number :returns: tuple of log(p) and log(1-p) :rtype: tuple r rr3)r'r!r$exp) rrrr%tlogpp1lplp1logmidplqlq1logmidqs rlog_poisson_obs_mid_pr@s dD' 2 2Eq$88G qB !*CfSkkBBF38,<,<(">>G G rct|||}|dks|dkrIt|||}|dkr|d}t|}n:|d}t|d}n tj| }|S)aCalculate the Z-value for measuring nobs observations given the expected value. The Z-value express the number of sigmas the observed value deviates from the expected value, and is based on the p-value calculation. The Lancaster midP correction is applied to take into account the effects of low statistics. If the uncertainty on the expected value is known the Linnemann method is used for the p-value calculation. Otherwise the Poisson distribution is used to estimate the p-value. :param int nobs: observed count :param float nexp: expected number :param float nexperr: uncertainty on the expected number :returns: Z-value :rtype: tuple rr Tr))r6r@r rr+r,r-s rpoisson_obs_mid_zrBs dG44G!||w!||%dD':: a<<8D!$''GG1XF!&D999GG:>>'*** Nrrvalues CI_methodc ~tj|j}tj|j}t|jdD]l}t|jdD]M}|||}|||z }|dd|f|z }||z |z |z } | dkr||z| z |||<t ||} t ||} t | |} tjt| |z| z dt| |z| z dzt| |||z| z dz|||<#tj|||<tj|||<On||fS)a Calculation of expected frequencies, based on the ABCD-method, i.e. independent frequency estimates. :param values: The contingency table. The table contains the observed number of occurrences in each category. :param string CI_method: method to be used for uncertainty calculation. poisson: normal poisson error. exact_poisson: error calculated from the asymmetric exact poisson interval :returns exp, experr: expected frequencies, error on the expected frequencies rr NrD) r!zerosshaperangesumget_uncertaintysqrtpownan) rCrDr8experrijAobsBCDsigmaBsigmaCsigmaDs r#get_independent_frequency_estimatesrZs (6< C Xfl # #F6<? # #&&v|A'' & &A!9Qz@outlier_significance_matrix_from_rebinned_df..saH}} &,,|Aq1!4l1oa6H6K%}}rNrF)r}r~)r~r rIwarningswarnr`r!rOkeysisindefsUFOFNaN value_counts sort_indexr}listsetgetattrappendwhererJlensetattrrurCpd DataFrame)rvrwrDrxryrzr{c0c1 df_datahista orig_valsmissingvvalsrprqoutlier_overviewrrs ` ` @@r,outlier_significance_matrix_from_rebinned_dfrs]4 FB)V^]K K k&7!7!7  "F2{'8';RARSTAUVV   v gY0**1 !!## # #[^00$'47DH1MNNNOPQR  3y>>C Q0G0G,H,HHIIGH @ @a 8 8 ;A >????s<?3344 D Da00T=>> >D KD ) ) )>iGW|{(+2E r Fdf interval_colsquantileretbinsverbosec t|jdkrtd|t|| }t |||\} } t | | d||\} }t | ||||||}| r||fS|S)a Calculate the significance matrix of excesses or deficits :param df: input data. DataFrame must contain exactly two columns :param interval_cols: columns with interval variables which need to be binned :param string CI_method: method to be used for uncertainty calculation. poisson: normal poisson error. exact_poisson: error calculated from the asymmetric exact poisson interval :param bins: number of bins, or a list of bin edges (same for all columns), or a dictionary where per column the bins are specified. (default=10) E.g.: bins = {'mileage':5, 'driver_age':[18,25,35,45,55,65,125]} :param ndecimals: number of decimals to use in labels of binned interval variables to specify bin edges (default=1) :param bool quantile: when the number of bins is specified, use uniform binning (False) or quantile binning (True) :param bool dropna: remove NaN values with True :param bool drop_underflow: do not take into account records in underflow bin when True (relevant when binning a numeric variable) :param bool drop_overflow: do not take into account records in overflow bin when True (relevant when binning a numeric variable) :param bool retbins: if true, function also returns dict with bin_edges of rebinned variables. :param bool verbose: if False, do not print all interval columns that are guessed :return: outlier significance matrix (pd.DataFrame) rGz"df should contain only two columnsNryTrbinsrrDrxryrzr{)rr~ ValueErrorrrr r)rrrDrxrrryrzr{rrdf_cleaninterval_cols_cleanrvrw os_matrixs routlier_significance_matrixrsD 2:!=>>>+B88 $; M&%%%!H!!)%t$!!!K=%#I',&& r combinationsc  |i}|stj|jd}g}t|D]P\} \} } t || | g||||||} || | | fQ|S)ah Calculate the significance matrix of excesses or deficits for all possible combinations of variables, or for those combinations specified using combinations. This functions could also be used instead of outlier_significance_matrices in case all variables are either categorical or ordinal, so no binning is required. :param data_binned: input data. Interval variables need to be binned. DataFrame must contain exactly two columns :param dict binning_dict: dictionary with bin edges for each binned interval variable. When no bin_edges are provided values are used as bin label. Otherwise, bin labels are constructed based on provided bin edge information. :param string CI_method: method to be used for uncertainty calculation. poisson: normal poisson error. exact_poisson: error calculated from the asymmetric exact poisson interval :param ndecimals: number of decimals to use in labels of binned interval variables to specify bin edges (default=1) :param combinations: in case you do not want to calculate an outlier significance matrix for all permutations of the available variables, you can specify a list of the required permutations here, in the format [(var1, var2), (var2, var4), etc] :param bool dropna: remove NaN values with True :param bool drop_underflow: do not take into account records in underflow bin when True (relevant when binninga numeric variable) :param bool drop_overflow: do not take into account records in overflow bin when True (relevant when binning a numeric variable) :return: dictionary with outlier significance matrices (pd.DataFrame) NrGr) itertoolsrr~ enumeratercopyr) rvrwrDrxrryrzr{outliers_overviewrQrrzvalues_overviews r.outlier_significance_matrices_from_rebinned_dfrs>  F -k.A1EE  .. = = 8BG R ! & & ( ( )'      "b*:!;<<<< rc |t|| }t|||\} } t| | d||\}}t|||||||| }d|D}| r||fS|S)a Calculate the significance matrix of excesses or deficits for all possible combinations of variables, or for those combinations specified using combinations :param df: input data :param interval_cols: columns with interval variables which need to be binned :param string CI_method: method to be used for uncertainty calculation. poisson: normal poisson error. exact_poisson: error calculated from the asymmetric exact poisson interval :param ndecimals: number of decimals to use in labels of binned interval variables to specify bin edges (default=1) :param bins: number of bins, or a list of bin edges (same for all columns), or a dictionary where per column the bins are specified. (default=10) E.g.: bins = {'mileage':5, 'driver_age':[18,25,35,45,55,65,125]} :param bool quantile: when the number of bins is specified, use uniform binning (False) or quantile binning (True) :param combinations: in case you do not want to calculate an outlier significance matrix for all permutations of the available variables, you can specify a list of the required permutations here, in the format [(var1, var2), (var2, var4), etc] :param bool dropna: remove NaN values with True :param bool drop_underflow: do not take into account records in underflow bin when True (relevant when binning a numeric variable) :param bool drop_overflow: do not take into account records in overflow bin when True (relevant when binning a numeric variable) :param bool retbins: if true, function also returns dict with bin_edges of rebinned variables. :param bool verbose: if False, do not print all interval columns that are guessed :return: dictionary with outlier significance matrices (pd.DataFrame) NrTr)rryrzr{cFi|]\}}}d||g|S):)join)rrrrs r z1outlier_significance_matrices..s0FFFYRQ388RH%%qFFFr)rrr r)rrrDrxrrrryrzr{rrrrrvrw os_matricess routlier_significance_matricesrHsN+B88 $; M&%%%!H!!)%t$!!!KA!%#   KGF+FFFK)L(( rynum_varsrc xt||} |t| | }t| |||||||| |  S)a Calculate the significance matrix of excesses or deficits of input x and input y. x and y can contain interval, ordinal or categorical data. Use the num_vars variable to indicate whether x and/or y contain interval data. :param list x: array-like input :param list y: array-like input :param list num_vars: list of variables which are numeric and need to be binned, either ['x'],['y'],or['x','y'] :param bins: number of bins, or a list of bin edges (same for all columns), or a dictionary where per column the bins are specified. (default=10) E.g.: bins = {'mileage':5, 'driver_age':[18,25,35,45,55,65,125]} :param bool quantile: when the number of bins is specified, use uniform binning (False) or quantile binning (True) :param ndecimals: number of decimals to use in labels of binned interval variables to specify bin edges (default=1) :param string CI_method: method to be used for uncertainty calculation. poisson: normal poisson error. exact_poisson: error calculated from the asymmetric exact poisson interval :param bool dropna: remove NaN values with True :param bool drop_underflow: do not take into account records in underflow bin when True (relevant when binning a numeric variable) :param bool drop_overflow: do not take into account records in overflow bin when True (relevant when binning a numeric variable) :param bool verbose: if False, do not print all interval columns that are guessed :return: outlier significance matrix (pd.DataFrame) N) rrrrxrDryrzr{r)rrr) rrrrrrxrDryrzr{rrs routlier_significance_from_arrayrs^F !A & &B&r733 &  %#    rcJt||}t|||||S)a$ Calculate the significance matrix of excesses or deficits of input x and input y. x and y can contain binned interval, ordinal or categorical data. :param list x: array-like input :param list y: array-like input :param string CI_method: method to be used for uncertainty calculation. poisson: normal poisson error. exact_poisson: error calculated from the asymmetric exact poisson interval :param bool dropna: remove NaN values with True :param bool drop_underflow: do not take into account records in underflow bin when True (relevant when binning a numeric variable) :param bool drop_overflow: do not take into account records in overflow bin when True (relevant when binning a numeric variable) :return: outlier significance matrix (pd.DataFrame) )rDryrzr{)rr)rrrDryrzr{rs r&outlier_significance_from_binned_arrayrs92 !A & &B & %#    r)r)r )rr TTT) Nrr rFTTTFT)Nrr rTTT) Nrr rFrTTTFT) NrFr rTTTT)rTTT)9__doc__typingrrrrnumpyr!pandasrrscipyr scipy.specialrphikrrbinningr r r statisticsr data_qualityrutilsrrintfloatrr'r1r6r@rBndarraystrrZrLr^r]rrrurdictboolrrrtuplerrSeriesrrrrrrs4  *)))))))))!!!!!!$$$$$$66666666############111111????????( ( 5( 5( U( ( ( ( V+C+u+u+ue|AT++++\55UB C u u     ,    %*  5%<    FCuuF*3'' J'#&' 2:rz !"''''TuU0)u)))))UU56 *.0j 2:rz !"6(1 *!$ 2:rz !",EEEEE E  E  EE\EEEET%) << <D><< <  < <<<<<<<<B') 333 e $ 3333p%) ')BB BD>BB B  Be $B BBBBBBBBP/133 RZry ()3 RZry ()33 T2:t+ , 3  3  33 3333\3333r !! RZry ()! RZry ()!!  !  !  !\!!!!!!r