M/Ph@p dZddlmZddlZdZdZdZd4dZd4d Z d5d Z d6d Z d7d Z dZ d7dZGddZdZGddZGddZedkreZeeeddlmZgddZgdZddgZe eeZeeej dZ!ej ddzZ"ej ddzdzZ#d evre e"\Z$Z%e e#\Z&Z'd!evr&ej(e"d"d#Z"ej(e#d"d#Z#e"j)dZ*e e"\Z$Z%e e#\Z&Z'e e$e&d $Z+e e$e&d%$Z,e e$e&d&$Z-eej.e+j/e+ej0e+1dk2d'd(gdZ3d)d*gdZ4e4ej56e*zZ7e3ej dd+zZ8e+e8z1de7zZ9e,e8z1de7zZ:e-e8z1de7zZ;ej<e9e+=Z>ej<e9e-=Z?e>j@ZAe?j@ZBej<e;e+=ZCej<e;e-=ZDeCj@ZEeDj@ZFejGHe-e+d,-dZIejGHe+e-d,-dZJeej.eJeFeEz eej.eIeAeBz ee+e-ZKeejLejMe+eKNe-z eejLejMe-eKOe+z eejLejMeFeKPeEz ejLejMeBeKPeAz e ee\ZQZRZSZTZUZVed.ed/eeSeeCWeRjXed0ed1eeUeeCWeTjXeCYeTejZej[d2ej\dfZ]e?j^d3dZ_e>WeKNe]jXZ`eejLejMe_e`z ee;e"e#ZaeeabjXeeabjceeadjeeeadjceeafddjXeeafddjXeeag dS)8afunctions to work with contrasts for multiple tests contrast matrices for comparing all pairs, all levels to reference level, ... extension to 2-way groups in progress TwoWay: class for bringing two-way analysis together and try out various helper functions Idea for second part - get all transformation matrices to move in between different full rank parameterizations - standardize to one parameterization to get all interesting effects. - multivariate normal distribution - exploit or expand what we have in LikelihoodResults, cov_params, f_test, t_test, example: resols_dropf_full.cov_params(C2) - connect to new multiple comparison for contrast matrices, based on multivariate normal or t distribution (Hothorn, Bretz, Westfall) ) assert_equalNcg}t|D]K}t|dz|D]5}tj|}d||<d||<||6Ltj|S)zcontrast or restriction matrix for all pairs of nm variables Parameters ---------- nm : int Returns ------- contr : ndarray, 2d, (nm*(nm-1)/2, nm) contrast matrix for all pairwise comparisons )rangenpzerosappendarray)nmcontrij contr_rows h/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/statsmodels/sandbox/stats/contrast_tools.pycontrast_allpairsrs E 2YY$$qsB $ $A IIaLIaL LL # # # #  $ 8E??ctjtj|dz tj|dz  f}|S)zcontrast or restriction matrix for all against first comparison Parameters ---------- nm : int Returns ------- contr : ndarray, 2d, (nm-1, nm) contrast matrix for all against first comparisons r)r column_stackoneseye)r r s rcontrast_all_oner5s7 ORWRT]]RVBqD\\M: ; ;E Lrc^tj|tj||f|z z S)zcontrast or restriction matrix for all against mean comparison Parameters ---------- nm : int Returns ------- contr : ndarray, 2d, (nm-1, nm) contrast matrix for all against mean comparisons )rrr)r s rcontrast_diff_meanrEs* 6"::B((+ ++rFc|dvr:|stj|dkrdndStj|dkrdndSt|S)N)rrrr+-)rsignstr)xnopluss rsignstrr#TsRH}} 0'!**a--33S 0Q22C /1vv rcn|rtdddntdfd|D}|S)Nrc vg|]5}ddt|D6S)rcJg|] \}}|dk t|d|!SrT)r"r#.0cvs r z.contrast_labels...csGDDD!aAFF"!D1116166.csg333!$ggDD%(e__R%8DDDEE333r)slice) contrastsr2reverselabelsr3s ` @rcontrast_labelsr8^s^ 4r " " 4[[33333(1333F Mrct|}t}fd|Dtjd|f}d|d<|s#tj|t | f}n"tj|t | f}tj|ddtj|} t| d} fd| D} tjd|f} d| d<|s#tj| t | f} n"tj| t | f} tjtj|| dd}fd|D}|7|5t|\}}t|\}}t||}nd}| | |||fS) abuild contrast matrices for products of two categorical variables this is an experimental script and should be converted to a class Parameters ---------- names1, names2 : lists of strings contains the list of level labels for each categorical variable intgroup1, intgroup2 : ndarrays TODO: this part not tested, finished yet categorical variable Notes ----- This creates a full rank matrix. It does not do all pairwise comparisons, parameterization is using contrast_all_one to get differences with first level. ? does contrast_all_pairs work as a plugin to get all pairs ? c(g|]}D] }|d| S)_)r*rrnames2s rr-z$contrast_product..s0===f==Q****====rr)rrNT)r6c |g|]8}ddt|dddD9S)rcJg|] \}}|dk t|d|!Sr'r(r)s rr-z/contrast_product...sG$O$O$O"%!AqAvv(/q'>'>'>$C$C$CGMvvrNrr.r*r1 names_prods rr-z$contrast_product..so;;;%(77$O$O),S*)=)=ddd)C$O$O$OPP;;;rc |g|]8}ddt|dddD9S)rcJg|] \}}|dk t|d|!Sr'r(r)s rr-z/contrast_product...sG%O%O%O"%!AqAvv)0$(?(?(?%D%D%DGMvvrNrr.r@s rr-z$contrast_product..so<<<%(GG%O%O),S*)=)=ddd)C%O%O%OPP<<>> x = np.array(['F', 'F', 'M', 'M', 'F', 'F', 'M', 'M', 'F', 'F', 'M', 'M'], dtype='|S1') >>> dummy_1d(x, varname='gender') (array([[1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1], [1, 0], [1, 0], [0, 1], [0, 1]]), ['gender_F', 'gender_M']) Ncg|]}d|zS)zlevel_%dr<)r*rs rr-zdummy_1d..s===Q*q.===rrc:g|]}dt|zzS)z_%s)r )r*rvarnames rr-zdummy_1d..s(@@@q'ECFFN*@@@r)rmaxrarangeastypeintunique)r!r_r7 grouplabelss ` rrGrGs\==%! *<*<===!!!D& 29QUUWWQY///77<>> d1 = np.array([[1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1]]) >>> dummy_limits(d1) (array([0, 4, 8]), array([ 4, 8, 12])) get group slices from an array >>> [np.arange(d1.shape[0])[b:e] for b,e in zip(*dummy_limits(d1))] [array([0, 1, 2, 3]), array([4, 5, 6, 7]), array([ 8, 9, 10, 11])] >>> [np.arange(d1.shape[0])[b:e] for b,e in zip(*dummy_limits(d1))] [array([0, 1, 2, 3]), array([4, 5, 6, 7]), array([ 8, 9, 10, 11])] raxisrrzdummy variable is not sorted)rlrnonzerodiffrarEallrm) dnobsnvarsstart1col1end1col1_ccstartends r dummy_limitsrs:'KD%:bgaQ///233LFD*RWQA...233KD% 5  BeQCI"$ ) ) + +9U557)+,2779997888 E1#vax- E %Q C #:rc|dkr|St|\}}t|\}}tj||}tj||}||k} || z} || z} |dkryt|ddddf|ddddf} tjtj|jdt|ddddf|dd| ff} n|dkryt|ddddf|ddddf}tjtj|jdt|ddddf|dd| ff} ntd| | | fS) aunfinished and incomplete mainly copy past dummy_product dummy variable from product of two dummy variables Parameters ---------- d1, d2 : ndarray two dummy variables, d2 is assumed to be nested in d1 Assumes full set for methods 'drop-last' and 'drop-first'. method : {'full', 'drop-last', 'drop-first'} 'full' returns the full product, which in this case is d2. The drop methods provide an effects encoding: (constant, main effects, subgroup effects). The first or last columns of the dummy variable (i.e. levels) are dropped to get full rank encoding. Returns ------- dummy : ndarray dummy variable for product, see method rfrhNrrrirrj) rrin1drHrrrlrcrm)rXrYrnrzr|start2end2firstlastequal col_dropf col_droplrorPrps r dummy_nestedr2s, ##LFD##LFD GFF # #E 74  D d]Ev Iuf Ib3B3iAAAcrcE33 _bgbhqk377AAAcrcEBqqq{OT U U <  R!""Xr!!!ABB$x00 _bgbhqk377AAAabbD2aaa k?S T T0111 y) ##rc0eZdZdZdZdZdZdZdZdS)DummyTransformaConversion between full rank dummy encodings y = X b + u b = C a a = C^{-1} b y = X C a + u define Z = X C, then y = Z a + u contrasts: R_b b = r R_a a = R_b C a = r where R_a = R_b C Here C is the transform matrix, with dot_left and dot_right as the main methods, and the same for the inverse transform matrix, C^{-1} Note: - The class was mainly written to keep left and right straight. - No checking is done. - not sure yet if method names make sense ctj||dd|_tj||dd|_dS)z\C such that d1 C = d2, with d1 = X, d2 = Z should be (x, z) in arguments ? rrcondrN)rlinalglstsq transf_matrixinvtransf_matrix)selfrXrYs r__init__zDummyTransform.__init__sL  Y__R2_>>qA " Bb A A! Drc6tj|j|S)z b = C a rdotr)ras rdot_leftzDummyTransform.dot_leftsvd(!,,,rc6tj||jS)z z = x C r)rr!s r dot_rightzDummyTransform.dot_rightsva+,,,rc6tj|j|S)z a = C^{-1} b rrr)rbs r inv_dot_leftzDummyTransform.inv_dot_leftsvd+Q///rc6tj||jS)z x = z C^{-1} r)rzs r inv_dot_rightzDummyTransform.inv_dot_rightsva.///rN) __name__ __module__ __qualname____doc__rrrrrr<rrrrasm@EEE--- --- 000 00000rrctj|}|jdz}tdgdg|dz zztdgz}|d||zddz|dz S)angroupmeans using dummy variables Parameters ---------- x : array_like, ndim data array, tested for 1,2 and 3 dimensions d : ndarray, 1d dummy variable, needs to have the same length as x in axis 0. Returns ------- groupmeans : ndarray, ndim-1 means for each group along axis 0, the levels of the groups are the last axis Notes ----- This will be memory intensive if there are many levels in the categorical variable, i.e. many columns in the dummy variable. In this case it is recommended to use a more efficient version. rN).Nr?)rasarrayndimr4sum)r!rwryslis r groupmean_drs}2 1 A FQJE ;;-4&%'* *eDkk] :C hK!C& % %a ( ( +AEE!HH 44rc8eZdZdZd dZdZdZdZdZdZ dS) TwoWayaa wrapper class for two way anova type of analysis with OLS currently mainly to bring things together Notes ----- unclear: adding multiple test might assume block design or orthogonality This estimates the full dummy version with OLS. The drop first dummy representation can be recovered through the transform method. TODO: add more methods, tests, pairwise, multiple, marginal effects try out what can be added for userfriendly access. missing: ANOVA table NcD|jd|_|d}d}n|\}}t||x\|_|_\}}t||x\|_|_\} } |jdx|_} | jdx|_} t|| } | \}}}}}}| \|_ |_ |_ |_ |_}t|| d}t|| d}t!|||_|jd|_||_t)j|||_|jj|_|j|j|_d| dz z| dz z|_|j|jz |_dS)Nrrrrrfrnri)rlrxrGrX d1_labelsrY d2_labelsnlevel1nlevel2r[ prod_labelC1C1_labelC2C2_labelrHr transformryexogsmOLSfitresolsparamsr params_dropfstart_interaction n_interaction)rendogfactor1factor2varnamesvname1vname2rXrrYrrrresprodlabrC1labrC2labr;dp_fulldp_dropfs rrzTwoWay.__init__sM!$  FFF%NFF2:7F2K2KK-"i2:7F2K2KK-"i!#!, w!#!, wy)44+.(UBqMPJ$-$-Bv666 R ==='::]1%  fUG,,0022 k( !N77 DD!"gk!2gk!B!Z$*@@rc|j}tjtj||j|z ftj|f}|j|}||_|S)z7contrast/restriction matrix for no interaction ) rrhstackr ryrrrR_nointer_transf)rnia R_nointerrs r r_nointerzTwoWay.r_nointersa Irxdjn(=>>s LMM >77 BB 0rc|j}tjtj||j|z ftj|f}|j|}||_|j |}|Sz1ttests for no-interaction terms are zero ) rrrr ryrrrrrt_test)rrrrt_ress rttest_interactionzTwoWay.ttest_interactionsv Irxdjn(=>>s LMM >77 BB 0 ""#344 rc^|}|j|Sr)rrf_test)rrs rftest_interactionzTwoWay.ftest_interactions+ >>++{!!"2333rc|dkr&|j|j|jfS|j|j|jfS)Nr)rrrrrr)r factorinds rttest_conditional_effectzTwoWay.ttest_conditional_effectsI >>;%%dg.. = =;%%dg.. = =rcddlm}|j|j|j}|j}|j}d}tdg|jz}||||||S)Nr) SimpleTablez!Estimated Coefficients by factorsz%#10.4g) data_fmts)titletxt_fmt) statsmodels.iolibrrrkrrrrdict)rr params_arrstubsheadersr table_fmts r summary_coeffzTwoWay.summary_coeffs111111[((t|DD .3" DL0222 {:wU#,... .rN) rrrrrrrrrrr<rrrrs&AAAAD      444 >>> . . . . .rrc&eZdZdZdZdZdZdS)TestContrastToolscgd|_ddg|_tjgdgdgdgdgdgdgdgdgdgdgdgdg |_dS)Na0a1a2b0b1)rrr)rrr)rrr)v1namev2namerr rX)rs rrzTestContrastTools.__init__.s((( Tl (III%II%II%II%II%II%II%II%II%II%II%II ' ( (rctjgdd}tjddgddgddgddgddgddgddgddgddgddgddgddgg ddg}}t|d \}}t||t||dS) N) FrMrrrrrrrrrz|S1)dtyperrgender_Fgender_Mgender)r_)rr rGr)rr!rwr7res_d res_labelss r test_dummy_1dzTestContrastTools.test_dummy_1d>s HQQQ   X1v !1v !1v !1v !1v !1v !1v !1v !1v !1v !1v !1v ' ( (+5j)A %Q999zUAZ(((((rct|j|j}dgdz}gd|d<tjgdgdgdgdg|d<gd |d <tjgd gd gd g|d<gd|d<t dD]=}tj||||t|>dS)Nr)a0_b0a0_b1a1_b0a1_b1a2_b0a2_b1)rrrr)rrrrrr)rrrrrr)rrrrrrr)z a1_b0-a0_b0z a1_b1-a0_b1z a2_b0-a0_b0z a2_b1-a0_b1r)rrrrrr)rrrrrr)rrrrrr)z a0_b1-a0_b0z a1_b1-a1_b0z a2_b1-a2_b0)err_msg) r[rrrr rtestingrr )rres_cpres_tiis rtest_contrast_productz'TestContrastTools.test_contrast_productQs!$+t{;;AIIIa8;;;999999999;<<aPOOaH<<<999999;<<aA@@a(( L LB J # #F2Jb 3r77 # K K K K L Lrct|j\}}t|tjgdt|tjgddS)N)rr)rr )rrXrrr )rres rtest_dummy_limitsz#TestContrastTools.test_dummy_limitsasW47##!Q++,,,Q../////rN)rrrrrrrr<rrrr,sS((( )))&LLL 00000rr__main__)smalllargeNrrrrrrrr r!rrrrrhrirg{Gz?gMbP?g?rrz" tvalues for no effect of factor 1z/each test is conditional on a level of factor 2z" tvalues for no effect of factor 2z/each test is conditional on a level of factor 1)rr)F)NNFr)rf)hr numpy.testingrnumpyrrrrr#r8r[rGrHrrrrrrrttrrrstatsmodels.apiapirexamplesrrrprintrayx1x2rXrrYrrepeatrlrxdd_fulldd_dropldd_dropfrTdiagrrv effect_size noise_scalerandomrandnnoisebeta ydata_full ydata_dropl ydata_dropfrrresols_full_fullresols_full_dropfr params_f_f params_f_dfresols_dropf_fullresols_dropf_dropf params_df_f params_df_dfrrtr_oftr_fo transf_f_dfr`absrrrrrrrrr;rtvalue cov_paramsrr rR_nointtvalues inter_direct inter_transftwrpvaluerfvaluerrr<rrrRs0'&&&&&, , , ,<E<E<E<ED4<4<4<4