
    M/Ph*                     @   d Z ddlmZ ddlZddlmZmZ ddlm	Z	 ddl
Z
ddlmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ  G d d          Z G d d          Z G d de          Z G d d          Z G d dee          Z G d dee          Z G d de          Z G d de          Ze
j                            dde            G d de                      Z! G d  d!e          Z"e
j                            d"de#           G d# d$e                      Z$dS )%a|	  Tests for gam.AdditiveModel and GAM with Polynomials compared to OLS and GLM


Created on Sat Nov 05 14:16:07 2011

Author: Josef Perktold
License: BSD


Notes
-----

TODO: TestGAMGamma: has test failure (GLM looks good),
        adding log-link did not help
        resolved: gamma does not fail anymore after tightening the
                  convergence criterium (rtol=1e-6)
TODO: TestGAMNegativeBinomial: rvs generation does not work,
        nbinom needs 2 parameters
TODO: TestGAMGaussianLogLink: test failure,
        but maybe precision issue, not completely off

        but something is wrong, either the testcase or with the link
        >>> tt3.__class__
        <class '__main__.TestGAMGaussianLogLink'>
        >>> tt3.res2.mu_pred.mean()
        3.5616368292650766
        >>> tt3.res1.mu_pred.mean()
        3.6144278964707679
        >>> tt3.mu_true.mean()
        34.821904835958122
        >>>
        >>> tt3.y_true.mean()
        2.685225067611543
        >>> tt3.res1.y_pred.mean()
        0.52991541684645616
        >>> tt3.res2.y_pred.mean()
        0.44626406889363229



one possible change
~~~~~~~~~~~~~~~~~~~
add average, integral based tests, instead of or additional to sup
    * for example mean squared error for mu and eta (predict, fittedvalues)
      or mean absolute error, what's the scale for this? required precision?
    * this will also work for real non-parametric tests

example: Gamma looks good in average bias and average RMSE (RMISE)

>>> tt3 = _estGAMGamma()
>>> np.mean((tt3.res2.mu_pred - tt3.mu_true))/tt3.mu_true.mean()
-0.0051829977497423706
>>> np.mean((tt3.res2.y_pred - tt3.y_true))/tt3.y_true.mean()
0.00015255264651864049
>>> np.mean((tt3.res1.y_pred - tt3.y_true))/tt3.y_true.mean()
0.00015255538823786711
>>> np.mean((tt3.res1.mu_pred - tt3.mu_true))/tt3.mu_true.mean()
-0.0051937668989744494
>>> np.sqrt(np.mean((tt3.res1.mu_pred - tt3.mu_true)**2))/tt3.mu_true.mean()
0.022946118520401692
>>> np.sqrt(np.mean((tt3.res2.mu_pred - tt3.mu_true)**2))/tt3.mu_true.mean()
0.022953913332599746
>>> maxabs = lambda x: np.max(np.abs(x))
>>> maxabs((tt3.res1.mu_pred - tt3.mu_true))/tt3.mu_true.mean()
0.079540546242707733
>>> maxabs((tt3.res2.mu_pred - tt3.mu_true))/tt3.mu_true.mean()
0.079578857986784574
>>> maxabs((tt3.res2.y_pred - tt3.y_true))/tt3.y_true.mean()
0.016282852522951426
>>> maxabs((tt3.res1.y_pred - tt3.y_true))/tt3.y_true.mean()
0.016288391235613865



    )lrangeN)assert_almost_equalassert_equal)stats)AdditiveModel)Model)familylinks)GLM)OLSc                       e Zd ZdS )DummyN)__name__
__module____qualname__     b/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/statsmodels/sandbox/tests/test_gam.pyr   r   Z   s        Dr   r   c                       e Zd Zd Zej                            ded          d             Zd Z	ej                            dde
          d	             Zd
S )CheckAMc                     t          | j        j        | j        j        d           t          | j        j        | j        j        d d         d           d S N   decimal
   )r   res1y_predres2y_predshortselfs    r   test_predictzCheckAM.test_predict_   se    DI, I,a	9 	9 	9 	9DI1 I,SbS11	> 	> 	> 	> 	> 	>r   z&Unknown, results do not match expectedT)reasonraisesstrictc                     t          | j        j        | j        j        d           t          | j        j        | j        j        d d         d           d S r   )r   r   r   r   fittedvaluesr    r!   s    r   test_fittedzCheckAM.test_fittede   sm     	DI, I2A	? 	? 	? 	?DI1 I23B37	D 	D 	D 	D 	D 	Dr   c                     t          | j        j        dd          | j        j        dd          d           t          | j        j        d         | j        j        d         d           d S )N   r   r   )r   r   paramsr   r!   s    r   test_paramszCheckAM.test_paramsn   s|     	DI,QRR0 I,QRR0!	= 	= 	= 	= 	DI,Q/ I,Q/	< 	< 	< 	< 	< 	<r   zres_ps attribute does not existr$   r&   r%   c                 ,   t          | j                                        | j        j                   t          | j                                        | j        j                   t          | j                                        | j        j                   d S N)r   res_psdf_modelr   df_fitdf_residr!   s    r   test_dfzCheckAM.test_dfw   ss     	T[))++TY-?@@@T[''))49+=>>>T[))++TY-?@@@@@r   N)r   r   r   r#   pytestmarkxfailAssertionErrorr)   r-   AttributeErrorr5   r   r   r   r   r   ]   s        > > > [F,T  ; ;D D; ;D< < < [?">  ; ;A A; ;A A Ar   r   c                       e Zd Zd Zd ZdS )CheckGAMc                 R    t          | j        j        | j        j        d           d S )Nr   r   )r   r   mu_predr   r!   s    r   test_muzCheckGAM.test_mu   s4    DI- I-q	: 	: 	: 	: 	: 	:r   c                 b    t          | j        j        | j        j        d d         d           d S )Nr   r   r   )r   r   r    r   r   r!   s    r   test_predictionzCheckGAM.test_prediction   s=    DI1 I,SbS11	> 	> 	> 	> 	> 	>r   N)r   r   r   r?   rA   r   r   r   r<   r<      s2        : : :
> > > > >r   r<   c                   $    e Zd Zed             ZdS )BaseAMc                    d}d}d\  }}t          j        |||          }t          j        d|z            }t          j        ||                                z  dz  d|z  f          }|d d d d d f         t          j        |dz             d d d d f         z                      |d          }t          |dz   dz            }	|	|dz   = |d d |	f         }
|                    d          }|| _	        |||
c| _
        | _        | _        d S )N      )g      rE   r   r+   g      ?)nplinspacesincolumn_stackmaxarangereshaper   sumnobsy_truexexog)clsorderrP   lbubx1x2rR   rS   idxexog_reducedrQ   s               r   setup_classzBaseAM.setup_class   s    B[R&&VAbD\\OR[]BrE233!!!AAAd(RYuQw//dAAA>>GGbQQeAgq[!!aLAAAcE{! &,a#
CE3888r   N)r   r   r   classmethodr\   r   r   r   rC   rC      s-        > > [> > >r   rC   c                   8     e Zd Ze fd            Z fdZ xZS )TestAdditiveModelc                 L   t                                                       | j        }| j        | j        | j        }}}t          j                            d           d}||t          j        	                    |          z  z   }t          |          }|                    |           |j        }t          ||                                          }	t                      x| _        }
|	x| _        }|                    |          |
_        |	j                            |	j        |          |_        |                    |d d                   |
_        d |j        D             }|j        t1          d |j        D                       z   }t          j        |g|z             |
_        d S )N) g?r   c                 6    g | ]}|j         d d         D ]}|S r+   Nr,   .0ssis      r   
<listcomp>z1TestAdditiveModel.setup_class.<locals>.<listcomp>   /    AAA29QRR=AAa!AAAAr   c                 (    g | ]}|j         d          S r+   rd   rf   rg   s     r   ri   z1TestAdditiveModel.setup_class.<locals>.<listcomp>       $H$H$HbRYq\$H$H$Hr   )superr\   rP   rQ   rR   rS   rH   randomseedrandnr   fitresultsr   r   r   r   predictr   modelr,   r    	smoothersalpharO   array)rT   rP   rQ   rR   rS   sigma_noiseymres_gamres_olsr   r   slopesconst	__class__s                 r   r\   zTestAdditiveModel.setup_class   sc   x*ceSX4
	w[29??4#8#888!	a)a,,""$$  ''!4!!4ooa((m++GNDAA"??1SbS622AAakAAA$H$HAK$H$H$H I IIhw/00r   c                 H    t                                                       d S r0   ro   r)   r"   r   s    r   r)   zTestAdditiveModel.test_fitted   s!     	r   )r   r   r   r]   r\   r)   __classcell__r   s   @r   r_   r_      s[        1 1 1 1 [1>        r   r_   c                   $    e Zd Zed             ZdS )BaseGAMc                 V   | j         }| j        | j        | j        }}}t	          | d          sd}n| j        }| j        }|j                            |          x| _	        }t          j                            d           	 |                     |||          }n'# t          $ r |                     ||          }Y nw xY wt          |||          }	|	                    |d           |	j        }
|
| _        |	| _        t)          |||                                          }t+                      x| _        }|x| _        }|j                            |j        |d	
          |_        |
                    |          |_        |
                    |d d                   |_        |j                            |j        |d
          |_        |
j        |_        d |	j        D             }|
j         tC          d |	j        D                       z   }t          j"        |g|z             |_        d S )Nscaler+   ra   )r   size)r   )r	   d   )maxiterlinear)whichr   meanc                 6    g | ]}|j         d d         D ]}|S rc   rd   re   s      r   ri   z BaseGAM.init.<locals>.<listcomp>   rj   r   c                 (    g | ]}|j         d          S rl   rd   rm   s     r   ri   z BaseGAM.init.<locals>.<listcomp>   rn   r   )#rP   rQ   rR   rS   hasattrr   r	   linkinversemu_truerH   rp   rq   rvs	TypeErrorGAMrs   rt   r}   mod_gamr   r   r   r   rv   ru   r,   r   r    r>   murw   rx   rO   ry   )rT   rP   rQ   rR   rS   r   fr   y_obsr|   r}   res_glmr   r   r   r   s                   r   initzBaseGAM.init   s   x*ceSX4sG$$ 	EEIEJ !v 6 66g
	w	0GGG5tG<<EE 	0 	0 	0GGG$G//EEE	0 q###	eS!!!)eT!,,,0022  ''!4!!4 m++GND+QQooa(("??1SbS622 },,W^T,PPz BAakAAA$H$HAK$H$H$H I IIhw/00s   ?B !B<;B<N)r   r   r   r]   r   r   r   r   r   r      s-        ,1 ,1 [,1 ,1 ,1r   r   c                   .     e Zd Ze fd            Z xZS )TestGAMPoissonc                     t                                                       t          j                    | _        t          j        j        | _        |                                  d S r0   )ro   r\   r	   Poissonr   poissonr   r   rT   r   s    r   r\   zTestGAMPoisson.setup_class   sE    ^%%
-#




r   r   r   r   r]   r\   r   r   s   @r   r   r      B            [    r   r   c                   .     e Zd Ze fd            Z xZS )TestGAMBinomialc                     t                                                       t          j                    | _        t          j        j        | _        |                                  d S r0   )ro   r\   r	   Binomialr   	bernoullir   r   r   s    r   r\   zTestGAMBinomial.setup_class  sE    _&&
/%




r   r   r   s   @r   r   r   	  r   r   r   z'Unknown, results do not match expected.Tr.   c                   .     e Zd Ze fd            Z xZS )TestGAMGaussianLogLinkc                     t                                                       t          j        t	          j                              | _        t          j        j        | _        d| _	        | 
                                 d S )N   )ro   r\   r	   Gaussianr
   Logr   normr   r   r   r   s    r   r\   z"TestGAMGaussianLogLink.setup_class   sS    _UY[[11
*.	




r   r   r   s   @r   r   r     sD             [    r   r   c                   .     e Zd Ze fd            Z xZS )TestGAMGammac                     t                                                       t          j        t	          j                              | _        t          j        j        | _        | 	                                 d S r0   )
ro   r\   r	   Gammar
   r   r   gammar   r   r   s    r   r\   zTestGAMGamma.setup_class-  sL    \%)++..
+/




r   r   r   s   @r   r   r   +  r   r   r   6Passing wrong number of args/kwargs to _parse_args_rvsc                        e Zd Ze fd            Zej                            dde           fd            Z	ej                            dde           fd            Z
 xZS )TestGAMNegativeBinomialc                     t                                                       t          j                    | _        t          j        j        | _        |                                  d S r0   )ro   r\   r	   NegativeBinomialr   nbinomr   r   r   s    r   r\   z#TestGAMNegativeBinomial.setup_class=  sF    ,..
,"




r   r   Tr.   c                 H    t                                                       d S r0   r   r   s    r   r)   z#TestGAMNegativeBinomial.test_fittedF  s!     	r   c                 H    t                                                       d S r0   )ro   r5   r   s    r   r5   zTestGAMNegativeBinomial.test_dfN  s     	r   )r   r   r   r]   r\   r6   r7   r8   r   r)   r5   r   r   s   @r   r   r   7  s             [ [ 3"9  6 6   6 6
 [ 3"9  6 6   6 6    r   r   )%__doc__statsmodels.compat.pythonr   numpyrH   numpy.testingr   r   scipyr   r6   statsmodels.sandbox.gamr   r   r   statsmodels.genmod.familiesr	   r
   +statsmodels.genmod.generalized_linear_modelr   #statsmodels.regression.linear_modelr   r   r   r<   rC   r_   r   r   r   r7   r8   r9   r   r   r   r   r   r   r   <module>r      s  J JV - , , , , ,     ; ; ; ; ; ; ; ;        1 1 1 1 1 1 0 0 0 0 0 0 5 5 5 5 5 5 5 5 ; ; ; ; ; ; 3 3 3 3 3 3	 	 	 	 	 	 	 	 A  A  A  A  A  A  A  AF
> 
> 
> 
> 
>w 
> 
> 
>> > > > > > > >.% % % % % % % %P/1 /1 /1 /1 /1fh /1 /1 /1d	 	 	 	 	W 	 	 		 	 	 	 	g 	 	 	 C~  7 7    W  7 7(	 	 	 	 	7 	 	 	  /y  2 2    g  2 2  r   