
    M/Ph-                         d Z ddlZddlmZmZ ddlZddlm	Z	m
Z
 ddlmZ ddlmZ ddlmZ ddlmZ  G d	 d
ee	          Z G d dee
          Z G d dee          Zd Zd Zd Zd Zd Zd Zd Zd Zd ZdS )z>
Created on Wed May 23 12:53:27 2018

Author: Josef Perktold

    N)assert_allcloseassert_equal)PoissonLogit)GLMfamily)PenalizedMixin)VariableScreeningc                       e Zd ZdS )PoissonPenalizedN__name__
__module____qualname__     e/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/statsmodels/base/tests/test_screening.pyr   r              Dr   r   c                       e Zd ZdS )LogitPenalizedNr   r   r   r   r   r      r   r   r   c                       e Zd ZdS )GLMPenalizedNr   r   r   r   r   r      r   r   r   c                     t           j                            d           d\  } }d}t           j                            | |          dt           j                            | d          dz
  z  z   dz  dz
  }|dz  }||                    d	          z
  |                    d	          z  }d|d d d	f<   t          j        |          }g d
}dt          j        d|dz             z  ||<   t          j        |          }|	                    |          }t          j
        |          }t           j                            |          }||||fS )N d              ?         ?   333333?r   r   r   ,      )nprandomseedrandmeanstdzerosarangesqrtdotexppoisson	nobsk_vars	k_nonzeroxbetaidx_nonzero_truelinpredmuys	            r   _get_poisson_datar?      s0   INN6LD&I	f	%	%	ry~~dA&&,	-
.12	356	7AHA	
QVVAYY!%%(("AAaaadG8FD...")Ay1}"="==D	74==DeeDkkG	B
	"Aa!4''r   c                     t           j                            d           t                      \  } }}}t	          |           }d |D             }d|d<   t          j        ||         |dg          }t          j        |d d |f         |          }t          | |                                          }|j	        |d<   t          | t          j        |          |d	z  
          }	t          |	          }
|d d dd f         }|
                    |d          }t          t          j        |j                  |           d |j        D             }d|d<   |j                            |           |j                                         t          |j        j        d         d           t          j        |j        j	        |d          }|                    |d          }t/          |d         |d         d           d S )Nr   c                     g | ]}d |z  S zvar%4dr   .0iis     r   
<listcomp>z*test_poisson_screening.<locals>.<listcomp>;       <<<R8b=<<<r   constr   trueindexcolumnsrL   oracler   
pen_weightr!   
   maxiterc                     g | ]}d |z  S rB   r   rC   s     r   rF   z*test_poisson_screening.<locals>.<listcomp>L       ===hm===r   xname	convergedTfinalrK   nameouterhowh㈵>atol)r)   r*   r+   r?   lenpd	DataFramer   fitparamsr   onesr   screen_exogr   sortidx_nonzeroresults_finalsummaryresults_penmle_retvalsSeriesjoinr   r>   r9   r;   r:   r6   xnames_true
parametersxframe_true
res_oraclemod_initialscreenerexog_candidates
res_screenxnamespss                  r   test_poisson_screeningr|   4   s   INN6#4#6#6 Aq
Dq66D<<+;<<<KKNd#34K'-h0 0 0J ,q$4!45{KKKKK((,,..J%,Jx"1bgdmmqIIIK --H122hO%%or%BBJ/002BCCC==j&<===FF1I $$6$222""$$$)5kBDIII	:+2&w	O	O	OB11JJx(*W*=DIIIIIIr   c                     t           j                            d           d\  t           j                            dz
            dt           j                            d          dz
  z  z   dz  dz
  } | dz  } | |                     d          z
  |                     d          z  } t          j        t          j                  | f          } dt          j        ddz             z  }t          j	        |          }| 
                    |          }t          j        |          }t           j                            |          }| d d dd f                             d          d d d f         | fd	}t          j                  }d|d dz  <   t          j        t          j                  |f          }d
D ]}t          ||d d d |f         dz            }	t          |	          }
d|
_        |
                     |                      }g d}t%          |j        |           t          j        ddgddgddgddggt           j                  }t%          |j        |           d S )Nr   )r   r   r!   r    r"   r#   r$   r   c               3     K   d} d}t          |          D ]}dz  t          j                            |           z   dt          j                            d          dz
  z  z   dz  dz
  }|dz  }|dz
  k     rd d |dz   f         |d d d	f<   ||                    d
          z
  |                    d
          z  }|V  d S )Nr      g?r    r!   r"   r#   r$   rQ   r   )ranger)   r*   r,   r-   r.   )r7   	n_batchesir9   commonr8   r6   	x_nonzeros       r   exog_iteratorz+test_screen_iterated.<locals>.exog_iteratoro   s      	y!! 	 	Af!=!==binnT1--34589:<=>AHA9q=  $QQQAX.!!!R%QVVAYY!%%((*AGGGG	 	r   r!   r#   r   rO      )var0_10var1_10var2_10var3_10rQ      )dtype)r)   r*   r+   r,   r-   r.   column_stackrg   r0   r1   r2   r3   r4   sumr   r   	k_max_addscreen_exog_iteratorr   exog_final_namesarrayint64idx_nonzero_batches)r9   r:   r<   r=   r>   r   dummy	exog_keepkrv   rw   rY   namesidx_fullr   r8   r6   r   s                 @@@@r   test_screen_iteratedr   Z   s   INN6OD)	i!m	,	,	binnT1%%+	,
-01	245	6AHA	
QVVAYY!%%(("A
*++A	!Y]+++D74==DeeDkkG	B
	"Aqqq!""uX\\!__QQQW%FI        GDMMEE*419* 677I : :&q)AAArrE*:tczRRR$[11--mmoo>><<<U+U3338q"X "X "X "X' /1h8 8 8 	U.9999: :r   c                     t                      \  } }}}t          |           }d |D             }d|d<   t          j        ||         |dg          }t          j        |d d |f         |          }t	          | |t          j                                                              }|j        |d<   t	          | t          j
        |          t          j                              }	t          |	          }
|d d d	d f         }|
                    |d
          }t          t          j        |j                  |           d |j        D             }d|d<   |j                            |           |j                                         t          |j        j        d         d           t          j        |j        j        |d          }|                    |d          }t-          |d         |d         d           d S )Nc                     g | ]}d |z  S rB   r   rC   s     r   rF   z-test_glmpoisson_screening.<locals>.<listcomp>   rG   r   rH   r   rI   rJ   rM   r   rN   r!   rQ   rR   c                     g | ]}d |z  S rB   r   rC   s     r   rF   z-test_glmpoisson_screening.<locals>.<listcomp>   rU   r   rV   rX   TrY   rZ   r\   r]   r_   r`   )r?   rb   rc   rd   r   r	   r   re   rf   r)   rg   r   rh   r   ri   rj   rk   rl   rm   rn   ro   rp   r   rq   s                  r   test_glmpoisson_screeningr      s   #4#6#6 Aq
Dq66D<<+;<<<KKNd#34KRXQYZZZJ,q$4!45{KKKKaV^5E5EFFFJJLLJ%,Jxq"'$--8H8HIIIK --H122hO%%or%BBJ/002BCCC==j&<===FF1I $$6$222""$$$)5kBDIII	:+2&w	O	O	OB11JJx(*W*=DIIIIIIr   c                     t           j                            d           d\  } }d}t           j                            | |          dt           j                            | d          dz
  z  z   dz  dz
  }|dz  }||                    d	          z
  |                    d	          z  }d|d d d	f<   t          j        |          }g d
}dt          j        d|dz             dz  z  ||<   t          j        |          }|	                    |          }ddt          j
        |           z   z  }t           j                            t          |                    |k                         t                    }||||fS )Nr   )r&   r   r   {Gz?r!   r"   r#   r$   r   r%   r    )r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   rb   astypeintr5   s	            r   _get_logit_datar      sZ   INN6LD&I	f	%	%	a((3.	/
034	578	9AHA	
QVVAYY!%%(("AAaaadG8FD...")Ay1}"="=s"BBD	74==DeeDkkG	
a"&'"""	#B	B	 	 2	%--c22Aa!4''r   c                     t                      \  } }}}t          |           }t          |dz  d          }d |D             }d|d<   t          j        ||         |dg          }t          j        |d d |f         |	          }t          | |                                          }	|	j        |d
<   t          | t          j
        |          |dz            }
t          |
fi |}d|_        |d d dd f         }|                    |d          }t          j        |j        j                  dk    }t!          t          j        |j        |                   |           t          j        g d          }t!          t          j        |j                  |           d |j        D             }d|d<   |j                            |           |j                                         t!          |j        j        d         d           t          j        |j        j        |d          }||d<   t1          |d
         |d         d           d S )Ngffffff?MbP?)rP   threshold_trimc                     g | ]}d |z  S rB   r   rC   s     r   rF   z(test_logit_screening.<locals>.<listcomp>   rG   r   rH   r   rI   rJ   rM   rN   r"   rO   r   r!   rR   皙?)r   J   r      r&   r'   r(   c                     g | ]}d |z  S rB   r   rC   s     r   rF   z(test_logit_screening.<locals>.<listcomp>   rU   r   rV   rX   TrY   rZ   {Gzt?r`   )r   rb   dictrc   rd   r   re   rf   r   r)   rg   r   r   rh   absrk   r   ri   rj   r   rl   rm   rn   ro   r   )r>   r9   r;   r:   r6   screener_kwdsrr   rs   rt   ru   rv   rw   rx   ry   maskidx_rrz   r{   s                     r   test_logit_screeningr      s[   #2#4#4 Aq
Dq66DD3JtDDDM<<+;<<<KKNd#34K'-h0 0 0J ,q$4!45{KKKKq+&&**,,J%,Jx BGDMMdSjIIIK >>>>HH122hO%%or%BBJ 6**122S8D/5668HIIIH55566E/00%888==j&<===FF1I $$6$222""$$$)5kBDIII	:+2&w	O	O	OB JwJx(*W*=EJJJJJJr   c                     t                      \  } }}}t          |           }t          |dz  dd          }d |D             }d|d<   t          j        ||         |dg	          }t          j        |d d |f         |
          }t          | |t          j                                                              }	|	j	        |d<   t          | t          j        |          t          j                              }
t          |
fi |}d|_        |d d dd f         }|                    |d          }|j         |j         d |j        D             }d|d<   |j                            |           |j                                         t'          |j        j        d         d           t          j        |j        j	        |d          }||d<   t-          |d         |d         d           d S )N      ?r   model.score_factorrP   r   ranking_attrc                     g | ]}d |z  S rB   r   rC   s     r   rF   z+test_glmlogit_screening.<locals>.<listcomp>   rG   r   rH   r   rI   rJ   rM   r   rN   rQ   r!   r   rR   c                     g | ]}d |z  S rB   r   rC   s     r   rF   z+test_glmlogit_screening.<locals>.<listcomp>  rU   r   rV   rX   TrY   rZ   r   r`   )r   rb   r   rc   rd   r   r	   Binomialre   rf   r)   rg   r   r   rh   rj   rk   rl   rm   r   rn   ro   r   )r>   r9   r;   r:   r6   r   rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   s                   r   test_glmlogit_screeningr      s   #2#4#4 Aq
Dq66D D4K&:< < <M =<+;<<<KKNd#34K'-h0 0 0J ,q$4!45{KKKKaV_5F5FGGGKKMMJ%,Jx q"'$--8I8IJJJK >>>>HH122hO%%or%BBJ==j&<===FF1I $$6$222""$$$)5kBDIII	:+2&w	O	O	OB JwJx(*W*=EJJJJJJr   c                     t           j                            d           d\  } }d}t           j                            | |          dt           j                            | d          dz
  z  z   dz  dz
  }|dz  }||                    d	          z
  |                    d	          z  }d|d d d	f<   t          j        |          }g d
}dt          j        d|dz             z  ||<   t          j        |          }|	                    |          }|dt           j        
                    t          |                    z  z   }||||fS )Nr   r   r   r   r!   r"   r#   r$   r   )r   r!   r&   r'   r(   r    r   )r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   randnrb   )r6   r7   r8   r9   r:   r;   r<   r>   s           r   _get_gaussian_datar   "  s1   INN6LD&I	f	%	%	a((3.	/
034	578	9AHA	
QVVAYY!%%(("AAaaadG8FD,,,")Ay1}"="==D	74==DeeDkkG#	G5555Aa!4''r   c                  d   t                      \  } }}}t          |           }| |                     d          z
  } t          |dz  dd          }d |D             }d|d<   t	          j        ||         |dg	          }t	          j        |d d |f         |
          }t          | |t          j                              	                                }	|	j
        |d<   dD ]N}
t          | |d d d |
f         t          j                              }t          |fi |}|d d |
d f         }|                    |d          }t          t          j        |j                  |           d |j        D             }d|d<   |j                            |           |j                                         t          |j        j        d         d           t	          j        |j        j
        |d          }|                    |d          }t/          |d         |d         d           |d= Pd S )Nr   r   r   r   r   c                     g | ]}d |z  S rB   r   rC   s     r   rF   z.test_glmgaussian_screening.<locals>.<listcomp>B  rG   r   rH   rI   rJ   rM   r   rN   r   r   rR   c                     g | ]}d |z  S rB   r   rC   s     r   rF   z.test_glmgaussian_screening.<locals>.<listcomp>S  s    AAAB(R-AAAr   rV   rX   TrY   rZ   r\   r]   gh㈵>r`   )r   rb   r-   r   rc   rd   r   r	   Gaussianre   rf   r   rh   r   r)   ri   rj   rk   rl   rm   rn   ro   rp   r   )r>   r9   r;   r:   r6   r   rr   rs   rt   ru   k_keeprv   rw   rx   ry   rz   r{   s                    r   test_glmgaussian_screeningr   7  s_   #5#7#7 Aq
Dq66D	AFF1IIA D4K&:< < <M =<+;<<<KKNd#34K'-h0 0 0J ,q$4!45{KKKKaV_5F5FGGGKKMMJ%,Jx    "1a7F7
mFO<M<MNNN$[BBMBBAAAvwwJ-))/2)FF
RWZ3446FGGGAA**@AAAq	 	 ((v(666&&(((Z-9+FMMMYz/6f7SSS__RW_55

8,j.AMMMMw+   r   )__doc__numpyr)   numpy.testingr   r   pandasrc   #statsmodels.discrete.discrete_modelr   r   +statsmodels.genmod.generalized_linear_modelr   statsmodels.genmod.familiesr	   statsmodels.base._penalizedr
   statsmodels.base._screeningr   r   r   r   r?   r|   r   r   r   r   r   r   r   r   r   r   <module>r      s        7 7 7 7 7 7 7 7     > > > > > > > > ; ; ; ; ; ; . . . . . . 6 6 6 6 6 6 9 9 9 9 9 9	 	 	 	 	~w 	 	 		 	 	 	 	^U 	 	 		 	 	 	 	>3 	 	 	( ( (*#J #J #JL2: 2: 2:j J  J  JF( ( (**K *K *KZ+K +K +K\( ( (*)  )  )  )  ) r   