# TODO: Determine which tests are valid for GLSAR, and under what conditions # TODO: Fix issue with constant and GLS # TODO: GLS: add options Iterative GLS, for iterative fgls if sigma is None # TODO: GLS: default if sigma is none should be two-step GLS # TODO: Check nesting when performing model based tests, lr, wald, lm """ This module implements standard regression models: Generalized Least Squares (GLS) Ordinary Least Squares (OLS) Weighted Least Squares (WLS) Generalized Least Squares with autoregressive error terms GLSAR(p) Models are specified with an endogenous response variable and an exogenous design matrix and are fit using their `fit` method. Subclasses that have more complicated covariance matrices should write over the 'whiten' method as the fit method prewhitens the response by calling 'whiten'. General reference for regression models: D. C. Montgomery and E.A. Peck. "Introduction to Linear Regression Analysis." 2nd. Ed., Wiley, 1992. Econometrics references for regression models: R. Davidson and J.G. MacKinnon. "Econometric Theory and Methods," Oxford, 2004. W. Green. "Econometric Analysis," 5th ed., Pearson, 2003. """ from __future__ import annotations from statsmodels.compat.pandas import Appender from statsmodels.compat.python import lrange, lzip from typing import Literal from collections.abc import Sequence import warnings import numpy as np from scipy import optimize, stats from scipy.linalg import cholesky, toeplitz from scipy.linalg.lapack import dtrtri import statsmodels.base.model as base import statsmodels.base.wrapper as wrap from statsmodels.emplike.elregress import _ELRegOpts # need import in module instead of lazily to copy `__doc__` from statsmodels.regression._prediction import PredictionResults from statsmodels.tools.decorators import cache_readonly, cache_writable from statsmodels.tools.sm_exceptions import InvalidTestWarning, ValueWarning from statsmodels.tools.tools import pinv_extended from statsmodels.tools.typing import Float64Array from statsmodels.tools.validation import bool_like, float_like, string_like from . import _prediction as pred __docformat__ = 'restructuredtext en' __all__ = ['GLS', 'WLS', 'OLS', 'GLSAR', 'PredictionResults', 'RegressionResultsWrapper'] _fit_regularized_doc =\ r""" Return a regularized fit to a linear regression model. Parameters ---------- method : str Either 'elastic_net' or 'sqrt_lasso'. alpha : scalar or array_like The penalty weight. If a scalar, the same penalty weight applies to all variables in the model. If a vector, it must have the same length as `params`, and contains a penalty weight for each coefficient. L1_wt : scalar The fraction of the penalty given to the L1 penalty term. Must be between 0 and 1 (inclusive). If 0, the fit is a ridge fit, if 1 it is a lasso fit. start_params : array_like Starting values for ``params``. profile_scale : bool If True the penalized fit is computed using the profile (concentrated) log-likelihood for the Gaussian model. Otherwise the fit uses the residual sum of squares. refit : bool If True, the model is refit using only the variables that have non-zero coefficients in the regularized fit. The refitted model is not regularized. **kwargs Additional keyword arguments that contain information used when constructing a model using the formula interface. Returns ------- statsmodels.base.elastic_net.RegularizedResults The regularized results. Notes ----- The elastic net uses a combination of L1 and L2 penalties. The implementation closely follows the glmnet package in R. The function that is minimized is: .. math:: 0.5*RSS/n + alpha*((1-L1\_wt)*|params|_2^2/2 + L1\_wt*|params|_1) where RSS is the usual regression sum of squares, n is the sample size, and :math:`|*|_1` and :math:`|*|_2` are the L1 and L2 norms. For WLS and GLS, the RSS is calculated using the whitened endog and exog data. Post-estimation results are based on the same data used to select variables, hence may be subject to overfitting biases. The elastic_net method uses the following keyword arguments: maxiter : int Maximum number of iterations cnvrg_tol : float Convergence threshold for line searches zero_tol : float Coefficients below this threshold are treated as zero. The square root lasso approach is a variation of the Lasso that is largely self-tuning (the optimal tuning parameter does not depend on the standard deviation of the regression errors). If the errors are Gaussian, the tuning parameter can be taken to be alpha = 1.1 * np.sqrt(n) * norm.ppf(1 - 0.05 / (2 * p)) where n is the sample size and p is the number of predictors. The square root lasso uses the following keyword arguments: zero_tol : float Coefficients below this threshold are treated as zero. The cvxopt module is required to estimate model using the square root lasso. References ---------- .. [*] Friedman, Hastie, Tibshirani (2008). Regularization paths for generalized linear models via coordinate descent. Journal of Statistical Software 33(1), 1-22 Feb 2010. .. [*] A Belloni, V Chernozhukov, L Wang (2011). Square-root Lasso: pivotal recovery of sparse signals via conic programming. Biometrika 98(4), 791-806. https://arxiv.org/pdf/1009.5689.pdf """ def _get_sigma(sigma, nobs): """ Returns sigma (matrix, nobs by nobs) for GLS and the inverse of its Cholesky decomposition. Handles dimensions and checks integrity. If sigma is None, returns None, None. Otherwise returns sigma, cholsigmainv. """ if sigma is None: return None, None sigma = np.asarray(sigma).squeeze() if sigma.ndim == 0: sigma = np.repeat(sigma, nobs) if sigma.ndim == 1: if sigma.shape != (nobs,): raise ValueError("Sigma must be a scalar, 1d of length %s or a 2d " "array of shape %s x %s" % (nobs, nobs, nobs)) cholsigmainv = 1/np.sqrt(sigma) else: if sigma.shape != (nobs, nobs): raise ValueError("Sigma must be a scalar, 1d of length %s or a 2d " "array of shape %s x %s" % (nobs, nobs, nobs)) cholsigmainv, info = dtrtri(cholesky(sigma, lower=True), lower=True, overwrite_c=True) if info > 0: raise np.linalg.LinAlgError('Cholesky decomposition of sigma ' 'yields a singular matrix') elif info < 0: raise ValueError('Invalid input to dtrtri (info = %d)' % info) return sigma, cholsigmainv class RegressionModel(base.LikelihoodModel): """ Base class for linear regression models. Should not be directly called. Intended for subclassing. """ def __init__(self, endog, exog, **kwargs): super().__init__(endog, exog, **kwargs) self.pinv_wexog: Float64Array | None = None self._data_attr.extend(['pinv_wexog', 'wendog', 'wexog', 'weights']) def initialize(self): """Initialize model components.""" self.wexog = self.whiten(self.exog) self.wendog = self.whiten(self.endog) # overwrite nobs from class Model: self.nobs = float(self.wexog.shape[0]) self._df_model = None self._df_resid = None self.rank = None @property def df_model(self): """ The model degree of freedom. The dof is defined as the rank of the regressor matrix minus 1 if a constant is included. """ if self._df_model is None: if self.rank is None: self.rank = np.linalg.matrix_rank(self.exog) self._df_model = float(self.rank - self.k_constant) return self._df_model @df_model.setter def df_model(self, value): self._df_model = value @property def df_resid(self): """ The residual degree of freedom. The dof is defined as the number of observations minus the rank of the regressor matrix. """ if self._df_resid is None: if self.rank is None: self.rank = np.linalg.matrix_rank(self.exog) self._df_resid = self.nobs - self.rank return self._df_resid @df_resid.setter def df_resid(self, value): self._df_resid = value def whiten(self, x): """ Whiten method that must be overwritten by individual models. Parameters ---------- x : array_like Data to be whitened. """ raise NotImplementedError("Subclasses must implement.") def fit( self, method: Literal["pinv", "qr"] = "pinv", cov_type: Literal[ "nonrobust", "fixed scale", "HC0", "HC1", "HC2", "HC3", "HAC", "hac-panel", "hac-groupsum", "cluster", ] = "nonrobust", cov_kwds=None, use_t: bool | None = None, **kwargs ): """ Full fit of the model. The results include an estimate of covariance matrix, (whitened) residuals and an estimate of scale. Parameters ---------- method : str, optional Can be "pinv", "qr". "pinv" uses the Moore-Penrose pseudoinverse to solve the least squares problem. "qr" uses the QR factorization. cov_type : str, optional See `regression.linear_model.RegressionResults` for a description of the available covariance estimators. cov_kwds : list or None, optional See `linear_model.RegressionResults.get_robustcov_results` for a description required keywords for alternative covariance estimators. use_t : bool, optional Flag indicating to use the Student's t distribution when computing p-values. Default behavior depends on cov_type. See `linear_model.RegressionResults.get_robustcov_results` for implementation details. **kwargs Additional keyword arguments that contain information used when constructing a model using the formula interface. Returns ------- RegressionResults The model estimation results. See Also -------- RegressionResults The results container. RegressionResults.get_robustcov_results A method to change the covariance estimator used when fitting the model. Notes ----- The fit method uses the pseudoinverse of the design/exogenous variables to solve the least squares minimization. """ if method == "pinv": if not (hasattr(self, 'pinv_wexog') and hasattr(self, 'normalized_cov_params') and hasattr(self, 'rank')): self.pinv_wexog, singular_values = pinv_extended(self.wexog) self.normalized_cov_params = np.dot( self.pinv_wexog, np.transpose(self.pinv_wexog)) # Cache these singular values for use later. self.wexog_singular_values = singular_values self.rank = np.linalg.matrix_rank(np.diag(singular_values)) beta = np.dot(self.pinv_wexog, self.wendog) elif method == "qr": if not (hasattr(self, 'exog_Q') and hasattr(self, 'exog_R') and hasattr(self, 'normalized_cov_params') and hasattr(self, 'rank')): Q, R = np.linalg.qr(self.wexog) self.exog_Q, self.exog_R = Q, R self.normalized_cov_params = np.linalg.inv(np.dot(R.T, R)) # Cache singular values from R. self.wexog_singular_values = np.linalg.svd(R, 0, 0) self.rank = np.linalg.matrix_rank(R) else: Q, R = self.exog_Q, self.exog_R # Needed for some covariance estimators, see GH #8157 self.pinv_wexog = np.linalg.pinv(self.wexog) # used in ANOVA self.effects = effects = np.dot(Q.T, self.wendog) beta = np.linalg.solve(R, effects) else: raise ValueError('method has to be "pinv" or "qr"') if self._df_model is None: self._df_model = float(self.rank - self.k_constant) if self._df_resid is None: self.df_resid = self.nobs - self.rank if isinstance(self, OLS): lfit = OLSResults( self, beta, normalized_cov_params=self.normalized_cov_params, cov_type=cov_type, cov_kwds=cov_kwds, use_t=use_t) else: lfit = RegressionResults( self, beta, normalized_cov_params=self.normalized_cov_params, cov_type=cov_type, cov_kwds=cov_kwds, use_t=use_t, **kwargs) return RegressionResultsWrapper(lfit) def predict(self, params, exog=None): """ Return linear predicted values from a design matrix. Parameters ---------- params : array_like Parameters of a linear model. exog : array_like, optional Design / exogenous data. Model exog is used if None. Returns ------- array_like An array of fitted values. Notes ----- If the model has not yet been fit, params is not optional. """ # JP: this does not look correct for GLMAR # SS: it needs its own predict method if exog is None: exog = self.exog return np.dot(exog, params) def get_distribution(self, params, scale, exog=None, dist_class=None): """ Construct a random number generator for the predictive distribution. Parameters ---------- params : array_like The model parameters (regression coefficients). scale : scalar The variance parameter. exog : array_like The predictor variable matrix. dist_class : class A random number generator class. Must take 'loc' and 'scale' as arguments and return a random number generator implementing an ``rvs`` method for simulating random values. Defaults to normal. Returns ------- gen Frozen random number generator object with mean and variance determined by the fitted linear model. Use the ``rvs`` method to generate random values. Notes ----- Due to the behavior of ``scipy.stats.distributions objects``, the returned random number generator must be called with ``gen.rvs(n)`` where ``n`` is the number of observations in the data set used to fit the model. If any other value is used for ``n``, misleading results will be produced. """ fit = self.predict(params, exog) if dist_class is None: from scipy.stats.distributions import norm dist_class = norm gen = dist_class(loc=fit, scale=np.sqrt(scale)) return gen class GLS(RegressionModel): __doc__ = r""" Generalized Least Squares {params} sigma : scalar or array The array or scalar `sigma` is the weighting matrix of the covariance. The default is None for no scaling. If `sigma` is a scalar, it is assumed that `sigma` is an n x n diagonal matrix with the given scalar, `sigma` as the value of each diagonal element. If `sigma` is an n-length vector, then `sigma` is assumed to be a diagonal matrix with the given `sigma` on the diagonal. This should be the same as WLS. {extra_params} Attributes ---------- pinv_wexog : ndarray `pinv_wexog` is the p x n Moore-Penrose pseudoinverse of `wexog`. cholsimgainv : ndarray The transpose of the Cholesky decomposition of the pseudoinverse. df_model : float p - 1, where p is the number of regressors including the intercept. of freedom. df_resid : float Number of observations n less the number of parameters p. llf : float The value of the likelihood function of the fitted model. nobs : float The number of observations n. normalized_cov_params : ndarray p x p array :math:`(X^{{T}}\Sigma^{{-1}}X)^{{-1}}` results : RegressionResults instance A property that returns the RegressionResults class if fit. sigma : ndarray `sigma` is the n x n covariance structure of the error terms. wexog : ndarray Design matrix whitened by `cholsigmainv` wendog : ndarray Response variable whitened by `cholsigmainv` See Also -------- WLS : Fit a linear model using Weighted Least Squares. OLS : Fit a linear model using Ordinary Least Squares. Notes ----- If sigma is a function of the data making one of the regressors a constant, then the current postestimation statistics will not be correct. Examples -------- >>> import statsmodels.api as sm >>> data = sm.datasets.longley.load() >>> data.exog = sm.add_constant(data.exog) >>> ols_resid = sm.OLS(data.endog, data.exog).fit().resid >>> res_fit = sm.OLS(ols_resid[1:], ols_resid[:-1]).fit() >>> rho = res_fit.params `rho` is a consistent estimator of the correlation of the residuals from an OLS fit of the longley data. It is assumed that this is the true rho of the AR process data. >>> from scipy.linalg import toeplitz >>> order = toeplitz(np.arange(16)) >>> sigma = rho**order `sigma` is an n x n matrix of the autocorrelation structure of the data. >>> gls_model = sm.GLS(data.endog, data.exog, sigma=sigma) >>> gls_results = gls_model.fit() >>> print(gls_results.summary()) """.format(params=base._model_params_doc, extra_params=base._missing_param_doc + base._extra_param_doc) def __init__(self, endog, exog, sigma=None, missing='none', hasconst=None, **kwargs): if type(self) is GLS: self._check_kwargs(kwargs) # TODO: add options igls, for iterative fgls if sigma is None # TODO: default if sigma is none should be two-step GLS sigma, cholsigmainv = _get_sigma(sigma, len(endog)) super().__init__(endog, exog, missing=missing, hasconst=hasconst, sigma=sigma, cholsigmainv=cholsigmainv, **kwargs) # store attribute names for data arrays self._data_attr.extend(['sigma', 'cholsigmainv']) def whiten(self, x): """ GLS whiten method. Parameters ---------- x : array_like Data to be whitened. Returns ------- ndarray The value np.dot(cholsigmainv,X). See Also -------- GLS : Fit a linear model using Generalized Least Squares. """ x = np.asarray(x) if self.sigma is None or self.sigma.shape == (): return x elif self.sigma.ndim == 1: if x.ndim == 1: return x * self.cholsigmainv else: return x * self.cholsigmainv[:, None] else: return np.dot(self.cholsigmainv, x) def loglike(self, params): r""" Compute the value of the Gaussian log-likelihood function at params. Given the whitened design matrix, the log-likelihood is evaluated at the parameter vector `params` for the dependent variable `endog`. Parameters ---------- params : array_like The model parameters. Returns ------- float The value of the log-likelihood function for a GLS Model. Notes ----- The log-likelihood function for the normal distribution is .. math:: -\frac{n}{2}\log\left(\left(Y-\hat{Y}\right)^{\prime} \left(Y-\hat{Y}\right)\right) -\frac{n}{2}\left(1+\log\left(\frac{2\pi}{n}\right)\right) -\frac{1}{2}\log\left(\left|\Sigma\right|\right) Y and Y-hat are whitened. """ # TODO: combine this with OLS/WLS loglike and add _det_sigma argument nobs2 = self.nobs / 2.0 SSR = np.sum((self.wendog - np.dot(self.wexog, params))**2, axis=0) llf = -np.log(SSR) * nobs2 # concentrated likelihood llf -= (1+np.log(np.pi/nobs2))*nobs2 # with likelihood constant if np.any(self.sigma): # FIXME: robust-enough check? unneeded if _det_sigma gets defined if self.sigma.ndim == 2: det = np.linalg.slogdet(self.sigma) llf -= .5*det[1] else: llf -= 0.5*np.sum(np.log(self.sigma)) # with error covariance matrix return llf def hessian_factor(self, params, scale=None, observed=True): """ Compute weights for calculating Hessian. Parameters ---------- params : ndarray The parameter at which Hessian is evaluated. scale : None or float If scale is None, then the default scale will be calculated. Default scale is defined by `self.scaletype` and set in fit. If scale is not None, then it is used as a fixed scale. observed : bool If True, then the observed Hessian is returned. If false then the expected information matrix is returned. Returns ------- ndarray A 1d weight vector used in the calculation of the Hessian. The hessian is obtained by `(exog.T * hessian_factor).dot(exog)`. """ if self.sigma is None or self.sigma.shape == (): return np.ones(self.exog.shape[0]) elif self.sigma.ndim == 1: return self.cholsigmainv else: return np.diag(self.cholsigmainv) @Appender(_fit_regularized_doc) def fit_regularized(self, method="elastic_net", alpha=0., L1_wt=1., start_params=None, profile_scale=False, refit=False, **kwargs): if not np.isscalar(alpha): alpha = np.asarray(alpha) # Need to adjust since RSS/n term in elastic net uses nominal # n in denominator if self.sigma is not None: if self.sigma.ndim == 2: var_obs = np.diag(self.sigma) elif self.sigma.ndim == 1: var_obs = self.sigma else: raise ValueError("sigma should be 1-dim or 2-dim") alpha = alpha * np.sum(1 / var_obs) / len(self.endog) rslt = OLS(self.wendog, self.wexog).fit_regularized( method=method, alpha=alpha, L1_wt=L1_wt, start_params=start_params, profile_scale=profile_scale, refit=refit, **kwargs) from statsmodels.base.elastic_net import ( RegularizedResults, RegularizedResultsWrapper, ) rrslt = RegularizedResults(self, rslt.params) return RegularizedResultsWrapper(rrslt) class WLS(RegressionModel): __doc__ = """ Weighted Least Squares The weights are presumed to be (proportional to) the inverse of the variance of the observations. That is, if the variables are to be transformed by 1/sqrt(W) you must supply weights = 1/W. {params} weights : array_like, optional A 1d array of weights. If you supply 1/W then the variables are pre- multiplied by 1/sqrt(W). If no weights are supplied the default value is 1 and WLS results are the same as OLS. {extra_params} Attributes ---------- weights : ndarray The stored weights supplied as an argument. See Also -------- GLS : Fit a linear model using Generalized Least Squares. OLS : Fit a linear model using Ordinary Least Squares. Notes ----- If the weights are a function of the data, then the post estimation statistics such as fvalue and mse_model might not be correct, as the package does not yet support no-constant regression. Examples -------- >>> import statsmodels.api as sm >>> Y = [1,3,4,5,2,3,4] >>> X = range(1,8) >>> X = sm.add_constant(X) >>> wls_model = sm.WLS(Y,X, weights=list(range(1,8))) >>> results = wls_model.fit() >>> results.params array([ 2.91666667, 0.0952381 ]) >>> results.tvalues array([ 2.0652652 , 0.35684428]) >>> print(results.t_test([1, 0])) >>> print(results.f_test([0, 1])) """.format(params=base._model_params_doc, extra_params=base._missing_param_doc + base._extra_param_doc) def __init__(self, endog, exog, weights=1., missing='none', hasconst=None, **kwargs): if type(self) is WLS: self._check_kwargs(kwargs) weights = np.array(weights) if weights.shape == (): if (missing == 'drop' and 'missing_idx' in kwargs and kwargs['missing_idx'] is not None): # patsy may have truncated endog weights = np.repeat(weights, len(kwargs['missing_idx'])) else: weights = np.repeat(weights, len(endog)) # handle case that endog might be of len == 1 if len(weights) == 1: weights = np.array([weights.squeeze()]) else: weights = weights.squeeze() super().__init__(endog, exog, missing=missing, weights=weights, hasconst=hasconst, **kwargs) nobs = self.exog.shape[0] weights = self.weights if weights.size != nobs and weights.shape[0] != nobs: raise ValueError('Weights must be scalar or same length as design') def whiten(self, x): """ Whitener for WLS model, multiplies each column by sqrt(self.weights). Parameters ---------- x : array_like Data to be whitened. Returns ------- array_like The whitened values sqrt(weights)*X. """ x = np.asarray(x) if x.ndim == 1: return x * np.sqrt(self.weights) elif x.ndim == 2: return np.sqrt(self.weights)[:, None] * x def loglike(self, params): r""" Compute the value of the gaussian log-likelihood function at params. Given the whitened design matrix, the log-likelihood is evaluated at the parameter vector `params` for the dependent variable `Y`. Parameters ---------- params : array_like The parameter estimates. Returns ------- float The value of the log-likelihood function for a WLS Model. Notes ----- .. math:: -\frac{n}{2}\log SSR -\frac{n}{2}\left(1+\log\left(\frac{2\pi}{n}\right)\right) +\frac{1}{2}\log\left(\left|W\right|\right) where :math:`W` is a diagonal weight matrix, :math:`\left|W\right|` is its determinant, and :math:`SSR=\left(Y-\hat{Y}\right)^\prime W \left(Y-\hat{Y}\right)` is the sum of the squared weighted residuals. """ nobs2 = self.nobs / 2.0 SSR = np.sum((self.wendog - np.dot(self.wexog, params))**2, axis=0) llf = -np.log(SSR) * nobs2 # concentrated likelihood llf -= (1+np.log(np.pi/nobs2))*nobs2 # with constant llf += 0.5 * np.sum(np.log(self.weights)) return llf def hessian_factor(self, params, scale=None, observed=True): """ Compute the weights for calculating the Hessian. Parameters ---------- params : ndarray The parameter at which Hessian is evaluated. scale : None or float If scale is None, then the default scale will be calculated. Default scale is defined by `self.scaletype` and set in fit. If scale is not None, then it is used as a fixed scale. observed : bool If True, then the observed Hessian is returned. If false then the expected information matrix is returned. Returns ------- ndarray A 1d weight vector used in the calculation of the Hessian. The hessian is obtained by `(exog.T * hessian_factor).dot(exog)`. """ return self.weights @Appender(_fit_regularized_doc) def fit_regularized(self, method="elastic_net", alpha=0., L1_wt=1., start_params=None, profile_scale=False, refit=False, **kwargs): # Docstring attached below if not np.isscalar(alpha): alpha = np.asarray(alpha) # Need to adjust since RSS/n in elastic net uses nominal n in # denominator alpha = alpha * np.sum(self.weights) / len(self.weights) rslt = OLS(self.wendog, self.wexog).fit_regularized( method=method, alpha=alpha, L1_wt=L1_wt, start_params=start_params, profile_scale=profile_scale, refit=refit, **kwargs) from statsmodels.base.elastic_net import ( RegularizedResults, RegularizedResultsWrapper, ) rrslt = RegularizedResults(self, rslt.params) return RegularizedResultsWrapper(rrslt) class OLS(WLS): __doc__ = """ Ordinary Least Squares {params} {extra_params} Attributes ---------- weights : scalar Has an attribute weights = array(1.0) due to inheritance from WLS. See Also -------- WLS : Fit a linear model using Weighted Least Squares. GLS : Fit a linear model using Generalized Least Squares. Notes ----- No constant is added by the model unless you are using formulas. Examples -------- >>> import statsmodels.api as sm >>> import numpy as np >>> duncan_prestige = sm.datasets.get_rdataset("Duncan", "carData") >>> Y = duncan_prestige.data['income'] >>> X = duncan_prestige.data['education'] >>> X = sm.add_constant(X) >>> model = sm.OLS(Y,X) >>> results = model.fit() >>> results.params const 10.603498 education 0.594859 dtype: float64 >>> results.tvalues const 2.039813 education 6.892802 dtype: float64 >>> print(results.t_test([1, 0])) Test for Constraints ============================================================================== coef std err t P>|t| [0.025 0.975] ------------------------------------------------------------------------------ c0 10.6035 5.198 2.040 0.048 0.120 21.087 ============================================================================== >>> print(results.f_test(np.identity(2))) """.format(params=base._model_params_doc, extra_params=base._missing_param_doc + base._extra_param_doc) def __init__(self, endog, exog=None, missing='none', hasconst=None, **kwargs): if "weights" in kwargs: msg = ("Weights are not supported in OLS and will be ignored" "An exception will be raised in the next version.") warnings.warn(msg, ValueWarning) super().__init__(endog, exog, missing=missing, hasconst=hasconst, **kwargs) if "weights" in self._init_keys: self._init_keys.remove("weights") if type(self) is OLS: self._check_kwargs(kwargs, ["offset"]) def loglike(self, params, scale=None): """ The likelihood function for the OLS model. Parameters ---------- params : array_like The coefficients with which to estimate the log-likelihood. scale : float or None If None, return the profile (concentrated) log likelihood (profiled over the scale parameter), else return the log-likelihood using the given scale value. Returns ------- float The likelihood function evaluated at params. """ nobs2 = self.nobs / 2.0 nobs = float(self.nobs) resid = self.endog - np.dot(self.exog, params) if hasattr(self, 'offset'): resid -= self.offset ssr = np.sum(resid**2) if scale is None: # profile log likelihood llf = -nobs2*np.log(2*np.pi) - nobs2*np.log(ssr / nobs) - nobs2 else: # log-likelihood llf = -nobs2 * np.log(2 * np.pi * scale) - ssr / (2*scale) return llf def whiten(self, x): """ OLS model whitener does nothing. Parameters ---------- x : array_like Data to be whitened. Returns ------- array_like The input array unmodified. See Also -------- OLS : Fit a linear model using Ordinary Least Squares. """ return x def score(self, params, scale=None): """ Evaluate the score function at a given point. The score corresponds to the profile (concentrated) log-likelihood in which the scale parameter has been profiled out. Parameters ---------- params : array_like The parameter vector at which the score function is computed. scale : float or None If None, return the profile (concentrated) log likelihood (profiled over the scale parameter), else return the log-likelihood using the given scale value. Returns ------- ndarray The score vector. """ if not hasattr(self, "_wexog_xprod"): self._setup_score_hess() xtxb = np.dot(self._wexog_xprod, params) sdr = -self._wexog_x_wendog + xtxb if scale is None: ssr = self._wendog_xprod - 2 * np.dot(self._wexog_x_wendog.T, params) ssr += np.dot(params, xtxb) return -self.nobs * sdr / ssr else: return -sdr / scale def _setup_score_hess(self): y = self.wendog if hasattr(self, 'offset'): y = y - self.offset self._wendog_xprod = np.sum(y * y) self._wexog_xprod = np.dot(self.wexog.T, self.wexog) self._wexog_x_wendog = np.dot(self.wexog.T, y) def hessian(self, params, scale=None): """ Evaluate the Hessian function at a given point. Parameters ---------- params : array_like The parameter vector at which the Hessian is computed. scale : float or None If None, return the profile (concentrated) log likelihood (profiled over the scale parameter), else return the log-likelihood using the given scale value. Returns ------- ndarray The Hessian matrix. """ if not hasattr(self, "_wexog_xprod"): self._setup_score_hess() xtxb = np.dot(self._wexog_xprod, params) if scale is None: ssr = self._wendog_xprod - 2 * np.dot(self._wexog_x_wendog.T, params) ssr += np.dot(params, xtxb) ssrp = -2*self._wexog_x_wendog + 2*xtxb hm = self._wexog_xprod / ssr - np.outer(ssrp, ssrp) / ssr**2 return -self.nobs * hm / 2 else: return -self._wexog_xprod / scale def hessian_factor(self, params, scale=None, observed=True): """ Calculate the weights for the Hessian. Parameters ---------- params : ndarray The parameter at which Hessian is evaluated. scale : None or float If scale is None, then the default scale will be calculated. Default scale is defined by `self.scaletype` and set in fit. If scale is not None, then it is used as a fixed scale. observed : bool If True, then the observed Hessian is returned. If false then the expected information matrix is returned. Returns ------- ndarray A 1d weight vector used in the calculation of the Hessian. The hessian is obtained by `(exog.T * hessian_factor).dot(exog)`. """ return np.ones(self.exog.shape[0]) @Appender(_fit_regularized_doc) def fit_regularized(self, method="elastic_net", alpha=0., L1_wt=1., start_params=None, profile_scale=False, refit=False, **kwargs): # In the future we could add support for other penalties, e.g. SCAD. if method not in ("elastic_net", "sqrt_lasso"): msg = "Unknown method '%s' for fit_regularized" % method raise ValueError(msg) # Set default parameters. defaults = {"maxiter": 50, "cnvrg_tol": 1e-10, "zero_tol": 1e-8} defaults.update(kwargs) if method == "sqrt_lasso": from statsmodels.base.elastic_net import ( RegularizedResults, RegularizedResultsWrapper, ) params = self._sqrt_lasso(alpha, refit, defaults["zero_tol"]) results = RegularizedResults(self, params) return RegularizedResultsWrapper(results) from statsmodels.base.elastic_net import fit_elasticnet if L1_wt == 0: return self._fit_ridge(alpha) # If a scale parameter is passed in, the non-profile # likelihood (residual sum of squares divided by -2) is used, # otherwise the profile likelihood is used. if profile_scale: loglike_kwds = {} score_kwds = {} hess_kwds = {} else: loglike_kwds = {"scale": 1} score_kwds = {"scale": 1} hess_kwds = {"scale": 1} return fit_elasticnet(self, method=method, alpha=alpha, L1_wt=L1_wt, start_params=start_params, loglike_kwds=loglike_kwds, score_kwds=score_kwds, hess_kwds=hess_kwds, refit=refit, check_step=False, **defaults) def _sqrt_lasso(self, alpha, refit, zero_tol): try: import cvxopt except ImportError: msg = 'sqrt_lasso fitting requires the cvxopt module' raise ValueError(msg) n = len(self.endog) p = self.exog.shape[1] h0 = cvxopt.matrix(0., (2*p+1, 1)) h1 = cvxopt.matrix(0., (n+1, 1)) h1[1:, 0] = cvxopt.matrix(self.endog, (n, 1)) G0 = cvxopt.spmatrix([], [], [], (2*p+1, 2*p+1)) for i in range(1, 2*p+1): G0[i, i] = -1 G1 = cvxopt.matrix(0., (n+1, 2*p+1)) G1[0, 0] = -1 G1[1:, 1:p+1] = self.exog G1[1:, p+1:] = -self.exog c = cvxopt.matrix(alpha / n, (2*p + 1, 1)) c[0] = 1 / np.sqrt(n) from cvxopt import solvers solvers.options["show_progress"] = False rslt = solvers.socp(c, Gl=G0, hl=h0, Gq=[G1], hq=[h1]) x = np.asarray(rslt['x']).flat bp = x[1:p+1] bn = x[p+1:] params = bp - bn if not refit: return params ii = np.flatnonzero(np.abs(params) > zero_tol) rfr = OLS(self.endog, self.exog[:, ii]).fit() params *= 0 params[ii] = rfr.params return params def _fit_ridge(self, alpha): """ Fit a linear model using ridge regression. Parameters ---------- alpha : scalar or array_like The penalty weight. If a scalar, the same penalty weight applies to all variables in the model. If a vector, it must have the same length as `params`, and contains a penalty weight for each coefficient. Notes ----- Equivalent to fit_regularized with L1_wt = 0 (but implemented more efficiently). """ u, s, vt = np.linalg.svd(self.exog, 0) v = vt.T q = np.dot(u.T, self.endog) * s s2 = s * s if np.isscalar(alpha): sd = s2 + alpha * self.nobs params = q / sd params = np.dot(v, params) else: alpha = np.asarray(alpha) vtav = self.nobs * np.dot(vt, alpha[:, None] * v) d = np.diag(vtav) + s2 np.fill_diagonal(vtav, d) r = np.linalg.solve(vtav, q) params = np.dot(v, r) from statsmodels.base.elastic_net import RegularizedResults return RegularizedResults(self, params) class GLSAR(GLS): __doc__ = """ Generalized Least Squares with AR covariance structure {params} rho : int The order of the autoregressive covariance. {extra_params} Notes ----- GLSAR is considered to be experimental. The linear autoregressive process of order p--AR(p)--is defined as: TODO Examples -------- >>> import statsmodels.api as sm >>> X = range(1,8) >>> X = sm.add_constant(X) >>> Y = [1,3,4,5,8,10,9] >>> model = sm.GLSAR(Y, X, rho=2) >>> for i in range(6): ... results = model.fit() ... print("AR coefficients: {{0}}".format(model.rho)) ... rho, sigma = sm.regression.yule_walker(results.resid, ... order=model.order) ... model = sm.GLSAR(Y, X, rho) ... AR coefficients: [ 0. 0.] AR coefficients: [-0.52571491 -0.84496178] AR coefficients: [-0.6104153 -0.86656458] AR coefficients: [-0.60439494 -0.857867 ] AR coefficients: [-0.6048218 -0.85846157] AR coefficients: [-0.60479146 -0.85841922] >>> results.params array([-0.66661205, 1.60850853]) >>> results.tvalues array([ -2.10304127, 21.8047269 ]) >>> print(results.t_test([1, 0])) >>> print(results.f_test(np.identity(2))) Or, equivalently >>> model2 = sm.GLSAR(Y, X, rho=2) >>> res = model2.iterative_fit(maxiter=6) >>> model2.rho array([-0.60479146, -0.85841922]) """.format(params=base._model_params_doc, extra_params=base._missing_param_doc + base._extra_param_doc) # TODO: Complete docstring def __init__(self, endog, exog=None, rho=1, missing='none', hasconst=None, **kwargs): # this looks strange, interpreting rho as order if it is int if isinstance(rho, (int, np.integer)): self.order = int(rho) self.rho = np.zeros(self.order, np.float64) else: self.rho = np.squeeze(np.asarray(rho)) if len(self.rho.shape) not in [0, 1]: raise ValueError("AR parameters must be a scalar or a vector") if self.rho.shape == (): self.rho.shape = (1,) self.order = self.rho.shape[0] if exog is None: # JP this looks wrong, should be a regression on constant # results for rho estimate now identical to yule-walker on y # super(AR, self).__init__(endog, add_constant(endog)) super().__init__(endog, np.ones((endog.shape[0], 1)), missing=missing, hasconst=None, **kwargs) else: super().__init__(endog, exog, missing=missing, **kwargs) def iterative_fit(self, maxiter=3, rtol=1e-4, **kwargs): """ Perform an iterative two-stage procedure to estimate a GLS model. The model is assumed to have AR(p) errors, AR(p) parameters and regression coefficients are estimated iteratively. Parameters ---------- maxiter : int, optional The number of iterations. rtol : float, optional Relative tolerance between estimated coefficients to stop the estimation. Stops if max(abs(last - current) / abs(last)) < rtol. **kwargs Additional keyword arguments passed to `fit`. Returns ------- RegressionResults The results computed using an iterative fit. """ # TODO: update this after going through example. converged = False i = -1 # need to initialize for maxiter < 1 (skip loop) history = {'params': [], 'rho': [self.rho]} for i in range(maxiter - 1): if hasattr(self, 'pinv_wexog'): del self.pinv_wexog self.initialize() results = self.fit() history['params'].append(results.params) if i == 0: last = results.params else: diff = np.max(np.abs(last - results.params) / np.abs(last)) if diff < rtol: converged = True break last = results.params self.rho, _ = yule_walker(results.resid, order=self.order, df=None) history['rho'].append(self.rho) # why not another call to self.initialize # Use kwarg to insert history if not converged and maxiter > 0: # maxiter <= 0 just does OLS if hasattr(self, 'pinv_wexog'): del self.pinv_wexog self.initialize() # if converged then this is a duplicate fit, because we did not # update rho results = self.fit(history=history, **kwargs) results.iter = i + 1 # add last fit to history, not if duplicate fit if not converged: results.history['params'].append(results.params) results.iter += 1 results.converged = converged return results def whiten(self, x): """ Whiten a series of columns according to an AR(p) covariance structure. Whitening using this method drops the initial p observations. Parameters ---------- x : array_like The data to be whitened. Returns ------- ndarray The whitened data. """ # TODO: notation for AR process x = np.asarray(x, np.float64) _x = x.copy() # the following loops over the first axis, works for 1d and nd for i in range(self.order): _x[(i + 1):] = _x[(i + 1):] - self.rho[i] * x[0:-(i + 1)] return _x[self.order:] def yule_walker(x, order=1, method="adjusted", df=None, inv=False, demean=True): """ Estimate AR(p) parameters from a sequence using the Yule-Walker equations. Adjusted or maximum-likelihood estimator (mle) Parameters ---------- x : array_like A 1d array. order : int, optional The order of the autoregressive process. Default is 1. method : str, optional Method can be 'adjusted' or 'mle' and this determines denominator in estimate of autocorrelation function (ACF) at lag k. If 'mle', the denominator is n=X.shape[0], if 'adjusted' the denominator is n-k. The default is adjusted. df : int, optional Specifies the degrees of freedom. If `df` is supplied, then it is assumed the X has `df` degrees of freedom rather than `n`. Default is None. inv : bool If inv is True the inverse of R is also returned. Default is False. demean : bool True, the mean is subtracted from `X` before estimation. Returns ------- rho : ndarray AR(p) coefficients computed using the Yule-Walker method. sigma : float The estimate of the residual standard deviation. See Also -------- burg : Burg's AR estimator. Notes ----- See https://en.wikipedia.org/wiki/Autoregressive_moving_average_model for further details. Examples -------- >>> import statsmodels.api as sm >>> from statsmodels.datasets.sunspots import load >>> data = load() >>> rho, sigma = sm.regression.yule_walker(data.endog, order=4, ... method="mle") >>> rho array([ 1.28310031, -0.45240924, -0.20770299, 0.04794365]) >>> sigma 16.808022730464351 """ # TODO: define R better, look back at notes and technical notes on YW. # First link here is useful # http://www-stat.wharton.upenn.edu/~steele/Courses/956/ResourceDetails/YuleWalkerAndMore.htm method = string_like( method, "method", options=("adjusted", "unbiased", "mle") ) if method == "unbiased": warnings.warn( "unbiased is deprecated in factor of adjusted to reflect that the " "term is adjusting the sample size used in the autocovariance " "calculation rather than estimating an unbiased autocovariance. " "After release 0.13, using 'unbiased' will raise.", FutureWarning, ) method = "adjusted" if method not in ("adjusted", "mle"): raise ValueError("ACF estimation method must be 'adjusted' or 'MLE'") x = np.array(x, dtype=np.float64) if demean: if not x.flags.writeable: x = np.require(x, requirements="W") x -= x.mean() n = df or x.shape[0] # this handles df_resid ie., n - p adj_needed = method == "adjusted" if x.ndim > 1 and x.shape[1] != 1: raise ValueError("expecting a vector to estimate AR parameters") r = np.zeros(order+1, np.float64) r[0] = (x ** 2).sum() / n for k in range(1, order+1): r[k] = (x[0:-k] * x[k:]).sum() / (n - k * adj_needed) R = toeplitz(r[:-1]) try: rho = np.linalg.solve(R, r[1:]) except np.linalg.LinAlgError as err: if 'Singular matrix' in str(err): warnings.warn("Matrix is singular. Using pinv.", ValueWarning) rho = np.linalg.pinv(R) @ r[1:] else: raise sigmasq = r[0] - (r[1:]*rho).sum() if not np.isnan(sigmasq) and sigmasq > 0: sigma = np.sqrt(sigmasq) else: sigma = np.nan if inv: return rho, sigma, np.linalg.inv(R) else: return rho, sigma def burg(endog, order=1, demean=True): """ Compute Burg's AP(p) parameter estimator. Parameters ---------- endog : array_like The endogenous variable. order : int, optional Order of the AR. Default is 1. demean : bool, optional Flag indicating to subtract the mean from endog before estimation. Returns ------- rho : ndarray The AR(p) coefficients computed using Burg's algorithm. sigma2 : float The estimate of the residual variance. See Also -------- yule_walker : Estimate AR parameters using the Yule-Walker method. Notes ----- AR model estimated includes a constant that is estimated using the sample mean (see [1]_). This value is not reported. References ---------- .. [1] Brockwell, P.J. and Davis, R.A., 2016. Introduction to time series and forecasting. Springer. Examples -------- >>> import statsmodels.api as sm >>> from statsmodels.datasets.sunspots import load >>> data = load() >>> rho, sigma2 = sm.regression.linear_model.burg(data.endog, order=4) >>> rho array([ 1.30934186, -0.48086633, -0.20185982, 0.05501941]) >>> sigma2 271.2467306963966 """ # Avoid circular imports from statsmodels.tsa.stattools import levinson_durbin_pacf, pacf_burg endog = np.squeeze(np.asarray(endog)) if endog.ndim != 1: raise ValueError('endog must be 1-d or squeezable to 1-d.') order = int(order) if order < 1: raise ValueError('order must be an integer larger than 1') if demean: endog = endog - endog.mean() pacf, sigma = pacf_burg(endog, order, demean=demean) ar, _ = levinson_durbin_pacf(pacf) return ar, sigma[-1] class RegressionResults(base.LikelihoodModelResults): r""" This class summarizes the fit of a linear regression model. It handles the output of contrasts, estimates of covariance, etc. Parameters ---------- model : RegressionModel The regression model instance. params : ndarray The estimated parameters. normalized_cov_params : ndarray The normalized covariance parameters. scale : float The estimated scale of the residuals. cov_type : str The covariance estimator used in the results. cov_kwds : dict Additional keywords used in the covariance specification. use_t : bool Flag indicating to use the Student's t in inference. **kwargs Additional keyword arguments used to initialize the results. Attributes ---------- pinv_wexog See model class docstring for implementation details. cov_type Parameter covariance estimator used for standard errors and t-stats. df_model Model degrees of freedom. The number of regressors `p`. Does not include the constant if one is present. df_resid Residual degrees of freedom. `n - p - 1`, if a constant is present. `n - p` if a constant is not included. het_scale adjusted squared residuals for heteroscedasticity robust standard errors. Is only available after `HC#_se` or `cov_HC#` is called. See HC#_se for more information. history Estimation history for iterative estimators. model A pointer to the model instance that called fit() or results. params The linear coefficients that minimize the least squares criterion. This is usually called Beta for the classical linear model. """ _cache = {} # needs to be a class attribute for scale setter? def __init__(self, model, params, normalized_cov_params=None, scale=1., cov_type='nonrobust', cov_kwds=None, use_t=None, **kwargs): super().__init__( model, params, normalized_cov_params, scale) self._cache = {} if hasattr(model, 'wexog_singular_values'): self._wexog_singular_values = model.wexog_singular_values else: self._wexog_singular_values = None self.df_model = model.df_model self.df_resid = model.df_resid if cov_type == 'nonrobust': self.cov_type = 'nonrobust' self.cov_kwds = { 'description': 'Standard Errors assume that the ' + 'covariance matrix of the errors is correctly ' + 'specified.'} if use_t is None: use_t = True # TODO: class default self.use_t = use_t else: if cov_kwds is None: cov_kwds = {} if 'use_t' in cov_kwds: # TODO: we want to get rid of 'use_t' in cov_kwds use_t_2 = cov_kwds.pop('use_t') if use_t is None: use_t = use_t_2 # TODO: warn or not? self.get_robustcov_results(cov_type=cov_type, use_self=True, use_t=use_t, **cov_kwds) for key in kwargs: setattr(self, key, kwargs[key]) def conf_int(self, alpha=.05, cols=None): """ Compute the confidence interval of the fitted parameters. Parameters ---------- alpha : float, optional The `alpha` level for the confidence interval. The default `alpha` = .05 returns a 95% confidence interval. cols : array_like, optional Columns to include in returned confidence intervals. Returns ------- array_like The confidence intervals. Notes ----- The confidence interval is based on Student's t-distribution. """ # keep method for docstring for now ci = super().conf_int(alpha=alpha, cols=cols) return ci @cache_readonly def nobs(self): """Number of observations n.""" return float(self.model.wexog.shape[0]) @cache_readonly def fittedvalues(self): """The predicted values for the original (unwhitened) design.""" return self.model.predict(self.params, self.model.exog) @cache_readonly def wresid(self): """ The residuals of the transformed/whitened regressand and regressor(s). """ return self.model.wendog - self.model.predict( self.params, self.model.wexog) @cache_readonly def resid(self): """The residuals of the model.""" return self.model.endog - self.model.predict( self.params, self.model.exog) # TODO: fix writable example @cache_writable() def scale(self): """ A scale factor for the covariance matrix. The Default value is ssr/(n-p). Note that the square root of `scale` is often called the standard error of the regression. """ wresid = self.wresid return np.dot(wresid, wresid) / self.df_resid @cache_readonly def ssr(self): """Sum of squared (whitened) residuals.""" wresid = self.wresid return np.dot(wresid, wresid) @cache_readonly def centered_tss(self): """The total (weighted) sum of squares centered about the mean.""" model = self.model weights = getattr(model, 'weights', None) sigma = getattr(model, 'sigma', None) if weights is not None: mean = np.average(model.endog, weights=weights) return np.sum(weights * (model.endog - mean)**2) elif sigma is not None: # Exactly matches WLS when sigma is diagonal iota = np.ones_like(model.endog) iota = model.whiten(iota) mean = model.wendog.dot(iota) / iota.dot(iota) err = model.endog - mean err = model.whiten(err) return np.sum(err**2) else: centered_endog = model.wendog - model.wendog.mean() return np.dot(centered_endog, centered_endog) @cache_readonly def uncentered_tss(self): """ Uncentered sum of squares. The sum of the squared values of the (whitened) endogenous response variable. """ wendog = self.model.wendog return np.dot(wendog, wendog) @cache_readonly def ess(self): """ The explained sum of squares. If a constant is present, the centered total sum of squares minus the sum of squared residuals. If there is no constant, the uncentered total sum of squares is used. """ if self.k_constant: return self.centered_tss - self.ssr else: return self.uncentered_tss - self.ssr @cache_readonly def rsquared(self): """ R-squared of the model. This is defined here as 1 - `ssr`/`centered_tss` if the constant is included in the model and 1 - `ssr`/`uncentered_tss` if the constant is omitted. """ if self.k_constant: return 1 - self.ssr/self.centered_tss else: return 1 - self.ssr/self.uncentered_tss @cache_readonly def rsquared_adj(self): """ Adjusted R-squared. This is defined here as 1 - (`nobs`-1)/`df_resid` * (1-`rsquared`) if a constant is included and 1 - `nobs`/`df_resid` * (1-`rsquared`) if no constant is included. """ return 1 - (np.divide(self.nobs - self.k_constant, self.df_resid) * (1 - self.rsquared)) @cache_readonly def mse_model(self): """ Mean squared error the model. The explained sum of squares divided by the model degrees of freedom. """ if np.all(self.df_model == 0.0): return np.full_like(self.ess, np.nan) return self.ess/self.df_model @cache_readonly def mse_resid(self): """ Mean squared error of the residuals. The sum of squared residuals divided by the residual degrees of freedom. """ if np.all(self.df_resid == 0.0): return np.full_like(self.ssr, np.nan) return self.ssr/self.df_resid @cache_readonly def mse_total(self): """ Total mean squared error. The uncentered total sum of squares divided by the number of observations. """ if np.all(self.df_resid + self.df_model == 0.0): return np.full_like(self.centered_tss, np.nan) if self.k_constant: return self.centered_tss / (self.df_resid + self.df_model) else: return self.uncentered_tss / (self.df_resid + self.df_model) @cache_readonly def fvalue(self): """ F-statistic of the fully specified model. Calculated as the mean squared error of the model divided by the mean squared error of the residuals if the nonrobust covariance is used. Otherwise computed using a Wald-like quadratic form that tests whether all coefficients (excluding the constant) are zero. """ if hasattr(self, 'cov_type') and self.cov_type != 'nonrobust': # with heteroscedasticity or correlation robustness k_params = self.normalized_cov_params.shape[0] mat = np.eye(k_params) const_idx = self.model.data.const_idx # TODO: What if model includes implicit constant, e.g. all # dummies but no constant regressor? # TODO: Restats as LM test by projecting orthogonalizing # to constant? if self.model.data.k_constant == 1: # if constant is implicit, return nan see #2444 if const_idx is None: return np.nan idx = lrange(k_params) idx.pop(const_idx) mat = mat[idx] # remove constant if mat.size == 0: # see #3642 return np.nan ft = self.f_test(mat) # using backdoor to set another attribute that we already have self._cache['f_pvalue'] = float(ft.pvalue) return float(ft.fvalue) else: # for standard homoscedastic case return self.mse_model/self.mse_resid @cache_readonly def f_pvalue(self): """The p-value of the F-statistic.""" # Special case for df_model 0 if self.df_model == 0: return np.full_like(self.fvalue, np.nan) return stats.f.sf(self.fvalue, self.df_model, self.df_resid) @cache_readonly def bse(self): """The standard errors of the parameter estimates.""" return np.sqrt(np.diag(self.cov_params())) @cache_readonly def aic(self): r""" Akaike's information criteria. For a model with a constant :math:`-2llf + 2(df\_model + 1)`. For a model without a constant :math:`-2llf + 2(df\_model)`. """ return self.info_criteria("aic") @cache_readonly def bic(self): r""" Bayes' information criteria. For a model with a constant :math:`-2llf + \log(n)(df\_model+1)`. For a model without a constant :math:`-2llf + \log(n)(df\_model)`. """ return self.info_criteria("bic") def info_criteria(self, crit, dk_params=0): """Return an information criterion for the model. Parameters ---------- crit : string One of 'aic', 'bic', 'aicc' or 'hqic'. dk_params : int or float Correction to the number of parameters used in the information criterion. By default, only mean parameters are included, the scale parameter is not included in the parameter count. Use ``dk_params=1`` to include scale in the parameter count. Returns ------- Value of information criterion. References ---------- Burnham KP, Anderson KR (2002). Model Selection and Multimodel Inference; Springer New York. """ crit = crit.lower() k_params = self.df_model + self.k_constant + dk_params if crit == "aic": return -2 * self.llf + 2 * k_params elif crit == "bic": bic = -2*self.llf + np.log(self.nobs) * k_params return bic elif crit == "aicc": from statsmodels.tools.eval_measures import aicc return aicc(self.llf, self.nobs, k_params) elif crit == "hqic": from statsmodels.tools.eval_measures import hqic return hqic(self.llf, self.nobs, k_params) @cache_readonly def eigenvals(self): """ Return eigenvalues sorted in decreasing order. """ if self._wexog_singular_values is not None: eigvals = self._wexog_singular_values ** 2 else: wx = self.model.wexog eigvals = np.linalg.eigvalsh(wx.T @ wx) return np.sort(eigvals)[::-1] @cache_readonly def condition_number(self): """ Return condition number of exogenous matrix. Calculated as ratio of largest to smallest singular value of the exogenous variables. This value is the same as the square root of the ratio of the largest to smallest eigenvalue of the inner-product of the exogenous variables. """ eigvals = self.eigenvals return np.sqrt(eigvals[0]/eigvals[-1]) # TODO: make these properties reset bse def _HCCM(self, scale): H = np.dot(self.model.pinv_wexog, scale[:, None] * self.model.pinv_wexog.T) return H def _abat_diagonal(self, a, b): # equivalent to np.diag(a @ b @ a.T) return np.einsum('ij,ik,kj->i', a, a, b) @cache_readonly def cov_HC0(self): """ Heteroscedasticity robust covariance matrix. See HC0_se. """ self.het_scale = self.wresid**2 cov_HC0 = self._HCCM(self.het_scale) return cov_HC0 @cache_readonly def cov_HC1(self): """ Heteroscedasticity robust covariance matrix. See HC1_se. """ self.het_scale = self.nobs/(self.df_resid)*(self.wresid**2) cov_HC1 = self._HCCM(self.het_scale) return cov_HC1 @cache_readonly def cov_HC2(self): """ Heteroscedasticity robust covariance matrix. See HC2_se. """ wexog = self.model.wexog h = self._abat_diagonal(wexog, self.normalized_cov_params) self.het_scale = self.wresid**2/(1-h) cov_HC2 = self._HCCM(self.het_scale) return cov_HC2 @cache_readonly def cov_HC3(self): """ Heteroscedasticity robust covariance matrix. See HC3_se. """ wexog = self.model.wexog h = self._abat_diagonal(wexog, self.normalized_cov_params) self.het_scale = (self.wresid / (1 - h))**2 cov_HC3 = self._HCCM(self.het_scale) return cov_HC3 @cache_readonly def HC0_se(self): """ White's (1980) heteroskedasticity robust standard errors. Notes ----- Defined as sqrt(diag(X.T X)^(-1)X.T diag(e_i^(2)) X(X.T X)^(-1) where e_i = resid[i]. When HC0_se or cov_HC0 is called the RegressionResults instance will then have another attribute `het_scale`, which is in this case is just resid**2. """ return np.sqrt(np.diag(self.cov_HC0)) @cache_readonly def HC1_se(self): """ MacKinnon and White's (1985) heteroskedasticity robust standard errors. Notes ----- Defined as sqrt(diag(n/(n-p)*HC_0). When HC1_se or cov_HC1 is called the RegressionResults instance will then have another attribute `het_scale`, which is in this case is n/(n-p)*resid**2. """ return np.sqrt(np.diag(self.cov_HC1)) @cache_readonly def HC2_se(self): """ MacKinnon and White's (1985) heteroskedasticity robust standard errors. Notes ----- Defined as (X.T X)^(-1)X.T diag(e_i^(2)/(1-h_ii)) X(X.T X)^(-1) where h_ii = x_i(X.T X)^(-1)x_i.T When HC2_se or cov_HC2 is called the RegressionResults instance will then have another attribute `het_scale`, which is in this case is resid^(2)/(1-h_ii). """ return np.sqrt(np.diag(self.cov_HC2)) @cache_readonly def HC3_se(self): """ MacKinnon and White's (1985) heteroskedasticity robust standard errors. Notes ----- Defined as (X.T X)^(-1)X.T diag(e_i^(2)/(1-h_ii)^(2)) X(X.T X)^(-1) where h_ii = x_i(X.T X)^(-1)x_i.T. When HC3_se or cov_HC3 is called the RegressionResults instance will then have another attribute `het_scale`, which is in this case is resid^(2)/(1-h_ii)^(2). """ return np.sqrt(np.diag(self.cov_HC3)) @cache_readonly def resid_pearson(self): """ Residuals, normalized to have unit variance. Returns ------- array_like The array `wresid` normalized by the sqrt of the scale to have unit variance. """ if not hasattr(self, 'resid'): raise ValueError('Method requires residuals.') eps = np.finfo(self.wresid.dtype).eps if np.sqrt(self.scale) < 10 * eps * self.model.endog.mean(): # do not divide if scale is zero close to numerical precision warnings.warn( "All residuals are 0, cannot compute normed residuals.", RuntimeWarning ) return self.wresid else: return self.wresid / np.sqrt(self.scale) def _is_nested(self, restricted): """ Parameters ---------- restricted : Result instance The restricted model is assumed to be nested in the current model. The result instance of the restricted model is required to have two attributes, residual sum of squares, `ssr`, residual degrees of freedom, `df_resid`. Returns ------- nested : bool True if nested, otherwise false Notes ----- A most nests another model if the regressors in the smaller model are spanned by the regressors in the larger model and the regressand is identical. """ if self.model.nobs != restricted.model.nobs: return False full_rank = self.model.rank restricted_rank = restricted.model.rank if full_rank <= restricted_rank: return False restricted_exog = restricted.model.wexog full_wresid = self.wresid scores = restricted_exog * full_wresid[:, None] score_l2 = np.sqrt(np.mean(scores.mean(0) ** 2)) # TODO: Could be improved, and may fail depending on scale of # regressors return np.allclose(score_l2, 0) def compare_lm_test(self, restricted, demean=True, use_lr=False): """ Use Lagrange Multiplier test to test a set of linear restrictions. Parameters ---------- restricted : Result instance The restricted model is assumed to be nested in the current model. The result instance of the restricted model is required to have two attributes, residual sum of squares, `ssr`, residual degrees of freedom, `df_resid`. demean : bool Flag indicating whether the demean the scores based on the residuals from the restricted model. If True, the covariance of the scores are used and the LM test is identical to the large sample version of the LR test. use_lr : bool A flag indicating whether to estimate the covariance of the model scores using the unrestricted model. Setting the to True improves the power of the test. Returns ------- lm_value : float The test statistic which has a chi2 distributed. p_value : float The p-value of the test statistic. df_diff : int The degrees of freedom of the restriction, i.e. difference in df between models. Notes ----- The LM test examines whether the scores from the restricted model are 0. If the null is true, and the restrictions are valid, then the parameters of the restricted model should be close to the minimum of the sum of squared errors, and so the scores should be close to zero, on average. """ from numpy.linalg import inv import statsmodels.stats.sandwich_covariance as sw if not self._is_nested(restricted): raise ValueError("Restricted model is not nested by full model.") wresid = restricted.wresid wexog = self.model.wexog scores = wexog * wresid[:, None] n = self.nobs df_full = self.df_resid df_restr = restricted.df_resid df_diff = (df_restr - df_full) s = scores.mean(axis=0) if use_lr: scores = wexog * self.wresid[:, None] demean = False if demean: scores = scores - scores.mean(0)[None, :] # Form matters here. If homoskedastics can be sigma^2 (X'X)^-1 # If Heteroskedastic then the form below is fine # If HAC then need to use HAC # If Cluster, should use cluster cov_type = getattr(self, 'cov_type', 'nonrobust') if cov_type == 'nonrobust': sigma2 = np.mean(wresid**2) xpx = np.dot(wexog.T, wexog) / n s_inv = inv(sigma2 * xpx) elif cov_type in ('HC0', 'HC1', 'HC2', 'HC3'): s_inv = inv(np.dot(scores.T, scores) / n) elif cov_type == 'HAC': maxlags = self.cov_kwds['maxlags'] s_inv = inv(sw.S_hac_simple(scores, maxlags) / n) elif cov_type == 'cluster': # cluster robust standard errors groups = self.cov_kwds['groups'] # TODO: Might need demean option in S_crosssection by group? s_inv = inv(sw.S_crosssection(scores, groups)) else: raise ValueError('Only nonrobust, HC, HAC and cluster are ' + 'currently connected') lm_value = n * (s @ s_inv @ s.T) p_value = stats.chi2.sf(lm_value, df_diff) return lm_value, p_value, df_diff def compare_f_test(self, restricted): """ Use F test to test whether restricted model is correct. Parameters ---------- restricted : Result instance The restricted model is assumed to be nested in the current model. The result instance of the restricted model is required to have two attributes, residual sum of squares, `ssr`, residual degrees of freedom, `df_resid`. Returns ------- f_value : float The test statistic which has an F distribution. p_value : float The p-value of the test statistic. df_diff : int The degrees of freedom of the restriction, i.e. difference in df between models. Notes ----- See mailing list discussion October 17, This test compares the residual sum of squares of the two models. This is not a valid test, if there is unspecified heteroscedasticity or correlation. This method will issue a warning if this is detected but still return the results under the assumption of homoscedasticity and no autocorrelation (sphericity). """ has_robust1 = getattr(self, 'cov_type', 'nonrobust') != 'nonrobust' has_robust2 = (getattr(restricted, 'cov_type', 'nonrobust') != 'nonrobust') if has_robust1 or has_robust2: warnings.warn('F test for comparison is likely invalid with ' + 'robust covariance, proceeding anyway', InvalidTestWarning) ssr_full = self.ssr ssr_restr = restricted.ssr df_full = self.df_resid df_restr = restricted.df_resid df_diff = (df_restr - df_full) f_value = (ssr_restr - ssr_full) / df_diff / ssr_full * df_full p_value = stats.f.sf(f_value, df_diff, df_full) return f_value, p_value, df_diff def compare_lr_test(self, restricted, large_sample=False): """ Likelihood ratio test to test whether restricted model is correct. Parameters ---------- restricted : Result instance The restricted model is assumed to be nested in the current model. The result instance of the restricted model is required to have two attributes, residual sum of squares, `ssr`, residual degrees of freedom, `df_resid`. large_sample : bool Flag indicating whether to use a heteroskedasticity robust version of the LR test, which is a modified LM test. Returns ------- lr_stat : float The likelihood ratio which is chisquare distributed with df_diff degrees of freedom. p_value : float The p-value of the test statistic. df_diff : int The degrees of freedom of the restriction, i.e. difference in df between models. Notes ----- The exact likelihood ratio is valid for homoskedastic data, and is defined as .. math:: D=-2\\log\\left(\\frac{\\mathcal{L}_{null}} {\\mathcal{L}_{alternative}}\\right) where :math:`\\mathcal{L}` is the likelihood of the model. With :math:`D` distributed as chisquare with df equal to difference in number of parameters or equivalently difference in residual degrees of freedom. The large sample version of the likelihood ratio is defined as .. math:: D=n s^{\\prime}S^{-1}s where :math:`s=n^{-1}\\sum_{i=1}^{n} s_{i}` .. math:: s_{i} = x_{i,alternative} \\epsilon_{i,null} is the average score of the model evaluated using the residuals from null model and the regressors from the alternative model and :math:`S` is the covariance of the scores, :math:`s_{i}`. The covariance of the scores is estimated using the same estimator as in the alternative model. This test compares the loglikelihood of the two models. This may not be a valid test, if there is unspecified heteroscedasticity or correlation. This method will issue a warning if this is detected but still return the results without taking unspecified heteroscedasticity or correlation into account. This test compares the loglikelihood of the two models. This may not be a valid test, if there is unspecified heteroscedasticity or correlation. This method will issue a warning if this is detected but still return the results without taking unspecified heteroscedasticity or correlation into account. is the average score of the model evaluated using the residuals from null model and the regressors from the alternative model and :math:`S` is the covariance of the scores, :math:`s_{i}`. The covariance of the scores is estimated using the same estimator as in the alternative model. """ # TODO: put into separate function, needs tests # See mailing list discussion October 17, if large_sample: return self.compare_lm_test(restricted, use_lr=True) has_robust1 = (getattr(self, 'cov_type', 'nonrobust') != 'nonrobust') has_robust2 = ( getattr(restricted, 'cov_type', 'nonrobust') != 'nonrobust') if has_robust1 or has_robust2: warnings.warn('Likelihood Ratio test is likely invalid with ' + 'robust covariance, proceeding anyway', InvalidTestWarning) llf_full = self.llf llf_restr = restricted.llf df_full = self.df_resid df_restr = restricted.df_resid lrdf = (df_restr - df_full) lrstat = -2*(llf_restr - llf_full) lr_pvalue = stats.chi2.sf(lrstat, lrdf) return lrstat, lr_pvalue, lrdf def get_robustcov_results(self, cov_type='HC1', use_t=None, **kwargs): """ Create new results instance with robust covariance as default. Parameters ---------- cov_type : str The type of robust sandwich estimator to use. See Notes below. use_t : bool If true, then the t distribution is used for inference. If false, then the normal distribution is used. If `use_t` is None, then an appropriate default is used, which is `True` if the cov_type is nonrobust, and `False` in all other cases. **kwargs Required or optional arguments for robust covariance calculation. See Notes below. Returns ------- RegressionResults This method creates a new results instance with the requested robust covariance as the default covariance of the parameters. Inferential statistics like p-values and hypothesis tests will be based on this covariance matrix. Notes ----- The following covariance types and required or optional arguments are currently available: - 'fixed scale' uses a predefined scale ``scale``: float, optional Argument to set the scale. Default is 1. - 'HC0', 'HC1', 'HC2', 'HC3': heteroscedasticity robust covariance - no keyword arguments - 'HAC': heteroskedasticity-autocorrelation robust covariance ``maxlags`` : integer, required number of lags to use ``kernel`` : {callable, str}, optional kernels currently available kernels are ['bartlett', 'uniform'], default is Bartlett ``use_correction``: bool, optional If true, use small sample correction - 'cluster': clustered covariance estimator ``groups`` : array_like[int], required : Integer-valued index of clusters or groups. ``use_correction``: bool, optional If True the sandwich covariance is calculated with a small sample correction. If False the sandwich covariance is calculated without small sample correction. ``df_correction``: bool, optional If True (default), then the degrees of freedom for the inferential statistics and hypothesis tests, such as pvalues, f_pvalue, conf_int, and t_test and f_test, are based on the number of groups minus one instead of the total number of observations minus the number of explanatory variables. `df_resid` of the results instance is also adjusted. When `use_t` is also True, then pvalues are computed using the Student's t distribution using the corrected values. These may differ substantially from p-values based on the normal is the number of groups is small. If False, then `df_resid` of the results instance is not adjusted. - 'hac-groupsum': Driscoll and Kraay, heteroscedasticity and autocorrelation robust covariance for panel data # TODO: more options needed here ``time`` : array_like, required index of time periods ``maxlags`` : integer, required number of lags to use ``kernel`` : {callable, str}, optional The available kernels are ['bartlett', 'uniform']. The default is Bartlett. ``use_correction`` : {False, 'hac', 'cluster'}, optional If False the the sandwich covariance is calculated without small sample correction. If `use_correction = 'cluster'` (default), then the same small sample correction as in the case of `covtype='cluster'` is used. ``df_correction`` : bool, optional The adjustment to df_resid, see cov_type 'cluster' above - 'hac-panel': heteroscedasticity and autocorrelation robust standard errors in panel data. The data needs to be sorted in this case, the time series for each panel unit or cluster need to be stacked. The membership to a time series of an individual or group can be either specified by group indicators or by increasing time periods. One of ``groups`` or ``time`` is required. # TODO: we need more options here ``groups`` : array_like[int] indicator for groups ``time`` : array_like[int] index of time periods ``maxlags`` : int, required number of lags to use ``kernel`` : {callable, str}, optional Available kernels are ['bartlett', 'uniform'], default is Bartlett ``use_correction`` : {False, 'hac', 'cluster'}, optional If False the sandwich covariance is calculated without small sample correction. ``df_correction`` : bool, optional Adjustment to df_resid, see cov_type 'cluster' above **Reminder**: ``use_correction`` in "hac-groupsum" and "hac-panel" is not bool, needs to be in {False, 'hac', 'cluster'}. .. todo:: Currently there is no check for extra or misspelled keywords, except in the case of cov_type `HCx` """ from statsmodels.base.covtype import descriptions, normalize_cov_type import statsmodels.stats.sandwich_covariance as sw cov_type = normalize_cov_type(cov_type) if 'kernel' in kwargs: kwargs['weights_func'] = kwargs.pop('kernel') if 'weights_func' in kwargs and not callable(kwargs['weights_func']): kwargs['weights_func'] = sw.kernel_dict[kwargs['weights_func']] # TODO: make separate function that returns a robust cov plus info use_self = kwargs.pop('use_self', False) if use_self: res = self else: res = self.__class__( self.model, self.params, normalized_cov_params=self.normalized_cov_params, scale=self.scale) res.cov_type = cov_type # use_t might already be defined by the class, and already set if use_t is None: use_t = self.use_t res.cov_kwds = {'use_t': use_t} # store for information res.use_t = use_t adjust_df = False if cov_type in ['cluster', 'hac-panel', 'hac-groupsum']: df_correction = kwargs.get('df_correction', None) # TODO: check also use_correction, do I need all combinations? if df_correction is not False: # i.e. in [None, True]: # user did not explicitely set it to False adjust_df = True res.cov_kwds['adjust_df'] = adjust_df # verify and set kwargs, and calculate cov # TODO: this should be outsourced in a function so we can reuse it in # other models # TODO: make it DRYer repeated code for checking kwargs if cov_type in ['fixed scale', 'fixed_scale']: res.cov_kwds['description'] = descriptions['fixed_scale'] res.cov_kwds['scale'] = scale = kwargs.get('scale', 1.) res.cov_params_default = scale * res.normalized_cov_params elif cov_type.upper() in ('HC0', 'HC1', 'HC2', 'HC3'): if kwargs: raise ValueError('heteroscedasticity robust covariance ' 'does not use keywords') res.cov_kwds['description'] = descriptions[cov_type.upper()] res.cov_params_default = getattr(self, 'cov_' + cov_type.upper()) elif cov_type.lower() == 'hac': # TODO: check if required, default in cov_hac_simple maxlags = kwargs['maxlags'] res.cov_kwds['maxlags'] = maxlags weights_func = kwargs.get('weights_func', sw.weights_bartlett) res.cov_kwds['weights_func'] = weights_func use_correction = kwargs.get('use_correction', False) res.cov_kwds['use_correction'] = use_correction res.cov_kwds['description'] = descriptions['HAC'].format( maxlags=maxlags, correction=['without', 'with'][use_correction]) res.cov_params_default = sw.cov_hac_simple( self, nlags=maxlags, weights_func=weights_func, use_correction=use_correction) elif cov_type.lower() == 'cluster': # cluster robust standard errors, one- or two-way groups = kwargs['groups'] if not hasattr(groups, 'shape'): groups = [np.squeeze(np.asarray(group)) for group in groups] groups = np.asarray(groups).T if groups.ndim >= 2: groups = groups.squeeze() res.cov_kwds['groups'] = groups use_correction = kwargs.get('use_correction', True) res.cov_kwds['use_correction'] = use_correction if groups.ndim == 1: if adjust_df: # need to find number of groups # duplicate work self.n_groups = n_groups = len(np.unique(groups)) res.cov_params_default = sw.cov_cluster( self, groups, use_correction=use_correction) elif groups.ndim == 2: if hasattr(groups, 'values'): groups = groups.values if adjust_df: # need to find number of groups # duplicate work n_groups0 = len(np.unique(groups[:, 0])) n_groups1 = len(np.unique(groups[:, 1])) self.n_groups = (n_groups0, n_groups1) n_groups = min(n_groups0, n_groups1) # use for adjust_df # Note: sw.cov_cluster_2groups has 3 returns res.cov_params_default = sw.cov_cluster_2groups( self, groups, use_correction=use_correction)[0] else: raise ValueError('only two groups are supported') res.cov_kwds['description'] = descriptions['cluster'] elif cov_type.lower() == 'hac-panel': # cluster robust standard errors res.cov_kwds['time'] = time = kwargs.get('time', None) res.cov_kwds['groups'] = groups = kwargs.get('groups', None) # TODO: nlags is currently required # nlags = kwargs.get('nlags', True) # res.cov_kwds['nlags'] = nlags # TODO: `nlags` or `maxlags` res.cov_kwds['maxlags'] = maxlags = kwargs['maxlags'] use_correction = kwargs.get('use_correction', 'hac') res.cov_kwds['use_correction'] = use_correction weights_func = kwargs.get('weights_func', sw.weights_bartlett) res.cov_kwds['weights_func'] = weights_func if groups is not None: groups = np.asarray(groups) tt = (np.nonzero(groups[:-1] != groups[1:])[0] + 1).tolist() nobs_ = len(groups) elif time is not None: time = np.asarray(time) # TODO: clumsy time index in cov_nw_panel tt = (np.nonzero(time[1:] < time[:-1])[0] + 1).tolist() nobs_ = len(time) else: raise ValueError('either time or groups needs to be given') groupidx = lzip([0] + tt, tt + [nobs_]) self.n_groups = n_groups = len(groupidx) res.cov_params_default = sw.cov_nw_panel( self, maxlags, groupidx, weights_func=weights_func, use_correction=use_correction ) res.cov_kwds['description'] = descriptions['HAC-Panel'] elif cov_type.lower() == 'hac-groupsum': # Driscoll-Kraay standard errors res.cov_kwds['time'] = time = kwargs['time'] # TODO: nlags is currently required # nlags = kwargs.get('nlags', True) # res.cov_kwds['nlags'] = nlags # TODO: `nlags` or `maxlags` res.cov_kwds['maxlags'] = maxlags = kwargs['maxlags'] use_correction = kwargs.get('use_correction', 'cluster') res.cov_kwds['use_correction'] = use_correction weights_func = kwargs.get('weights_func', sw.weights_bartlett) res.cov_kwds['weights_func'] = weights_func if adjust_df: # need to find number of groups tt = (np.nonzero(time[1:] < time[:-1])[0] + 1) self.n_groups = n_groups = len(tt) + 1 res.cov_params_default = sw.cov_nw_groupsum( self, maxlags, time, weights_func=weights_func, use_correction=use_correction) res.cov_kwds['description'] = descriptions['HAC-Groupsum'] else: raise ValueError('cov_type not recognized. See docstring for ' + 'available options and spelling') if adjust_df: # Note: df_resid is used for scale and others, add new attribute res.df_resid_inference = n_groups - 1 return res @Appender(pred.get_prediction.__doc__) def get_prediction(self, exog=None, transform=True, weights=None, row_labels=None, **kwargs): return pred.get_prediction( self, exog=exog, transform=transform, weights=weights, row_labels=row_labels, **kwargs) def summary( self, yname: str | None = None, xname: Sequence[str] | None = None, title: str | None = None, alpha: float = 0.05, slim: bool = False, ): """ Summarize the Regression Results. Parameters ---------- yname : str, optional Name of endogenous (response) variable. The Default is `y`. xname : list[str], optional Names for the exogenous variables. Default is `var_##` for ## in the number of regressors. Must match the number of parameters in the model. title : str, optional Title for the top table. If not None, then this replaces the default title. alpha : float, optional The significance level for the confidence intervals. slim : bool, optional Flag indicating to produce reduced set or diagnostic information. Default is False. Returns ------- Summary Instance holding the summary tables and text, which can be printed or converted to various output formats. See Also -------- statsmodels.iolib.summary.Summary : A class that holds summary results. """ from statsmodels.stats.stattools import ( durbin_watson, jarque_bera, omni_normtest, ) alpha = float_like(alpha, "alpha", optional=False) slim = bool_like(slim, "slim", optional=False, strict=True) jb, jbpv, skew, kurtosis = jarque_bera(self.wresid) omni, omnipv = omni_normtest(self.wresid) eigvals = self.eigenvals condno = self.condition_number # TODO: Avoid adding attributes in non-__init__ self.diagn = dict(jb=jb, jbpv=jbpv, skew=skew, kurtosis=kurtosis, omni=omni, omnipv=omnipv, condno=condno, mineigval=eigvals[-1]) # TODO not used yet # diagn_left_header = ['Models stats'] # diagn_right_header = ['Residual stats'] # TODO: requiring list/iterable is a bit annoying # need more control over formatting # TODO: default do not work if it's not identically spelled top_left = [('Dep. Variable:', None), ('Model:', None), ('Method:', ['Least Squares']), ('Date:', None), ('Time:', None), ('No. Observations:', None), ('Df Residuals:', None), ('Df Model:', None), ] if hasattr(self, 'cov_type'): top_left.append(('Covariance Type:', [self.cov_type])) rsquared_type = '' if self.k_constant else ' (uncentered)' top_right = [('R-squared' + rsquared_type + ':', ["%#8.3f" % self.rsquared]), ('Adj. R-squared' + rsquared_type + ':', ["%#8.3f" % self.rsquared_adj]), ('F-statistic:', ["%#8.4g" % self.fvalue]), ('Prob (F-statistic):', ["%#6.3g" % self.f_pvalue]), ('Log-Likelihood:', None), ('AIC:', ["%#8.4g" % self.aic]), ('BIC:', ["%#8.4g" % self.bic]) ] if slim: slimlist = ['Dep. Variable:', 'Model:', 'No. Observations:', 'Covariance Type:', 'R-squared:', 'Adj. R-squared:', 'F-statistic:', 'Prob (F-statistic):'] diagn_left = diagn_right = [] top_left = [elem for elem in top_left if elem[0] in slimlist] top_right = [elem for elem in top_right if elem[0] in slimlist] top_right = top_right + \ [("", [])] * (len(top_left) - len(top_right)) else: diagn_left = [('Omnibus:', ["%#6.3f" % omni]), ('Prob(Omnibus):', ["%#6.3f" % omnipv]), ('Skew:', ["%#6.3f" % skew]), ('Kurtosis:', ["%#6.3f" % kurtosis]) ] diagn_right = [('Durbin-Watson:', ["%#8.3f" % durbin_watson(self.wresid)] ), ('Jarque-Bera (JB):', ["%#8.3f" % jb]), ('Prob(JB):', ["%#8.3g" % jbpv]), ('Cond. No.', ["%#8.3g" % condno]) ] if title is None: title = self.model.__class__.__name__ + ' ' + "Regression Results" # create summary table instance from statsmodels.iolib.summary import Summary smry = Summary() smry.add_table_2cols(self, gleft=top_left, gright=top_right, yname=yname, xname=xname, title=title) smry.add_table_params(self, yname=yname, xname=xname, alpha=alpha, use_t=self.use_t) if not slim: smry.add_table_2cols(self, gleft=diagn_left, gright=diagn_right, yname=yname, xname=xname, title="") # add warnings/notes, added to text format only etext = [] if not self.k_constant: etext.append( "R² is computed without centering (uncentered) since the " "model does not contain a constant." ) if hasattr(self, 'cov_type'): etext.append(self.cov_kwds['description']) if self.model.exog.shape[0] < self.model.exog.shape[1]: wstr = "The input rank is higher than the number of observations." etext.append(wstr) if eigvals[-1] < 1e-10: wstr = "The smallest eigenvalue is %6.3g. This might indicate " wstr += "that there are\n" wstr += "strong multicollinearity problems or that the design " wstr += "matrix is singular." wstr = wstr % eigvals[-1] etext.append(wstr) elif condno > 1000: # TODO: what is recommended? wstr = "The condition number is large, %6.3g. This might " wstr += "indicate that there are\n" wstr += "strong multicollinearity or other numerical " wstr += "problems." wstr = wstr % condno etext.append(wstr) if etext: etext = [f"[{i + 1}] {text}" for i, text in enumerate(etext)] etext.insert(0, "Notes:") smry.add_extra_txt(etext) return smry def summary2( self, yname: str | None = None, xname: Sequence[str] | None = None, title: str | None = None, alpha: float = 0.05, float_format: str = "%.4f", ): """ Experimental summary function to summarize the regression results. Parameters ---------- yname : str The name of the dependent variable (optional). xname : list[str], optional Names for the exogenous variables. Default is `var_##` for ## in the number of regressors. Must match the number of parameters in the model. title : str, optional Title for the top table. If not None, then this replaces the default title. alpha : float The significance level for the confidence intervals. float_format : str The format for floats in parameters summary. Returns ------- Summary Instance holding the summary tables and text, which can be printed or converted to various output formats. See Also -------- statsmodels.iolib.summary2.Summary A class that holds summary results. """ # Diagnostics from statsmodels.stats.stattools import ( durbin_watson, jarque_bera, omni_normtest, ) jb, jbpv, skew, kurtosis = jarque_bera(self.wresid) omni, omnipv = omni_normtest(self.wresid) dw = durbin_watson(self.wresid) eigvals = self.eigenvals condno = self.condition_number diagnostic = dict([ ('Omnibus:', "%.3f" % omni), ('Prob(Omnibus):', "%.3f" % omnipv), ('Skew:', "%.3f" % skew), ('Kurtosis:', "%.3f" % kurtosis), ('Durbin-Watson:', "%.3f" % dw), ('Jarque-Bera (JB):', "%.3f" % jb), ('Prob(JB):', "%.3f" % jbpv), ('Condition No.:', "%.0f" % condno) ]) # Summary from statsmodels.iolib import summary2 smry = summary2.Summary() smry.add_base(results=self, alpha=alpha, float_format=float_format, xname=xname, yname=yname, title=title) smry.add_dict(diagnostic) etext = [] if not self.k_constant: etext.append( "R² is computed without centering (uncentered) since the \ model does not contain a constant." ) if hasattr(self, 'cov_type'): etext.append(self.cov_kwds['description']) if self.model.exog.shape[0] < self.model.exog.shape[1]: wstr = "The input rank is higher than the number of observations." etext.append(wstr) # Warnings if eigvals[-1] < 1e-10: warn = "The smallest eigenvalue is %6.3g. This might indicate that\ there are strong multicollinearity problems or that the design\ matrix is singular." % eigvals[-1] etext.append(warn) elif condno > 1000: warn = "The condition number is large, %6.3g. This might indicate\ that there are strong multicollinearity or other numerical\ problems." % condno etext.append(warn) if etext: etext = [f"[{i + 1}] {text}" for i, text in enumerate(etext)] etext.insert(0, "Notes:") for line in etext: smry.add_text(line) return smry class OLSResults(RegressionResults): """ Results class for for an OLS model. Parameters ---------- model : RegressionModel The regression model instance. params : ndarray The estimated parameters. normalized_cov_params : ndarray The normalized covariance parameters. scale : float The estimated scale of the residuals. cov_type : str The covariance estimator used in the results. cov_kwds : dict Additional keywords used in the covariance specification. use_t : bool Flag indicating to use the Student's t in inference. **kwargs Additional keyword arguments used to initialize the results. See Also -------- RegressionResults Results store for WLS and GLW models. Notes ----- Most of the methods and attributes are inherited from RegressionResults. The special methods that are only available for OLS are: - get_influence - outlier_test - el_test - conf_int_el """ def get_influence(self): """ Calculate influence and outlier measures. Returns ------- OLSInfluence The instance containing methods to calculate the main influence and outlier measures for the OLS regression. See Also -------- statsmodels.stats.outliers_influence.OLSInfluence A class that exposes methods to examine observation influence. """ from statsmodels.stats.outliers_influence import OLSInfluence return OLSInfluence(self) def outlier_test(self, method='bonf', alpha=.05, labels=None, order=False, cutoff=None): """ Test observations for outliers according to method. Parameters ---------- method : str The method to use in the outlier test. Must be one of: - `bonferroni` : one-step correction - `sidak` : one-step correction - `holm-sidak` : - `holm` : - `simes-hochberg` : - `hommel` : - `fdr_bh` : Benjamini/Hochberg - `fdr_by` : Benjamini/Yekutieli See `statsmodels.stats.multitest.multipletests` for details. alpha : float The familywise error rate (FWER). labels : None or array_like If `labels` is not None, then it will be used as index to the returned pandas DataFrame. See also Returns below. order : bool Whether or not to order the results by the absolute value of the studentized residuals. If labels are provided they will also be sorted. cutoff : None or float in [0, 1] If cutoff is not None, then the return only includes observations with multiple testing corrected p-values strictly below the cutoff. The returned array or dataframe can be empty if t. Returns ------- array_like Returns either an ndarray or a DataFrame if labels is not None. Will attempt to get labels from model_results if available. The columns are the Studentized residuals, the unadjusted p-value, and the corrected p-value according to method. Notes ----- The unadjusted p-value is stats.t.sf(abs(resid), df) where df = df_resid - 1. """ from statsmodels.stats.outliers_influence import outlier_test return outlier_test(self, method, alpha, labels=labels, order=order, cutoff=cutoff) def el_test(self, b0_vals, param_nums, return_weights=0, ret_params=0, method='nm', stochastic_exog=1): """ Test single or joint hypotheses using Empirical Likelihood. Parameters ---------- b0_vals : 1darray The hypothesized value of the parameter to be tested. param_nums : 1darray The parameter number to be tested. return_weights : bool If true, returns the weights that optimize the likelihood ratio at b0_vals. The default is False. ret_params : bool If true, returns the parameter vector that maximizes the likelihood ratio at b0_vals. Also returns the weights. The default is False. method : str Can either be 'nm' for Nelder-Mead or 'powell' for Powell. The optimization method that optimizes over nuisance parameters. The default is 'nm'. stochastic_exog : bool When True, the exogenous variables are assumed to be stochastic. When the regressors are nonstochastic, moment conditions are placed on the exogenous variables. Confidence intervals for stochastic regressors are at least as large as non-stochastic regressors. The default is True. Returns ------- tuple The p-value and -2 times the log-likelihood ratio for the hypothesized values. Examples -------- >>> import statsmodels.api as sm >>> data = sm.datasets.stackloss.load() >>> endog = data.endog >>> exog = sm.add_constant(data.exog) >>> model = sm.OLS(endog, exog) >>> fitted = model.fit() >>> fitted.params >>> array([-39.91967442, 0.7156402 , 1.29528612, -0.15212252]) >>> fitted.rsquared >>> 0.91357690446068196 >>> # Test that the slope on the first variable is 0 >>> fitted.el_test([0], [1]) >>> (27.248146353888796, 1.7894660442330235e-07) """ params = np.copy(self.params) opt_fun_inst = _ELRegOpts() # to store weights if len(param_nums) == len(params): llr = opt_fun_inst._opt_nuis_regress( [], param_nums=param_nums, endog=self.model.endog, exog=self.model.exog, nobs=self.model.nobs, nvar=self.model.exog.shape[1], params=params, b0_vals=b0_vals, stochastic_exog=stochastic_exog) pval = 1 - stats.chi2.cdf(llr, len(param_nums)) if return_weights: return llr, pval, opt_fun_inst.new_weights else: return llr, pval x0 = np.delete(params, param_nums) args = (param_nums, self.model.endog, self.model.exog, self.model.nobs, self.model.exog.shape[1], params, b0_vals, stochastic_exog) if method == 'nm': llr = optimize.fmin(opt_fun_inst._opt_nuis_regress, x0, maxfun=10000, maxiter=10000, full_output=1, disp=0, args=args)[1] if method == 'powell': llr = optimize.fmin_powell(opt_fun_inst._opt_nuis_regress, x0, full_output=1, disp=0, args=args)[1] pval = 1 - stats.chi2.cdf(llr, len(param_nums)) if ret_params: return llr, pval, opt_fun_inst.new_weights, opt_fun_inst.new_params elif return_weights: return llr, pval, opt_fun_inst.new_weights else: return llr, pval def conf_int_el(self, param_num, sig=.05, upper_bound=None, lower_bound=None, method='nm', stochastic_exog=True): """ Compute the confidence interval using Empirical Likelihood. Parameters ---------- param_num : float The parameter for which the confidence interval is desired. sig : float The significance level. Default is 0.05. upper_bound : float The maximum value the upper limit can be. Default is the 99.9% confidence value under OLS assumptions. lower_bound : float The minimum value the lower limit can be. Default is the 99.9% confidence value under OLS assumptions. method : str Can either be 'nm' for Nelder-Mead or 'powell' for Powell. The optimization method that optimizes over nuisance parameters. The default is 'nm'. stochastic_exog : bool When True, the exogenous variables are assumed to be stochastic. When the regressors are nonstochastic, moment conditions are placed on the exogenous variables. Confidence intervals for stochastic regressors are at least as large as non-stochastic regressors. The default is True. Returns ------- lowerl : float The lower bound of the confidence interval. upperl : float The upper bound of the confidence interval. See Also -------- el_test : Test parameters using Empirical Likelihood. Notes ----- This function uses brentq to find the value of beta where test_beta([beta], param_num)[1] is equal to the critical value. The function returns the results of each iteration of brentq at each value of beta. The current function value of the last printed optimization should be the critical value at the desired significance level. For alpha=.05, the value is 3.841459. To ensure optimization terminated successfully, it is suggested to do el_test([lower_limit], [param_num]). If the optimization does not terminate successfully, consider switching optimization algorithms. If optimization is still not successful, try changing the values of start_int_params. If the current function value repeatedly jumps from a number between 0 and the critical value and a very large number (>50), the starting parameters of the interior minimization need to be changed. """ r0 = stats.chi2.ppf(1 - sig, 1) if upper_bound is None: upper_bound = self.conf_int(.01)[param_num][1] if lower_bound is None: lower_bound = self.conf_int(.01)[param_num][0] def f(b0): return self.el_test(np.array([b0]), np.array([param_num]), method=method, stochastic_exog=stochastic_exog)[0] - r0 lowerl = optimize.brenth(f, lower_bound, self.params[param_num]) upperl = optimize.brenth(f, self.params[param_num], upper_bound) # ^ Seems to be faster than brentq in most cases return (lowerl, upperl) class RegressionResultsWrapper(wrap.ResultsWrapper): _attrs = { 'chisq': 'columns', 'sresid': 'rows', 'weights': 'rows', 'wresid': 'rows', 'bcov_unscaled': 'cov', 'bcov_scaled': 'cov', 'HC0_se': 'columns', 'HC1_se': 'columns', 'HC2_se': 'columns', 'HC3_se': 'columns', 'norm_resid': 'rows', } _wrap_attrs = wrap.union_dicts(base.LikelihoodResultsWrapper._attrs, _attrs) _methods = {} _wrap_methods = wrap.union_dicts( base.LikelihoodResultsWrapper._wrap_methods, _methods) wrap.populate_wrapper(RegressionResultsWrapper, RegressionResults)