Coverage for /home/martinb/.local/share/virtualenvs/camcops/lib/python3.6/site-packages/statsmodels/base/l1_solvers_common.py : 8%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Holds common functions for l1 solvers.
3"""
5import numpy as np
7from statsmodels.tools.sm_exceptions import ConvergenceWarning
10def qc_results(params, alpha, score, qc_tol, qc_verbose=False):
11 """
12 Theory dictates that one of two conditions holds:
13 i) abs(score[i]) == alpha[i] and params[i] != 0
14 ii) abs(score[i]) <= alpha[i] and params[i] == 0
15 qc_results checks to see that (ii) holds, within qc_tol
17 qc_results also checks for nan or results of the wrong shape.
19 Parameters
20 ----------
21 params : ndarray
22 model parameters. Not including the added variables x_added.
23 alpha : ndarray
24 regularization coefficients
25 score : function
26 Gradient of unregularized objective function
27 qc_tol : float
28 Tolerance to hold conditions (i) and (ii) to for QC check.
29 qc_verbose : bool
30 If true, print out a full QC report upon failure
32 Returns
33 -------
34 passed : bool
35 True if QC check passed
36 qc_dict : Dictionary
37 Keys are fprime, alpha, params, passed_array
39 Prints
40 ------
41 Warning message if QC check fails.
42 """
43 ## Check for fatal errors
44 assert not np.isnan(params).max()
45 assert (params == params.ravel('F')).min(), \
46 "params should have already been 1-d"
48 ## Start the theory compliance check
49 fprime = score(params)
50 k_params = len(params)
52 passed_array = np.array([True] * k_params)
53 for i in range(k_params):
54 if alpha[i] > 0:
55 # If |fprime| is too big, then something went wrong
56 if (abs(fprime[i]) - alpha[i]) / alpha[i] > qc_tol:
57 passed_array[i] = False
58 qc_dict = dict(
59 fprime=fprime, alpha=alpha, params=params, passed_array=passed_array)
60 passed = passed_array.min()
61 if not passed:
62 num_failed = (~passed_array).sum()
63 message = 'QC check did not pass for %d out of %d parameters' % (
64 num_failed, k_params)
65 message += '\nTry increasing solver accuracy or number of iterations'\
66 ', decreasing alpha, or switch solvers'
67 if qc_verbose:
68 message += _get_verbose_addon(qc_dict)
70 import warnings
71 warnings.warn(message, ConvergenceWarning)
73 return passed
76def _get_verbose_addon(qc_dict):
77 alpha = qc_dict['alpha']
78 params = qc_dict['params']
79 fprime = qc_dict['fprime']
80 passed_array = qc_dict['passed_array']
82 addon = '\n------ verbose QC printout -----------------'
83 addon = '\n------ Recall the problem was rescaled by 1 / nobs ---'
84 addon += '\n|%-10s|%-10s|%-10s|%-10s|' % (
85 'passed', 'alpha', 'fprime', 'param')
86 addon += '\n--------------------------------------------'
87 for i in range(len(alpha)):
88 addon += '\n|%-10s|%-10.3e|%-10.3e|%-10.3e|' % (
89 passed_array[i], alpha[i], fprime[i], params[i])
90 return addon
93def do_trim_params(params, k_params, alpha, score, passed, trim_mode,
94 size_trim_tol, auto_trim_tol):
95 """
96 Trims (set to zero) params that are zero at the theoretical minimum.
97 Uses heuristics to account for the solver not actually finding the minimum.
99 In all cases, if alpha[i] == 0, then do not trim the ith param.
100 In all cases, do nothing with the added variables.
102 Parameters
103 ----------
104 params : ndarray
105 model parameters. Not including added variables.
106 k_params : Int
107 Number of parameters
108 alpha : ndarray
109 regularization coefficients
110 score : Function.
111 score(params) should return a 1-d vector of derivatives of the
112 unpenalized objective function.
113 passed : bool
114 True if the QC check passed
115 trim_mode : 'auto, 'size', or 'off'
116 If not 'off', trim (set to zero) parameters that would have been zero
117 if the solver reached the theoretical minimum.
118 If 'auto', trim params using the Theory above.
119 If 'size', trim params if they have very small absolute value
120 size_trim_tol : float or 'auto' (default = 'auto')
121 For use when trim_mode === 'size'
122 auto_trim_tol : float
123 For sue when trim_mode == 'auto'. Use
124 qc_tol : float
125 Print warning and do not allow auto trim when (ii) in "Theory" (above)
126 is violated by this much.
128 Returns
129 -------
130 params : ndarray
131 Trimmed model parameters
132 trimmed : ndarray of booleans
133 trimmed[i] == True if the ith parameter was trimmed.
134 """
135 ## Trim the small params
136 trimmed = [False] * k_params
138 if trim_mode == 'off':
139 trimmed = np.array([False] * k_params)
140 elif trim_mode == 'auto' and not passed:
141 import warnings
142 msg = "Could not trim params automatically due to failed QC check. " \
143 "Trimming using trim_mode == 'size' will still work."
144 warnings.warn(msg, ConvergenceWarning)
145 trimmed = np.array([False] * k_params)
146 elif trim_mode == 'auto' and passed:
147 fprime = score(params)
148 for i in range(k_params):
149 if alpha[i] != 0:
150 if (alpha[i] - abs(fprime[i])) / alpha[i] > auto_trim_tol:
151 params[i] = 0.0
152 trimmed[i] = True
153 elif trim_mode == 'size':
154 for i in range(k_params):
155 if alpha[i] != 0:
156 if abs(params[i]) < size_trim_tol:
157 params[i] = 0.0
158 trimmed[i] = True
159 else:
160 raise ValueError(
161 "trim_mode == %s, which is not recognized" % (trim_mode))
163 return params, np.asarray(trimmed)