Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" 

2State Space Representation and Kalman Filter, Smoother 

3 

4Author: Chad Fulton 

5License: Simplified-BSD 

6""" 

7 

8import numpy as np 

9 

10from statsmodels.tsa.statespace.representation import OptionWrapper 

11from statsmodels.tsa.statespace.kalman_filter import (KalmanFilter, 

12 FilterResults) 

13from statsmodels.tsa.statespace.tools import ( 

14 reorder_missing_matrix, reorder_missing_vector, copy_index_matrix) 

15from statsmodels.tsa.statespace import tools 

16 

17SMOOTHER_STATE = 0x01 # Durbin and Koopman (2012), Chapter 4.4.2 

18SMOOTHER_STATE_COV = 0x02 # ibid., Chapter 4.4.3 

19SMOOTHER_DISTURBANCE = 0x04 # ibid., Chapter 4.5 

20SMOOTHER_DISTURBANCE_COV = 0x08 # ibid., Chapter 4.5 

21SMOOTHER_STATE_AUTOCOV = 0x10 # ibid., Chapter 4.7 

22SMOOTHER_ALL = ( 

23 SMOOTHER_STATE | SMOOTHER_STATE_COV | SMOOTHER_DISTURBANCE | 

24 SMOOTHER_DISTURBANCE_COV | SMOOTHER_STATE_AUTOCOV 

25) 

26 

27SMOOTH_CONVENTIONAL = 0x01 

28SMOOTH_CLASSICAL = 0x02 

29SMOOTH_ALTERNATIVE = 0x04 

30SMOOTH_UNIVARIATE = 0x08 

31 

32 

33class KalmanSmoother(KalmanFilter): 

34 r""" 

35 State space representation of a time series process, with Kalman filter 

36 and smoother. 

37 

38 Parameters 

39 ---------- 

40 k_endog : {array_like, int} 

41 The observed time-series process :math:`y` if array like or the 

42 number of variables in the process if an integer. 

43 k_states : int 

44 The dimension of the unobserved state process. 

45 k_posdef : int, optional 

46 The dimension of a guaranteed positive definite covariance matrix 

47 describing the shocks in the measurement equation. Must be less than 

48 or equal to `k_states`. Default is `k_states`. 

49 results_class : class, optional 

50 Default results class to use to save filtering output. Default is 

51 `SmootherResults`. If specified, class must extend from 

52 `SmootherResults`. 

53 **kwargs 

54 Keyword arguments may be used to provide default values for state space 

55 matrices, for Kalman filtering options, or for Kalman smoothing 

56 options. See `Representation` for more details. 

57 """ 

58 

59 smoother_outputs = [ 

60 'smoother_state', 'smoother_state_cov', 'smoother_state_autocov', 

61 'smoother_disturbance', 'smoother_disturbance_cov', 'smoother_all', 

62 ] 

63 

64 smoother_state = OptionWrapper('smoother_output', SMOOTHER_STATE) 

65 smoother_state_cov = OptionWrapper('smoother_output', SMOOTHER_STATE_COV) 

66 smoother_disturbance = ( 

67 OptionWrapper('smoother_output', SMOOTHER_DISTURBANCE) 

68 ) 

69 smoother_disturbance_cov = ( 

70 OptionWrapper('smoother_output', SMOOTHER_DISTURBANCE_COV) 

71 ) 

72 smoother_state_autocov = ( 

73 OptionWrapper('smoother_output', SMOOTHER_STATE_AUTOCOV) 

74 ) 

75 smoother_all = OptionWrapper('smoother_output', SMOOTHER_ALL) 

76 

77 smooth_methods = [ 

78 'smooth_conventional', 'smooth_alternative', 'smooth_classical' 

79 ] 

80 

81 smooth_conventional = OptionWrapper('smooth_method', SMOOTH_CONVENTIONAL) 

82 """ 

83 (bool) Flag for conventional (Durbin and Koopman, 2012) Kalman smoothing. 

84 """ 

85 smooth_alternative = OptionWrapper('smooth_method', SMOOTH_ALTERNATIVE) 

86 """ 

87 (bool) Flag for alternative (modified Bryson-Frazier) smoothing. 

88 """ 

89 smooth_classical = OptionWrapper('smooth_method', SMOOTH_CLASSICAL) 

90 """ 

91 (bool) Flag for classical (see e.g. Anderson and Moore, 1979) smoothing. 

92 """ 

93 smooth_univariate = OptionWrapper('smooth_method', SMOOTH_UNIVARIATE) 

94 """ 

95 (bool) Flag for univariate smoothing (uses modified Bryson-Frazier timing). 

96 """ 

97 

98 # Default smoother options 

99 smoother_output = SMOOTHER_ALL 

100 smooth_method = 0 

101 

102 def __init__(self, k_endog, k_states, k_posdef=None, results_class=None, 

103 kalman_smoother_classes=None, **kwargs): 

104 # Set the default results class 

105 if results_class is None: 

106 results_class = SmootherResults 

107 

108 super(KalmanSmoother, self).__init__( 

109 k_endog, k_states, k_posdef, results_class=results_class, **kwargs 

110 ) 

111 

112 # Options 

113 self.prefix_kalman_smoother_map = ( 

114 kalman_smoother_classes 

115 if kalman_smoother_classes is not None 

116 else tools.prefix_kalman_smoother_map.copy()) 

117 

118 # Setup the underlying Kalman smoother storage 

119 self._kalman_smoothers = {} 

120 

121 # Set the smoother options 

122 self.set_smoother_output(**kwargs) 

123 self.set_smooth_method(**kwargs) 

124 

125 def _clone_kwargs(self, endog, **kwargs): 

126 # See Representation._clone_kwargs for docstring 

127 kwargs = super(KalmanSmoother, self)._clone_kwargs(endog, **kwargs) 

128 

129 # Get defaults for options 

130 kwargs.setdefault('smoother_output', self.smoother_output) 

131 kwargs.setdefault('smooth_method', self.smooth_method) 

132 

133 return kwargs 

134 

135 @property 

136 def _kalman_smoother(self): 

137 prefix = self.prefix 

138 if prefix in self._kalman_smoothers: 

139 return self._kalman_smoothers[prefix] 

140 return None 

141 

142 def _initialize_smoother(self, smoother_output=None, smooth_method=None, 

143 prefix=None, **kwargs): 

144 if smoother_output is None: 

145 smoother_output = self.smoother_output 

146 if smooth_method is None: 

147 smooth_method = self.smooth_method 

148 

149 # Make sure we have the required Kalman filter 

150 prefix, dtype, create_filter, create_statespace = ( 

151 self._initialize_filter(prefix, **kwargs) 

152 ) 

153 

154 # Determine if we need to (re-)create the smoother 

155 # (definitely need to recreate if we recreated the filter) 

156 create_smoother = (create_filter or 

157 prefix not in self._kalman_smoothers) 

158 if not create_smoother: 

159 kalman_smoother = self._kalman_smoothers[prefix] 

160 

161 create_smoother = (kalman_smoother.kfilter is not 

162 self._kalman_filters[prefix]) 

163 

164 # If the dtype-specific _kalman_smoother does not exist (or if we 

165 # need to re-create it), create it 

166 if create_smoother: 

167 # Setup the smoother 

168 cls = self.prefix_kalman_smoother_map[prefix] 

169 self._kalman_smoothers[prefix] = cls( 

170 self._statespaces[prefix], self._kalman_filters[prefix], 

171 smoother_output, smooth_method 

172 ) 

173 # Otherwise, update the smoother parameters 

174 else: 

175 self._kalman_smoothers[prefix].set_smoother_output( 

176 smoother_output, False) 

177 self._kalman_smoothers[prefix].set_smooth_method(smooth_method) 

178 

179 return prefix, dtype, create_smoother, create_filter, create_statespace 

180 

181 def set_smoother_output(self, smoother_output=None, **kwargs): 

182 """ 

183 Set the smoother output 

184 

185 The smoother can produce several types of results. The smoother output 

186 variable controls which are calculated and returned. 

187 

188 Parameters 

189 ---------- 

190 smoother_output : int, optional 

191 Bitmask value to set the smoother output to. See notes for details. 

192 **kwargs 

193 Keyword arguments may be used to influence the smoother output by 

194 setting individual boolean flags. See notes for details. 

195 

196 Notes 

197 ----- 

198 The smoother output is defined by a collection of boolean flags, and 

199 is internally stored as a bitmask. The methods available are: 

200 

201 SMOOTHER_STATE = 0x01 

202 Calculate and return the smoothed states. 

203 SMOOTHER_STATE_COV = 0x02 

204 Calculate and return the smoothed state covariance matrices. 

205 SMOOTHER_STATE_AUTOCOV = 0x10 

206 Calculate and return the smoothed state lag-one autocovariance 

207 matrices. 

208 SMOOTHER_DISTURBANCE = 0x04 

209 Calculate and return the smoothed state and observation 

210 disturbances. 

211 SMOOTHER_DISTURBANCE_COV = 0x08 

212 Calculate and return the covariance matrices for the smoothed state 

213 and observation disturbances. 

214 SMOOTHER_ALL 

215 Calculate and return all results. 

216 

217 If the bitmask is set directly via the `smoother_output` argument, then 

218 the full method must be provided. 

219 

220 If keyword arguments are used to set individual boolean flags, then 

221 the lowercase of the method must be used as an argument name, and the 

222 value is the desired value of the boolean flag (True or False). 

223 

224 Note that the smoother output may also be specified by directly 

225 modifying the class attributes which are defined similarly to the 

226 keyword arguments. 

227 

228 The default smoother output is SMOOTHER_ALL. 

229 

230 If performance is a concern, only those results which are needed should 

231 be specified as any results that are not specified will not be 

232 calculated. For example, if the smoother output is set to only include 

233 SMOOTHER_STATE, the smoother operates much more quickly than if all 

234 output is required. 

235 

236 Examples 

237 -------- 

238 >>> import statsmodels.tsa.statespace.kalman_smoother as ks 

239 >>> mod = ks.KalmanSmoother(1,1) 

240 >>> mod.smoother_output 

241 15 

242 >>> mod.set_smoother_output(smoother_output=0) 

243 >>> mod.smoother_state = True 

244 >>> mod.smoother_output 

245 1 

246 >>> mod.smoother_state 

247 True 

248 """ 

249 if smoother_output is not None: 

250 self.smoother_output = smoother_output 

251 for name in KalmanSmoother.smoother_outputs: 

252 if name in kwargs: 

253 setattr(self, name, kwargs[name]) 

254 

255 def set_smooth_method(self, smooth_method=None, **kwargs): 

256 r""" 

257 Set the smoothing method 

258 

259 The smoothing method can be used to override the Kalman smoother 

260 approach used. By default, the Kalman smoother used depends on the 

261 Kalman filter method. 

262 

263 Parameters 

264 ---------- 

265 smooth_method : int, optional 

266 Bitmask value to set the filter method to. See notes for details. 

267 **kwargs 

268 Keyword arguments may be used to influence the filter method by 

269 setting individual boolean flags. See notes for details. 

270 

271 Notes 

272 ----- 

273 The smoothing method is defined by a collection of boolean flags, and 

274 is internally stored as a bitmask. The methods available are: 

275 

276 SMOOTH_CONVENTIONAL = 0x01 

277 Default Kalman smoother, as presented in Durbin and Koopman, 2012 

278 chapter 4. 

279 SMOOTH_CLASSICAL = 0x02 

280 Classical Kalman smoother, as presented in Anderson and Moore, 1979 

281 or Durbin and Koopman, 2012 chapter 4.6.1. 

282 SMOOTH_ALTERNATIVE = 0x04 

283 Modified Bryson-Frazier Kalman smoother method; this is identical 

284 to the conventional method of Durbin and Koopman, 2012, except that 

285 an additional intermediate step is included. 

286 SMOOTH_UNIVARIATE = 0x08 

287 Univariate Kalman smoother, as presented in Durbin and Koopman, 

288 2012 chapter 6, except with modified Bryson-Frazier timing. 

289 

290 Practically speaking, these methods should all produce the same output 

291 but different computational implications, numerical stability 

292 implications, or internal timing assumptions. 

293 

294 Note that only the first method is available if using a Scipy version 

295 older than 0.16. 

296 

297 If the bitmask is set directly via the `smooth_method` argument, then 

298 the full method must be provided. 

299 

300 If keyword arguments are used to set individual boolean flags, then 

301 the lowercase of the method must be used as an argument name, and the 

302 value is the desired value of the boolean flag (True or False). 

303 

304 Note that the filter method may also be specified by directly modifying 

305 the class attributes which are defined similarly to the keyword 

306 arguments. 

307 

308 The default filtering method is SMOOTH_CONVENTIONAL. 

309 

310 Examples 

311 -------- 

312 >>> mod = sm.tsa.statespace.SARIMAX(range(10)) 

313 >>> mod.smooth_method 

314 1 

315 >>> mod.filter_conventional 

316 True 

317 >>> mod.filter_univariate = True 

318 >>> mod.smooth_method 

319 17 

320 >>> mod.set_smooth_method(filter_univariate=False, 

321 filter_collapsed=True) 

322 >>> mod.smooth_method 

323 33 

324 >>> mod.set_smooth_method(smooth_method=1) 

325 >>> mod.filter_conventional 

326 True 

327 >>> mod.filter_univariate 

328 False 

329 >>> mod.filter_collapsed 

330 False 

331 >>> mod.filter_univariate = True 

332 >>> mod.smooth_method 

333 17 

334 """ 

335 if smooth_method is not None: 

336 self.smooth_method = smooth_method 

337 for name in KalmanSmoother.smooth_methods: 

338 if name in kwargs: 

339 setattr(self, name, kwargs[name]) 

340 

341 def _smooth(self, smoother_output=None, smooth_method=None, prefix=None, 

342 complex_step=False, results=None, **kwargs): 

343 # Initialize the smoother 

344 prefix, dtype, create_smoother, create_filter, create_statespace = ( 

345 self._initialize_smoother( 

346 smoother_output, smooth_method, prefix=prefix, **kwargs 

347 )) 

348 

349 # Check that the filter and statespace weren't just recreated 

350 if create_filter or create_statespace: 

351 raise ValueError('Passed settings forced re-creation of the' 

352 ' Kalman filter. Please run `_filter` before' 

353 ' running `_smooth`.') 

354 

355 # Get the appropriate smoother 

356 smoother = self._kalman_smoothers[prefix] 

357 

358 # Run the smoother 

359 smoother() 

360 

361 return smoother 

362 

363 def smooth(self, smoother_output=None, smooth_method=None, results=None, 

364 run_filter=True, prefix=None, complex_step=False, 

365 **kwargs): 

366 """ 

367 Apply the Kalman smoother to the statespace model. 

368 

369 Parameters 

370 ---------- 

371 smoother_output : int, optional 

372 Determines which Kalman smoother output calculate. Default is all 

373 (including state, disturbances, and all covariances). 

374 results : class or object, optional 

375 If a class, then that class is instantiated and returned with the 

376 result of both filtering and smoothing. 

377 If an object, then that object is updated with the smoothing data. 

378 If None, then a SmootherResults object is returned with both 

379 filtering and smoothing results. 

380 run_filter : bool, optional 

381 Whether or not to run the Kalman filter prior to smoothing. Default 

382 is True. 

383 prefix : str 

384 The prefix of the datatype. Usually only used internally. 

385 

386 Returns 

387 ------- 

388 SmootherResults object 

389 """ 

390 

391 # Run the filter 

392 kfilter = self._filter(**kwargs) 

393 

394 # Create the results object 

395 results = self.results_class(self) 

396 results.update_representation(self) 

397 results.update_filter(kfilter) 

398 

399 # Run the smoother 

400 if smoother_output is None: 

401 smoother_output = self.smoother_output 

402 smoother = self._smooth(smoother_output, results=results, **kwargs) 

403 

404 # Update the results 

405 results.update_smoother(smoother) 

406 

407 return results 

408 

409 

410class SmootherResults(FilterResults): 

411 r""" 

412 Results from applying the Kalman smoother and/or filter to a state space 

413 model. 

414 

415 Parameters 

416 ---------- 

417 model : Representation 

418 A Statespace representation 

419 

420 Attributes 

421 ---------- 

422 nobs : int 

423 Number of observations. 

424 k_endog : int 

425 The dimension of the observation series. 

426 k_states : int 

427 The dimension of the unobserved state process. 

428 k_posdef : int 

429 The dimension of a guaranteed positive definite covariance matrix 

430 describing the shocks in the measurement equation. 

431 dtype : dtype 

432 Datatype of representation matrices 

433 prefix : str 

434 BLAS prefix of representation matrices 

435 shapes : dictionary of name:tuple 

436 A dictionary recording the shapes of each of the representation 

437 matrices as tuples. 

438 endog : ndarray 

439 The observation vector. 

440 design : ndarray 

441 The design matrix, :math:`Z`. 

442 obs_intercept : ndarray 

443 The intercept for the observation equation, :math:`d`. 

444 obs_cov : ndarray 

445 The covariance matrix for the observation equation :math:`H`. 

446 transition : ndarray 

447 The transition matrix, :math:`T`. 

448 state_intercept : ndarray 

449 The intercept for the transition equation, :math:`c`. 

450 selection : ndarray 

451 The selection matrix, :math:`R`. 

452 state_cov : ndarray 

453 The covariance matrix for the state equation :math:`Q`. 

454 missing : array of bool 

455 An array of the same size as `endog`, filled with boolean values that 

456 are True if the corresponding entry in `endog` is NaN and False 

457 otherwise. 

458 nmissing : array of int 

459 An array of size `nobs`, where the ith entry is the number (between 0 

460 and k_endog) of NaNs in the ith row of the `endog` array. 

461 time_invariant : bool 

462 Whether or not the representation matrices are time-invariant 

463 initialization : str 

464 Kalman filter initialization method. 

465 initial_state : array_like 

466 The state vector used to initialize the Kalamn filter. 

467 initial_state_cov : array_like 

468 The state covariance matrix used to initialize the Kalamn filter. 

469 filter_method : int 

470 Bitmask representing the Kalman filtering method 

471 inversion_method : int 

472 Bitmask representing the method used to invert the forecast error 

473 covariance matrix. 

474 stability_method : int 

475 Bitmask representing the methods used to promote numerical stability in 

476 the Kalman filter recursions. 

477 conserve_memory : int 

478 Bitmask representing the selected memory conservation method. 

479 tolerance : float 

480 The tolerance at which the Kalman filter determines convergence to 

481 steady-state. 

482 loglikelihood_burn : int 

483 The number of initial periods during which the loglikelihood is not 

484 recorded. 

485 converged : bool 

486 Whether or not the Kalman filter converged. 

487 period_converged : int 

488 The time period in which the Kalman filter converged. 

489 filtered_state : ndarray 

490 The filtered state vector at each time period. 

491 filtered_state_cov : ndarray 

492 The filtered state covariance matrix at each time period. 

493 predicted_state : ndarray 

494 The predicted state vector at each time period. 

495 predicted_state_cov : ndarray 

496 The predicted state covariance matrix at each time period. 

497 kalman_gain : ndarray 

498 The Kalman gain at each time period. 

499 forecasts : ndarray 

500 The one-step-ahead forecasts of observations at each time period. 

501 forecasts_error : ndarray 

502 The forecast errors at each time period. 

503 forecasts_error_cov : ndarray 

504 The forecast error covariance matrices at each time period. 

505 loglikelihood : ndarray 

506 The loglikelihood values at each time period. 

507 collapsed_forecasts : ndarray 

508 If filtering using collapsed observations, stores the one-step-ahead 

509 forecasts of collapsed observations at each time period. 

510 collapsed_forecasts_error : ndarray 

511 If filtering using collapsed observations, stores the one-step-ahead 

512 forecast errors of collapsed observations at each time period. 

513 collapsed_forecasts_error_cov : ndarray 

514 If filtering using collapsed observations, stores the one-step-ahead 

515 forecast error covariance matrices of collapsed observations at each 

516 time period. 

517 standardized_forecast_error : ndarray 

518 The standardized forecast errors 

519 smoother_output : int 

520 Bitmask representing the generated Kalman smoothing output 

521 scaled_smoothed_estimator : ndarray 

522 The scaled smoothed estimator at each time period. 

523 scaled_smoothed_estimator_cov : ndarray 

524 The scaled smoothed estimator covariance matrices at each time period. 

525 smoothing_error : ndarray 

526 The smoothing error covariance matrices at each time period. 

527 smoothed_state : ndarray 

528 The smoothed state at each time period. 

529 smoothed_state_cov : ndarray 

530 The smoothed state covariance matrices at each time period. 

531 smoothed_state_autocov : ndarray 

532 The smoothed state lago-one autocovariance matrices at each time 

533 period: :math:`Cov(\alpha_{t+1}, \alpha_t)`. 

534 smoothed_measurement_disturbance : ndarray 

535 The smoothed measurement at each time period. 

536 smoothed_state_disturbance : ndarray 

537 The smoothed state at each time period. 

538 smoothed_measurement_disturbance_cov : ndarray 

539 The smoothed measurement disturbance covariance matrices at each time 

540 period. 

541 smoothed_state_disturbance_cov : ndarray 

542 The smoothed state disturbance covariance matrices at each time period. 

543 """ 

544 

545 _smoother_attributes = [ 

546 'smoother_output', 'scaled_smoothed_estimator', 

547 'scaled_smoothed_estimator_cov', 'smoothing_error', 

548 'smoothed_state', 'smoothed_state_cov', 'smoothed_state_autocov', 

549 'smoothed_measurement_disturbance', 'smoothed_state_disturbance', 

550 'smoothed_measurement_disturbance_cov', 

551 'smoothed_state_disturbance_cov' 

552 ] 

553 

554 _smoother_options = KalmanSmoother.smoother_outputs 

555 

556 _attributes = FilterResults._model_attributes + _smoother_attributes 

557 

558 def update_representation(self, model, only_options=False): 

559 """ 

560 Update the results to match a given model 

561 

562 Parameters 

563 ---------- 

564 model : Representation 

565 The model object from which to take the updated values. 

566 only_options : bool, optional 

567 If set to true, only the smoother and filter options are updated, 

568 and the state space representation is not updated. Default is 

569 False. 

570 

571 Notes 

572 ----- 

573 This method is rarely required except for internal usage. 

574 """ 

575 super(SmootherResults, self).update_representation(model, only_options) 

576 

577 # Save the options as boolean variables 

578 for name in self._smoother_options: 

579 setattr(self, name, getattr(model, name, None)) 

580 

581 # Initialize holders for smoothed forecasts 

582 self._smoothed_forecasts = None 

583 self._smoothed_forecasts_error = None 

584 self._smoothed_forecasts_error_cov = None 

585 

586 def update_smoother(self, smoother): 

587 """ 

588 Update the smoother results 

589 

590 Parameters 

591 ---------- 

592 smoother : KalmanSmoother 

593 The model object from which to take the updated values. 

594 

595 Notes 

596 ----- 

597 This method is rarely required except for internal usage. 

598 """ 

599 # Copy the appropriate output 

600 attributes = [] 

601 

602 # Since update_representation will already have been called, we can 

603 # use the boolean options smoother_* and know they match the smoother 

604 # itself 

605 if self.smoother_state or self.smoother_disturbance: 

606 attributes.append('scaled_smoothed_estimator') 

607 if self.smoother_state_cov or self.smoother_disturbance_cov: 

608 attributes.append('scaled_smoothed_estimator_cov') 

609 if self.smoother_state: 

610 attributes.append('smoothed_state') 

611 if self.smoother_state_cov: 

612 attributes.append('smoothed_state_cov') 

613 if self.smoother_state_autocov: 

614 attributes.append('smoothed_state_autocov') 

615 if self.smoother_disturbance: 

616 attributes += [ 

617 'smoothing_error', 

618 'smoothed_measurement_disturbance', 

619 'smoothed_state_disturbance' 

620 ] 

621 if self.smoother_disturbance_cov: 

622 attributes += [ 

623 'smoothed_measurement_disturbance_cov', 

624 'smoothed_state_disturbance_cov' 

625 ] 

626 

627 has_missing = np.sum(self.nmissing) > 0 

628 for name in self._smoother_attributes: 

629 if name == 'smoother_output': 

630 pass 

631 elif name in attributes: 

632 if name in ['smoothing_error', 

633 'smoothed_measurement_disturbance']: 

634 vector = getattr(smoother, name, None) 

635 if vector is not None and has_missing: 

636 vector = np.array(reorder_missing_vector( 

637 vector, self.missing, prefix=self.prefix)) 

638 else: 

639 vector = np.array(vector, copy=True) 

640 setattr(self, name, vector) 

641 elif name == 'smoothed_measurement_disturbance_cov': 

642 matrix = getattr(smoother, name, None) 

643 if matrix is not None and has_missing: 

644 matrix = reorder_missing_matrix( 

645 matrix, self.missing, reorder_rows=True, 

646 reorder_cols=True, prefix=self.prefix) 

647 # In the missing data case, we want to set the missing 

648 # components equal to their unconditional distribution 

649 copy_index_matrix( 

650 self.obs_cov, matrix, self.missing, 

651 index_rows=True, index_cols=True, inplace=True, 

652 prefix=self.prefix) 

653 else: 

654 matrix = np.array(matrix, copy=True) 

655 setattr(self, name, matrix) 

656 else: 

657 setattr(self, name, 

658 np.array(getattr(smoother, name, None), copy=True)) 

659 else: 

660 setattr(self, name, None) 

661 

662 # Diffuse objects 

663 self.scaled_smoothed_diffuse_estimator = None 

664 self.scaled_smoothed_diffuse1_estimator_cov = None 

665 self.scaled_smoothed_diffuse2_estimator_cov = None 

666 if self.nobs_diffuse > 0: 

667 self.scaled_smoothed_diffuse_estimator = np.array( 

668 smoother.scaled_smoothed_diffuse_estimator, copy=True) 

669 self.scaled_smoothed_diffuse1_estimator_cov = np.array( 

670 smoother.scaled_smoothed_diffuse1_estimator_cov, copy=True) 

671 self.scaled_smoothed_diffuse2_estimator_cov = np.array( 

672 smoother.scaled_smoothed_diffuse2_estimator_cov, copy=True) 

673 

674 # Adjustments 

675 

676 # For r_t (and similarly for N_t), what was calculated was 

677 # r_T, ..., r_{-1}. We only want r_0, ..., r_T 

678 # so exclude the appropriate element so that the time index is 

679 # consistent with the other returned output 

680 # r_t stored such that scaled_smoothed_estimator[0] == r_{-1} 

681 start = 1 

682 end = None 

683 if 'scaled_smoothed_estimator' in attributes: 

684 self.scaled_smoothed_estimator = ( 

685 self.scaled_smoothed_estimator[:, start:end] 

686 ) 

687 if 'scaled_smoothed_estimator_cov' in attributes: 

688 self.scaled_smoothed_estimator_cov = ( 

689 self.scaled_smoothed_estimator_cov[:, :, start:end] 

690 ) 

691 

692 # Clear the smoothed forecasts 

693 self._smoothed_forecasts = None 

694 self._smoothed_forecasts_error = None 

695 self._smoothed_forecasts_error_cov = None 

696 

697 # Note: if we concentrated out the scale, need to adjust the 

698 # loglikelihood values and all of the covariance matrices and the 

699 # values that depend on the covariance matrices 

700 if self.filter_concentrated and self.model._scale is None: 

701 self.smoothed_state_cov *= self.scale 

702 self.smoothed_state_autocov *= self.scale 

703 self.smoothed_state_disturbance_cov *= self.scale 

704 self.smoothed_measurement_disturbance_cov *= self.scale 

705 self.scaled_smoothed_estimator /= self.scale 

706 self.scaled_smoothed_estimator_cov /= self.scale 

707 self.smoothing_error /= self.scale 

708 

709 def _get_smoothed_forecasts(self): 

710 if self._smoothed_forecasts is None: 

711 # Initialize empty arrays 

712 self._smoothed_forecasts = np.zeros(self.forecasts.shape, 

713 dtype=self.dtype) 

714 self._smoothed_forecasts_error = ( 

715 np.zeros(self.forecasts_error.shape, dtype=self.dtype) 

716 ) 

717 self._smoothed_forecasts_error_cov = ( 

718 np.zeros(self.forecasts_error_cov.shape, dtype=self.dtype) 

719 ) 

720 

721 for t in range(self.nobs): 

722 design_t = 0 if self.design.shape[2] == 1 else t 

723 obs_cov_t = 0 if self.obs_cov.shape[2] == 1 else t 

724 obs_intercept_t = 0 if self.obs_intercept.shape[1] == 1 else t 

725 

726 mask = ~self.missing[:, t].astype(bool) 

727 # We can recover forecasts 

728 self._smoothed_forecasts[:, t] = np.dot( 

729 self.design[:, :, design_t], self.smoothed_state[:, t] 

730 ) + self.obs_intercept[:, obs_intercept_t] 

731 if self.nmissing[t] > 0: 

732 self._smoothed_forecasts_error[:, t] = np.nan 

733 self._smoothed_forecasts_error[mask, t] = ( 

734 self.endog[mask, t] - self._smoothed_forecasts[mask, t] 

735 ) 

736 self._smoothed_forecasts_error_cov[:, :, t] = np.dot( 

737 np.dot(self.design[:, :, design_t], 

738 self.smoothed_state_cov[:, :, t]), 

739 self.design[:, :, design_t].T 

740 ) + self.obs_cov[:, :, obs_cov_t] 

741 

742 return ( 

743 self._smoothed_forecasts, 

744 self._smoothed_forecasts_error, 

745 self._smoothed_forecasts_error_cov 

746 ) 

747 

748 @property 

749 def smoothed_forecasts(self): 

750 return self._get_smoothed_forecasts()[0] 

751 

752 @property 

753 def smoothed_forecasts_error(self): 

754 return self._get_smoothed_forecasts()[1] 

755 

756 @property 

757 def smoothed_forecasts_error_cov(self): 

758 return self._get_smoothed_forecasts()[2]