Coverage for emd/sift.py: 77%
578 statements
« prev ^ index » next coverage.py v7.6.10, created at 2025-01-09 10:07 +0000
« prev ^ index » next coverage.py v7.6.10, created at 2025-01-09 10:07 +0000
1#!/usr/bin/python
3# vim: set expandtab ts=4 sw=4:
5"""
6Implementations of the sift algorithm for Empirical Mode Decomposition.
8Main Routines:
9 sift - The classic sift algorithm
10 ensemble_sift - Noise-assisted sift algorithm
11 complete_ensemble_sift - Adapeted noise-assisted sift algorithm
12 mask_sift - Sift with masks to separate very sparse or nonlinear components
13 iterated_mask_sift - Sift which automatically identifies optimal masks
14 sift_second_layer - Apply sift to amplitude envlope of a set of IMFs
16Sift Helper Routines:
17 get_next_imf
18 get_next_imf_mask
19 get_mask_freqs
20 energy_difference
21 stop_imf_energy
22 stop_imf_sd
23 stop_imf_rilling
24 stop_imf_fixed_iter
26Sift Config:
27 get_config
28 SiftConfig
30"""
32import collections
33import functools
34import inspect
35import logging
36import sys
38import numpy as np
39import yaml
40from scipy.stats import zscore
42from ._sift_core import (_find_extrema, get_padded_extrema, interp_envelope,
43 zero_crossing_count)
44from .logger import sift_logger, wrap_verbose
45from .spectra import frequency_transform
46from .support import (EMDSiftCovergeError, ensure_1d_with_singleton, ensure_2d,
47 ensure_equal_dims, run_parallel)
49# Housekeeping for logging
50logger = logging.getLogger(__name__)
53##################################################################
54# Basic SIFT
56# Utilities
58def get_next_imf(X, env_step_size=1, max_iters=1000, energy_thresh=50,
59 stop_method='sd', sd_thresh=.1, rilling_thresh=(0.05, 0.5, 0.05),
60 envelope_opts=None, extrema_opts=None):
61 """Compute the next IMF from a data set.
63 This is a helper function used within the more general sifting functions.
65 Parameters
66 ----------
67 X : ndarray [nsamples x 1]
68 1D input array containing the time-series data to be decomposed
69 env_step_size : float
70 Scaling of envelope prior to removal at each iteration of sift. The
71 average of the upper and lower envelope is muliplied by this value
72 before being subtracted from the data. Values should be between
73 0 > x >= 1 (Default value = 1)
74 max_iters : int > 0
75 Maximum number of iterations to compute before throwing an error
76 energy_thresh : float > 0
77 Threshold for energy difference (in decibels) between IMF and residual
78 to suggest stopping overall sift. (Default is None, recommended value is 50)
79 stop_method : {'sd','rilling','fixed'}
80 Flag indicating which metric to use to stop sifting and return an IMF.
81 sd_thresh : float
82 Used if 'stop_method' is 'sd'. The threshold at which the sift of each
83 IMF will be stopped. (Default value = .1)
84 rilling_thresh : tuple
85 Used if 'stop_method' is 'rilling', needs to contain three values (sd1, sd2, alpha).
86 An evaluation function (E) is defined by dividing the residual by the
87 mode amplitude. The sift continues until E < sd1 for the fraction
88 (1-alpha) of the data, and E < sd2 for the remainder.
89 See section 3.2 of http://perso.ens-lyon.fr/patrick.flandrin/NSIP03.pdf
91 Returns
92 -------
93 proto_imf : ndarray
94 1D vector containing the next IMF extracted from X
95 continue_flag : bool
96 Boolean indicating whether the sift can be continued beyond this IMF
98 Other Parameters
99 ----------------
100 envelope_opts : dict
101 Optional dictionary of keyword arguments to be passed to emd.interp_envelope
102 extrema_opts : dict
103 Optional dictionary of keyword options to be passed to emd.get_padded_extrema
105 See Also
106 --------
107 emd.sift.sift
108 emd.sift.interp_envelope
110 """
111 X = ensure_1d_with_singleton([X], ['X'], 'get_next_imf')
113 if envelope_opts is None:
114 envelope_opts = {}
116 proto_imf = X.copy()
118 continue_imf = True # TODO - assess this properly here, return input if already passing!
120 continue_flag = True
121 niters = 0
122 while continue_imf:
124 if stop_method != 'fixed':
125 if niters == 3*max_iters//4:
126 logger.debug('Sift reached {0} iterations, taking a long time to coverge'.format(niters))
127 elif niters > max_iters:
128 msg = 'Sift failed. No covergence after {0} iterations'.format(niters)
129 raise EMDSiftCovergeError(msg)
130 niters += 1
132 # Compute envelopes, local mean and next proto imf
133 upper, lower = interp_envelope(proto_imf, mode='both',
134 **envelope_opts, extrema_opts=extrema_opts)
136 # If upper or lower are None we should stop sifting altogether
137 if upper is None or lower is None:
138 continue_flag = False
139 continue_imf = False
140 logger.debug('Finishing sift: IMF has no extrema')
141 continue
143 # Find local mean
144 avg = np.mean([upper, lower], axis=0)[:, None]
146 # Remove local mean estimate from proto imf
147 #x1 = proto_imf - avg
148 next_proto_imf = proto_imf - (env_step_size*avg)
150 # Evaluate if we should stop the sift - methods are very different in
151 # requirements here...
153 # Stop sifting if we pass threshold
154 if stop_method == 'sd':
155 # Cauchy criterion
156 stop, _ = stop_imf_sd(proto_imf, next_proto_imf, sd=sd_thresh, niters=niters)
157 elif stop_method == 'rilling':
158 # Rilling et al 2003 - this actually evaluates proto_imf NOT next_proto_imf
159 stop, _ = stop_imf_rilling(upper, lower, niters=niters,
160 sd1=rilling_thresh[0],
161 sd2=rilling_thresh[1],
162 tol=rilling_thresh[2])
163 if stop:
164 next_proto_imf = proto_imf
165 elif stop_method == 'energy':
166 # Rato et al 2008
167 # Compare energy of signal at start of sift with energy of envelope average
168 stop, _ = stop_imf_energy(X, avg, thresh=energy_thresh, niters=niters)
169 elif stop_method == 'fixed':
170 stop = stop_imf_fixed_iter(niters, max_iters)
171 else:
172 raise ValueError("stop_method '{0}' not recogised".format(stop_method))
174 proto_imf = next_proto_imf
176 if stop:
177 continue_imf = False
178 continue
180 if proto_imf.ndim == 1:
181 proto_imf = proto_imf[:, None]
183 return proto_imf, continue_flag
186def _energy_difference(imf, residue):
187 """Compute energy change in IMF during a sift.
189 Parameters
190 ----------
191 imf : ndarray
192 IMF to be evaluated
193 residue : ndarray
194 Remaining signal after IMF removal
196 Returns
197 -------
198 float
199 Energy difference in decibels
201 Notes
202 -----
203 This function is used during emd.sift.stop_imf_energy to implement the
204 energy-difference sift-stopping method defined in section 3.2.4 of
205 https://doi.org/10.1016/j.ymssp.2007.11.028
207 """
208 sumsqr = np.sum(imf**2)
209 imf_energy = 20 * np.log10(sumsqr, where=sumsqr > 0)
210 sumsqr = np.sum(residue ** 2)
211 resid_energy = 20 * np.log10(sumsqr, where=sumsqr > 0)
212 return imf_energy-resid_energy
215def stop_imf_energy(imf, residue, thresh=50, niters=None):
216 """Compute energy change in IMF during a sift.
218 The energy in the IMFs are compared to the energy at the start of sifting.
219 The sift terminates once this ratio reaches a predefined threshold.
221 Parameters
222 ----------
223 imf : ndarray
224 IMF to be evaluated
225 residue : ndarray
226 Average of the upper and lower envelopes
227 thresh : float
228 Energy ratio threshold (default=50)
229 niters : int
230 Number of sift iterations currently completed
232 Returns
233 -------
234 bool
235 A flag indicating whether to stop siftingg
236 float
237 Energy difference in decibels
239 Notes
240 -----
241 This function implements the energy-difference sift-stopping method defined
242 in section 3.2.4 of https://doi.org/10.1016/j.ymssp.2007.11.028
244 """
245 diff = _energy_difference(imf, residue)
246 stop = bool(diff > thresh)
248 if stop:
249 logger.debug('Sift stopped by Energy Ratio in {0} iters with difference of {1}dB'.format(niters, diff))
250 else:
251 logger.debug('Energy Ratio evaluated at iter {0} is : {1}dB'.format(niters, diff))
253 return stop, diff
256def stop_imf_sd(proto_imf, prev_imf, sd=0.2, niters=None):
257 """Compute the sd sift stopping metric.
259 Parameters
260 ----------
261 proto_imf : ndarray
262 A signal which may be an IMF
263 prev_imf : ndarray
264 The previously identified IMF
265 sd : float
266 The stopping threshold
267 niters : int
268 Number of sift iterations currently completed
269 niters : int
270 Number of sift iterations currently completed
272 Returns
273 -------
274 bool
275 A flag indicating whether to stop siftingg
276 float
277 The SD metric value
279 """
280 metric = np.sum((prev_imf - proto_imf)**2) / np.sum(prev_imf**2)
282 stop = metric < sd
284 if stop:
285 logger.verbose('Sift stopped by SD-thresh in {0} iters with sd {1}'.format(niters, metric))
286 else:
287 logger.debug('SD-thresh stop metric evaluated at iter {0} is : {1}'.format(niters, metric))
289 return stop, metric
292def stop_imf_rilling(upper_env, lower_env, sd1=0.05, sd2=0.5, tol=0.05, niters=None):
293 """Compute the Rilling et al 2003 sift stopping metric.
295 This metric tries to guarantee globally small fluctuations in the IMF mean
296 while taking into account locally large excursions that may occur in noisy
297 signals.
299 Parameters
300 ----------
301 upper_env : ndarray
302 The upper envelope of a proto-IMF
303 lower_env : ndarray
304 The lower envelope of a proto-IMF
305 sd1 : float
306 The maximum threshold for globally small differences from zero-mean
307 sd2 : float
308 The maximum threshold for locally large differences from zero-mean
309 tol : float (0 < tol < 1)
310 (1-tol) defines the proportion of time which may contain large deviations
311 from zero-mean
312 niters : int
313 Number of sift iterations currently completed
315 Returns
316 -------
317 bool
318 A flag indicating whether to stop siftingg
319 float
320 The SD metric value
322 Notes
323 -----
324 This method is described in section 3.2 of:
325 Rilling, G., Flandrin, P., & Goncalves, P. (2003, June). On empirical mode
326 decomposition and its algorithms. In IEEE-EURASIP workshop on nonlinear
327 signal and image processing (Vol. 3, No. 3, pp. 8-11). NSIP-03, Grado (I).
328 http://perso.ens-lyon.fr/patrick.flandrin/NSIP03.pdf
330 """
331 avg_env = (upper_env+lower_env)/2
332 amp = np.abs(upper_env-lower_env)/2
334 eval_metric = np.abs(avg_env)/amp
336 metric = np.mean(eval_metric > sd1)
337 continue1 = metric > tol
338 continue2 = np.any(eval_metric > sd2)
340 stop = (continue1 or continue2) == False # noqa: E712
342 if stop:
343 logger.verbose('Sift stopped by Rilling-metric in {0} iters (val={1})'.format(niters, metric))
344 else:
345 logger.debug('Rilling stop metric evaluated at iter {0} is : {1}'.format(niters, metric))
347 return stop, metric
350def stop_imf_fixed_iter(niters, max_iters):
351 """Compute the fixed-iteraiton sift stopping metric.
353 Parameters
354 ----------
355 niters : int
356 Number of sift iterations currently completed
357 max_iters : int
358 Maximum number of sift iterations to be completed
360 Returns
361 -------
362 bool
363 A flag indicating whether to stop siftingg
365 """
366 stop = bool(niters == max_iters)
368 if stop:
369 logger.debug('Sift stopped at fixed number of {0} iterations'.format(niters))
371 return stop
374def _nsamples_warn(N, max_imfs):
375 if max_imfs is None:
376 return
377 if N < 2**(max_imfs+1):
378 msg = 'Inputs samples ({0}) is small for specified max_imfs ({1})'
379 msg += ' very likely that {2} or fewer imfs are returned'
380 logger.warning(msg.format(N, max_imfs, np.floor(np.log2(N)).astype(int)-1))
383def _set_rilling_defaults(rilling_thresh):
384 rilling_thresh = (0.05, 0.5, 0.05) if rilling_thresh is True else rilling_thresh
385 return rilling_thresh
388# SIFT implementation
390@wrap_verbose
391@sift_logger('sift')
392def sift(X, sift_thresh=1e-8, energy_thresh=50, rilling_thresh=None,
393 max_imfs=None, verbose=None, return_residual=True,
394 imf_opts=None, envelope_opts=None, extrema_opts=None):
395 """Compute Intrinsic Mode Functions from an input data vector.
397 This function implements the original sift algorithm [1]_.
399 Parameters
400 ----------
401 X : ndarray
402 1D input array containing the time-series data to be decomposed
403 sift_thresh : float
404 The threshold at which the overall sifting process will stop. (Default value = 1e-8)
405 max_imfs : int
406 The maximum number of IMFs to compute. (Default value = None)
408 Returns
409 -------
410 imf: ndarray
411 2D array [samples x nimfs] containing he Intrisic Mode Functions from the decomposition of X.
413 Other Parameters
414 ----------------
415 imf_opts : dict
416 Optional dictionary of keyword options to be passed to emd.get_next_imf
417 envelope_opts : dict
418 Optional dictionary of keyword options to be passed to emd.interp_envelope
419 extrema_opts : dict
420 Optional dictionary of keyword options to be passed to emd.get_padded_extrema
421 verbose : {None,'CRITICAL','WARNING','INFO','DEBUG'}
422 Option to override the EMD logger level for a call to this function.
424 See Also
425 --------
426 emd.sift.get_next_imf
427 emd.sift.get_config
429 Notes
430 -----
431 The classic sift is computed by passing an input vector with all options
432 left to default
434 >>> imf = emd.sift.sift(x)
436 The sift can be customised by passing additional options, here we only
437 compute the first four IMFs.
439 >>> imf = emd.sift.sift(x, max_imfs=4)
441 More detailed options are passed as dictionaries which are passed to the
442 relevant lower-level functions. For instance `imf_opts` are passed to
443 `get_next_imf`.
445 >>> imf_opts = {'env_step_size': 1/3, 'stop_method': 'rilling'}
446 >>> imf = emd.sift.sift(x, max_imfs=4, imf_opts=imf_opts)
448 A modified dictionary of all options can be created using `get_config`.
449 This can be modified and used by unpacking the options into a `sift` call.
451 >>> conf = emd.sift.get_config('sift')
452 >>> conf['max_imfs'] = 4
453 >>> conf['imf_opts'] = imf_opts
454 >>> imfs = emd.sift.sift(x, **conf)
456 References
457 ----------
458 .. [1] Huang, N. E., Shen, Z., Long, S. R., Wu, M. C., Shih, H. H., Zheng,
459 Q., … Liu, H. H. (1998). The empirical mode decomposition and the Hilbert
460 spectrum for nonlinear and non-stationary time series analysis. Proceedings
461 of the Royal Society of London. Series A: Mathematical, Physical and
462 Engineering Sciences, 454(1971), 903–995.
463 https://doi.org/10.1098/rspa.1998.0193
465 """
466 if not imf_opts:
467 imf_opts = {'env_step_size': 1,
468 'sd_thresh': .1}
469 rilling_thresh = _set_rilling_defaults(rilling_thresh)
471 X = ensure_1d_with_singleton([X], ['X'], 'sift')
473 _nsamples_warn(X.shape[0], max_imfs)
475 layer = 0
476 # Only evaluate peaks and if already an IMF if rilling is specified.
477 continue_sift = check_sift_continue(X, X, layer,
478 max_imfs=max_imfs,
479 sift_thresh=None,
480 energy_thresh=None,
481 rilling_thresh=rilling_thresh,
482 envelope_opts=envelope_opts,
483 extrema_opts=extrema_opts,
484 merge_tests=True)
486 proto_imf = X.copy()
488 while continue_sift:
490 logger.info('sifting IMF : {0}'.format(layer))
492 next_imf, continue_sift = get_next_imf(proto_imf,
493 envelope_opts=envelope_opts,
494 extrema_opts=extrema_opts,
495 **imf_opts)
497 if layer == 0:
498 imf = next_imf
499 else:
500 imf = np.concatenate((imf, next_imf), axis=1)
502 proto_imf = X - imf.sum(axis=1)[:, None]
503 layer += 1
505 # Check if sifting should continue - all metrics whose thresh is not
506 # None will be assessed and sifting will stop if any metric says so
507 continue_sift = check_sift_continue(X, proto_imf, layer,
508 max_imfs=max_imfs,
509 sift_thresh=sift_thresh,
510 energy_thresh=energy_thresh,
511 rilling_thresh=rilling_thresh,
512 envelope_opts=envelope_opts,
513 extrema_opts=extrema_opts,
514 merge_tests=True)
516 # Append final residual as last mode - unless its empty
517 if np.sum(np.abs(proto_imf)) != 0:
518 imf = np.c_[imf, proto_imf]
520 return imf
523def check_sift_continue(X, residual, layer, max_imfs=None, sift_thresh=1e-8, energy_thresh=50,
524 rilling_thresh=None, envelope_opts=None, extrema_opts=None,
525 merge_tests=True):
526 """Run checks to see if siftiing should continue into another layer.
528 Parameters
529 ----------
530 X : ndarray
531 1D array containing the data being decomposed
532 residual : ndarray
533 1D array containing the current residuals (X - imfs so far)
534 layer : int
535 Current IMF number being decomposed
536 max_imf : int
537 Largest number of IMFs to compute
538 sift_thresh : float
539 The threshold at which the overall sifting process will stop.
540 (Default value = 1e-8)
541 energy_thresh : float
542 The difference in energy between the raw data and the residuals in
543 decibels at which we stop sifting (default = 50).
544 rilling_thresh : tuple or None
545 Tuple (or tuple-like) containing three values (sd1, sd2, alpha).
546 An evaluation function (E) is defined by dividing the residual by the
547 mode amplitude. The sift continues until E < sd1 for the fraction
548 (1-alpha) of the data, and E < sd2 for the remainder.
549 See section 3.2 of http://perso.ens-lyon.fr/patrick.flandrin/NSIP03.pdf
550 envelope_opts : dict or None
551 Optional dictionary of keyword options to be passed to emd.interp_envelope
552 extrema_opts : dict or None
553 Optional dictionary of keyword options to be passed to emd.get_padded_extrema
555 Returns
556 -------
557 bool
558 Flag indicating whether to stop sifting.
560 """
561 continue_sift = [None, None, None, None, None]
563 # Check if we've reached the pre-specified number of IMFs
564 if max_imfs is not None and layer == max_imfs:
565 logger.info('Finishing sift: reached max number of imfs ({0})'.format(layer))
566 continue_sift[0] = False
567 else:
568 continue_sift[0] = True
570 # Check if residual has enough peaks to sift again
571 pks, _ = _find_extrema(residual)
572 trs, _ = _find_extrema(-residual)
573 if len(pks) < 2 or len(trs) < 2:
574 logger.info('Finishing sift: {0} peaks {1} trough in residual'.format(len(pks), len(trs)))
575 continue_sift[1] = False
576 else:
577 continue_sift[1] = True
579 # Optional: Check if the sum-sqr of the resduals is below the sift_thresh
580 sumsq_resid = np.abs(residual).sum()
581 if sift_thresh is not None and sumsq_resid < sift_thresh:
582 logger.info('Finishing sift: reached threshold {0}'.format(sumsq_resid))
583 continue_sift[2] = False
584 else:
585 continue_sift[2] = True
587 # Optional: Check if energy_ratio of residual to original signal is below thresh
588 energy_ratio = _energy_difference(X, residual)
589 if energy_thresh is not None and energy_ratio > energy_thresh:
590 logger.info('Finishing sift: reached energy ratio {0}'.format(energy_ratio))
591 continue_sift[3] = False
592 else:
593 continue_sift[3] = True
595 # Optional: Check if the residual is already an IMF with Rilling method -
596 # only run if we have enough extrema
597 if rilling_thresh is not None and continue_sift[1]:
598 upper, lower = interp_envelope(residual, mode='both',
599 **envelope_opts, extrema_opts=extrema_opts)
600 rilling_continue_sift, rilling_metric = stop_imf_rilling(upper, lower, niters=-1)
601 if rilling_continue_sift is False:
602 logger.info('Finishing sift: reached rilling {0}'.format(rilling_metric))
603 continue_sift[4] = False
604 else:
605 continue_sift[4] = True
607 if merge_tests:
608 # Merge tests that aren't none - return False for any Falses
609 return np.any([x == False for x in continue_sift if x is not None]) == False # noqa: E712
610 else:
611 return continue_sift
614##################################################################
615# Ensemble SIFT variants
617# Utilities
619def _sift_with_noise(X, noise_scaling=None, noise=None, noise_mode='single',
620 sift_thresh=1e-8, max_imfs=None, job_ind=1,
621 imf_opts=None, envelope_opts=None, extrema_opts=None):
622 """Apply white noise to a signal prior to computing a sift.
624 Parameters
625 ----------
626 X : ndarray
627 1D input array containing the time-series data to be decomposed
628 noise_scaling : float
629 Standard deviation of noise to add to each ensemble (Default value =
630 None)
631 noise : ndarray
632 array of noise values the same size as X to add prior to sift (Default value = None)
633 noise_mode : {'single','flip'}
634 Flag indicating whether to compute each ensemble with noise once or
635 twice with the noise and sign-flipped noise (Default value = 'single')
636 sift_thresh : float
637 The threshold at which the overall sifting process will stop. (Default value = 1e-8)
638 max_imfs : int
639 The maximum number of IMFs to compute. (Default value = None)
640 job_ind : 1
641 Optional job index value for display in logger (Default value = 1)
643 Returns
644 -------
645 imf: ndarray
646 2D array [samples x nimfs] containing he Intrisic Mode Functions from the decomposition of X.
648 Other Parameters
649 ----------------
650 imf_opts : dict
651 Optional dictionary of arguments to be passed to emd.get_next_imf
652 envelope_opts : dict
653 Optional dictionary of keyword options to be passed to emd.interp_envelope
654 extrema_opts : dict
655 Optional dictionary of keyword options to be passed to emd.get_padded_extrema
657 See Also
658 --------
659 emd.sift.ensemble_sift
660 emd.sift.complete_ensemble_sift
661 emd.sift.get_next_imf
663 """
664 if job_ind is not None:
665 logger.info('Starting SIFT Ensemble: {0}'.format(job_ind))
667 if noise is None:
668 noise = np.random.randn(*X.shape)
670 X = ensure_1d_with_singleton([X], ['X'], 'sift')
671 ensure_equal_dims([X, noise], ['X', 'noise'], '_sift_with_noise', dim=0)
673 if noise_scaling is not None:
674 noise = noise * noise_scaling
676 ensX = X.copy() + noise
677 imf = sift(ensX, sift_thresh=sift_thresh, max_imfs=max_imfs,
678 imf_opts=imf_opts, envelope_opts=envelope_opts, extrema_opts=extrema_opts)
680 if noise_mode == 'single':
681 return imf
682 elif noise_mode == 'flip':
683 ensX = X.copy() - noise
684 imf += sift(ensX, sift_thresh=sift_thresh, max_imfs=max_imfs,
685 imf_opts=imf_opts, envelope_opts=envelope_opts, extrema_opts=extrema_opts)
686 return imf / 2
689# Implementation
691@wrap_verbose
692@sift_logger('ensemble_sift')
693def ensemble_sift(X, nensembles=4, ensemble_noise=.2, noise_mode='single',
694 noise_seed=None, nprocesses=1, sift_thresh=1e-8, max_imfs=None, verbose=None,
695 imf_opts=None, envelope_opts=None, extrema_opts=None):
696 """Compute Intrinsic Mode Functions with the ensemble EMD.
698 This function implements the ensemble empirical model decomposition
699 algorithm defined in [1]_. This approach sifts an ensemble of signals with
700 white-noise added and treats the mean IMFs as the result. The resulting
701 IMFs from the ensemble sift resembles a dyadic filter [2]_.
703 Parameters
704 ----------
705 X : ndarray
706 1D input array containing the time-series data to be decomposed
707 nensembles : int
708 Integer number of different ensembles to compute the sift across.
709 ensemble_noise : float
710 Standard deviation of noise to add to each ensemble (Default value = .2)
711 noise_mode : {'single','flip'}
712 Flag indicating whether to compute each ensemble with noise once or
713 twice with the noise and sign-flipped noise (Default value = 'single')
714 noise_seed : int
715 seed value to use for random noise generation.
716 nprocesses : int
717 Integer number of parallel processes to compute. Each process computes
718 a single realisation of the total ensemble (Default value = 1)
719 sift_thresh : float
720 The threshold at which the overall sifting process will stop. (Default value = 1e-8)
721 max_imfs : int
722 The maximum number of IMFs to compute. (Default value = None)
724 Returns
725 -------
726 imf : ndarray
727 2D array [samples x nimfs] containing he Intrisic Mode Functions from the decomposition of X.
729 Other Parameters
730 ----------------
731 imf_opts : dict
732 Optional dictionary of keyword options to be passed to emd.get_next_imf.
733 envelope_opts : dict
734 Optional dictionary of keyword options to be passed to emd.interp_envelope
735 extrema_opts : dict
736 Optional dictionary of keyword options to be passed to emd.get_padded_extrema
737 verbose : {None,'CRITICAL','WARNING','INFO','DEBUG'}
738 Option to override the EMD logger level for a call to this function.
740 See Also
741 --------
742 emd.sift.get_next_imf
744 References
745 ----------
746 .. [1] Wu, Z., & Huang, N. E. (2009). Ensemble Empirical Mode Decomposition:
747 A Noise-Assisted Data Analysis Method. Advances in Adaptive Data Analysis,
748 1(1), 1–41. https://doi.org/10.1142/s1793536909000047
749 .. [2] Wu, Z., & Huang, N. E. (2004). A study of the characteristics of
750 white noise using the empirical mode decomposition method. Proceedings of
751 the Royal Society of London. Series A: Mathematical, Physical and
752 Engineering Sciences, 460(2046), 1597–1611.
753 https://doi.org/10.1098/rspa.2003.1221
756 """
757 if noise_mode not in ['single', 'flip']:
758 raise ValueError(
759 'noise_mode: {0} not recognised, please use \'single\' or \'flip\''.format(noise_mode))
761 X = ensure_1d_with_singleton([X], ['X'], 'sift')
763 _nsamples_warn(X.shape[0], max_imfs)
765 # Noise is defined with respect to variance in the data
766 noise_scaling = X.std() * ensemble_noise
768 if noise_seed is not None:
769 np.random.seed(noise_seed)
771 # Create partial function containing everything we need to run one iteration
772 pfunc = functools.partial(_sift_with_noise, X, noise_scaling=noise_scaling,
773 noise=None, noise_mode=noise_mode, sift_thresh=sift_thresh,
774 max_imfs=max_imfs, imf_opts=imf_opts, envelope_opts=envelope_opts,
775 extrema_opts=extrema_opts)
777 # Run the actual sifting - in parallel if requested
778 args = [[] for ii in range(nensembles)]
779 res = run_parallel(pfunc, args, nprocesses=nprocesses)
781 # Keep largest group of ensembles with matching number of imfs.
782 nimfs = [r.shape[1] for r in res]
783 uni, unic = np.unique(nimfs, return_counts=True)
784 target_imfs = uni[np.argmax(unic)]
786 # Adjust for max_imfs if it was defined
787 if (max_imfs is not None) and (target_imfs > max_imfs):
788 target_imfs = max_imfs
790 msg = 'Retaining {0} ensembles ({1}%) each with {2} IMFs'
791 logger.info(msg.format(np.max(unic), 100*(np.max(unic)/nensembles), target_imfs))
793 # Take average across ensembles
794 imfs = np.zeros((X.shape[0], target_imfs))
795 for ii in range(target_imfs):
796 imfs[:, ii] = np.array([r[:, ii] for r in res if r.shape[1] >= target_imfs]).mean(axis=0)
798 return imfs
801@wrap_verbose
802@sift_logger('complete_ensemble_sift')
803def complete_ensemble_sift(X, nensembles=4, ensemble_noise=.2,
804 nprocesses=1, noise_seed=None,
805 sift_thresh=1e-8, energy_thresh=50,
806 rilling_thresh=None, max_imfs=None, verbose=None,
807 imf_opts=None, envelope_opts=None,
808 extrema_opts=None):
809 """Compute Intrinsic Mode Functions with complete ensemble EMD.
811 This function implements the complete ensemble empirical model
812 decomposition algorithm defined in [1]_. This approach sifts an ensemble of
813 signals with white-noise added taking a single IMF across all ensembles at
814 before moving to the next IMF.
816 Parameters
817 ----------
818 X : ndarray
819 1D input array containing the time-series data to be decomposed
820 nensembles : int
821 Integer number of different ensembles to compute the sift across.
822 ensemble_noise : float
823 Standard deviation of noise to add to each ensemble (Default value = .2)
824 noise_mode : {'single','flip'}
825 Flag indicating whether to compute each ensemble with noise once or
826 twice with the noise and sign-flipped noise (Default value = 'single')
827 nprocesses : int
828 Integer number of parallel processes to compute. Each process computes
829 a single realisation of the total ensemble (Default value = 1)
830 sift_thresh : float
831 The threshold at which the overall sifting process will stop. (Default value = 1e-8)
832 max_imfs : int
833 The maximum number of IMFs to compute. (Default value = None)
835 Returns
836 -------
837 imf: ndarray
838 2D array [samples x nimfs] containing he Intrisic Mode Functions from the decomposition of X.
839 noise: array_like
840 The Intrisic Mode Functions from the decomposition of X.
842 Other Parameters
843 ----------------
844 imf_opts : dict
845 Optional dictionary of keyword options to be passed to emd.get_next_imf.
846 envelope_opts : dict
847 Optional dictionary of keyword options to be passed to emd.interp_envelope
848 extrema_opts : dict
849 Optional dictionary of keyword options to be passed to emd.get_padded_extrema
850 verbose : {None,'CRITICAL','WARNING','INFO','DEBUG'}
851 Option to override the EMD logger level for a call to this function.
853 See Also
854 --------
855 emd.sift.get_next_imf
857 References
858 ----------
859 .. [1] Torres, M. E., Colominas, M. A., Schlotthauer, G., & Flandrin, P.
860 (2011). A complete ensemble empirical mode decomposition with adaptive
861 noise. In 2011 IEEE International Conference on Acoustics, Speech and
862 Signal Processing (ICASSP). IEEE.
863 https://doi.org/10.1109/icassp.2011.5947265
865 """
866 X = ensure_1d_with_singleton([X], ['X'], 'sift')
868 imf_opts = {} if imf_opts is None else imf_opts
869 envelope_opts = {} if envelope_opts is None else envelope_opts
871 _nsamples_warn(X.shape[0], max_imfs)
873 # Work with normalised units internally - easier for noise scaling
874 Xstd = X.std()
875 X = X / Xstd
877 # Compute white noise
878 if noise_seed is not None:
879 np.random.seed(noise_seed)
880 white_noise = zscore(np.random.randn(nensembles, X.shape[0]), axis=1)
882 # Compute white noise modes - sift each to completion
883 modes_white_noise = [sift(white_noise[ii, :],
884 imf_opts=imf_opts,
885 envelope_opts=envelope_opts,
886 extrema_opts=extrema_opts) for ii in range(nensembles)]
888 # Define the core sifting func and options - this is applied to compute
889 # successive IMFs in the main loop
890 pfunc = functools.partial(get_next_imf,
891 envelope_opts=envelope_opts,
892 extrema_opts=extrema_opts,
893 **imf_opts)
895 # Wrapper to return local mean terms rather than IMFs - could make this an
896 # option in get_next_imf in future
897 def get_next_local_mean(X):
898 X = ensure_1d_with_singleton([X], ['X'], 'get_next_local_mean')
899 imf, flag = pfunc(X)
900 return X - imf, flag
902 # Get first local mean from across ensemble
903 args = []
904 for ii in range(nensembles):
905 scaled_noise = ensemble_noise*modes_white_noise[ii][:, 0]/modes_white_noise[ii][:, 0].std()
906 args.append([X + scaled_noise[:, np.newaxis]])
907 res = run_parallel(get_next_local_mean, args, nprocesses=nprocesses)
908 # Finaly local mean is average across all
909 local_mean = np.array([r[0] for r in res]).mean(axis=0)
911 # IMF is data minus final local mean
912 imf = X - local_mean
913 residue = local_mean
915 # Prep for loop
916 layer = 1
917 # continue_sift = _ceemdan_check_continue(local_mean, sift_thresh)
918 continue_sift = check_sift_continue(X, local_mean, layer,
919 max_imfs=max_imfs,
920 sift_thresh=sift_thresh,
921 energy_thresh=energy_thresh,
922 rilling_thresh=rilling_thresh,
923 envelope_opts=envelope_opts,
924 extrema_opts=extrema_opts,
925 merge_tests=True)
926 snrflag = 1
928 while continue_sift:
930 # Prepare noise for ensembles
931 args = []
932 for ii in range(nensembles):
933 noise = modes_white_noise[ii][:, layer].copy()
934 if snrflag == 2:
935 noise = noise / noise.std()
936 noise = ensemble_noise * noise
938 # Sift current local-mean + each noise process
939 args.append([local_mean[:, 0]+noise*local_mean.std()])
940 res = run_parallel(get_next_local_mean, args, nprocesses=nprocesses)
942 # New local mean is the mean of local means (resid_i - imf_i) across ensemble
943 local_mean = np.array([r[0] for r in res]).mean(axis=0)
945 # New IMF is current residue minus new local mean
946 imf = np.c_[imf, (residue[:, -1] - local_mean[:, 0])[:, None]]
948 # Next residue is current new local mean
949 residue = np.c_[residue, local_mean]
951 # Check if sifting should continue - all metrics whose thresh is not
952 # None will be assessed and sifting will stop if any metric says so
953 continue_sift = check_sift_continue(X, local_mean, layer,
954 max_imfs=max_imfs,
955 sift_thresh=sift_thresh,
956 energy_thresh=energy_thresh,
957 rilling_thresh=rilling_thresh,
958 envelope_opts=envelope_opts,
959 extrema_opts=extrema_opts,
960 merge_tests=True)
962 layer += 1
964 # Concatenate final IMF
965 imf = np.c_[imf, local_mean]
967 # Reinstate original variance
968 imf = imf * Xstd
970 return imf
973##################################################################
974# Mask SIFT implementations
976# Utilities
979def get_next_imf_mask(X, z, amp, nphases=4, nprocesses=1,
980 imf_opts=None, envelope_opts=None, extrema_opts=None):
981 """Compute the next IMF from a data set a mask sift.
983 This is a helper function used within the more general sifting functions.
985 Parameters
986 ----------
987 X : ndarray
988 1D input array containing the time-series data to be decomposed
989 z : float
990 Mask frequency as a proportion of the sampling rate, values between 0->z->.5
991 amp : float
992 Mask amplitude
993 nphases : int > 0
994 The number of separate sinusoidal masks to apply for each IMF, the
995 phase of masks are uniformly spread across a 0<=p<2pi range
996 (Default=4).
997 nprocesses : int
998 Integer number of parallel processes to compute. Each process computes
999 an IMF from the signal plus a mask. nprocesses should be less than or
1000 equal to nphases, no additional benefit from setting nprocesses > nphases
1001 (Default value = 1)
1003 Returns
1004 -------
1005 proto_imf : ndarray
1006 1D vector containing the next IMF extracted from X
1007 continue_sift : bool
1008 Boolean indicating whether the sift can be continued beyond this IMF
1010 Other Parameters
1011 ----------------
1012 imf_opts : dict
1013 Optional dictionary of keyword arguments to be passed to emd.get_next_imf
1014 envelope_opts : dict
1015 Optional dictionary of keyword options to be passed to emd.interp_envelope
1016 extrema_opts : dict
1017 Optional dictionary of keyword options to be passed to emd.get_padded_extrema
1019 See Also
1020 --------
1021 emd.sift.mask_sift
1022 emd.sift.get_next_imf
1024 """
1025 X = ensure_1d_with_singleton([X], ['X'], 'get_next_imf_mask')
1027 if imf_opts is None:
1028 imf_opts = {}
1030 logger.info("Defining masks with freq {0} and amp {1} at {2} phases".format(z, amp, nphases))
1032 # Create normalised freq
1033 zf = z * 2 * np.pi
1034 # Create time matrix including mask phase-shifts
1035 t = np.repeat(np.arange(X.shape[0])[:, np.newaxis], nphases, axis=1)
1036 phases = np.linspace(0, (2*np.pi), nphases+1)[:nphases]
1037 # Create masks
1038 m = amp * np.cos(zf * t + phases)
1040 # Work with a partial function to make the parallel loop cleaner
1041 # This partial function contains all the settings which will be constant across jobs.
1042 pfunc = functools.partial(get_next_imf, **imf_opts,
1043 envelope_opts=envelope_opts,
1044 extrema_opts=extrema_opts)
1046 args = [[X+m[:, ii, np.newaxis]] for ii in range(nphases)]
1047 res = run_parallel(pfunc, args, nprocesses=nprocesses)
1049 # Collate results
1050 imfs = [r[0] for r in res]
1051 continue_flags = [r[1] for r in res]
1053 # star map should preserve the order of outputs so we can remove masks easily
1054 imfs = np.concatenate(imfs, axis=1) - m
1056 logger.verbose('Averaging across {0} proto IMFs'.format(imfs.shape[1]))
1058 return imfs.mean(axis=1)[:, np.newaxis], np.any(continue_flags)
1061def get_mask_freqs(X, first_mask_mode='zc', imf_opts=None):
1062 """Determine mask frequencies for a sift.
1064 Parameters
1065 ----------
1066 X : ndarray
1067 Vector time-series
1068 first_mask_mode : (str, float<0.5)
1069 Either a string denoting a method {'zc', 'if'} or a float determining
1070 and initial frequency. See notes for more details.
1071 imf_opts : dict
1072 Options to be passed to get_next_imf if first_mask_mode is 'zc' or 'if'.
1074 Returns
1075 -------
1076 float
1077 Frequency for the first mask in normalised units.
1079 """
1080 if imf_opts is None:
1081 imf_opts = {}
1083 if first_mask_mode in ('zc', 'if'):
1084 logger.info('Computing first mask frequency with method {0}'.format(first_mask_mode))
1085 logger.info('Getting first IMF with no mask')
1086 # First IMF is computed normally
1087 imf, _ = get_next_imf(X, **imf_opts)
1089 # Compute first mask frequency from first IMF
1090 if first_mask_mode == 'zc':
1091 num_zero_crossings = zero_crossing_count(imf)[0, 0]
1092 z = num_zero_crossings / imf.shape[0] / 2
1093 logger.info('Found first mask frequency of {0}'.format(z))
1094 elif first_mask_mode == 'if':
1095 _, IF, IA = frequency_transform(imf[:, 0, None], 1, 'nht',
1096 smooth_phase=3)
1097 z = np.average(IF, weights=IA)
1098 logger.info('Found first mask frequency of {0}'.format(z))
1099 elif isinstance(first_mask_mode, (int, float)):
1100 if first_mask_mode <= 0 or first_mask_mode > .5:
1101 raise ValueError("The frequency of the first mask must be 0 <= x < 0.5")
1102 logger.info('Using specified first mask frequency of {0}'.format(first_mask_mode))
1103 z = first_mask_mode
1105 return z
1108# Implementation
1110@wrap_verbose
1111@sift_logger('mask_sift')
1112def mask_sift(X, mask_amp=1, mask_amp_mode='ratio_sig', mask_freqs='zc',
1113 mask_step_factor=2, ret_mask_freq=False, max_imfs=9, sift_thresh=1e-8,
1114 nphases=4, nprocesses=1, verbose=None,
1115 imf_opts=None, envelope_opts=None, extrema_opts=None):
1116 """Compute Intrinsic Mode Functions using a mask sift.
1118 This function implements a masked sift from a dataset using a set of
1119 masking signals to reduce mixing of components between modes [1]_, multiple
1120 masks of different phases can be applied when isolating each IMF [2]_.
1122 This function can either compute the mask frequencies based on the fastest
1123 dynamics in the data (the properties of the first IMF from a standard sift)
1124 or apply a pre-specified set of masks.
1126 Parameters
1127 ----------
1128 X : ndarray
1129 1D input array containing the time-series data to be decomposed
1130 mask_amp : float or array_like
1131 Amplitude of mask signals as specified by mask_amp_mode. If float the
1132 same value is applied to all IMFs, if an array is passed each value is
1133 applied to each IMF in turn (Default value = 1)
1134 mask_amp_mode : {'abs','ratio_imf','ratio_sig'}
1135 Method for computing mask amplitude. Either in absolute units ('abs'),
1136 or as a ratio of the standard deviation of the input signal
1137 ('ratio_sig') or previous imf ('ratio_imf') (Default value = 'ratio_imf')
1138 mask_freqs : {'zc','if',float,,array_like}
1139 Define the set of mask frequencies to use. If 'zc' or 'if' are passed,
1140 the frequency of the first mask is taken from either the zero-crossings
1141 or instantaneous frequnecy the first IMF of a standard sift on the
1142 data. If a float is passed this is taken as the first mask frequency.
1143 Subsequent masks are defined by the mask_step_factor. If an array_like
1144 vector is passed, the values in the vector will specify the mask
1145 frequencies.
1146 mask_step_factor : float
1147 Step in frequency between successive masks (Default value = 2)
1148 mask_type : {'all','sine','cosine'}
1149 Which type of masking signal to use. 'sine' or 'cosine' options return
1150 the average of a +ve and -ve flipped wave. 'all' applies four masks:
1151 sine and cosine with +ve and -ve sign and returns the average of all
1152 four.
1153 nphases : int > 0
1154 The number of separate sinusoidal masks to apply for each IMF, the
1155 phase of masks are uniformly spread across a 0<=p<2pi range
1156 (Default=4).
1157 ret_mask_freq : bool
1158 Boolean flag indicating whether mask frequencies are returned (Default value = False)
1159 max_imfs : int
1160 The maximum number of IMFs to compute. (Default value = None)
1161 sift_thresh : float
1162 The threshold at which the overall sifting process will stop. (Default value = 1e-8)
1164 Returns
1165 -------
1166 imf : ndarray
1167 2D array [samples x nimfs] containing he Intrisic Mode Functions from the decomposition of X.
1168 mask_freqs : ndarray
1169 1D array of mask frequencies, if ret_mask_freq is set to True.
1171 Other Parameters
1172 ----------------
1173 imf_opts : dict
1174 Optional dictionary of keyword arguments to be passed to emd.get_next_imf
1175 envelope_opts : dict
1176 Optional dictionary of keyword options to be passed to emd.interp_envelope
1177 extrema_opts : dict
1178 Optional dictionary of keyword options to be passed to emd.get_padded_extrema
1179 verbose : {None,'CRITICAL','WARNING','INFO','DEBUG'}
1180 Option to override the EMD logger level for a call to this function.
1182 Notes
1183 -----
1184 Here are some example mask_sift variants you can run:
1186 A mask sift in which the mask frequencies are determined with
1187 zero-crossings and mask amplitudes by a ratio with the amplitude of the
1188 previous IMF (note - this is also the default):
1190 >>> imf = emd.sift.mask_sift(X, mask_amp_mode='ratio_imf', mask_freqs='zc')
1192 A mask sift in which the first mask is set at .4 of the sampling rate and
1193 subsequent masks found by successive division of this mask_freq by 3:
1195 >>> imf = emd.sift.mask_sift(X, mask_freqs=.4, mask_step_factor=3)
1197 A mask sift using user specified frequencies and amplitudes:
1199 >>> mask_freqs = np.array([.4,.2,.1,.05,.025,0])
1200 >>> mask_amps = np.array([2,2,1,1,.5,.5])
1201 >>> imf = emd.sift.mask_sift(X, mask_freqs=mask_freqs, mask_amp=mask_amps, mask_amp_mode='abs')
1203 See Also
1204 --------
1205 emd.sift.get_next_imf
1206 emd.sift.get_next_imf_mask
1208 References
1209 ----------
1210 .. [1] Ryan Deering, & James F. Kaiser. (2005). The Use of a Masking Signal
1211 to Improve Empirical Mode Decomposition. In Proceedings. (ICASSP ’05). IEEE
1212 International Conference on Acoustics, Speech, and Signal Processing, 2005.
1213 IEEE. https://doi.org/10.1109/icassp.2005.1416051
1214 .. [2] Tsai, F.-F., Fan, S.-Z., Lin, Y.-S., Huang, N. E., & Yeh, J.-R.
1215 (2016). Investigating Power Density and the Degree of Nonlinearity in
1216 Intrinsic Components of Anesthesia EEG by the Hilbert-Huang Transform: An
1217 Example Using Ketamine and Alfentanil. PLOS ONE, 11(12), e0168108.
1218 https://doi.org/10.1371/journal.pone.0168108
1220 """
1221 X = ensure_1d_with_singleton([X], ['X'], 'sift')
1223 # if first mask is if or zc - compute first imf as normal and get freq
1224 if isinstance(mask_freqs, (list, tuple, np.ndarray)):
1225 logger.info('Using user specified masks')
1226 if len(mask_freqs) < max_imfs:
1227 max_imfs = len(mask_freqs)
1228 logger.info("Reducing max_imfs to {0} as len(mask_freqs) < max_imfs".format(max_imfs))
1229 elif mask_freqs in ['zc', 'if'] or isinstance(mask_freqs, float):
1230 z = get_mask_freqs(X, mask_freqs, imf_opts=imf_opts)
1231 mask_freqs = np.array([z/mask_step_factor**ii for ii in range(max_imfs)])
1233 _nsamples_warn(X.shape[0], max_imfs)
1235 # Initialise mask amplitudes
1236 if mask_amp_mode == 'ratio_imf':
1237 sd = X.std() # Take ratio of input signal for first IMF
1238 elif mask_amp_mode == 'ratio_sig':
1239 sd = X.std()
1240 elif mask_amp_mode == 'abs':
1241 sd = 1
1243 continue_sift = True
1244 imf_layer = 0
1245 proto_imf = X.copy()
1246 imf = []
1247 while continue_sift:
1249 # Update mask amplitudes if needed
1250 if mask_amp_mode == 'ratio_imf' and imf_layer > 0:
1251 sd = imf[:, -1].std()
1253 if isinstance(mask_amp, (int, float)):
1254 amp = mask_amp * sd
1255 else:
1256 # Should be array_like if not a single number
1257 amp = mask_amp[imf_layer] * sd
1259 logger.info('Sifting IMF-{0}'.format(imf_layer))
1261 next_imf, continue_sift = get_next_imf_mask(proto_imf, mask_freqs[imf_layer], amp,
1262 nphases=nphases,
1263 nprocesses=nprocesses,
1264 imf_opts=imf_opts,
1265 envelope_opts=envelope_opts,
1266 extrema_opts=extrema_opts)
1268 if imf_layer == 0:
1269 imf = next_imf
1270 else:
1271 imf = np.concatenate((imf, next_imf), axis=1)
1273 proto_imf = X - imf.sum(axis=1)[:, None]
1275 if max_imfs is not None and imf_layer == max_imfs-1:
1276 logger.info('Finishing sift: reached max number of imfs ({0})'.format(imf.shape[1]))
1277 continue_sift = False
1279 if np.abs(next_imf).sum() < sift_thresh:
1280 continue_sift = False
1282 imf_layer += 1
1284 if ret_mask_freq:
1285 return imf, mask_freqs
1286 else:
1287 return imf
1290@wrap_verbose
1291@sift_logger('iterated_mask_sift')
1292def iterated_mask_sift(X,
1293 # Iterated mask sift arguments
1294 mask_0='zc', w_method='power', max_iter=15, iter_th=0.1,
1295 N_avg=1, exclude_edges=False, sample_rate=1.0,
1296 seed=None,
1297 # Standard mask sift arguments - specify a couple which need defaults.
1298 max_imfs=6, ret_mask_freq=False, mask_amp_mode='ratio_imf',
1299 **kwargs):
1300 """Compute Intrinsic Mode Functions using an iterated mask sift.
1302 This function implements a masked sift from a dataset using a set of
1303 masking signals to reduce mixing of components between modes [1]_, multiple
1304 masks of different phases can be applied when isolating each IMF [2]_.
1306 Mask frequencies are determined automatically by an iterative process [3]_.
1307 The iteration can be started with either a random mask, a mask based on the
1308 fastest dynamics (same as 'zc' in mask_sift), or a pre-specified mask.
1310 Parameters
1311 ----------
1312 X : ndarray
1313 1D input array containing the time-series data to be decomposed
1314 mask_0 : {array_like, 'zc', 'random'}
1315 Initial mask for the iteration process, can be one of:
1317 * 'zc' or 'if' initialises with the masks chosen by the zero-crossing
1318 count or instantaneous frequency method in the standard mask sift.
1320 * 'random' chooses random integers between 0 and sample_rate/4 as the starting mask.
1321 seed=int can be optionally passed to control the random seed in numpy.
1323 * array-like needs to be in normalised units, i.e. divided by the sample rate.
1324 (Default value = 'zc')
1325 w_method : {'amplitude', 'power', float, None}
1326 Weighting method to use in the iteration process. 'amplitude' weights
1327 frequencies by the instantaneous amplitude, 'power' by its square. If
1328 a float is passed, the amplitude is raised to that exponent before averaging.
1329 None performs a simple average without weighting.
1330 (Default value = 'power')
1331 max_imfs : int
1332 The maximum number of IMFs to compute. (Default value = 6)
1333 max_iter : int
1334 The maximum number of iterations to compute. (Default value = 15)
1335 iter_th : float
1336 Relative mask variability threshold below which iteration is stopped.
1337 (Default value = 0.1)
1338 N_avg : int
1339 Number of iterations to average after convergence is reached. (Default value = 1)
1340 exlude_edges : bool
1341 If True, excludes first and last 2.5% of frequency data during the iteration
1342 process to avoid edge effects. (Default value = False)
1343 sample_rate : float
1344 Sampling rate of the data in Hz (Default value = 1.0)
1345 seed : int or None
1346 Random seed to use for random initial mask selection when mask_0 = 'random'
1347 **kwargs
1348 Any additional arguments for the standard emd.sift.mask_sift can be
1349 specified - see the documentation for emd.sift.mask_sift for more
1350 details.
1352 Returns
1353 -------
1354 imf : ndarray
1355 2D array [samples x nimfs] containing he Intrisic Mode Functions from
1356 the decomposition of X.
1357 mask_freqs : ndarray
1358 1D array of mask frequencies, if ret_mask_freq is set to True.
1360 Notes
1361 -----
1362 Here are some example iterated_mask_sift variants you can run:
1364 An iterated mask sift in which the mask frequencies are determined with
1365 zero-crossings and iteration stop at 15 iterations or if masks
1366 stabilize to within 10% (note - this is also the default):
1368 >>> imf = emd.sift.iterated_mask_sift(X, sample_rate, mask_0='zc',
1369 max_iter=15, iter_th=0.1)
1371 An iterated mask sift in which a custom initial mask is used and after convergence
1372 5 further iterations are averaged:
1374 >>> imf = emd.sift.iterated_mask_sift(X, sample_rate,
1375 mask_0=[10, 5, 3, 1]/sample_rate,
1376 N_avg=5)
1378 An iterated mask sift weighted by instantaneous amplitude that also returns
1379 the automatically determined mask and excludes 5% of edge data to avoid
1380 edge effectd:
1382 >>> imf, mask = emd.sift.iterated_mask_sift(X, sample_rate, w_method='amplitude',
1383 exclude_edges=True, ret_mask_freq=True)
1385 See Also
1386 --------
1387 emd.sift.mask_sift
1388 emd.sift.get_next_imf_mask
1390 References
1391 ----------
1392 .. [1] Ryan Deering, & James F. Kaiser. (2005). The Use of a Masking Signal
1393 to Improve Empirical Mode Decomposition. In Proceedings. (ICASSP ’05). IEEE
1394 International Conference on Acoustics, Speech, and Signal Processing, 2005.
1395 IEEE. https://doi.org/10.1109/icassp.2005.1416051
1396 .. [2] Tsai, F.-F., Fan, S.-Z., Lin, Y.-S., Huang, N. E., & Yeh, J.-R.
1397 (2016). Investigating Power Density and the Degree of Nonlinearity in
1398 Intrinsic Components of Anesthesia EEG by the Hilbert-Huang Transform: An
1399 Example Using Ketamine and Alfentanil. PLOS ONE, 11(12), e0168108.
1400 https://doi.org/10.1371/journal.pone.0168108
1401 .. [3] Marco S. Fabus, Andrew J. Quinn, Catherine E. Warnaby,
1402 and Mark W. Woolrich (2021). Automatic decomposition of
1403 electrophysiological data into distinct nonsinusoidal oscillatory modes.
1404 Journal of Neurophysiology 2021 126:5, 1670-1684.
1405 https://doi.org/10.1152/jn.00315.2021
1407 """
1408 # Housekeeping
1409 X = ensure_1d_with_singleton([X], ['X'], 'sift')
1410 _nsamples_warn(X.shape[0], max_imfs)
1411 nsamples = X.shape[0]
1413 # Add explicitly specified mask_sift kwargs into full dict for use later
1414 kwargs['max_imfs'] = max_imfs
1415 kwargs['mask_amp_mode'] = mask_amp_mode
1417 # Main switch initialising the mask frequency set
1418 if isinstance(mask_0, (list, tuple, np.ndarray)):
1419 # User has provided a full set of masks
1420 logger.info('Initialising masks with user specified frequencies')
1421 if len(mask_0) < max_imfs:
1422 max_imfs = len(mask_0)
1423 logger.info("Reducing max_imfs to {0} as len(mask_freqs) < max_imfs".format(max_imfs))
1424 mask = mask_0
1425 elif isinstance(mask_0, (int, float)):
1426 logger.info('Initialising masks with user specified single frequency')
1427 mask = mask_0
1428 elif mask_0 in ('zc', 'if'):
1429 logger.info('Initialising masks with mask_sift default mask_freqs={0}'.format(mask_0))
1430 # if first mask is if or zc - compute first imf as normal and get freq
1431 _, mask = mask_sift(X, mask_freqs=mask_0, ret_mask_freq=True, **kwargs)
1432 mask = mask
1433 elif mask_0 == 'random':
1434 logger.info('Initialising masks with random values')
1435 if seed is not None:
1436 np.random.seed(seed)
1437 mask = np.random.randint(0, sample_rate/4, size=max_imfs) / sample_rate
1438 else:
1439 raise ValueError("'mask_0' input {0} not recognised - cannot initialise mask frequencies".format(mask_0))
1441 # Preallocate arrays for loop process
1442 mask_all = np.zeros((max_iter+N_avg, max_imfs))
1443 imf_all = np.zeros((max_iter+N_avg, nsamples, max_imfs))
1445 # Start counters
1446 niters = 0
1447 niters_c = 0
1448 maxiter_flag = 0
1449 continue_iter = True
1450 converged = False
1452 # Main loop
1453 while continue_iter:
1454 if not converged:
1455 logger.info('Computing iteration number ' + str(niters))
1456 else:
1457 logger.info('Converged, averaging... ' + str(niters_c) + ' / ' + str(N_avg))
1459 # Update masks
1460 mask_prev = mask.copy()
1461 mask_all[niters+niters_c, :len(mask)] = mask.copy()
1463 # Compute mask sift
1464 imf = mask_sift(X, mask_freqs=mask, **kwargs)
1465 imf_all[niters+niters_c, :, :imf.shape[1]] = imf
1467 # Compute IMF frequencies
1468 IP, IF, IA = frequency_transform(imf, sample_rate, 'nht')
1470 # Trim IMF edges if requested - avoids edge effects distorting IF average
1471 if exclude_edges:
1472 logger.info('Excluding 5% of edge frequencies in mask estimation.')
1473 ex = int(0.025*nsamples)
1474 samples_included = list(range(ex, nsamples-ex)) # Edge effects ignored
1475 else:
1476 samples_included = list(range(nsamples)) # All, default
1478 # find weighted IF average as the next mask
1479 if w_method == 'amplitude':
1480 # IF weighed by amplitude values in IA
1481 IF_weighted = np.average(IF[samples_included, :], 0, weights=IA[samples_included, :])
1482 elif w_method == 'power':
1483 # IF weighed by power values from IA**2
1484 IF_weighted = np.average(IF[samples_included, :], 0, weights=IA[samples_included, :]**2)
1485 elif isinstance(w_method, float):
1486 # IF weighed by amplitude raised to user specified power
1487 IF_weighted = np.average(IF[samples_included, :], 0, weights=IA[samples_included, :]**w_method)
1488 elif w_method == 'avg':
1489 # IF average not weighted
1490 IF_weighted = np.mean(IF[samples_included, :], axis=0)
1491 else:
1492 raise ValueError("w_method '{0}' not recognised".format(w_method))
1494 # Compute new mask frequencies and variances
1495 mask = IF_weighted/sample_rate
1496 l = min(len(mask), len(mask_prev))
1497 mask_variance = np.abs((mask[:l] - mask_prev[:l]) / mask_prev[:l])
1499 # Check convergence
1500 if np.all(mask_variance[~np.isnan(mask_variance)] < iter_th) or converged:
1501 converged = True
1502 logger.info('Finishing iteration process: convergence reached in {0} iterations '.format(niters))
1503 if niters_c < N_avg:
1504 niters_c += 1
1505 else:
1506 continue_iter = False
1508 if not converged:
1509 niters += 1
1511 if niters >= max_iter:
1512 logger.info('Finishing iteration process: reached max number of iterations: {0}'.format(max_iter))
1513 maxiter_flag = 1
1514 continue_iter = False
1516 # Average IMFs across iterations after convergence
1517 imf_final = np.nanmean(imf_all[niters:niters+N_avg, :, :], axis=0)
1518 IF_final = np.nanmean(mask_all[niters:niters+N_avg, :], axis=0)*sample_rate
1519 IF_std_final = np.nanstd(mask_all[niters:niters+N_avg, :], axis=0)*sample_rate
1521 if maxiter_flag:
1522 imf_final = imf_all[niters-1, :, :]
1523 IF_final = mask
1524 IF_std_final = mask_variance
1526 # If we are not averaging, output relative change from last mask instead
1527 if N_avg == 1:
1528 IF_std_final = mask_variance
1530 N_imf_final = int(np.sum(~np.isnan(mask_all[niters-1, :])))
1531 imf_final = imf_final[:, :N_imf_final]
1532 IF_final = IF_final[:N_imf_final]
1533 IF_std_final = IF_std_final[:N_imf_final]
1534 imf = imf_final
1536 logger.info('Final mask variability: %s', str(IF_std_final))
1537 logger.info('COMPLETED: iterated mask sift')
1539 if ret_mask_freq:
1540 return imf, IF_final
1541 else:
1542 return imf
1545##################################################################
1546# Second Layer SIFT
1549@sift_logger('second_layer')
1550def sift_second_layer(IA, sift_func=sift, sift_args=None):
1551 """Compute second layer intrinsic mode functions.
1553 This function implements a second-layer sift to be appliede to the
1554 amplitude envelopes of a set of first layer IMFs [1]_.
1556 Parameters
1557 ----------
1558 IA : ndarray
1559 Input array containing a set of first layer IMFs
1560 sift_func : function
1561 Sift function to apply
1562 sift_args : dict
1563 Dictionary of sift options to be passed into sift_func
1565 Returns
1566 -------
1567 imf2 : ndarray
1568 3D array [samples x first layer imfs x second layer imfs ] containing
1569 the second layer IMFs
1571 References
1572 ----------
1573 .. [1] Huang, N. E., Hu, K., Yang, A. C. C., Chang, H.-C., Jia, D., Liang,
1574 W.-K., … Wu, Z. (2016). On Holo-Hilbert spectral analysis: a full
1575 informational spectral representation for nonlinear and non-stationary
1576 data. Philosophical Transactions of the Royal Society A: Mathematical,
1577 Physical and Engineering Sciences, 374(2065), 20150206.
1578 https://doi.org/10.1098/rsta.2015.0206
1580 """
1581 IA = ensure_2d([IA], ['IA'], 'sift_second_layer')
1583 if (sift_args is None) or ('max_imfs' not in sift_args):
1584 max_imfs = IA.shape[1]
1585 elif 'max_imfs' in sift_args:
1586 max_imfs = sift_args['max_imfs']
1588 imf2 = np.zeros((IA.shape[0], IA.shape[1], max_imfs))
1590 for ii in range(max_imfs):
1591 tmp = sift_func(IA[:, ii], **sift_args)
1592 imf2[:, ii, :tmp.shape[1]] = tmp
1594 return imf2
1597@sift_logger('mask_sift_second_layer')
1598def mask_sift_second_layer(IA, mask_freqs, sift_args=None):
1599 """Compute second layer IMFs using a mask sift.
1601 Second layer IMFs are computed from the amplitude envelopes of a set of
1602 first layer IMFs [1]_.A single set of masks is applied across all IMFs with
1603 the highest frequency mask dropped for each successive first level IMF.
1605 Parameters
1606 ----------
1607 IA : ndarray
1608 Input array containing a set of first layer IMFs
1609 mask_freqs : function
1610 Sift function to apply
1611 sift_args : dict
1612 Dictionary of sift options to be passed into sift_func
1614 Returns
1615 -------
1616 imf2 : ndarray
1617 3D array [samples x first layer imfs x second layer imfs ] containing
1618 the second layer IMFs
1620 References
1621 ----------
1622 .. [1] Huang, N. E., Hu, K., Yang, A. C. C., Chang, H.-C., Jia, D., Liang,
1623 W.-K., … Wu, Z. (2016). On Holo-Hilbert spectral analysis: a full
1624 informational spectral representation for nonlinear and non-stationary
1625 data. Philosophical Transactions of the Royal Society A: Mathematical,
1626 Physical and Engineering Sciences, 374(2065), 20150206.
1627 https://doi.org/10.1098/rsta.2015.0206
1629 """
1630 IA = ensure_2d([IA], ['IA'], 'sift_second_layer')
1632 if (sift_args is None):
1633 sift_args = {'max_imfs': IA.shape[1]}
1634 elif ('max_imfs' not in sift_args):
1635 sift_args['max_imfs'] = IA.shape[1]
1637 imf2 = np.zeros((IA.shape[0], IA.shape[1], sift_args['max_imfs']))
1639 for ii in range(IA.shape[1]):
1640 sift_args['mask_freqs'] = mask_freqs[ii:]
1641 tmp = mask_sift(IA[:, ii], **sift_args)
1642 imf2[:, ii, :tmp.shape[1]] = tmp
1643 return imf2
1646##################################################################
1647# SIFT Estimation Utilities
1649##################################################################
1650# SIFT Config Utilities
1653class SiftConfig(collections.abc.MutableMapping):
1654 """A dictionary-like object specifying keyword arguments configuring a sift."""
1656 def __init__(self, name='sift', *args, **kwargs):
1657 """Specify keyword arguments configuring a sift."""
1658 self.store = dict()
1659 self.sift_type = name
1660 self.update(dict(*args, **kwargs)) # use the free update to set keys
1662 def __getitem__(self, key):
1663 """Return an item from the internal store."""
1664 key = self.__keytransform__(key)
1665 if isinstance(key, list):
1666 if len(key) == 2:
1667 return self.store[key[0]][key[1]]
1668 elif len(key) == 3:
1669 return self.store[key[0]][key[1]][key[2]]
1670 else:
1671 return self.store[key]
1673 def __setitem__(self, key, value):
1674 """Set or change the value of an item in the internal store."""
1675 key = self.__keytransform__(key)
1676 if isinstance(key, list):
1677 if len(key) == 2:
1678 self.store[key[0]][key[1]] = value
1679 elif len(key) == 3:
1680 self.store[key[0]][key[1]][key[2]] = value
1681 else:
1682 self.store[key] = value
1684 def __delitem__(self, key):
1685 """Remove an item from the internal store."""
1686 key = self.__keytransform__(key)
1687 if isinstance(key, list):
1688 if len(key) == 2:
1689 del self.store[key[0]][key[1]]
1690 elif len(key) == 3:
1691 del self.store[key[0]][key[1]][key[2]]
1692 else:
1693 del self.store[key]
1695 def __iter__(self):
1696 """Iterate through items in the internal store."""
1697 return iter(self.store)
1699 def __str__(self):
1700 """Print summary of internal store."""
1701 out = []
1702 lower_level = ['imf_opts', 'envelope_opts', 'extrema_opts']
1703 for stage in self.store.keys():
1704 if stage not in lower_level:
1705 out.append('{0} : {1}'.format(stage, self.store[stage]))
1706 else:
1707 out.append(stage + ':')
1708 for key in self.store[stage].keys():
1709 out.append(' {0} : {1}'.format(key, self.store[stage][key]))
1711 return '%s %s\n%s' % (self.sift_type, self.__class__, '\n'.join(out))
1713 def __repr__(self):
1714 """Print summary of internal store."""
1715 return "<{0} ({1})>".format(self.__module__ + '.' + type(self).__name__, self.sift_type)
1717 def _repr_html_(self):
1718 _str_html = "<h3><b>%s %s</b></h3><hr><ul>" % (self.sift_type, self.__class__)
1719 lower_level = ['imf_opts', 'envelope_opts', 'extrema_opts']
1720 for stage in self.store.keys():
1721 if stage not in lower_level:
1722 _str_html += '<li><b>{0}</b> : {1}</li>'.format(stage, self.store[stage])
1723 else:
1724 outer_list = '<li><b>{0}</b></li>%s'.format(stage)
1725 inner_list = '<ul>'
1726 for key in self.store[stage].keys():
1727 inner_list += '<li><i>{0}</i> : {1}</li>'.format(key, self.store[stage][key])
1728 _str_html += outer_list % (inner_list + '</ul>')
1729 return _str_html + '</ul>'
1731 def __len__(self):
1732 """Return number of items in internal store."""
1733 return len(self.store)
1735 def __keytransform__(self, key):
1736 """Split a merged dictionary key into separate levels."""
1737 key = key.split('/')
1738 if len(key) == 1:
1739 return key[0]
1740 else:
1741 if len(key) > 3:
1742 raise ValueError("Requested key is nested too deep. Should be a \
1743 maximum of three levels separated by '/'")
1744 return key
1746 def _get_yamlsafe_dict(self):
1747 """Return copy of internal store with values prepped for saving into yaml format."""
1748 conf = self.store.copy()
1749 conf = _array_or_tuple_to_list(conf)
1750 return [{'sift_type': self.sift_type}, conf]
1752 def to_yaml_text(self):
1753 """Return a copy of the internal store in yaml-text format."""
1754 return yaml.dump(self._get_yamlsafe_dict(), sort_keys=False)
1756 def to_yaml_file(self, fname):
1757 """Save a copy of the internal store in a specified yaml file."""
1758 with open(fname, 'w') as f:
1759 yaml.dump_all(self._get_yamlsafe_dict(), f, sort_keys=False)
1760 logger.info("Saved SiftConfig ({0}) to {1}".format(self, fname))
1762 @classmethod
1763 def from_yaml_file(cls, fname):
1764 """Create and return a new SiftConfig object with options loaded from a yaml file."""
1765 ret = cls()
1766 with open(fname, 'r') as f:
1767 cfg = [d for d in yaml.load_all(f, Loader=yaml.FullLoader)]
1768 if len(cfg) == 1:
1769 ret.store = cfg[0]
1770 ret.sift_type = 'Unknown'
1771 else:
1772 ret.sift_type = cfg[0]['sift_type']
1773 ret.store = cfg[1]
1774 logger.info("Loaded SiftConfig ({0}) from {1}".format(ret, fname))
1776 return ret
1778 @classmethod
1779 def from_yaml_stream(cls, stream):
1780 """Create and return a new SiftConfig object with options loaded from a yaml stream."""
1781 ret = cls()
1782 ret.store = yaml.load(stream, Loader=yaml.FullLoader)
1783 return ret
1785 def get_func(self):
1786 """Get a partial-function coded with the options from this config."""
1787 mod = sys.modules[__name__]
1788 func = getattr(mod, self.sift_type)
1789 return functools.partial(func, **self.store)
1792def get_config(siftname='sift'):
1793 """Return a SiftConfig with default options for a specified sift variant.
1795 Helper function for specifying config objects specifying parameters to be
1796 used in a sift. The functions used during the sift areinspected
1797 automatically and default values are populated into a nested dictionary
1798 which can be modified and used as input to one of the sift functions.
1800 Parameters
1801 ----------
1802 siftname : str
1803 Name of the sift function to find configuration from
1805 Returns
1806 -------
1807 SiftConfig
1808 A modified dictionary containing the sift specification
1810 Notes
1811 -----
1812 The sift config acts as a nested dictionary which can be modified to
1813 specify parameters for different parts of the sift. This is initialised
1814 using this function
1816 >>> config = emd.sift.get_config()
1818 The first level of the dictionary contains three sub-dicts configuring
1819 different parts of the algorithm:
1821 >>> config['imf_opts'] # options passed to `get_next_imf`
1822 >>> config['envelope_opts'] # options passed to interp_envelope
1823 >>> config['extrema_opts'] # options passed to get_padded_extrema
1825 Specific values can be modified in the dictionary
1827 >>> config['extrema_opts']['parabolic_extrema'] = True
1829 or using this shorthand
1831 >>> config['imf_opts/env_step_factor'] = 1/3
1833 Finally, the SiftConfig dictionary should be nested before being passed as
1834 keyword arguments to a sift function.
1836 >>> imfs = emd.sift.sift(X, **config)
1838 """
1839 # Extrema padding opts are hard-coded for the moment, these run through
1840 # np.pad which has a complex signature
1841 mag_pad_opts = {'mode': 'median', 'stat_length': 1}
1842 loc_pad_opts = {'mode': 'reflect', 'reflect_type': 'odd'}
1844 # Get defaults for extrema detection and padding
1845 extrema_opts = _get_function_opts(get_padded_extrema, ignore=['X', 'mag_pad_opts',
1846 'loc_pad_opts',
1847 'mode'])
1849 # Get defaults for envelope interpolation
1850 envelope_opts = _get_function_opts(interp_envelope, ignore=['X', 'extrema_opts', 'mode', 'ret_extrema', 'trim'])
1852 # Get defaults for computing IMFs
1853 imf_opts = _get_function_opts(get_next_imf, ignore=['X', 'envelope_opts', 'extrema_opts'])
1855 # Get defaults for the given sift variant
1856 sift_types = ['sift', 'ensemble_sift', 'complete_ensemble_sift',
1857 'mask_sift', 'iterated_mask_sift']
1858 if siftname in sift_types:
1859 mod = sys.modules[__name__]
1860 sift_opts = _get_function_opts(getattr(mod, siftname), ignore=['X', 'imf_opts'
1861 'envelope_opts',
1862 'extrema_opts',
1863 'kwargs'])
1864 if siftname == 'iterated_mask_sift':
1865 # Add options for mask sift as well
1866 mask_opts = _get_function_opts(getattr(mod, 'mask_sift'), ignore=['X', 'imf_opts'
1867 'envelope_opts',
1868 'extrema_opts',
1869 'mask_freqs',
1870 'mask_step_factor'])
1871 sift_opts = {**sift_opts, **mask_opts}
1872 else:
1873 raise AttributeError('Sift siftname not recognised: please use one of {0}'.format(sift_types))
1875 out = SiftConfig(siftname)
1876 for key in sift_opts:
1877 out[key] = sift_opts[key]
1878 out['imf_opts'] = imf_opts
1879 out['envelope_opts'] = envelope_opts
1880 out['extrema_opts'] = extrema_opts
1881 out['extrema_opts/mag_pad_opts'] = mag_pad_opts
1882 out['extrema_opts/loc_pad_opts'] = loc_pad_opts
1884 return out
1887def _get_function_opts(func, ignore=None):
1888 """Inspect a function and extract its keyword arguments and their default values.
1890 Parameters
1891 ----------
1892 func : function
1893 handle for the function to be inspected
1894 ignore : {None or list}
1895 optional list of keyword argument names to be ignored in function
1896 signature
1898 Returns
1899 -------
1900 dict
1901 Dictionary of keyword arguments with keyword keys and default value
1902 values.
1904 """
1905 if ignore is None:
1906 ignore = []
1907 out = {}
1908 sig = inspect.signature(func)
1909 for p in sig.parameters:
1910 if p not in out.keys() and p not in ignore:
1911 out[p] = sig.parameters[p].default
1912 return out
1915def _array_or_tuple_to_list(conf):
1916 """Convert an input array or tuple to list (for yaml_safe dict creation."""
1917 for key, val in conf.items():
1918 if isinstance(val, np.ndarray):
1919 conf[key] = val.tolist()
1920 elif isinstance(val, dict):
1921 conf[key] = _array_or_tuple_to_list(conf[key])
1922 elif isinstance(val, tuple):
1923 conf[key] = list(val)
1924 return conf