Coverage for utils/histogram_processor.py: 81%

254 statements  

« prev     ^ index     » next       coverage.py v7.0.4, created at 2023-01-10 09:27 -0600

1""" 

2Copyright 1999 Illinois Institute of Technology 

3 

4Permission is hereby granted, free of charge, to any person obtaining 

5a copy of this software and associated documentation files (the 

6"Software"), to deal in the Software without restriction, including 

7without limitation the rights to use, copy, modify, merge, publish, 

8distribute, sublicense, and/or sell copies of the Software, and to 

9permit persons to whom the Software is furnished to do so, subject to 

10the following conditions: 

11 

12The above copyright notice and this permission notice shall be 

13included in all copies or substantial portions of the Software. 

14 

15THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 

16EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 

17MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 

18IN NO EVENT SHALL ILLINOIS INSTITUTE OF TECHNOLOGY BE LIABLE FOR ANY 

19CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 

20TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 

21SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 

22 

23Except as contained in this notice, the name of Illinois Institute 

24of Technology shall not be used in advertising or otherwise to promote 

25the sale, use or other dealings in this Software without prior written 

26authorization from Illinois Institute of Technology. 

27""" 

28 

29import numpy as np 

30 

31def convexHull(hist, start_p = 0, end_p = 99999999, ignore = None): 

32 """ 

33 Apply 1D Convex hull to a histogram from left to right 

34 :param hist: input histogram (list or numpy array) 

35 :param start_p: start position of applying. The value of indexes before start_p will be 0 (int) 

36 :param end_p: end position of applying (int) 

37 :param ignore: specify ignore indexes in case the histogram has valleys from pilatus lines (list of boolean) 

38 :return: a histogram after convex hull is applied (list) 

39 """ 

40 start_p = int(round(start_p)) 

41 end_p = int(round(end_p)) 

42 

43 if end_p - start_p < 5 and (start_p !=0 or end_p != 99999999): 

44 return np.array(hist) 

45 

46 hist = np.array(hist) 

47 

48 if end_p > len(hist) : 

49 end_p = len(hist) 

50 

51 hist_x = list(range(start_p, end_p)) 

52 hist_y = np.array(hist[hist_x], dtype=np.float32) 

53 

54 if len(hist_x) < 5: 

55 return np.array(hist) 

56 

57 hist_y2 = hist_y.copy() 

58 if ignore is not None: 

59 ignore2 = ignore[hist_x] 

60 hist_y2[ignore2] = int(max(hist_y2)*1.5) 

61 

62 hull_x, hull_y = getHull(hist_x, hist_y2) 

63 hull = getSubtractedHist(hist_x, hist_y, hull_x, hull_y) 

64 

65 ret = list(np.zeros(start_p)) 

66 ret.extend(hull) 

67 ret.extend(np.zeros(len(hist)-end_p)) 

68 

69 if ignore is not None: 

70 sub = np.array(ret) 

71 sub[ignore] = 0 

72 ret = list(sub) 

73 

74 return ret 

75 

76def getHull(x_data, y_data): 

77 """ 

78 Get Hull from histogram 

79 :param x_data: x values of the histogram (list) 

80 :param y_data: y values of the histogram (list) 

81 :return: x and y values of hull (list) 

82 """ 

83 xhull = [] 

84 yhull = [] 

85 if len(x_data) == 0 or len(y_data) == 0: 

86 return xhull, yhull 

87 xhull.append(x_data[0]) 

88 yhull.append(y_data[0]) 

89 

90 lasthullindex = 0 

91 

92 points = len(y_data) 

93 while lasthullindex < points - 1: 

94 slope = (y_data[lasthullindex + 1] - y_data[lasthullindex]) / ( 

95 x_data[lasthullindex + 1] - x_data[lasthullindex]) 

96 currenthullindex = lasthullindex + 1 

97 currenthully = y_data[lasthullindex] 

98 

99 for i in range(currenthullindex + 1, points): 

100 extrapolation = currenthully + slope * (x_data[i] - x_data[lasthullindex]) 

101 if y_data[i] < extrapolation: 

102 slope = ((y_data[i] - y_data[lasthullindex]) / (x_data[i] - x_data[lasthullindex])) 

103 currenthullindex = i 

104 

105 # Store the hull points to be used for a spline fit 

106 xhull.append(x_data[currenthullindex]) 

107 yhull.append(y_data[currenthullindex]) 

108 lasthullindex = currenthullindex 

109 

110 return xhull, yhull 

111 

112def getSubtractedHist(xdata, ydata, xhull, yhull): 

113 """ 

114 Apply Subtraction to original histogram by using a pchip line created from hull 

115 :param xdata: x values of original histogram (list) 

116 :param ydata: y values of original histogram (list) 

117 :param xhull: x values of hull (list) 

118 :param yhull: y values of hull (list) 

119 :return: Backgound subtracted histogram 

120 """ 

121 if len(xdata) < 2 or len(ydata) < 2 or len(xhull) < 2 or len(yhull) < 2: 

122 return ydata 

123 

124 if len(xhull) < 3 or len(yhull) < 3: 

125 segmentlx = xhull[0] 

126 segmentrx = xhull[1] 

127 leftindex = xdata.index(segmentlx) 

128 rightindex = xdata.index(segmentrx) 

129 segmently = ydata[leftindex] 

130 segmentry = ydata[rightindex] 

131 slope = (float(segmently) - float(segmentry)) / (float(leftindex) - float(rightindex)) 

132 y_pchip = [segmently] 

133 for i in range(1, len(xdata)): 

134 val = segmently + slope * (i - leftindex) 

135 y_pchip.append(val) 

136 else: 

137 y_pchip = pchip(xhull, yhull, xdata) # Create a pchip line (curve) from hull 

138 

139 suby = [] 

140 for i in range(len(xdata)): 

141 val = ydata[i] - y_pchip[i] 

142 if val > 0: 

143 suby.append(ydata[i] - y_pchip[i]) 

144 else: 

145 suby.append(0) 

146 return suby 

147 

148def pchip(x, y, u): 

149 """ 

150 Calculate the first derivative at each section 

151 there will be len(x)-1 

152 """ 

153 h = [] 

154 h0 = x[0] 

155 for h1 in x[1:]: 

156 h.append(h1 - h0) 

157 h0 = h1 

158 

159 delta = [] 

160 for (j, f) in enumerate(h): 

161 delta.append((y[j + 1] - y[j]) / f) 

162 

163 d = [] 

164 d.append(pchipend(h[0], h[1], delta[0], delta[1])) 

165 for k in range(1, len(x) - 1): 

166 d.append(pchipslopes(h[k - 1], h[k], delta[k - 1], delta[k])) 

167 

168 d.append(pchipend(h[-1], h[-2], delta[-1], delta[-2])) 

169 

170 # evaluate function 

171 pchipy = [] 

172 segmentlx = x[0] 

173 segmently = y[0] 

174 for (i, e) in enumerate(delta): 

175 segmentrx = x[i + 1] 

176 segmentry = y[i + 1] 

177 leftindex = u.index(segmentlx) 

178 rightindex = u.index(segmentrx) 

179 c = (3 * e - 2 * d[i] - d[i + 1]) / h[i] 

180 b = (d[i] - 2 * e + d[i + 1]) / (h[i] ** 2) 

181 dfloat = d[i] 

182 for j in u[leftindex:rightindex]: 

183 j = j - u[leftindex] 

184 pchipy.append(segmently + j * (dfloat + j * (c + j * b))) 

185 segmentlx = segmentrx 

186 segmently = segmentry 

187 

188 # append the last point 

189 pchipy.append(y[-1]) 

190 return pchipy 

191 

192def pchipslopes(hm, h, deltam, delta): 

193 """ 

194 PCHIPSLOPES Slopes for shape-preserving Hermite cubic 

195 pchipslopes(h,delta) computes d(k) = P(x(k)). 

196 """ 

197 # Slopes at interior points 

198 # delta = diff(y)./diff(x). 

199 # d(k) = 0 if delta(k-1) and delta(k) have opposites 

200 # signs or either is zero. 

201 # d(k) = weighted harmonic mean of delta(k-1) and 

202 # delta(k) if they have the same sign. 

203 

204 if sign(deltam) * sign(delta) > 0: 

205 w1 = 2 * h + hm 

206 w2 = h + 2 * hm 

207 return (w1 + w2) / (w1 / deltam + w2 / delta) 

208 else: 

209 return 0.0 

210 

211def pchipend(h1, h2, del1, del2): 

212 """ 

213 Noncentered, shape-preserving, three-point formula. 

214 """ 

215 d = ((2 * h1 + h2) * del1 - h1 * del2) / (h1 + h2) 

216 if sign(d) != sign(del1): 

217 d = 0 

218 elif (sign(del1) != sign(del2)) and (abs(d) > abs(3 * del1)): 

219 d = 3 * del1 

220 return d 

221 

222def sign(d): 

223 """ 

224 Return the sign of d 

225 """ 

226 if d > 0: 

227 return 1 

228 if d == 0: 

229 return 0 

230 if d < 0: 

231 return -1 

232 return None 

233 

234######################################################### 

235 

236def getPeaksFromHist(orig_hist, width_thres=0, height_thres=0): 

237 """ 

238 Find peaks from histogram 

239 :param orig_hist: input histogram 

240 :param width_thres: Width threshold (default = 5) 

241 :param height_thres: Height threshold (default = mean value of histogram) 

242 :return: sorted peak list 

243 """ 

244 if len(orig_hist) < 10: 

245 return [] 

246 if width_thres == 0: 

247 width_thres = 5 

248 if height_thres == 0: 

249 height_thres = np.mean(orig_hist) 

250 

251 hist = smooth(np.array(orig_hist), 20) 

252 hist[hist < height_thres] = height_thres 

253 hist = hist - min(hist) 

254 

255 peak_hist1 = np.zeros((len(hist), 1)) 

256 peak_list = [] 

257 for i in range(4, len(hist) - 4): 

258 peak_hist1[i] = hist[i + 1] - hist[i - 1] 

259 

260 for i in range(width_thres, len(peak_hist1) - width_thres): 

261 

262 

263 if all(peak_hist1[i - width_thres:i] > 0) and all(peak_hist1[i + 1:i + width_thres] < 0): 

264 peak_list.append(i) 

265 

266 if len(peak_list) != 0: 

267 new_list = [] 

268 derivate = np.zeros((len(peak_list), 1)) 

269 for i in range(1, len(peak_list)): 

270 derivate[i] = peak_list[i] - peak_list[i - 1] 

271 i = 0 

272 while i < len(derivate) - 1: 

273 s = peak_list[i] 

274 c = 0 

275 if derivate[i + 1] == 1 and not i + 1 == len(derivate) - 1: 

276 j = i + 1 

277 while j <= len(derivate): 

278 if derivate[j] == 1 and not j + 1 == len(derivate): 

279 s += peak_list[j] 

280 c += 1 

281 j += 1 

282 else: 

283 s /= (c + 1) 

284 i = j 

285 break 

286 else: 

287 i += 1 

288 new_list.append(s) 

289 if i != len(derivate) - 1: 

290 new_list.append(peak_list[i]) 

291 peak_list = new_list 

292 

293 peak_list.sort() 

294 return peak_list 

295 

296def movePeaks(hist, peaks, dist=20): 

297 """ 

298 Move peak to the local maximum point 

299 :param hist: input histogram 

300 :param peaks: approximate peak locations 

301 :param dist: maximum distance considered as local range 

302 :return: moved peaks 

303 """ 

304 peakList = [] 

305 smooth_hist = smooth(hist) 

306 for pk in peaks: 

307 p = int(round(pk)) 

308 while True: 

309 start = int(round(max(0, p - dist))) 

310 end = int(round(min(len(hist), p + dist))) 

311 if end < start: 

312 new_peak = p 

313 break 

314 new_peak = start + np.argmax(hist[int(start):int(end)]) 

315 

316 # if the local maximum is not far from initital peak, break 

317 if abs(p - new_peak) <= 5: # 

318 break 

319 else: 

320 left = min(p, new_peak) 

321 right = max(p, new_peak) 

322 

323 # Check if between initial peak and local maximum has valley 

324 if all(smooth_hist[left + 1:right] > smooth_hist[p]): 

325 break 

326 dist = dist / 2 

327 peakList.append(new_peak) 

328 return sorted(list(peakList)) 

329 

330def getFirstVallay(hist): 

331 """ 

332 Find first valley from the radial histogram. This assumes that the first peak is the maximum peak. 

333 :param hist: input histogram 

334 :return: firstvalley 

335 """ 

336 hist = smooth(hist) 

337 safe_range = (len(hist)//100, len(hist)//4) 

338 start = max(int(round(np.argmax(hist[:int(len(hist)/4)]))), 10) # Start searching from maximum peak 

339 start = min(start, safe_range[1]) # start point should be smaller than 50 pixel 

340 start = max(safe_range[0], start) # start point should be bigger than 20 

341 limit = min(int(start * 1.5), 100) # Set limit of the first valley as 150% of maximum peak location 

342 max_val = 0 

343 for i in range(start, limit): 

344 if hist[i] > max_val: 

345 max_val = hist[i] 

346 if hist[i] <= max_val*0.5: 

347 return i 

348 return limit 

349 

350def getCentroid(hist, p, intersections): 

351 """ 

352 Get centroid 

353 :param hist: input histogram (list) 

354 :param p: peak location (int) 

355 :param intersections: intersections of histogram and baseline at peak (tuple) 

356 :return: centroid 

357 """ 

358 xs = np.arange(intersections[0], intersections[1] + 1) 

359 ys = hist[xs] 

360 numerator = np.dot(xs, ys) 

361 denominator = sum(ys) 

362 if denominator != 0: 

363 cent = numerator / denominator 

364 else: 

365 cent = p 

366 return cent 

367 

368def getWidth(hist, max_loc, baseline): 

369 """ 

370 Get peak width 

371 :param hist: input histogram 

372 :param max_loc: peak location 

373 :param baseline: baseline of peak 

374 :return: peak width 

375 """ 

376 l = 1 

377 r = 1 

378 while max_loc - l > 0 and hist[max_loc - l] > baseline: 

379 l += 1 

380 while max_loc + r < len(hist)-1 and hist[max_loc + r] > baseline: 

381 r += 1 

382 return l+r, (max_loc - l, max_loc + r) 

383 

384def getPeakInformations(hist, peaks, baselines): 

385 """ 

386 Get all peak informations including centroid, width, intersection with baseline, area (intensity) 

387 :param hist: input histogram (list) 

388 :param peaks: peak locations (list) 

389 :param baselines: baselines of peaks (list) 

390 :return: result with infomations (dict) 

391 """ 

392 centroids = [] 

393 widths = [] 

394 intersectionsList = [] 

395 areas = [] 

396 

397 for (i, p) in enumerate(peaks): 

398 baseline = baselines[i] 

399 width, intersections = getWidth(hist, p, baseline) 

400 cent = getCentroid(hist, p, intersections) 

401 areas.append(hist[p] * width / (2.35 * 0.3989)) 

402 centroids.append(cent) 

403 widths.append(width) 

404 intersectionsList.append(intersections) 

405 results = {} 

406 results["centroids"] = centroids 

407 results["widths"] = widths 

408 results["intersections"] = intersectionsList 

409 results["areas"] = areas 

410 return results 

411 

412def smooth(x, window_len=10, window='hanning'): # From http://scipy-cookbook.readthedocs.io/items/SignalSmooth.html 

413 """ 

414 Smooth the data using a window with requested size. 

415 

416 This method is based on the convolution of a scaled window with the signal. 

417 The signal is prepared by introducing reflected copies of the signal 

418 (with the window size) in both ends so that transient parts are minimized 

419 in the begining and end part of the output signal. 

420 

421 input: 

422 x: the input signal 

423 window_len: the dimension of the smoothing window 

424 window: the type of window from 'flat', 'hanning', 'hamming', 'bartlett', 'blackman' 

425 flat window will produce a moving average smoothing. 

426 

427 output: 

428 the smoothed signal 

429 

430 example: 

431 

432 import numpy as np 

433 t = np.linspace(-2,2,0.1) 

434 x = np.sin(t)+np.random.randn(len(t))*0.1 

435 y = smooth(x) 

436 

437 see also: 

438 

439 numpy.hanning, numpy.hamming, numpy.bartlett, numpy.blackman, numpy.convolve 

440 scipy.signal.lfilter 

441 

442 TODO: the window parameter could be the window itself if an array instead of a string 

443 """ 

444 x = np.array(x) 

445 if x.ndim != 1: 

446 raise ValueError("smooth only accepts 1 dimension arrays.") 

447 

448 if x.size < window_len: 

449 window_len = int(x.size/3) 

450 # raise ValueError, "Input vector needs to be bigger than window size." 

451 

452 if window_len < 3: 

453 return x 

454 

455 if window not in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']: 

456 raise ValueError("Window is on of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'") 

457 

458 s=np.r_[2*x[0]-x[window_len:1:-1], x, 2*x[-1]-x[-1:-window_len:-1]] 

459 

460 if window == 'flat': #moving average 

461 w = np.ones(window_len,'d') 

462 else: 

463 w = getattr(np, window)(window_len) 

464 y = np.convolve(w/w.sum(), s, mode='same') 

465 return y[window_len-1:-window_len+1]