Coverage for utils/hdf5_manager.py: 4%

538 statements  

« prev     ^ index     » next       coverage.py v7.0.4, created at 2023-01-10 09:27 -0600

1""" 

2Copyright 1999 Illinois Institute of Technology 

3 

4Permission is hereby granted, free of charge, to any person obtaining 

5a copy of this software and associated documentation files (the 

6"Software"), to deal in the Software without restriction, including 

7without limitation the rights to use, copy, modify, merge, publish, 

8distribute, sublicense, and/or sell copies of the Software, and to 

9permit persons to whom the Software is furnished to do so, subject to 

10the following conditions: 

11 

12The above copyright notice and this permission notice shall be 

13included in all copies or substantial portions of the Software. 

14 

15THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 

16EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 

17MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 

18IN NO EVENT SHALL ILLINOIS INSTITUTE OF TECHNOLOGY BE LIABLE FOR ANY 

19CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 

20TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 

21SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 

22 

23Except as contained in this notice, the name of Illinois Institute 

24of Technology shall not be used in advertising or otherwise to promote 

25the sale, use or other dealings in this Software without prior written 

26authorization from Illinois Institute of Technology. 

27""" 

28 

29# File from BioXTAS RAW named originally SASFileIO.py but modified to serve the purpose of this software 

30 

31import os 

32import re 

33import traceback 

34import fabio 

35 

36############################ 

37#--- ## Load image files: ## 

38############################ 

39 

40def loadFabio(filename, hdf5_file=None, next_image=None): 

41 if hdf5_file is None: 

42 fabio_img = fabio.open(filename) 

43 else: 

44 fabio_img = hdf5_file 

45 

46 num_frames = fabio_img.nframes 

47 

48 if num_frames == 1: 

49 data = fabio_img.data 

50 hdr = fabio_img.getheader() 

51 

52 img = [data] 

53 img_hdr = [hdr] 

54 

55 else: 

56 if next_image is None: 

57 img = [None for i in range(fabio_img.nframes)] 

58 img_hdr = [None for i in range(fabio_img.nframes)] 

59 

60 img[0] = fabio_img.data 

61 img_hdr[0] = fabio_img.getheader() 

62 

63 for i in range(1,fabio_img.nframes): 

64 fabio_img = fabio_img.next() 

65 img[i] = fabio_img.data 

66 img_hdr[i] = fabio_img.getheader() 

67 

68 else: 

69 frame = fabio_img.get_frame(next_image) 

70 data = frame.data 

71 hdr = frame.header 

72 

73 img = [data] 

74 img_hdr = [hdr] 

75 

76 if next_image is None or next_image == fabio_img.nframes -1: 

77 fabio_img.close() 

78 

79 return img, img_hdr, num_frames 

80 

81########################################## 

82#--- ## Parse Counter Files and Headers ## 

83########################################## 

84 

85def parseCHESSF2CTSfile(filename, new_filename=None): 

86 

87 timeMonitorPattern = re.compile('\d*-second\s[a-z]*\s[()A-Z,]*\s\d*\s\d*') 

88 closedShutterCountPattern = re.compile('closed\s\d*') 

89 datePattern = re.compile('#D\s.*\n') 

90 

91 

92 with open(filename[:-3] + 'cts', 'rU') as f: 

93 

94 mon1, mon2, exposure_time, closed_shutter_count = None, None, None, None 

95 

96 for line in f: 

97 timeMonitor_match = timeMonitorPattern.search(line) 

98 closedShutterCount_match = closedShutterCountPattern.search(line) 

99 date_match = datePattern.search(line) 

100 

101 if timeMonitor_match: 

102 exposure_time = int(timeMonitor_match.group().split('-')[0]) 

103 mon1 = int(timeMonitor_match.group().split(' ')[3]) 

104 mon2 = int(timeMonitor_match.group().split(' ')[4]) 

105 

106 if closedShutterCount_match: 

107 closed_shutter_count = int(closedShutterCount_match.group().split(' ')[1]) 

108 

109 if date_match: 

110 try: 

111 date = date_match.group()[3:-1] 

112 except Exception: 

113 date = 'Error loading date' 

114 

115 background = closed_shutter_count * exposure_time 

116 

117 counters = {'closedShutterCnt' : closed_shutter_count, 

118 'mon1': mon1, 

119 'mon2': mon2, 

120 'bgcount' : background, 

121 'exposureTime': exposure_time, 

122 'date': date} 

123 

124 return counters 

125 

126def parseCHESSEIGER4MCountFile(filename, new_filename=None): 

127 ''' Loads information from the counter file at CHESS, id7a from 

128 the image filename. EIGER .h5 files with 1-based frame numbers ''' 

129 dir, file = os.path.split(filename) 

130 underscores = file.split('_') 

131 

132 countFile = underscores[0] 

133 

134 filenumber = int(underscores[-3]) 

135 

136 try: 

137 frame_number = int(underscores[-1].split('.')[0]) 

138 except Exception: 

139 frame_number = 0 

140 

141 # REG: if user root file name contains underscores, include those 

142 # note: must start at -3 to leave out "data" in image name 

143 

144 if len(underscores)>3: 

145 for each in underscores[1:-3]: 

146 countFile += '_' + each 

147 

148 countFilename = os.path.join(dir, countFile) 

149 

150 with open(countFilename,'rU') as f: 

151 allLines = f.readlines() 

152 

153 line_num = 0 

154 start_found = False 

155 start_idx = None 

156 label_idx = None 

157 date_idx = None 

158 

159 for eachLine in allLines: 

160 splitline = eachLine.split() 

161 

162 if len(splitline) > 1: 

163 if splitline[0] == '#S' and splitline[1] == str(filenumber): 

164 start_found = True 

165 start_idx = line_num 

166 

167 if splitline[0] == '#D' and start_found: 

168 date_idx = line_num 

169 

170 if splitline[0] == '#L' and start_found: 

171 label_idx = line_num 

172 break 

173 

174 line_num = line_num + 1 

175 

176 counters = {} 

177 try: 

178 if start_idx and label_idx: 

179 labels = allLines[label_idx].split() 

180 # REG: hdf5 indices start at 1 not 0 as was our Pilatus convention! 

181 vals = allLines[label_idx+0+frame_number].split() 

182 

183 for idx in range(0,len(vals)): 

184 counters[labels[idx+1]] = vals[idx] 

185 

186 if date_idx: 

187 counters['date'] = allLines[date_idx][3:-1] 

188 

189 except: 

190 print('Error loading CHESS id7a counter file') 

191 

192 return counters 

193 

194def parseCHESSG1CountFile(filename, new_filename=None): 

195 ''' Loads information from the counter file at CHESS, G1 from 

196 the image filename ''' 

197 dir, file = os.path.split(filename) 

198 underscores = file.split('_') 

199 

200 countFile = underscores[0] 

201 

202 filenumber = int(underscores[-2].strip('scan')) 

203 

204 try: 

205 frame_number = int(underscores[-1].split('.')[0]) 

206 except Exception: 

207 frame_number = 0 

208 

209 

210 if len(underscores)>3: 

211 for each in underscores[1:-2]: 

212 countFile += '_' + each 

213 

214 countFilename = os.path.join(dir, countFile) 

215 

216 with open(countFilename,'rU') as f: 

217 allLines = f.readlines() 

218 

219 line_num = 0 

220 start_found = False 

221 start_idx = None 

222 label_idx = None 

223 date_idx = None 

224 

225 for eachLine in allLines: 

226 splitline = eachLine.split() 

227 

228 if len(splitline) > 1: 

229 if splitline[0] == '#S' and splitline[1] == str(filenumber): 

230 start_found = True 

231 start_idx = line_num 

232 

233 if splitline[0] == '#D' and start_found: 

234 date_idx = line_num 

235 

236 if splitline[0] == '#L' and start_found: 

237 label_idx = line_num 

238 break 

239 

240 line_num = line_num + 1 

241 

242 counters = {} 

243 try: 

244 if start_idx and label_idx: 

245 labels = allLines[label_idx].split() 

246 vals = allLines[label_idx+1+frame_number].split() 

247 

248 for idx in range(0,len(vals)): 

249 counters[labels[idx+1]] = vals[idx] 

250 

251 if date_idx: 

252 counters['date'] = allLines[date_idx][3:-1] 

253 

254 except Exception: 

255 print('Error loading G1 header') 

256 

257 return counters 

258 

259def parseCHESSG1CountFileWAXS(filename, new_filename=None): 

260 ''' Loads information from the counter file at CHESS, G1 from 

261 the image filename ''' 

262 

263 dir, file = os.path.split(filename) 

264 underscores = file.split('_') 

265 

266 countFile = underscores[0] 

267 

268 filenumber = int(underscores[-2].strip('scan')) 

269 

270 try: 

271 frame_number = int(underscores[-1].split('.')[0]) 

272 except Exception: 

273 frame_number = 0 

274 

275 

276 if len(underscores)>3: 

277 for each in underscores[1:-3]: 

278 countFile += '_' + each 

279 

280 countFilename = os.path.join(dir, countFile) 

281 

282 with open(countFilename,'rU') as f: 

283 allLines = f.readlines() 

284 

285 line_num = 0 

286 start_found = False 

287 start_idx = None 

288 label_idx = None 

289 date_idx = None 

290 

291 for eachLine in allLines: 

292 splitline = eachLine.split() 

293 

294 if len(splitline) > 1: 

295 if splitline[0] == '#S' and splitline[1] == str(filenumber): 

296 start_found = True 

297 start_idx = line_num 

298 

299 if splitline[0] == '#D' and start_found: 

300 date_idx = line_num 

301 

302 if splitline[0] == '#L' and start_found: 

303 label_idx = line_num 

304 break 

305 

306 line_num = line_num + 1 

307 

308 counters = {} 

309 try: 

310 if start_idx and label_idx: 

311 labels = allLines[label_idx].split() 

312 vals = allLines[label_idx+1+frame_number].split() 

313 

314 for idx in range(0,len(vals)): 

315 counters[labels[idx+1]] = vals[idx] 

316 

317 if date_idx: 

318 counters['date'] = allLines[date_idx][3:-1] 

319 

320 except Exception: 

321 print('Error loading G1 header') 

322 

323 

324 return counters 

325 

326def parseCHESSG1CountFileEiger(filename, new_filename=None): 

327 ''' Loads information from the counter file at CHESS, G1 from 

328 the image filename ''' 

329 

330 dirname, file = os.path.split(filename) 

331 

332 dirname = os.path.dirname(dirname) 

333 underscores = file.split('_') 

334 

335 countFile = underscores[0] 

336 

337 filenumber = int(underscores[-3].strip('scan')) 

338 

339 try: 

340 frame_number = int(underscores[-1].split('.')[0])-1 

341 except Exception: 

342 frame_number = 0 

343 

344 

345 if len(underscores)>3: 

346 for each in underscores[1:-3]: 

347 countFile += '_' + each 

348 

349 countFilename = os.path.join(dirname, countFile) 

350 

351 with open(countFilename,'rU') as f: 

352 allLines = f.readlines() 

353 

354 line_num = 0 

355 start_found = False 

356 start_idx = None 

357 label_idx = None 

358 date_idx = None 

359 

360 for eachLine in allLines: 

361 splitline = eachLine.split() 

362 

363 if len(splitline) > 1: 

364 if splitline[0] == '#S' and splitline[1] == str(filenumber): 

365 start_found = True 

366 start_idx = line_num 

367 

368 if splitline[0] == '#D' and start_found: 

369 date_idx = line_num 

370 

371 if splitline[0] == '#L' and start_found: 

372 label_idx = line_num 

373 break 

374 

375 line_num = line_num + 1 

376 

377 counters = {} 

378 

379 try: 

380 if start_idx and label_idx: 

381 labels = allLines[label_idx].split() 

382 vals = allLines[label_idx+1+frame_number].split() 

383 

384 for idx in range(0,len(vals)): 

385 counters[labels[idx+1]] = vals[idx] 

386 

387 if date_idx: 

388 counters['date'] = allLines[date_idx][3:-1] 

389 

390 except Exception: 

391 print('Error loading G1 header') 

392 

393 return counters 

394 

395def parseMAXLABI911HeaderFile(filename, new_filename=None): 

396 

397 filepath, ext = os.path.splitext(filename) 

398 hdr_file = filename + '.hdr' 

399 

400 with open(hdr_file,'rU') as f: 

401 all_lines = f.readlines() 

402 

403 counters = {} 

404 

405 for each_line in all_lines: 

406 split_lines = each_line.split('=') 

407 key = split_lines[0] 

408 counters[key] = split_lines[-1][:-1] 

409 

410 return counters 

411 

412def parseMAXLABI77HeaderFile(filename, new_filename=None): 

413 

414 filepath, ext = os.path.splitext(filename) 

415 hdr_file = filename + '.hdr' 

416 

417 with open(hdr_file,'rU') as f: 

418 all_lines = f.readlines() 

419 

420 counters = {} 

421 

422 for each_line in all_lines: 

423 

424 split_lines = each_line.split() 

425 key = split_lines[0] 

426 

427 if key == 'Start:': 

428 counters['date'] = " ".join(split_lines[1:6]) 

429 counters['end_time'] = split_lines[-1] 

430 elif key == 'Sample:': 

431 counters['sample'] = split_lines[1] 

432 counters['code'] = split_lines[-1] 

433 elif key == 'MAXII': 

434 counters['current_begin'] = split_lines[4] 

435 counters['current_end'] = split_lines[6] 

436 counters['current_mean'] = split_lines[-1] 

437 elif key == 'SampleTemperature:': 

438 counters['temp_begin'] = split_lines[2] 

439 counters['temp_end'] = split_lines[2] 

440 counters['temp_mean'] = split_lines[6] 

441 elif key == 'SampleDiode:': 

442 counters['SmpDiode_begin'] = split_lines[2] 

443 counters['SmpDiode_end'] = split_lines[2] 

444 counters['SmpDiode_mean'] = split_lines[6] 

445 elif key == 'BeamstopDiode:': 

446 counters['BmStpDiode_avg'] = split_lines[2] 

447 elif key == 'IonChamber:': 

448 counters['IonDiode_avg'] = split_lines[2] 

449 elif key == 'Tube': 

450 counters['vacuum'] = split_lines[-1] 

451 elif key == 'ExposureTime:': 

452 counters['exposureTime'] = split_lines[1] 

453 elif key == 'MarCCD': 

454 counters['diameter'] = split_lines[4] 

455 counters['binning'] = split_lines[-1] 

456 elif key == 'BeamCenterX:': 

457 counters['xCenter'] = split_lines[1] 

458 counters['yCenter'] = split_lines[3] 

459 

460 

461 return counters 

462 

463def parseBioCATlogfile(filename, new_filename=None): 

464 datadir, fname = os.path.split(filename) 

465 

466 if new_filename is not None: 

467 #BioCAT Eiger 

468 

469 sname_val = int(new_filename.split('.')[0].split('_')[-1]) 

470 searchName = '.'.join(fname.split('.')[:-1]) 

471 

472 if fname.endswith('master.h5'): 

473 countFilename=os.path.join(datadir, '_'.join(fname.split('_')[:-1])+'.log') 

474 searchName = '_'.join(searchName.split('_')[:-1]) + '_{:06d}'.format(sname_val) 

475 else: 

476 countFilename=os.path.join(datadir, '_'.join(fname.split('_')[:-2])+'.log') 

477 searchName = '_'.join(searchName.split('_')[:-2]) + '_{:06d}'.format(sname_val) 

478 

479 else: 

480 #BioCAT Pilatus 

481 countFilename=os.path.join(datadir, '_'.join(fname.split('_')[:-1])+'.log') 

482 searchName='.'.join(fname.split('.')[:-1]) 

483 

484 with open(countFilename,'rU') as f: 

485 allLines=f.readlines() 

486 

487 line_num=0 

488 

489 counters = {} 

490 

491 offset = 0 

492 

493 for i, line in enumerate(allLines): 

494 if line.startswith('#'): 

495 if line.startswith('#Filename') or line.startswith('#image'): 

496 labels = line.strip('#').split('\t') 

497 offset = i 

498 else: 

499 key = line.strip('#').split(':')[0].strip() 

500 val = ':'.join(line.strip('#').split(':')[1:]) 

501 if key in counters: 

502 counters[key] = counters[key] + '\n' + val.strip() 

503 else: 

504 counters[key] = val.strip() 

505 else: 

506 break 

507 

508 test_idx = int(searchName.split('_')[-1]) + offset 

509 

510 if test_idx < len(allLines) and searchName in allLines[test_idx]: 

511 line_num = test_idx 

512 else: 

513 for a in range(1,len(allLines)): 

514 if searchName in allLines[a]: 

515 line_num=a 

516 

517 if line_num>0: 

518 vals=allLines[line_num].split('\t') 

519 

520 for a in range(len(labels)): 

521 counters[labels[a].strip()] = vals[a].strip() 

522 

523 else: 

524 counters = {} 

525 

526 return counters 

527 

528def parseBL19U2HeaderFile(filename, new_filename=None): 

529 fname, ext = os.path.splitext(filename) 

530 

531 countFilename=fname + '.txt' 

532 

533 counters = {} 

534 

535 with open(countFilename, 'rU') as f: 

536 for line in f: 

537 name = line.split(':')[0] 

538 value = ':'.join(line.split(':')[1:]) 

539 counters[name.strip()] = value.strip() 

540 

541 return counters 

542 

543def parsePetraIIIP12EigerFile(filename, new_filename = None): 

544 if new_filename: 

545 fnum = int(new_filename.split('_')[-1].split('.')[0]) 

546 else: 

547 fnum = 1 

548 

549 data_path, data_name = os.path.split(filename) 

550 

551 header_name = '_'.join(data_name.split('_')[:2])+'_%05i.txt' %(fnum) 

552 

553 header_path = os.path.join(os.path.split(data_path)[0], 'header') 

554 

555 countFilename = os.path.join(header_path, header_name) 

556 

557 counters = {} 

558 

559 with open(countFilename, 'rU') as f: 

560 for line in f: 

561 name = line.split(':')[0] 

562 value = ':'.join(line.split(':')[1:]) 

563 counters[name.strip()] = value.strip() 

564 

565 return counters 

566 

567################################################################# 

568#--- ** Header and Image formats ** 

569################################################################# 

570# To add new header types, write a parse function and append the 

571# dictionary header_types below 

572################################################################# 

573 

574all_header_types = {'None' : None, 

575 'F2, CHESS' : parseCHESSF2CTSfile, 

576 'G1, CHESS' : parseCHESSG1CountFile, 

577 'CHESS EIGER 4M' : parseCHESSEIGER4MCountFile, 

578 'G1 WAXS, CHESS' : parseCHESSG1CountFileWAXS, 

579 'G1 Eiger, CHESS' : parseCHESSG1CountFileEiger, 

580 'I711, MaxLab' : parseMAXLABI77HeaderFile, 

581 'I911-4 Maxlab' : parseMAXLABI911HeaderFile, 

582 'BioCAT, APS' : parseBioCATlogfile, 

583 'BL19U2, SSRF' : parseBL19U2HeaderFile, 

584 'P12 Eiger, Petra III' : parsePetraIIIP12EigerFile} 

585 

586all_image_types = { 

587 'Pilatus' : loadFabio, 

588 'CBF' : loadFabio, 

589 'ADSC Quantum' : loadFabio, 

590 'Bruker' : loadFabio, 

591 'Gatan Digital Micrograph' : loadFabio, 

592 'EDNA-XML' : loadFabio, 

593 'ESRF EDF' : loadFabio, 

594 'Nonius KappaCCD' : loadFabio, 

595 'Fit2D spreadsheet' : loadFabio, 

596 'General Electric' : loadFabio, 

597 'Hamamatsu CCD' : loadFabio, 

598 'HDF5 (Hierarchical data format)' : loadFabio, 

599 'MarCCD 165' : loadFabio, 

600 'Mar345' : loadFabio, 

601 'Numpy 2D Array' : loadFabio, 

602 'Oxford Diffraction' : loadFabio, 

603 'Pixi' : loadFabio, 

604 'Portable aNy Map' : loadFabio, 

605 'Rigaku SAXS format' : loadFabio, 

606 '16 bit TIF' : loadFabio, 

607 'MPA (multiwire)' : loadFabio, 

608 'Eiger' : loadFabio, 

609 'Rigaku HiPix' : loadFabio, 

610 # 'NeXus' : loadNeXusFile, 

611 } 

612 

613def loadAllHeaders(filename, image_type, header_type): 

614 ''' returns the image header and the info from the header file only. ''' 

615 

616 try: 

617 file_type = checkFileType(filename) 

618 # print file_type 

619 except IOError: 

620 raise 

621 except Exception as msg: 

622 print(str(msg)) 

623 file_type = None 

624 

625 if file_type == 'hdf5': 

626 try: 

627 hdf5_file = fabio.open(filename) 

628 file_type = 'image' 

629 except Exception: 

630 pass 

631 else: 

632 hdf5_file = None 

633 

634 if hdf5_file is not None: 

635 img, imghdr, num_frames = loadImage(filename, hdf5_file, 

636 next_image=0) 

637 else: 

638 img, imghdr, num_frames = loadImage(filename, next_image=0) 

639 

640 if len(img) > 1 or num_frames > 1 or hdf5_file is not None: 

641 temp_filename = os.path.split(filename)[1].split('.') 

642 

643 if len(temp_filename) > 1: 

644 temp_filename[-2] = temp_filename[-2] + '_%05i' %(1) 

645 else: 

646 temp_filename[0] = temp_filename[0] + '_%05i' %(1) 

647 

648 new_filename = '.'.join(temp_filename) 

649 else: 

650 new_filename = os.path.split(filename)[1] 

651 

652 if header_type != 'None': 

653 hdr = loadHeader(filename, new_filename, header_type) 

654 else: 

655 hdr = None 

656 

657 masks = {'BeamStopMask' : [None, None, None], 

658 'TransparentBSMask': [None, None, None], 

659 } 

660 tbs_mask = masks['TransparentBSMask'][0] 

661 

662 if tbs_mask is not None: 

663 if isinstance(img, list): 

664 roi_counter = img[0][tbs_mask==1].sum() #In the case of multiple images in the same file, load the ROI for the first one 

665 else: 

666 roi_counter = img[tbs_mask==1].sum() 

667 

668 if hdr is None: 

669 hdr = {'roi_counter': roi_counter} 

670 else: 

671 hdr['roi_counter'] = roi_counter 

672 

673 return imghdr, hdr 

674 

675def loadHeader(filename, new_filename, header_type): 

676 ''' returns header information based on the *image* filename 

677 and the type of headerfile ''' 

678 

679 if header_type != 'None': 

680 try: 

681 if new_filename != os.path.split(filename)[1]: 

682 hdr = all_header_types[header_type](filename, new_filename) 

683 else: 

684 hdr = all_header_types[header_type](filename) 

685 except IOError as io: 

686 print(str(io).replace("u'",'')) 

687 # traceback.print_exc() 

688 except Exception as e: 

689 # print(e) 

690 # traceback.print_exc() 

691 print('Header file for : ' + str(filename) + ' could not be read or contains incorrectly formatted data. ') 

692 else: 

693 hdr = {} 

694 

695 #Clean up headers by removing spaces in header names and non-unicode characters) 

696 if hdr is not None: 

697 hdr = {key.replace(' ', '_').translate(str.maketrans('', '', '()[]')) 

698 if isinstance(key, str) else key: hdr[key] for key in hdr} 

699 # hdr = { key : str(hdr[key], errors='ignore') if isinstance(hdr[key], str) 

700 # else hdr[key] for key in hdr} 

701 

702 return hdr 

703 

704def loadImage(filename, hdf5_file=None, next_image=None): 

705 ''' returns the loaded image based on the image filename 

706 and image type. ''' 

707 image_type = 'Pilatus' 

708 

709 num_frames = 1 

710 

711 try: 

712 if all_image_types[image_type] == loadFabio: 

713 img, imghdr, num_frames = all_image_types[image_type](filename, 

714 hdf5_file, next_image) 

715 else: 

716 img, imghdr = all_image_types[image_type](filename) 

717 except (ValueError, TypeError, KeyError, fabio.fabioutils.NotGoodReader, Exception) as msg: 

718 # traceback.print_exc() 

719 print('Error loading image, ' + str(msg)) 

720 

721 if not isinstance(img, list): 

722 img = [img] 

723 if not isinstance(imghdr, list): 

724 imghdr = [imghdr] 

725 

726 #Clean up headers by removing spaces in header names and non-unicode characters) 

727 for hdr in imghdr: 

728 if hdr is not None: 

729 hdr = {key.replace(' ', '_').translate(str.maketrans('', '', '()[]')) 

730 if isinstance(key, str) else key: hdr[key] for key in hdr} 

731 # hdr = { key : str(hdr[key], errors='ignore') if isinstance(hdr[key], str) 

732 # else hdr[key] for key in hdr} 

733 

734 return img, imghdr, num_frames 

735 

736################################# 

737#--- ** MAIN LOADING FUNCTION ** 

738################################# 

739 

740def loadFile(filename, no_processing=False, return_all_images=True): 

741 ''' Loads a file an returns a SAS Measurement Object (SASM) and the full image if the 

742 selected file was an Image file 

743 

744 NB: This is the function used to load any type of file in RAW 

745 ''' 

746 try: 

747 file_type = checkFileType(filename) 

748 # print file_type 

749 except IOError: 

750 raise 

751 except Exception as msg: 

752 print(str(msg)) 

753 file_type = None 

754 

755 if file_type == 'hdf5': 

756 try: 

757 hdf5_file = fabio.open(filename) 

758 file_type = 'image' 

759 except Exception: 

760 pass 

761 else: 

762 hdf5_file = None 

763 

764 if file_type == 'image': 

765 try: 

766 sasm, img = loadImageFile(filename, hdf5_file, 

767 return_all_images) 

768 except (ValueError, AttributeError) as msg: 

769 print('No data could be retrieved from the file, unknown format.') 

770 traceback.print_exc() 

771 # except Exception: 

772 # traceback.print_exc() 

773 # raise 

774 

775 #Always do some post processing for image files 

776 if not isinstance(sasm, list): 

777 sasm = [sasm] 

778 

779 if not isinstance(sasm, list) and (sasm is None or len(sasm.i) == 0): 

780 print('No data could be retrieved from the file, unknown format.') 

781 

782 return sasm, img 

783 

784def loadImageFile(filename, hdf5_file=None, return_all_images=True): 

785 hdr_fmt = 'None' 

786 

787 if hdf5_file is not None: 

788 is_hdf5 = True 

789 else: 

790 is_hdf5 = False 

791 

792 load_one_frame = False 

793 

794 if is_hdf5 and hdf5_file.nframes > 1: 

795 num_frames = hdf5_file.nframes 

796 load_one_frame = True 

797 else: 

798 num_frames = 1 

799 

800 

801 loaded_data = [] 

802 sasm_list = [] 

803 

804 for file_num in range(num_frames): 

805 if load_one_frame: 

806 new_data, new_hdr, _ = loadImage(filename, hdf5_file, file_num) 

807 

808 else: 

809 new_data, new_hdr, _ = loadImage(filename, hdf5_file) 

810 

811 if return_all_images: 

812 loaded_data.extend(new_data) 

813 elif not return_all_images and file_num == 0: 

814 loaded_data.append(new_data[0]) 

815 

816 offset = 0 

817 

818 #Process all loaded images into sasms 

819 for i in range(len(new_data)): 

820 img = new_data[i] 

821 img_hdr = new_hdr[i] 

822 

823 if i == 0 and (len(new_data) > 1 or is_hdf5): 

824 

825 temp_filename = os.path.split(filename)[1].split('.') 

826 

827 if len(temp_filename) > 1: 

828 temp_filename[-2] = temp_filename[-2] + '_%05i' %(i+1) 

829 else: 

830 temp_filename[0] = temp_filename[0] + '_%05i' %(i+1) 

831 

832 new_filename = '.'.join(temp_filename) 

833 

834 base_hdr = hdrfile_info = loadHeader(filename, new_filename, hdr_fmt) 

835 

836 if not filename.endswith('master.h5'): 

837 sname_offset = int(os.path.splitext(filename)[0].split('_')[-1])-1 

838 else: 

839 sname_offset = 0 

840 

841 if 'Number_of_images_per_file' in base_hdr: 

842 mult = int(base_hdr['Number_of_images_per_file']) 

843 else: 

844 mult = len(new_data) 

845 

846 offset = sname_offset*mult 

847 

848 if len(new_data) > 1 or is_hdf5: 

849 temp_filename = os.path.split(filename)[1].split('.') 

850 

851 if len(temp_filename) > 1: 

852 temp_filename[-2] = temp_filename[-2] + '_%05i' %(i+file_num+offset+1) 

853 else: 

854 temp_filename[0] = temp_filename[0] + '_%05i' %(i+file_num+offset+1) 

855 

856 new_filename = '.'.join(temp_filename) 

857 else: 

858 new_filename = os.path.split(filename)[1] 

859 

860 hdrfile_info = loadHeader(filename, new_filename, hdr_fmt) 

861 

862 parameters = {'imageHeader' : img_hdr, 

863 'counters' : hdrfile_info, 

864 'filename' : new_filename, 

865 'load_path' : filename} 

866 

867 sasm = parameters['filename'] 

868 

869 sasm_list.append(sasm) 

870 

871 return sasm_list, loaded_data 

872 

873def checkFileType(filename): 

874 ''' Tries to find out what file type it is and reports it back ''' 

875 

876 path, ext = os.path.splitext(filename) 

877 

878 if ext == '.fit': 

879 return 'fit' 

880 elif ext == '.fir': 

881 return 'fir' 

882 elif ext == '.abs': 

883 return 'abs' 

884 elif ext == '.out': 

885 return 'out' 

886 elif ext == '.nxs': #Nexus file 

887 return 'image' 

888 elif ext == '.edf': 

889 return 'image' 

890 elif ext == '.ccdraw': 

891 return 'image' 

892 elif ext == '.int': 

893 return 'int' 

894 elif ext == '.img' or ext == '.imx_0' or ext == '.dkx_0' or ext == '.dkx_1' or ext == '.png' or ext == '.mpa': 

895 return 'image' 

896 elif ext == '.dat' or ext == '.sub' or ext =='.txt': 

897 return 'primus' 

898 elif ext == '.mar1200' or ext == '.mar2400' or ext == '.mar2300' or ext == '.mar3600': 

899 return 'image' 

900 elif (ext == '.img' or ext == '.sfrm' or ext == '.dm3' or ext == '.edf' or ext == '.xml' or ext == '.cbf' or ext == '.kccd' or 

901 ext == '.msk' or ext == '.spr' or ext == '.tif' or ext == '.mccd' or ext == '.mar3450' or ext =='.npy' or 

902 ext == '.pnm' or ext == '.No'): 

903 return 'image' 

904 elif ext == '.ift': 

905 return 'ift' 

906 elif ext == '.csv': 

907 return 'txt' 

908 elif ext == '.h5': 

909 return 'hdf5' 

910 else: 

911 try: 

912 f = fabio.open(filename) 

913 return 'image' 

914 except Exception: 

915 try: 

916 float(ext.strip('.')) 

917 except Exception: 

918 return 'txt' 

919 return 'csv'