Coverage for /Users/Newville/Codes/xraylarch/larch/xrmmap/xrm_mapfile.py: 6%

2273 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-09 10:08 -0600

1import os 

2import sys 

3import uuid 

4import time 

5import h5py 

6import numpy as np 

7import scipy.stats as stats 

8import json 

9import multiprocessing as mp 

10from functools import partial 

11 

12import larch 

13from larch.utils import debugtime, isotime 

14from larch.utils.strutils import fix_varname, fix_filename, bytes2str, version_ge 

15 

16from larch.io import (nativepath, new_filename, read_xrf_netcdf, 

17 read_xsp3_hdf5, read_xrd_netcdf, read_xrd_hdf5) 

18 

19from larch.xrf import MCA, ROI 

20from larch.xrd import (XRD, E_from_lambda, integrate_xrd_row, q_from_twth, 

21 q_from_d, lambda_from_E, read_xrd_data, read_poni) 

22 

23from larch.math.tomography import tomo_reconstruction, reshape_sinogram, trim_sinogram 

24 

25from .configfile import FastMapConfig 

26from .asciifiles import (readASCII, readMasterFile, readROIFile, 

27 readEnvironFile, parseEnviron) 

28 

29from .gsexrm_utils import (GSEXRM_MCADetector, GSEXRM_Area, GSEXRM_Exception, 

30 GSEXRM_MapRow, GSEXRM_FileStatus) 

31 

32 

33DEFAULT_XRAY_ENERGY = 39987.0 # probably means x-ray energy was not found in meta data 

34NINIT = 64 

35COMPRESSION_OPTS = 2 

36COMPRESSION = 'gzip' 

37#COMPRESSION = 'lzf' 

38DEFAULT_ROOTNAME = 'xrmmap' 

39VALID_ROOTNAMES = ('xrmmap', 'xrfmap') 

40EXTRA_DETGROUPS = ('scalars', 'work', 'xrd1d', 'xrd2d') 

41NOT_OWNER = "Not Owner of HDF5 file %s" 

42READ_ONLY = "HDF5 file %s is open read-only" 

43QSTEPS = 2048 

44 

45H5ATTRS = {'Type': 'XRM 2D Map', 

46 'Version': '2.1.0', 

47 # 'Version': '2.0.0', 

48 # 'Version': '1.0.1', 

49 'Title': 'Epics Scan Data', 

50 'Beamline': 'GSECARS, 13-IDE / APS', 

51 'Start_Time': '', 

52 'Stop_Time': '', 

53 'Map_Folder': '', 

54 'Dimension': 2, 

55 'Process_Machine': '', 

56 'Process_ID': 0, 

57 'Compression': ''} 

58 

59def h5str(obj): 

60 '''strings stored in an HDF5 from Python2 may look like 

61 "b'xxx'", that is containg "b". strip these out here 

62 ''' 

63 out = str(obj) 

64 if out.startswith("b'") and out.endswith("'"): 

65 out = out[2:-1] 

66 return out 

67 

68def get_machineid(): 

69 "machine id / MAC address, independent of hostname" 

70 return hex(uuid.getnode())[2:] 

71 

72def strlist(alist): 

73 return [a.encode('utf-8') for a in alist] 

74 

75def isGSEXRM_MapFolder(fname): 

76 "return whether folder a valid Scan Folder (raw data)" 

77 if (fname is None or not os.path.exists(fname) or 

78 not os.path.isdir(fname)): 

79 return False 

80 flist = os.listdir(fname) 

81 for f in ('Master.dat', 'Environ.dat', 'Scan.ini'): 

82 if f not in flist: 

83 return False 

84 

85 has_xrmdata = False 

86 

87 header, rows = readMasterFile(os.path.join(fname, 'Master.dat')) 

88 try: 

89 for f in rows[0]: 

90 if f in flist: 

91 has_xrmdata = True 

92 except: 

93 pass 

94 return has_xrmdata 

95 

96 

97def test_h5group(group, folder=None): 

98 "test h5 group as a XRM Map" 

99 valid = ('config' in group and 'roimap' in group) 

100 for attr in ('Version', 'Map_Folder', 'Dimension', 'Start_Time'): 

101 valid = valid and attr in group.attrs 

102 if not valid: 

103 return None, None 

104 status = GSEXRM_FileStatus.hasdata 

105 vers = h5str(group.attrs.get('Version','')) 

106 fullpath = group.attrs.get('Map_Folder','') 

107 _parent, _folder = os.path.split(fullpath) 

108 if folder is not None and folder != _folder: 

109 status = GSEXRM_FileStatus.wrongfolder 

110 return status, vers 

111 

112 

113def create_xrmmap(h5root, root=None, dimension=2, folder='', start_time=None): 

114 '''creates a skeleton '/xrmmap' group in an open HDF5 file 

115 

116 This is left as a function, not method of GSEXRM_MapFile below 

117 because it may be called by the mapping collection program 

118 (ie, from collector.py) when a map is started 

119 

120 This leaves a structure to be filled in by 

121 GSEXRM_MapFile.init_xrmmap(), 

122 ''' 

123 attrs = {} 

124 attrs.update(H5ATTRS) 

125 if start_time is None: 

126 start_time = time.ctime() 

127 

128 attrs.update({'Dimension':dimension, 'Start_Time':start_time, 

129 'Map_Folder': folder, 'Last_Row': -1}) 

130 if root in ('', None): 

131 root = DEFAULT_ROOTNAME 

132 xrmmap = h5root.create_group(root) 

133 

134 for key, val in attrs.items(): 

135 xrmmap.attrs[key] = str(val) 

136 

137 g = xrmmap.create_group('roimap') 

138 g.attrs['type'] = 'roi maps' 

139 g.attrs['desc'] = 'ROI data, including summed and deadtime corrected maps' 

140 

141 g = xrmmap.create_group('config') 

142 g.attrs['type'] = 'scan config' 

143 g.attrs['desc'] = '''scan configuration, including scan definitions, 

144 ROI definitions, MCA calibration, Environment Data, etc''' 

145 

146 g = xrmmap.create_group('areas') 

147 g.attrs['type'] = 'areas' 

148 

149 g = xrmmap.create_group('positions') 

150 g.attrs['type'] = 'position arrays' 

151 

152 g = xrmmap.create_group('scalars') 

153 g.attrs['type'] = 'scalar detectors' 

154 

155 g = xrmmap.create_group('work') 

156 g.attrs['type'] = 'virtual detectors for work/analysis arrays' 

157 

158 conf = xrmmap['config'] 

159 for name in ('scan', 'general', 'environ', 'positioners', 'notes', 

160 'motor_controller', 'rois', 'mca_settings', 'mca_calib'): 

161 conf.create_group(name) 

162 h5root.flush() 

163 

164def ensure_subgroup(subgroup, group, dtype='virtual detector'): 

165 if subgroup not in group.keys(): 

166 g = group.create_group(subgroup) 

167 g.attrs['type'] = dtype 

168 return g 

169 else: 

170 g = group[subgroup] 

171 if 'type' not in g.attrs: 

172 g.attrs['type'] = dtype 

173 return g 

174 

175def toppath(pname, n=4): 

176 words = [] 

177 for i in range(n): 

178 pname, f = os.path.split(pname) 

179 words.append(f) 

180 return '/'.join(words) 

181 

182 

183def remove_zigzag(map, zigzag=0): 

184 if zigzag == 0: 

185 return map 

186 nrows, ncols = map.shape 

187 tmp = 1.0 * map 

188 even = 0 

189 if zigzag < 0: 

190 even = 1 

191 zigzag = -zigzag 

192 for i in range(nrows): 

193 if (i % 2) == even: 

194 tmp[i, :] = map[i, :] 

195 else: 

196 tmp[i, zigzag:] = map[i, :-zigzag] 

197 return tmp 

198 

199 

200class GSEXRM_MapFile(object): 

201 ''' 

202 Access to GSECARS X-ray Microprobe Map File: 

203 

204 The GSEXRM Map file is an HDF5 file built from a folder containing 

205 'raw' data from a set of sources 

206 xmap: XRF spectra saved to NetCDF by the Epics MCA detector 

207 struck: a multichannel scaler, saved as ASCII column data 

208 xps: stage positions, saved as ASCII file from the Newport XPS 

209 

210 The object here is intended to expose an HDF5 file that: 

211 a) watches the corresponding folder and auto-updates when new 

212 data is available, as for on-line collection 

213 b) stores locking information (Machine Name/Process ID) in the top-level 

214 

215 For extracting data from a GSEXRM Map File, use: 

216 

217 >>> from epicscollect.io import GSEXRM_MapFile 

218 >>> map = GSEXRM_MapFile('MyMap.001') 

219 >>> fe = map.get_roimap('Fe', det='mca2') 

220 >>> as = map.get_roimap('As Ka', det='mca1', dtcorrect=True) 

221 >>> rgb = map.get_rgbmap('Fe', 'Ca', 'Zn', dtcorrect=True, scale_each=False) 

222 >>> en = map.get_energy(det=1) 

223 

224 All these take the following options: 

225 

226 det: which detector element to use (1, 2, 3, 4, None), [None] 

227 None means to use the sum of all detectors 

228 dtcorrect: whether to return dead-time corrected spectra [True] 

229 

230 ''' 

231 

232 ScanFile = 'Scan.ini' 

233 EnvFile = 'Environ.dat' 

234 ROIFile = 'ROI.dat' 

235 XRDCALFile = 'XRD.poni' 

236 MasterFile = 'Master.dat' 

237 

238 def __init__(self, filename=None, folder=None, create_empty=False, 

239 hotcols=False, zigzag=0, dtcorrect=True, root=None, 

240 chunksize=None, xrdcal=None, xrd2dmask=None, xrd2dbkgd=None, 

241 xrd1dbkgd=None, azwdgs=0, qstps=QSTEPS, flip=True, 

242 bkgdscale=1., has_xrf=True, has_xrd1d=False, has_xrd2d=False, 

243 compression=COMPRESSION, compression_opts=COMPRESSION_OPTS, 

244 facility='APS', beamline='13-ID-E', run='', proposal='', 

245 user='', scandb=None, all_mcas=False, **kws): 

246 

247 self.filename = filename 

248 self.folder = folder 

249 self.root = root 

250 self.chunksize = chunksize 

251 # whether to remove first and last columns from data 

252 self.hotcols = hotcols 

253 # whether to shift rows to fix zig-zag 

254 self.zigzag = zigzag 

255 self.dtcorrect = dtcorrect 

256 self.scandb = scandb 

257 self.envvar = None 

258 self.write_access = True 

259 self.status = GSEXRM_FileStatus.err_notfound 

260 self.dimension = None 

261 self.nmca = None 

262 self.npts = None 

263 self.start_time = None 

264 self.xrmmap = None 

265 self.h5root = None 

266 self.last_row = -1 

267 self.rowdata = [] 

268 self.roi_names = {} 

269 self.roi_slices = None 

270 self._pixeltime = None 

271 self.masterfile = None 

272 self.force_no_dtc = False 

273 self.all_mcas = all_mcas 

274 self.detector_list = None 

275 

276 self.compress_args = {'compression': compression} 

277 if compression != 'lzf': 

278 self.compress_args['compression_opts'] = compression_opts 

279 

280 self.incident_energy = None 

281 self.has_xrf = has_xrf 

282 self.has_xrd1d = has_xrd1d 

283 self.has_xrd2d = has_xrd2d 

284 self.pos_desc = [] 

285 self.pos_addr = [] 

286 ## used for XRD 

287 self.bkgd_xrd2d = None 

288 self.bkgd_xrd1d = None 

289 self.mask_xrd2d = None 

290 self.xrdcalfile = None 

291 self.xrd2dmaskfile = None 

292 self.xrd2dbkgdfile = None 

293 self.xrd1dbkgdfile = None 

294 self.bkgdscale = bkgdscale if bkgdscale > 0 else 1. 

295 self.azwdgs = 0 if azwdgs > 36 or azwdgs < 2 else int(azwdgs) 

296 self.qstps = int(qstps) 

297 self.flip = flip 

298 self.master_modtime = -1 

299 

300 ## used for tomography orientation 

301 self.x = None 

302 self.reshape = None 

303 self.notes = {'facility' : facility, 

304 'beamline' : beamline, 

305 'run' : run, 

306 'proposal' : proposal, 

307 'user' : user} 

308 

309 nmaster = -1 

310 # initialize from filename or folder 

311 if self.filename is not None: 

312 self.getFileStatus(root=root) 

313 # print("Get File Status ", root, self.status) 

314 if self.status == GSEXRM_FileStatus.empty: 

315 ftmp = open(self.filename, 'r') 

316 self.folder = ftmp.readlines()[0][:-1].strip() 

317 if '/' in self.folder: 

318 self.folder = self.folder.split('/')[-1] 

319 ftmp.close() 

320 os.unlink(self.filename) 

321 

322 if (self.status==GSEXRM_FileStatus.err_notfound and 

323 self.filename is not None and self.folder is None and 

324 isGSEXRM_MapFolder(self.filename)): 

325 self.folder = self.filename 

326 self.filename = None 

327 

328 if isGSEXRM_MapFolder(self.folder): 

329 nmaster = self.read_master() 

330 if self.filename is None: 

331 raise GSEXRM_Exception( 

332 "'%s' is not a valid GSEXRM Map folder" % self.folder) 

333 self.getFileStatus(root=root) 

334 

335 # for existing file, read initial settings 

336 if self.status in (GSEXRM_FileStatus.hasdata, 

337 GSEXRM_FileStatus.created): 

338 self.open(self.filename, root=self.root, check_status=True) 

339 self.reset_flags() 

340 return 

341 

342 # file exists but is not hdf5 

343 if self.status == GSEXRM_FileStatus.err_nothdf5: 

344 raise GSEXRM_Exception( 

345 "'%s' is not a readable HDF5 file" % self.filename) 

346 

347 # file has no write permission 

348 # if self.status == GSEXRM_FileStatus.err_nowrite: 

349 # raise GSEXRM_Exception( 

350 # "'%s' does not have write access" % self.filename) 

351 

352 # create empty HDF5 if needed 

353 if self.status == GSEXRM_FileStatus.empty and os.path.exists(self.filename): 

354 try: 

355 flines = open(self.filename, 'r').readlines() 

356 if len(flines) < 3: 

357 os.unlink(self.filename) 

358 self.status = GSEXRM_FileStatus.err_notfound 

359 except (IOError, ValueError): 

360 pass 

361 

362 if (self.status in (GSEXRM_FileStatus.err_notfound, 

363 GSEXRM_FileStatus.wrongfolder) and 

364 self.folder is not None and isGSEXRM_MapFolder(self.folder)): 

365 

366 if nmaster < 1: 

367 self.read_master() 

368 if self.status == GSEXRM_FileStatus.wrongfolder: 

369 self.filename = new_filename(self.filename) 

370 cfile = FastMapConfig() 

371 cfile.Read(os.path.join(self.folder, self.ScanFile)) 

372 cfile.config['scan']['filename'] = self.filename 

373 # cfile.Save(os.path.join(self.folder, self.ScanFile)) 

374 print("Create HDF5 File ") 

375 self.h5root = h5py.File(self.filename, 'w') 

376 self.write_access = True 

377 if self.dimension is None and isGSEXRM_MapFolder(self.folder): 

378 if nmaster < 1: 

379 self.read_master() 

380 create_xrmmap(self.h5root, root=self.root, dimension=self.dimension, 

381 folder=self.folder, start_time=self.start_time) 

382 

383 # cfile = FastMapConfig() 

384 # self.add_map_config(cfile.config, nmca=nmca) 

385 

386 self.notes['h5_create_time'] = isotime() 

387 self.status = GSEXRM_FileStatus.created 

388 self.open(self.filename, root=self.root, check_status=False) 

389 

390 for xkey, xval in zip(self.xrmmap.attrs.keys(), self.xrmmap.attrs.values()): 

391 if xkey == 'Version': 

392 self.version = xval 

393 

394 self.add_XRDfiles(xrdcalfile=xrdcal, 

395 xrd2dmaskfile=xrd2dmask, 

396 xrd2dbkgdfile=xrd2dbkgd, 

397 xrd1dbkgdfile=xrd1dbkgd) 

398 elif (self.filename is not None and 

399 self.status == GSEXRM_FileStatus.err_notfound and create_empty): 

400 print("Create HDF5 File") 

401 self.h5root = h5py.File(self.filename, 'w') 

402 self.write_access = True 

403 create_xrmmap(self.h5root, root=None, dimension=2, start_time=self.start_time) 

404 self.notes['h5_create_time'] = isotime() 

405 self.xrmmap = self.h5root[DEFAULT_ROOTNAME] 

406 self.take_ownership() 

407 # self.status = GSEXRM_FileStatus.created 

408 cfile = FastMapConfig() 

409 self.add_map_config(cfile.config, nmca=1) 

410 self.open(self.filename, root=self.root, check_status=False) 

411 self.reset_flags() 

412 

413 else: 

414 raise GSEXRM_Exception('GSEXMAP Error: could not locate map file or folder') 

415 print("Initialized done ", self.status, self.version, self.root) 

416 

417 def __repr__(self): 

418 fname = '' 

419 if self.filename is not None: 

420 fpath, fname = os.path.split(self.filename) 

421 return "GSEXRM_MapFile('%s')" % fname 

422 

423 

424 def getFileStatus(self, filename=None, root=None, folder=None): 

425 '''return status, top-level group, and version''' 

426 

427 if filename is not None: 

428 self.filename = filename 

429 filename = self.filename 

430 folder = self.folder 

431 # print("getFileStatus 0 ", filename, folder) 

432 if folder is not None: 

433 folder = os.path.abspath(folder) 

434 parent, folder = os.path.split(folder) 

435 self.status = GSEXRM_FileStatus.err_notfound 

436 self.root, self.version = '', '' 

437 if root not in ('', None): 

438 self.root = root 

439 # see if file exists: 

440 if not (os.path.exists(filename) and os.path.isfile(filename)): 

441 return 

442 # see if file is empty/too small(signifies "read from folder") 

443 if os.stat(filename).st_size < 1024: 

444 self.status = GSEXRM_FileStatus.empty 

445 return 

446 

447 if not os.access(filename, os.W_OK): 

448 self.write_access = False 

449 # print("getFileStatus 1 ", filename, self.write_access) 

450 # see if file is an H5 file 

451 try: 

452 fh = h5py.File(filename, 'r') 

453 except IOError: 

454 self.status = GSEXRM_FileStatus.err_nothdf5 

455 return 

456 

457 status = GSEXRM_FileStatus.no_xrfmap 

458 if root is not None and root in fh: 

459 stat, vers = test_h5group(fh[root], folder=folder) 

460 if stat is not None: 

461 self.status = stat 

462 self.root, self.version = root, vers 

463 else: 

464 for root, group in fh.items(): 

465 stat, vers = test_h5group(group, folder=folder) 

466 if stat is not None: 

467 self.status = stat 

468 self.root, self.version = root, vers 

469 break 

470 # print("getFileStatus 2 ", self.status) 

471 fh.close() 

472 return 

473 

474 def get_det(self, index): 

475 return GSEXRM_MCADetector(self.xrmmap, index=index) 

476 

477 def area_obj(self, index, det=None): 

478 return GSEXRM_Area(self.xrmmap, index, det=det) 

479 

480 def get_coarse_stages(self): 

481 '''return coarse stage positions for map''' 

482 stages = [] 

483 env_addrs = [h5str(s) for s in self.xrmmap['config/environ/address']] 

484 env_vals = [h5str(s) for s in self.xrmmap['config/environ/value']] 

485 for addr, pname in self.xrmmap['config/positioners'].items(): 

486 name = h5str(pname[()]) 

487 addr = h5str(addr) 

488 val = '' 

489 if not addr.endswith('.VAL'): 

490 addr = '%s.VAL' % addr 

491 if addr in env_addrs: 

492 val = env_vals[env_addrs.index(addr)] 

493 

494 stages.append((addr, val, name)) 

495 

496 return stages 

497 

498 def open(self, filename, root=None, check_status=True): 

499 '''open GSEXRM HDF5 File : 

500 with check_status=False, this **must** be called 

501 for an existing, valid GSEXRM HDF5 File!! 

502 ''' 

503 if root in ('', None): 

504 root = DEFAULT_ROOTNAME 

505 if check_status: 

506 self.getFileStatus(filename, root=root) 

507 if self.status not in (GSEXRM_FileStatus.hasdata, 

508 GSEXRM_FileStatus.created): 

509 raise GSEXRM_Exception( 

510 "'%s' is not a valid GSEXRM HDF5 file" % self.filename) 

511 self.filename = filename 

512 # print("OPEN ", filename, self.write_access, self.h5root) 

513 if self.h5root is None: 

514 mode = 'a' if self.write_access else 'r' 

515 try: 

516 self.h5root = h5py.File(self.filename, mode) 

517 except PermissionError: 

518 self.write_access = False 

519 self.h5root = h5py.File(self.filename, 'r') 

520 print("Warning : file opened as read only") 

521 self.xrmmap = self.h5root[root] 

522 if self.folder is None: 

523 self.folder = bytes2str(self.xrmmap.attrs.get('Map_Folder','')) 

524 self.last_row = int(self.xrmmap.attrs.get('Last_Row',0)) 

525 

526 try: 

527 self.dimension = self.xrmmap['config/scan/dimension'][()] 

528 except: 

529 pass 

530 

531 if (len(self.rowdata) < 1 or 

532 (self.dimension is None and isGSEXRM_MapFolder(self.folder))): 

533 self.read_master() 

534 

535 if self.nmca is None: 

536 self.nmca = self.xrmmap.attrs.get('N_Detectors', 1) 

537 

538 def close(self): 

539 if self.check_hostid() and self.write_access: 

540 self.xrmmap.attrs['Process_Machine'] = '' 

541 self.xrmmap.attrs['Process_ID'] = 0 

542 self.xrmmap.attrs['Last_Row'] = self.last_row 

543 try: 

544 self.h5root.close() 

545 except RuntimeError: 

546 print("Got Runtime Error ") 

547 print(sys.exc_info()) 

548 

549 self.h5root = None 

550 

551 def add_XRDfiles(self, flip=None, xrdcalfile=None, xrd2dmaskfile=None, 

552 xrd2dbkgdfile=None, xrd1dbkgdfile=None): 

553 ''' 

554 adds mask file to exisiting '/xrmmap' group in an open HDF5 file 

555 mkak 2018.02.01 

556 ''' 

557 xrd1dgrp = ensure_subgroup('xrd1d', self.xrmmap) 

558 

559 if xrdcalfile is not None: 

560 self.xrdcalfile = xrdcalfile 

561 if os.path.exists(str(self.xrdcalfile)): 

562 print('Calibration file loaded: %s' % self.xrdcalfile) 

563 xrd1dgrp.attrs['calfile'] = str(self.xrdcalfile) 

564 

565 

566 self.flip = flip if flip is not None else self.flip 

567 

568 if xrd1dbkgdfile is not None: 

569 self.xrd1dbkgdfile= xrd1dbkgdfile 

570 if os.path.exists(str(self.xrd1dbkgdfile)): 

571 print('xrd1d background file loaded: %s' % self.xrd1dbkgdfile) 

572 xrd1dgrp.attrs['1Dbkgdfile'] = '%s' % (self.xrd1dbkgdfile) 

573 self.bkgd_xrd1d = read_xrd_data(self.xrd1dbkgdfile)*self.bkgdscale 

574 

575 if xrd2dbkgdfile is not None: 

576 self.xrd2dbkgdfile= xrd2dbkgdfile 

577 if os.path.exists(str(self.xrd2dbkgdfile)): 

578 print('2DXRD background file loaded: %s' % self.xrd2dbkgdfile) 

579 xrd1dgrp.attrs['2Dbkgdfile'] = '%s' % (self.xrd2dbkgdfile) 

580 self.bkgd_xrd2d = read_xrd_data(self.xrd2dbkgdfile)*self.bkgdscale 

581 

582 if xrd2dmaskfile is not None: 

583 self.xrd2dmaskfile= xrd2dmaskfile 

584 if os.path.exists(str(self.xrd2dmaskfile)): 

585 print('Mask file loaded: %s' % self.xrd2dmaskfile) 

586 xrd1dgrp.attrs['maskfile'] = '%s' % (self.xrd2dmaskfile) 

587 self.mask_xrd2d = read_xrd_data(self.xrd2dmaskfile) 

588 

589 self.h5root.flush() 

590 

591 def add_data(self, group, name, data, attrs=None, **kws): 

592 ''' creata an hdf5 dataset, replacing existing dataset if needed''' 

593 if not self.check_hostid(): 

594 raise GSEXRM_Exception(NOT_OWNER % self.filename) 

595 if not self.write_access: 

596 raise GSEXRM_Exception(READ_ONLY % self.filename) 

597 

598 kws.update(self.compress_args) 

599 if name in group: 

600 del group[name] 

601 d = group.create_dataset(name, data=data, **kws) 

602 if isinstance(attrs, dict): 

603 for key, val in attrs.items(): 

604 d.attrs[key] = val 

605 return d 

606 

607 def add_map_config(self, config, nmca=None): 

608 '''add configuration from Map Folder to HDF5 file 

609 ROI, DXP Settings, and Config data 

610 ''' 

611 if not self.check_hostid(): 

612 raise GSEXRM_Exception(NOT_OWNER % self.filename) 

613 if not self.write_access: 

614 raise GSEXRM_Exception(READ_ONLY % self.filename) 

615 

616 group = self.xrmmap['config'] 

617 for name, sect in (('scan', 'scan'), 

618 ('general', 'general'), 

619 ('positioners', 'slow_positioners'), 

620 ('motor_controller', 'xps')): 

621 for key, val in config[sect].items(): 

622 if key in group[name]: 

623 del group[name][key] 

624 group[name].create_dataset(key, data=val) 

625 

626 scanfile = os.path.join(self.folder, self.ScanFile) 

627 if os.path.exists(scanfile): 

628 scantext = open(scanfile, 'r').read() 

629 else: 

630 scantext = '' 

631 if 'text' in group['scan']: 

632 del group['scan']['text'] 

633 group['scan'].create_dataset('text', data=scantext) 

634 

635 roifile = os.path.join(self.folder, self.ROIFile) 

636 self.nmca = 0 

637 if nmca is not None: 

638 self.nmca = nmca 

639 

640 if os.path.exists(roifile): 

641 roidat, calib, extra = readROIFile(roifile) 

642 self.xrmmap.attrs['N_Detectors'] = self.nmca = len(calib['slope']) 

643 roi_desc, roi_addr, roi_lim = [], [], [] 

644 roi_slices = [] 

645 

646 for iroi, label, lims in roidat: 

647 roi_desc.append(label) 

648 try: 

649 xrf_prefix = config['xrf']['prefix'] 

650 except KeyError: # very old map files 

651 xrf_prefix = 'xrf_det:' 

652 roi_addr.append("%smca%%i.R%i" % (xrf_prefix, iroi)) 

653 roi_lim.append([lims[i] for i in range(self.nmca)]) 

654 roi_slices.append([slice(lims[i][0], lims[i][1]) for i in range(self.nmca)]) 

655 roi_lim = np.array(roi_lim) 

656 self.add_data(group['rois'], 'name', strlist(roi_desc)) 

657 self.add_data(group['rois'], 'address', strlist(roi_addr)) 

658 self.add_data(group['rois'], 'limits', roi_lim) 

659 

660 for key, val in calib.items(): 

661 self.add_data(group['mca_calib'], key, val) 

662 

663 for key, val in extra.items(): 

664 try: 

665 self.add_data(group['mca_settings'], key, val) 

666 except TypeError: 

667 pass 

668 

669 self.roi_desc = roi_desc 

670 self.roi_addr = roi_addr 

671 self.roi_slices = roi_slices 

672 self.calib = calib 

673 else: 

674 nmca = self.nmca 

675 roilims = np.array([ [[0, 1]] for i in range(nmca)]) 

676 self.add_data(group['rois'], 'name', [b'roi1']*nmca) 

677 self.add_data(group['rois'], 'address', [b'addr%i.ROI1']*nmca) 

678 self.add_data(group['rois'], 'limits', roilims) 

679 self.add_data(group['mca_calib'], 'offset', [0.00]*nmca) 

680 self.add_data(group['mca_calib'], 'slope', [0.01]*nmca) 

681 self.add_data(group['mca_calib'], 'quad', [0.00]*nmca) 

682 

683 # add env data 

684 envfile = os.path.join(self.folder, self.EnvFile) 

685 if os.path.exists(envfile): 

686 envdat = readEnvironFile(envfile) 

687 else: 

688 envdat = ['Facility.Ring_Current (UnknownPV) = 0'] 

689 env_desc, env_addr, env_val = parseEnviron(envdat) 

690 

691 self.add_data(group['environ'], 'name', strlist(env_desc)) 

692 self.add_data(group['environ'], 'address', strlist(env_addr)) 

693 self.add_data(group['environ'], 'value', strlist(env_val)) 

694 

695 cmprstr = '%s' % self.compress_args['compression'] 

696 if self.compress_args['compression'] != 'lzf': 

697 cmprstr = '%s-%s' % (cmprstr,self.compress_args['compression_opts']) 

698 self.xrmmap.attrs['Compression'] = cmprstr 

699 

700 self.h5root.flush() 

701 

702 def initialize_xrmmap(self, callback=None): 

703 ''' initialize '/xrmmap' group in HDF5 file, generally 

704 possible once at least 1 row of raw data is available 

705 in the scan folder. 

706 ''' 

707 self.starttime = time.time() 

708 if self.status == GSEXRM_FileStatus.hasdata: 

709 return 

710 if self.status != GSEXRM_FileStatus.created: 

711 print( 'Warning, cannot initialize xrmmap yet.') 

712 return 

713 

714 if not self.check_hostid(): 

715 raise GSEXRM_Exception(NOT_OWNER % self.filename) 

716 if not self.write_access: 

717 raise GSEXRM_Exception(READ_ONLY % self.filename) 

718 

719 if (len(self.rowdata) < 1 or 

720 (self.dimension is None and isGSEXRM_MapFolder(self.folder))): 

721 self.read_master() 

722 if len(self.rowdata) < 1: 

723 return 

724 

725 self.last_row = -1 

726 self.add_map_config(self.mapconf) 

727 

728 self.process_row(0, flush=True, callback=None, 

729 nrows_expected=len(self.rowdata)) 

730 

731 self.status = GSEXRM_FileStatus.hasdata 

732 

733 def process_row(self, irow, flush=False, complete=False, offset=None, 

734 nrows_expected=None, callback=None): 

735 row = self.read_rowdata(irow, offset=offset) 

736 if irow == 0: 

737 nmca, nchan = 0, 2048 

738 if row.counts is not None: 

739 nmca, xnpts, nchan = row.counts.shape 

740 xrd2d_shape = None 

741 if row.xrd2d is not None: 

742 xrd2d_shape = rows.xrd2d.shape 

743 self.build_schema(row.npts, nmca=nmca, nchan=nchan, 

744 scaler_names=row.scaler_names, 

745 scaler_addrs=row.scaler_addrs, 

746 xrd2d_shape=xrd2d_shape, verbose=True, 

747 nrows_expected=nrows_expected) 

748 if row.read_ok: 

749 self.add_rowdata(row, callback=callback) 

750 

751 if flush or complete: 

752 # print("Flush, ", irow, self.last_row, flush, complete) 

753 self.resize_arrays(self.last_row+1, force_shrink=True) 

754 self.h5root.flush() 

755 if self._pixeltime is None: 

756 self.calc_pixeltime() 

757 if callable(callback): 

758 status = 'complete' if complete else 'flush' 

759 callback(filename=self.filename, status=status) 

760 

761 

762 def process(self, maxrow=None, force=False, callback=None, offset=None, 

763 force_no_dtc=False, all_mcas=None): 

764 "look for more data from raw folder, process if needed" 

765 self.force_no_dtc = force_no_dtc 

766 if all_mcas is not None: 

767 self.all_mcas = all_mcas 

768 

769 if not self.check_hostid(): 

770 raise GSEXRM_Exception(NOT_OWNER % self.filename) 

771 if not self.write_access: 

772 raise GSEXRM_Exception(READ_ONLY % self.filename) 

773 

774 self.reset_flags() 

775 if self.status == GSEXRM_FileStatus.created: 

776 self.initialize_xrmmap(callback=callback) 

777 if (force or len(self.rowdata) < 1 or 

778 (self.dimension is None and isGSEXRM_MapFolder(self.folder))): 

779 self.read_master() 

780 

781 nrows = len(self.rowdata) 

782 if maxrow is not None: 

783 nrows = min(nrows, maxrow) 

784 

785 if force or self.folder_has_newdata(): 

786 irow = self.last_row + 1 

787 while irow < nrows: 

788 flush = irow < 2 or (irow % 64 == 0) 

789 complete = irow >= nrows-1 

790 self.process_row(irow, flush=flush, offset=offset, 

791 complete=complete, callback=callback) 

792 irow = irow + 1 

793 if callable(callback): 

794 callback(filename=self.filename, status='complete') 

795 

796 

797 def set_roidata(self, row_start=0, row_end=None): 

798 if row_end is None: 

799 row_end = self.last_row 

800 

801 # print("Process ROIs for rows %d to %d " % (row_start+1, row_end+1)) 

802 rows = slice(row_start, row_end+1) 

803 roigrp = self.xrmmap['roimap'] 

804 conf = self.xrmmap['config'] 

805 roi_names = [h5str(s) for s in conf['rois/name']] 

806 roi_limits = conf['rois/limits'][()] 

807 # print("roi names ", roi_names, roi_limits) 

808 

809 for roiname, lims in zip(roi_names, roi_limits): 

810 dt = debugtime() 

811 roi_slice = lims[0] 

812 # dt.add('get slice') 

813 sumraw = roigrp['mcasum'][roiname]['raw'][rows,] 

814 # dt.add('get sum raw') 

815 sumcor = roigrp['mcasum'][roiname]['cor'][rows,] 

816 # dt.add('get sum cor') 

817 

818 for detname in self.mca_dets: 

819 mcaraw = self.xrmmap[detname]['counts'][rows,:,roi_slice].sum(axis=2) 

820 # dt.add(" mcaraw %s " % detname) 

821 mcacor = mcaraw*self.xrmmap[detname]['dtfactor'][rows,:] 

822 # dt.add(" mcacor %s " % detname) 

823 roigrp[detname][roiname]['raw'][rows,] = mcaraw 

824 roigrp[detname][roiname]['cor'][rows,] = mcacor 

825 # dt.add(" set roigrps for %s " % detname) 

826 sumraw += mcaraw 

827 sumcor += mcacor 

828 # dt.add(" sum %s " % detname) 

829 #dt.show() 

830 

831 roigrp['mcasum'][roiname]['raw'][rows,] = sumraw 

832 roigrp['mcasum'][roiname]['cor'][rows,] = sumcor 

833 self.h5root.flush() 

834 

835 def calc_pixeltime(self): 

836 scanconf = self.xrmmap['config/scan'] 

837 rowtime = float(scanconf['time1'][()]) 

838 start = float(scanconf['start1'][()]) 

839 stop = float(scanconf['stop1'][()]) 

840 step = float(scanconf['step1'][()]) 

841 npts = int((abs(stop - start) + 1.1*step)/step) 

842 self._pixeltime = rowtime/(npts-1) 

843 return self._pixeltime 

844 

845 @property 

846 def pixeltime(self): 

847 """Return the pixel time""" 

848 if self._pixeltime is None: 

849 self.calc_pixeltime() 

850 return self._pixeltime 

851 

852 def read_rowdata(self, irow, offset=None): 

853 '''read a row worth of raw data from the Map Folder 

854 returns arrays of data 

855 ''' 

856 if self.dimension is None or irow > len(self.rowdata): 

857 self.read_master() 

858 

859 if self.folder is None or irow >= len(self.rowdata): 

860 return 

861 

862 if self.has_xrd1d and self.xrdcalfile is None: 

863 self.xrdcalfile = bytes2str(self.xrmmap['xrd1d'].attrs.get('calfile','')) 

864 if self.xrdcalfile in (None, ''): 

865 calfile = os.path.join(nativepath(self.folder), self.XRDCALFile) 

866 if os.path.exists(calfile): 

867 self.xrdcalfile = calfile 

868 

869 scan_version = getattr(self, 'scan_version', 1.00) 

870 print(" read row data, scan version ", scan_version, self.xrdcalfile) 

871 xrdcal_dat = bytes2str(self.xrmmap['xrd1d'].attrs.get('caldata','{}')) 

872 if self.xrdcalfile is not None and len(xrdcal_dat) < 10: 

873 xrdcal_dat = read_poni(self.xrdcalfile) 

874 self.xrmmap['xrd1d'].attrs['caldata'] = json.dumps(xrdcal_dat) 

875 

876 # if not self.has_xrf and not self.has_xrd2d and not self.has_xrd1d: 

877 # raise IOError('No XRF or XRD flags provided.') 

878 # return 

879 

880 if scan_version > 1.35 or self.has_xrd2d or self.has_xrd1d: 

881 yval, xrff, sisf, xpsf, xrdf, etime = self.rowdata[irow] 

882 if xrff.startswith('None'): 

883 xrff = xrff.replace('None', 'xsp3') 

884 if sisf.startswith('None'): 

885 sisf = sisf.replace('None', 'struck') 

886 if xpsf.startswith('None'): 

887 xpsf = xpsf.replace('None', 'xps') 

888 if xrdf.startswith('None'): 

889 xrdf = xrdf.replace('None', 'pexrd') 

890 else: 

891 yval, xrff, sisf, xpsf, etime = self.rowdata[irow] 

892 xrdf = '_unused_' 

893 

894 if '_unused_' in xrdf: 

895 self.has_xrd1d = False 

896 self.has_xrd2d = False 

897 

898 # eiger XRD maps with 1D data 

899 if (xrdf.startswith('eig') and xrdf.endswith('.h5') or 

900 xrdf.startswith('pexrd')): 

901 self.has_xrd2d = False 

902 self.has_xrd1d = True 

903 

904 if '_unused_' in xrff: 

905 self.has_xrf = False 

906 

907 reverse = True 

908 ioffset = 0 

909 if scan_version > 1.35: 

910 ioffset = 1 

911 if scan_version >= 2.0: 

912 reverse = False 

913 if offset is not None: 

914 ioffset = offset 

915 self.has_xrf = self.has_xrf and xrff != '_unused_' 

916 return GSEXRM_MapRow(yval, xrff, xrdf, xpsf, sisf, self.folder, 

917 irow=irow, nrows_expected=self.nrows_expected, 

918 ixaddr=0, dimension=self.dimension, 

919 npts=self.npts, 

920 reverse=reverse, 

921 ioffset=ioffset, 

922 force_no_dtc=self.force_no_dtc, 

923 masterfile=self.masterfile, flip=self.flip, 

924 xrdcal=self.xrdcalfile, 

925 xrd2dmask=self.mask_xrd2d, 

926 xrd2dbkgd=self.bkgd_xrd2d, wdg=self.azwdgs, 

927 steps=self.qstps, has_xrf=self.has_xrf, 

928 has_xrd2d=self.has_xrd2d, 

929 has_xrd1d=self.has_xrd1d) 

930 

931 

932 def add_rowdata(self, row, callback=None, flush=True): 

933 '''adds a row worth of real data''' 

934 dt = debugtime() 

935 if not self.check_hostid(): 

936 raise GSEXRM_Exception(NOT_OWNER % self.filename) 

937 if not self.write_access: 

938 raise GSEXRM_Exception(READ_ONLY % self.filename) 

939 

940 thisrow = self.last_row + 1 

941 if hasattr(callback, '__call__'): 

942 callback(row=(thisrow+1), maxrow=len(self.rowdata), filename=self.filename) 

943 

944 pform = 'Add row %4i, yval=%s' % (thisrow+1, row.yvalue) 

945 if self.has_xrf: 

946 pform = '%s, xrffile=%s' % (pform, row.xrffile) 

947 if self.has_xrd2d or self.has_xrd1d: 

948 pform = '%s, xrdfile=%s' % (pform, row.xrdfile) 

949 print(pform) 

950 

951 dt.add(" ran callback, print, version %s" %self.version) 

952 

953 if version_ge(self.version, '2.0.0'): 

954 

955 mcasum_raw,mcasum_cor = [],[] 

956 nrows = 0 

957 map_items = sorted(self.xrmmap.keys()) 

958 dt.add(" get %d map items" % len(map_items)) 

959 for gname in map_items: 

960 g = self.xrmmap[gname] 

961 if bytes2str(g.attrs.get('type', '')).startswith('scalar detect'): 

962 first_det = list(g.keys())[0] 

963 nrows, npts = g[first_det].shape 

964 

965 dt.add(" got %d map items" % len(map_items)) 

966 if thisrow >= nrows: 

967 self.resize_arrays(NINIT*(1+nrows/NINIT), force_shrink=False) 

968 

969 dt.add(" resized ") 

970 sclrgrp = self.xrmmap['scalars'] 

971 for ai, aname in enumerate(row.scaler_names): 

972 sclrgrp[aname][thisrow, :npts] = row.sisdata[:npts].transpose()[ai] 

973 dt.add(" add scaler group") 

974 if self.has_xrf: 

975 

976 npts = min([len(p) for p in row.posvals]) 

977 pos = self.xrmmap['positions/pos'] 

978 rowpos = np.array([p[:npts] for p in row.posvals]) 

979 

980 tpos = rowpos.transpose() 

981 pos[thisrow, :npts, :] = tpos[:npts, :] 

982 nmca, xnpts, nchan = row.counts.shape 

983 mca_dets = [] 

984 dt.add(" map xrf 1") 

985 for gname in map_items: 

986 g = self.xrmmap[gname] 

987 if bytes2str(g.attrs.get('type', '')).startswith('mca detect'): 

988 mca_dets.append(gname) 

989 nrows, npts, nchan = g['counts'].shape 

990 dt.add(" map xrf 2") 

991 _nr, npts, nchan = self.xrmmap['mcasum']['counts'].shape 

992 npts = min(npts, xnpts, self.npts) 

993 dt.add(" map xrf 3") 

994 # print("ADD ROW ", self.all_mcas, mca_dets, self.nmca) 

995 if self.all_mcas: 

996 for idet, gname in enumerate(mca_dets): 

997 grp = self.xrmmap[gname] 

998 grp['counts'][thisrow, :npts, :] = row.counts[idet, :npts, :] 

999 grp['dtfactor'][thisrow, :npts] = row.dtfactor[idet, :npts] 

1000 grp['realtime'][thisrow, :npts] = row.realtime[idet, :npts] 

1001 grp['livetime'][thisrow, :npts] = row.livetime[idet, :npts] 

1002 grp['inpcounts'][thisrow, :npts] = row.inpcounts[idet, :npts] 

1003 grp['outcounts'][thisrow, :npts] = row.outcounts[idet, :npts] 

1004 

1005 livetime = np.zeros(npts, dtype=np.float64) 

1006 realtime = np.zeros(npts, dtype=np.float64) 

1007 dtfactor = np.zeros(npts, dtype=np.float32) 

1008 inpcounts = np.zeros(npts, dtype=np.float32) 

1009 outcounts = np.zeros(npts, dtype=np.float32) 

1010 dt.add(" map xrf 4a: alloc ") 

1011 # print("ADD ", inpcounts.dtype, row.inpcounts.dtype) 

1012 for idet in range(self.nmca): 

1013 realtime += row.realtime[idet, :npts] 

1014 livetime += row.livetime[idet, :npts] 

1015 inpcounts += row.inpcounts[idet, :npts] 

1016 outcounts += row.outcounts[idet, :npts] 

1017 livetime /= (1.0*self.nmca) 

1018 realtime /= (1.0*self.nmca) 

1019 dt.add(" map xrf 4b: time sums") 

1020 

1021 sumgrp = self.xrmmap['mcasum'] 

1022 sumgrp['counts'][thisrow, :npts, :nchan] = row.total[:npts, :nchan] 

1023 dt.add(" map xrf 4b: set counts") 

1024 # print("add realtime ", sumgrp['realtime'].shape, self.xrmmap['roimap/det_raw'].shape, thisrow) 

1025 sumgrp['realtime'][thisrow, :npts] = realtime 

1026 sumgrp['livetime'][thisrow, :npts] = livetime 

1027 sumgrp['dtfactor'][thisrow, :npts] = row.total_dtfactor[:npts] 

1028 sumgrp['inpcounts'][thisrow, :npts] = inpcounts 

1029 sumgrp['outcounts'][thisrow, :npts] = outcounts 

1030 dt.add(" map xrf 4c: set time data ") 

1031 

1032 if version_ge(self.version, '2.1.0'): # version 2.1 

1033 det_raw = self.xrmmap['roimap/det_raw'] 

1034 det_cor = self.xrmmap['roimap/det_cor'] 

1035 sum_raw = self.xrmmap['roimap/sum_raw'] 

1036 sum_cor = self.xrmmap['roimap/sum_cor'] 

1037 

1038 detraw = list(row.sisdata[:npts].transpose()) 

1039 detcor = detraw[:] 

1040 sumraw = detraw[:] 

1041 sumcor = detraw[:] 

1042 

1043 if self.roi_slices is None: 

1044 lims = self.xrmmap['config/rois/limits'][()] 

1045 nrois, nmca, nx = lims.shape 

1046 

1047 self.roi_slices = [] 

1048 for iroi in range(nrois): 

1049 x = [slice(lims[iroi, i, 0], 

1050 lims[iroi, i, 1]) for i in range(nmca)] 

1051 self.roi_slices.append(x) 

1052 

1053 for slices in self.roi_slices: 

1054 iraw = [row.counts[i, :npts, slices[i]].sum(axis=1) 

1055 for i in range(nmca)] 

1056 icor = [row.counts[i, :npts, slices[i]].sum(axis=1)*row.dtfactor[i, :npts] 

1057 for i in range(nmca)] 

1058 detraw.extend(iraw) 

1059 detcor.extend(icor) 

1060 sumraw.append(np.array(iraw).sum(axis=0)) 

1061 sumcor.append(np.array(icor).sum(axis=0)) 

1062 dt.add(" map xrf 5a: got simple ROIS") 

1063 det_raw[thisrow, :npts, :] = np.array(detraw).transpose() 

1064 det_cor[thisrow, :npts, :] = np.array(detcor).transpose() 

1065 sum_raw[thisrow, :npts, :] = np.array(sumraw).transpose() 

1066 sum_cor[thisrow, :npts, :] = np.array(sumcor).transpose() 

1067 

1068 

1069 else: # version 2.0 

1070 roigrp = self.xrmmap['roimap'] 

1071 en = self.xrmmap['mcasum']['energy'][:] 

1072 for roiname in roigrp['mcasum'].keys(): 

1073 en_lim = roigrp['mcasum'][roiname]['limits'][:] 

1074 roi_slice = slice(np.abs(en-en_lim[0]).argmin(), 

1075 np.abs(en-en_lim[1]).argmin()) 

1076 sumraw = roigrp['mcasum'][roiname]['raw'][thisrow,] 

1077 sumcor = roigrp['mcasum'][roiname]['cor'][thisrow,] 

1078 for detname in mca_dets: 

1079 mcaraw = self.xrmmap[detname]['counts'][thisrow,][:,roi_slice].sum(axis=1) 

1080 mcacor = mcaraw*self.xrmmap[detname]['dtfactor'][thisrow,] 

1081 roigrp[detname][roiname]['raw'][thisrow,] = mcaraw 

1082 roigrp[detname][roiname]['cor'][thisrow,] = mcacor 

1083 sumraw += mcaraw 

1084 sumcor += mcacor 

1085 roigrp['mcasum'][roiname]['raw'][thisrow,] = sumraw 

1086 roigrp['mcasum'][roiname]['cor'][thisrow,] = sumcor 

1087 dt.add(" map xrf 6") 

1088 else: # version 1.0.1 

1089 if self.has_xrf: 

1090 nmca, xnpts, nchan = row.counts.shape 

1091 xrm_dets = [] 

1092 

1093 nrows = 0 

1094 map_items = sorted(self.xrmmap.keys()) 

1095 for gname in map_items: 

1096 g = self.xrmmap[gname] 

1097 if bytes2str(g.attrs.get('type', '')).startswith('mca detect'): 

1098 xrm_dets.append(g) 

1099 nrows, npts, nchan = g['counts'].shape 

1100 

1101 if thisrow >= nrows: 

1102 self.resize_arrays(NINIT*(1+nrows/NINIT), force_shrink=False) 

1103 

1104 _nr, npts, nchan = xrm_dets[0]['counts'].shape 

1105 npts = min(npts, xnpts, self.npts) 

1106 for idet, grp in enumerate(xrm_dets): 

1107 grp['dtfactor'][thisrow, :npts] = row.dtfactor[idet, :npts] 

1108 grp['realtime'][thisrow, :npts] = row.realtime[idet, :npts] 

1109 grp['livetime'][thisrow, :npts] = row.livetime[idet, :npts] 

1110 grp['inpcounts'][thisrow, :npts] = row.inpcounts[idet, :npts] 

1111 grp['outcounts'][thisrow, :npts] = row.outcounts[idet, :npts] 

1112 grp['counts'][thisrow, :npts, :] = row.counts[idet, :npts, :] 

1113 

1114 # here, we add the total dead-time-corrected data to detsum. 

1115 self.xrmmap['detsum']['counts'][thisrow, :npts, :nchan] = row.total[:npts, :nchan] 

1116 

1117 pos = self.xrmmap['positions/pos'] 

1118 rowpos = np.array([p[:npts] for p in row.posvals]) 

1119 

1120 tpos = rowpos.transpose() 

1121 

1122 pos[thisrow, :npts, :] = tpos[:npts, :] 

1123 

1124 # now add roi map data 

1125 roimap = self.xrmmap['roimap'] 

1126 det_raw = roimap['det_raw'] 

1127 det_cor = roimap['det_cor'] 

1128 sum_raw = roimap['sum_raw'] 

1129 sum_cor = roimap['sum_cor'] 

1130 

1131 detraw = list(row.sisdata[:npts].transpose()) 

1132 

1133 detcor = detraw[:] 

1134 sumraw = detraw[:] 

1135 sumcor = detraw[:] 

1136 

1137 if self.roi_slices is None: 

1138 lims = self.xrmmap['config/rois/limits'][()] 

1139 nrois, nmca, nx = lims.shape 

1140 

1141 self.roi_slices = [] 

1142 for iroi in range(nrois): 

1143 x = [slice(lims[iroi, i, 0], 

1144 lims[iroi, i, 1]) for i in range(nmca)] 

1145 self.roi_slices.append(x) 

1146 

1147 for slices in self.roi_slices: 

1148 iraw = [row.counts[i, :npts, slices[i]].sum(axis=1) 

1149 for i in range(nmca)] 

1150 icor = [row.counts[i, :npts, slices[i]].sum(axis=1)*row.dtfactor[i, :npts] 

1151 for i in range(nmca)] 

1152 detraw.extend(iraw) 

1153 detcor.extend(icor) 

1154 sumraw.append(np.array(iraw).sum(axis=0)) 

1155 sumcor.append(np.array(icor).sum(axis=0)) 

1156 

1157 det_raw[thisrow, :npts, :] = np.array(detraw).transpose() 

1158 det_cor[thisrow, :npts, :] = np.array(detcor).transpose() 

1159 sum_raw[thisrow, :npts, :] = np.array(sumraw).transpose() 

1160 sum_cor[thisrow, :npts, :] = np.array(sumcor).transpose() 

1161 

1162 if self.has_xrd1d and row.xrdq is not None: 

1163 if thisrow < 2: 

1164 if len(row.xrdq.shape) == 1: 

1165 self.xrmmap['xrd1d/q'][:] = row.xrdq 

1166 else: 

1167 self.xrmmap['xrd1d/q'][:] = row.xrdq[0] 

1168 

1169 if self.bkgd_xrd1d is not None: 

1170 self.xrmmap['xrd1d/counts'][thisrow,] = row.xrd1d - self.bkgd_xrd1d 

1171 else: 

1172 _ni, _nc, _nq = self.xrmmap['xrd1d/counts'].shape 

1173 _rc, _rq = row.xrd1d.shape 

1174 _nc = min(_nc, _rc) 

1175 _nq = min(_nq, _rq) 

1176 self.xrmmap['xrd1d/counts'][thisrow, :_nc, :_nq] = row.xrd1d[:_nc,:_nq] 

1177 

1178 if self.azwdgs > 1 and row.xrd1d_wdg is not None: 

1179 for iwdg,wdggrp in enumerate(self.xrmmap['work/xrdwedge'].values()): 

1180 try: 

1181 wdggrp['q'] = row.xrdq_wdg[0,:,iwdg] 

1182 except: 

1183 pass 

1184 

1185 ## does not yet subtract a background here BECAUSE q-range different 

1186 ## for each wedge - won't be same size or shape array 

1187 ## mkak 2018.02.26 

1188 wdggrp['counts'][thisrow,] = row.xrd1d_wdg[:,:,iwdg] 

1189 

1190 

1191 if self.has_xrd2d and row.xrd2d is not None: 

1192 self.xrmmap['xrd2d/counts'][thisrow,] = row.xrd2d 

1193 dt.add("xrd done") 

1194 self.last_row = thisrow 

1195 self.xrmmap.attrs['Last_Row'] = thisrow 

1196 #self.h5root.flush() 

1197 # dt.add("flushed h5 file") 

1198 # dt.show() 

1199 

1200 

1201 def build_schema(self, npts, nmca=1, nchan=2048, scaler_names=None, 

1202 scaler_addrs=None, xrd2d_shape=None, nrows_expected=None, 

1203 verbose=False): 

1204 '''build schema for detector and scan data''' 

1205 self.t0 = time.time() 

1206 if not self.check_hostid(): 

1207 raise GSEXRM_Exception(NOT_OWNER % self.filename) 

1208 if not self.write_access: 

1209 raise GSEXRM_Exception(READ_ONLY % self.filename) 

1210 

1211 xrmmap = self.xrmmap 

1212 for xkey, xval in xrmmap.attrs.items(): 

1213 if xkey == 'Version': 

1214 self.version = xval 

1215 

1216 self.xrmmap.attrs['N_Detectors'] = nmca 

1217 if scaler_names is None: 

1218 scaler_names = [] 

1219 if scaler_addrs is None: 

1220 scaler_addrs = [''] * len(scaler_names) 

1221 

1222 conf = xrmmap['config'] 

1223 for key in self.notes: 

1224 conf['notes'].attrs[key] = self.notes[key] 

1225 

1226 if self.npts is None: 

1227 self.npts = npts 

1228 

1229 if self.chunksize is None: 

1230 self.chunksize = (1, min(2048, npts), nchan) 

1231 

1232 if nrows_expected is not None: 

1233 NSTART = max(NINIT, nrows_expected) 

1234 NSTART = NINIT*2 

1235 

1236 # positions 

1237 pos = xrmmap['positions'] 

1238 for pname in ('mca realtime', 'mca livetime'): 

1239 self.pos_desc.append(pname) 

1240 self.pos_addr.append(pname) 

1241 npos = len(self.pos_desc) 

1242 self.add_data(pos, 'name', strlist(self.pos_desc)) 

1243 self.add_data(pos, 'address', strlist(self.pos_addr)) 

1244 pos.create_dataset('pos', (NSTART, npts, npos), np.float32, 

1245 maxshape=(None, npts, npos), **self.compress_args) 

1246 

1247 # cfile = FastMapConfig() 

1248 # print(" build_schema -> mapconfig") 

1249 # self.add_map_config(cfile.config, nmca=nmca) 

1250 conf = xrmmap['config'] 

1251 

1252 offset = conf['mca_calib/offset'][()] 

1253 slope = conf['mca_calib/slope'][()] 

1254 quad = np.array([0.0]*len(offset)) 

1255 if 'quad' in conf['mca_calib']: 

1256 quad = conf['mca_calib/quad'][()] 

1257 

1258 if len(offset) != nmca: 

1259 raise GSEXRM_Exception("incorrect XRF calibration data: need %d MCAs, not %d" % (nmca, len(offset))) 

1260 

1261 

1262 _ex = np.arange(nchan, dtype=np.float64) 

1263 enarr = [] 

1264 for i in range(len(offset)): 

1265 enarr.append(offset[i] + slope[i]*_ex + quad[i]*_ex**2) 

1266 

1267 if version_ge(self.version, '2.0.0'): 

1268 sismap = xrmmap['scalars'] 

1269 sismap.attrs['type'] = 'scalar detector' 

1270 for aname in scaler_names: 

1271 sismap.create_dataset(aname, (NSTART, npts), np.float32, 

1272 chunks=self.chunksize[:-1], 

1273 maxshape=(None, npts), **self.compress_args) 

1274 

1275 roishape = conf['rois/name'].shape 

1276 if roishape[0] > 0: 

1277 roi_names = [h5str(s) for s in conf['rois/name']] 

1278 roi_limits = np.einsum('jik->ijk', conf['rois/limits'][()]) 

1279 else: 

1280 roi_names = ['_'] 

1281 roi_limits = np.array([[[0, 2]]]) 

1282 

1283 if verbose: 

1284 allmcas = 'with all mcas' if self.all_mcas else 'with only mca sum' 

1285 msg = '--- Build XRF Schema: %i ---- MCA: (%i, %i) %s' 

1286 print(msg % (npts, nmca, nchan, allmcas)) 

1287 

1288 ## mca1 to mcaN 

1289 if self.all_mcas: 

1290 for i in range(nmca): 

1291 imca = "mca%d" % (i+1) 

1292 for grp in (xrmmap, xrmmap['roimap']): 

1293 dgrp = grp.create_group(imca) 

1294 dgrp.attrs['type'] = 'mca detector' 

1295 dgrp.attrs['desc'] = imca 

1296 

1297 dgrp = xrmmap[imca] 

1298 self.add_data(dgrp, 'energy', enarr[i], 

1299 attrs={'cal_offset':offset[i], 

1300 'cal_slope': slope[i], 

1301 'cal_quad': quad[i]}) 

1302 dgrp.create_dataset('counts', (NSTART, npts, nchan), np.uint32, 

1303 chunks=self.chunksize, 

1304 maxshape=(None, npts, nchan), **self.compress_args) 

1305 

1306 for name, dtype in (('realtime', np.int64), 

1307 ('livetime', np.int64), 

1308 ('dtfactor', np.float32), 

1309 ('inpcounts', np.float32), 

1310 ('outcounts', np.float32)): 

1311 dgrp.create_dataset(name, (NSTART, npts), dtype, 

1312 maxshape=(None, npts), **self.compress_args) 

1313 

1314 dgrp = xrmmap['roimap'][imca] 

1315 for rname, rlimit in zip(roi_names,roi_limits[i]): 

1316 rgrp = dgrp.create_group(rname) 

1317 for aname,dtype in (('raw', np.uint32), 

1318 ('cor', np.float32)): 

1319 rgrp.create_dataset(aname, (1, npts), dtype, 

1320 chunks=self.chunksize[:-1], 

1321 maxshape=(None, npts), **self.compress_args) 

1322 

1323 rlimit = [max(0, rlimit[0]), min(len(enarr[i])-1, rlimit[1])] 

1324 lmtgrp = rgrp.create_dataset('limits', data=enarr[i][rlimit]) 

1325 lmtgrp.attrs['type'] = 'energy' 

1326 lmtgrp.attrs['units'] = 'keV' 

1327 

1328 ## mcasum 

1329 for grp in (xrmmap, xrmmap['roimap']): 

1330 dgrp = grp.create_group('mcasum') 

1331 dgrp.attrs['type'] = 'virtual mca detector' 

1332 dgrp.attrs['desc'] = 'sum of detectors' 

1333 

1334 dgrp = xrmmap['mcasum'] 

1335 if len(enarr) == 0: 

1336 enarr = [np.zeros(nchan)] 

1337 counts = [np.zeros(nchan)] 

1338 offset = slope = quad = [0.0] 

1339 self.add_data(dgrp, 'energy', enarr[0], 

1340 attrs={'cal_offset':offset[0], 

1341 'cal_slope': slope[0], 

1342 'cal_quad': quad[0]}) 

1343 dgrp.create_dataset('counts', (NSTART, npts, nchan), np.float64, 

1344 chunks=self.chunksize, 

1345 maxshape=(None, npts, nchan), **self.compress_args) 

1346 

1347 for name, dtype in (('realtime', np.int64), 

1348 ('livetime', np.int64), 

1349 ('dtfactor', np.float32), 

1350 ('inpcounts', np.float32), 

1351 ('outcounts', np.float32)): 

1352 dgrp.create_dataset(name, (NSTART, npts), dtype, 

1353 maxshape=(None, npts), **self.compress_args) 

1354 

1355 

1356 dgrp = xrmmap['roimap']['mcasum'] 

1357 for rname, rlimit in zip(roi_names, roi_limits[0]): 

1358 rgrp = dgrp.create_group(rname) 

1359 for aname,dtype in (('raw', np.uint32), 

1360 ('cor', np.float32)): 

1361 rgrp.create_dataset(aname, (1, npts), dtype, 

1362 chunks=self.chunksize[:-1], 

1363 maxshape=(None, npts), **self.compress_args) 

1364 rlimit = [max(0, rlimit[0]), min(len(enarr[0])-1, rlimit[1])] 

1365 

1366 lmtgrp = rgrp.create_dataset('limits', data=enarr[0][rlimit], 

1367 **self.compress_args) 

1368 lmtgrp.attrs['type'] = 'energy' 

1369 lmtgrp.attrs['units'] = 'keV' 

1370 

1371 if version_ge(self.version, '2.1.0'): # NEW ROI RAW DAT 

1372 rdat = xrmmap['roimap'] 

1373 det_addr = [i.strip() for i in scaler_addrs] 

1374 det_desc = [i.strip() for i in scaler_names] 

1375 for addr in conf['rois/address']: 

1376 det_addr.extend([h5str(addr) % (i+1) for i in range(nmca)]) 

1377 

1378 for nam in roi_names: 

1379 det_desc.extend(["%s (mca%i)" % (nam, i+1) for i in range(nmca)]) 

1380 

1381 sums_map = {} 

1382 sums_desc = [] 

1383 nsum = 0 

1384 for idet, addr in enumerate(det_desc): 

1385 if '(mca' in addr: 

1386 addr = addr.split('(mca')[0].strip() 

1387 

1388 if addr not in sums_map: 

1389 sums_map[addr] = [] 

1390 sums_desc.append(addr) 

1391 sums_map[addr].append(idet) 

1392 nsum = max([len(s) for s in sums_map.values()]) 

1393 sums_list = [] 

1394 for sname in sums_desc: 

1395 slist = sums_map[sname] 

1396 if len(slist) < nsum: 

1397 slist.extend([-1]*(nsum-len(slist))) 

1398 sums_list.append(slist) 

1399 

1400 nsum = len(sums_list) 

1401 nsca = len(det_desc) 

1402 sums_list = np.array(sums_list) 

1403 

1404 self.add_data(rdat, 'det_name', strlist(det_desc)) 

1405 self.add_data(rdat, 'det_address', strlist(det_addr)) 

1406 self.add_data(rdat, 'sum_name', strlist(sums_desc)) 

1407 self.add_data(rdat, 'sum_list', sums_list) 

1408 

1409 for name, nx, dtype in (('det_raw', nsca, np.uint32), 

1410 ('det_cor', nsca, np.float32), 

1411 ('sum_raw', nsum, np.uint32), 

1412 ('sum_cor', nsum, np.float32) 

1413 ): 

1414 rdat.create_dataset(name, (NSTART, npts, nx), dtype, 

1415 chunks=(2, npts, nx), 

1416 maxshape=(None, npts, nx), **self.compress_args) 

1417 

1418 else: # version 1.* 

1419 if verbose: 

1420 msg = '--- Build XRF Schema: {:d} MCA: ({:d}, {:d})' 

1421 print(msg.format(npts, nmca, nchan)) 

1422 

1423 roi_names = [h5str(s) for s in conf['rois/name']] 

1424 roi_addrs = [h5str(s) for s in conf['rois/address']] 

1425 roi_limits = conf['rois/limits'][()] 

1426 for imca in range(nmca): 

1427 dname = 'det%i' % (imca+1) 

1428 dgrp = xrmmap.create_group(dname) 

1429 dgrp.attrs['type'] = 'mca detector' 

1430 dgrp.attrs['desc'] = 'mca%i' % (imca+1) 

1431 self.add_data(dgrp, 'energy', enarr[imca], 

1432 attrs={'cal_offset':offset[imca], 

1433 'cal_slope': slope[imca], 

1434 'cal_quad': quad[imca]}) 

1435 self.add_data(dgrp, 'roi_name', strlist(roi_names)) 

1436 self.add_data(dgrp, 'roi_address', strlist([s % (imca+1) for s in roi_addrs])) 

1437 self.add_data(dgrp, 'roi_limits', roi_limits[:,imca,:]) 

1438 

1439 dgrp.create_dataset('counts', (NSTART, npts, nchan), np.uint32, 

1440 chunks=self.chunksize, 

1441 maxshape=(None, npts, nchan), **self.compress_args) 

1442 for name, dtype in (('realtime', np.int64), 

1443 ('livetime', np.int64), 

1444 ('dtfactor', np.float32), 

1445 ('inpcounts', np.float32), 

1446 ('outcounts', np.float32)): 

1447 dgrp.create_dataset(name, (NSTART, npts), dtype, 

1448 maxshape=(None, npts), **self.compress_args) 

1449 

1450 # add 'virtual detector' for corrected sum: 

1451 dgrp = xrmmap.create_group('detsum') 

1452 dgrp.attrs['type'] = 'virtual mca' 

1453 dgrp.attrs['desc'] = 'deadtime corrected sum of detectors' 

1454 self.add_data(dgrp, 'energy', enarr[0], 

1455 attrs={'cal_offset':offset[0], 

1456 'cal_slope': slope[0], 

1457 'cal_quad': quad[0]}) 

1458 self.add_data(dgrp, 'roi_name', strlist(roi_names)) 

1459 self.add_data(dgrp, 'roi_address', strlist((s % 1) for s in roi_addrs)) 

1460 self.add_data(dgrp, 'roi_limits', roi_limits[: ,0, :]) 

1461 dgrp.create_dataset('counts', (NSTART, npts, nchan), np.uint32, 

1462 chunks=self.chunksize, 

1463 maxshape=(None, npts, nchan), **self.compress_args) 

1464 # roi map data 

1465 scan = xrmmap['roimap'] 

1466 det_addr = [i.strip() for i in scaler_addrs] 

1467 det_desc = [i.strip() for i in scaler_names] 

1468 for addr in roi_addrs: 

1469 det_addr.extend([addr % (i+1) for i in range(nmca)]) 

1470 

1471 for desc in roi_names: 

1472 det_desc.extend(["%s (mca%i)" % (desc, i+1) 

1473 for i in range(nmca)]) 

1474 

1475 sums_map = {} 

1476 sums_desc = [] 

1477 nsum = 0 

1478 for idet, addr in enumerate(det_desc): 

1479 if '(mca' in addr: 

1480 addr = addr.split('(mca')[0].strip() 

1481 

1482 if addr not in sums_map: 

1483 sums_map[addr] = [] 

1484 sums_desc.append(addr) 

1485 sums_map[addr].append(idet) 

1486 nsum = max([len(s) for s in sums_map.values()]) 

1487 sums_list = [] 

1488 for sname in sums_desc: 

1489 slist = sums_map[sname] 

1490 if len(slist) < nsum: 

1491 slist.extend([-1]*(nsum-len(slist))) 

1492 sums_list.append(slist) 

1493 

1494 nsum = len(sums_list) 

1495 nsca = len(det_desc) 

1496 

1497 sums_list = np.array(sums_list) 

1498 

1499 self.add_data(scan, 'det_name', strlist(det_desc)) 

1500 self.add_data(scan, 'det_address', strlist(det_addr)) 

1501 self.add_data(scan, 'sum_name', strlist(sum_desc)) 

1502 self.add_data(scan, 'sum_list', sums_list) 

1503 

1504 nxx = min(nsca, 8) 

1505 for name, nx, dtype in (('det_raw', nsca, np.uint32), 

1506 ('det_cor', nsca, np.float32), 

1507 ('sum_raw', nsum, np.uint32), 

1508 ('sum_cor', nsum, np.float32)): 

1509 scan.create_dataset(name, (NSTART, npts, nx), dtype, 

1510 chunks=(2, npts, nx), 

1511 maxshape=(None, npts, nx), **self.compress_args) 

1512 

1513 

1514 if self.has_xrd2d or self.has_xrd1d: 

1515 if self.has_xrd2d: 

1516 xrdpts, xpixx, xpixy = xrd2d_shape # row.xrd2d.shape 

1517 if verbose: 

1518 prtxt = '--- Build XRD Schema: %i ---- 2D XRD: (%i, %i)' 

1519 print(prtxt % (npts, xpixx, xpixy)) 

1520 

1521 xrdgrp = ensure_subgroup('xrd2d', xrmmap, dtype='2DXRD') 

1522 

1523 xrdgrp.attrs['type'] = 'xrd2d detector' 

1524 xrdgrp.attrs['desc'] = '' #'add detector name eventually' 

1525 

1526 xrdgrp.create_dataset('mask', (xpixx, xpixy), np.uint16, **self.compress_args) 

1527 xrdgrp.create_dataset('background', (xpixx, xpixy), np.uint16, **self.compress_args) 

1528 

1529 chunksize_2DXRD = (1, npts, xpixx, xpixy) 

1530 xrdgrp.create_dataset('counts', (NSTART, npts, xpixx, xpixy), np.uint32, 

1531 chunks = chunksize_2DXRD, 

1532 maxshape=(None, npts, xpixx, xpixy), **self.compress_args) 

1533 

1534 if self.has_xrd1d: 

1535 xrdgrp = ensure_subgroup('xrd1d', xrmmap) 

1536 xrdgrp.attrs['type'] = 'xrd1d detector' 

1537 xrdgrp.attrs['desc'] = 'pyFAI calculation from xrd2d data' 

1538 

1539 xrdgrp.create_dataset('q', (self.qstps,), np.float32, **self.compress_args) 

1540 xrdgrp.create_dataset('background', (self.qstps,), np.float32, **self.compress_args) 

1541 

1542 chunksize_xrd1d = (1, npts, self.qstps) 

1543 xrdgrp.create_dataset('counts', 

1544 (NSTART, npts, self.qstps), 

1545 np.float32, 

1546 chunks = chunksize_xrd1d, 

1547 maxshape=(None, npts, self.qstps), **self.compress_args) 

1548 

1549 if self.azwdgs > 1: 

1550 xrmmap['work'].create_group('xrdwedge') 

1551 for azi in range(self.azwdgs): 

1552 wdggrp = xrmmap['work/xrdwedge'].create_group('wedge_%02d' % azi) 

1553 

1554 wdggrp.create_dataset('q', (self.qstps,), np.float32, **self.compress_args) 

1555 

1556 wdggrp.create_dataset('counts', 

1557 (NSTART, npts, self.qstps), 

1558 np.float32, 

1559 chunks = chunksize_xrd1d, 

1560 maxshape=(None, npts, self.qstps), **self.compress_args) 

1561 

1562 #wdggrp.create_dataset('limits', (2,), np.float32) 

1563 wdg_sz = 360./self.azwdgs 

1564 wdg_lmts = np.array([azi*wdg_sz, (azi+1)*wdg_sz]) - 180 

1565 wdggrp.create_dataset('limits', data=wdg_lmts) 

1566 

1567 try: 

1568 print('\nStart: %s' % isotime(self.starttime)) 

1569 except: 

1570 pass 

1571 

1572 self.h5root.flush() 

1573 

1574 def add_xrd1d(self, qstps=None): 

1575 xrd1dgrp = ensure_subgroup('xrd1d',self.xrmmap) 

1576 xrdcalfile = bytes2str(xrd1dgrp.attrs.get('calfile', '')) 

1577 if os.path.exists(xrdcalfile): 

1578 print('Using calibration file : %s' % xrdcalfile) 

1579 try: 

1580 nrows, npts , xpixx, xpixy = self.xrmmap['xrd2d/counts'].shape 

1581 except: 

1582 return 

1583 

1584 if qstps is not None: self.qstps = qstps 

1585 

1586 pform ='\n--- Build XRD1D Schema (%i, %i, %i) from 2D XRD (%i, %i, %i, %i) ---' 

1587 print(pform % (nrows, npts, self.qstps, nrows, npts, xpixx, xpixy)) 

1588 

1589 try: 

1590 xrd1dgrp.attrs['type'] = 'xrd1d detector' 

1591 xrd1dgrp.attrs['desc'] = 'pyFAI calculation from xrd2d data' 

1592 xrd1dgrp.create_dataset('q', (self.qstps,), np.float32) 

1593 xrd1dgrp.create_dataset('background', (self.qstps,), np.float32) 

1594 

1595 chunksize_xrd1d = (1, npts, self.qstps) 

1596 xrd1dgrp.create_dataset('counts', 

1597 (nrows, npts, self.qstps), 

1598 np.float32, 

1599 chunks = chunksize_xrd1d) 

1600 

1601 attrs = {'steps':self.qstps,'mask':self.xrd2dmaskfile,'flip':self.flip} 

1602 print('\nStart: %s' % isotime()) 

1603 for i in np.arange(nrows): 

1604 rowq, row1d = integrate_xrd_row(self.xrmmap['xrd2d/counts'][i],xrdcalfile,**attrs) 

1605 if i == 0: 

1606 self.xrmmap['xrd1d/q'][:] = rowq[0] 

1607 self.xrmmap['xrd1d/counts'][i,] = row1d 

1608 

1609 self.has_xrd1d = True 

1610 # print('End: %s' % isotime()) 

1611 except: 

1612 print('xrd1d data already in file.') 

1613 return 

1614 

1615 def get_slice_y(self): 

1616 for name, val in zip([h5str(a) for a in self.xrmmap['config/environ/name']], 

1617 [h5str(a) for a in self.xrmmap['config/environ/value']]): 

1618 name = str(name).lower() 

1619 if name.startswith('sample'): 

1620 name = name.replace('samplestage.', '') 

1621 if name.lower() == 'fine y' or name.lower() == 'finey': 

1622 return float(val) 

1623 

1624 def get_datapath_list(self, remove='raw'): 

1625 def find_detector(group): 

1626 sub_list = [] 

1627 if 'counts' in group.keys(): 

1628 sub_list += [group['counts'].name] 

1629 elif 'scal' in group.name: 

1630 for key, val in dict(group).items(): 

1631 sub_list += [group[key].name] 

1632 return sub_list 

1633 

1634 dlist = [] 

1635 for det in self.get_detector_list(): 

1636 for idet in find_detector(self.xrmmap[det]): 

1637 if not (remove in idet): 

1638 dlist.append(idet) 

1639 

1640 return dlist 

1641 

1642 def get_roi_list(self, det_name, force=False): 

1643 """ 

1644 get a list of rois from detector 

1645 """ 

1646 detname = self.get_detname(det_name) 

1647 if not force and (detname not in EXTRA_DETGROUPS): 

1648 roilist = self.roi_names.get(detname, None) 

1649 if roilist is not None: 

1650 return roilist 

1651 

1652 roigrp = ensure_subgroup('roimap', self.xrmmap, dtype='roi maps') 

1653 def sort_roi_limits(roidetgrp): 

1654 roi_name, roi_limits = [],[] 

1655 for name in roidetgrp.keys(): 

1656 roi_name.append(name) 

1657 roi_limits.append(roidetgrp[name]['limits'][0]) 

1658 return [y for (x,y) in sorted(zip(roi_limits,roi_name))] 

1659 

1660 def sort_orderby(det, orderby): 

1661 roi_name, roi_order = [], [] 

1662 for name in det.keys(): 

1663 roi_name.append(name) 

1664 roi_order.append(det[name].attrs.get(orderby, 9999)) 

1665 return [y for (x, y) in sorted(zip(roi_order, roi_name))] 

1666 

1667 rois = [] 

1668 if version_ge(self.version, '2.0.0'): 

1669 if detname in roigrp.keys(): 

1670 rois = sort_roi_limits(roigrp[detname]) 

1671 else: 

1672 det = self.xrmmap[detname] 

1673 if (detname in EXTRA_DETGROUPS or 

1674 'detector' in det.attrs.get('type')): 

1675 rois = list(det.keys()) 

1676 orderby = det.attrs.get('orderby', None) 

1677 if orderby is not None: 

1678 rois = sort_orderby(det, orderby) 

1679 else: 

1680 if detname in EXTRA_DETGROUPS: 

1681 rois = list(self.xrmmap[detname].keys()) 

1682 elif detname in self.xrmmap.keys(): 

1683 rois = list(roigrp['sum_name']) + rois 

1684 try: 

1685 rois = sort_roi_limits(roigrp[detname]) + rois 

1686 except: 

1687 pass 

1688 rois.append('1') 

1689 self.roi_names[detname] = [h5str(a) for a in rois] 

1690 return self.roi_names[detname] 

1691 

1692 def get_detector_list(self, use_cache=False): 

1693 """get a list of detector groups, 

1694 ['mcasum', 'mca1', ..., 'scalars', 'work', 'xrd1d', ...] 

1695 """ 

1696 workgroup = ensure_subgroup('work', self.xrmmap) 

1697 if use_cache and self.detector_list is not None: 

1698 return self.detector_list 

1699 def build_dlist(group): 

1700 detlist, sumslist = [], [] 

1701 for key, grp in group.items(): 

1702 if ('det' in bytes2str(grp.attrs.get('type', '')) or 

1703 'mca' in bytes2str(grp.attrs.get('type', ''))): 

1704 if 'sum' in key.lower(): 

1705 sumslist.append(key) 

1706 else: 

1707 detlist.append(key) 

1708 return sumslist + detlist 

1709 

1710 xrmmap = self.xrmmap 

1711 det_list = [] 

1712 if version_ge(self.version, '2.0.0'): 

1713 det_list = build_dlist(xrmmap['roimap']) 

1714 else: 

1715 det_list = build_dlist(xrmmap) 

1716 for det in build_dlist(xrmmap['roimap']): 

1717 if det not in det_list: 

1718 det_list.append(det) 

1719 

1720 # add any other groups with 'detector' in the `type` attribute: 

1721 for det, grp in xrmmap.items(): 

1722 attrs = getattr(grp, 'attrs', {'type': ''}) 

1723 if det not in det_list and 'detector' in h5str(attrs.get('type', '')): 

1724 det_list.append(det) 

1725 self.detector_list = det_list 

1726 if len(det_list) < 1: 

1727 det_list = [''] 

1728 self.detector_list = None 

1729 return det_list 

1730 

1731 def reset_flags(self): 

1732 ''' 

1733 Reads hdf5 file for data and sets the flags. 

1734 mkak 2016.08.30 // rewritten mkak 2017.08.03 // rewritten mkak 2017.12.05 

1735 ''' 

1736 for det in self.get_detector_list(): 

1737 if det in self.xrmmap: 

1738 

1739 detgrp = self.xrmmap[det] 

1740 

1741 dettype = bytes2str(detgrp.attrs.get('type', '')).lower() 

1742 if 'mca' in dettype: 

1743 self.has_xrf = 'counts' in detgrp 

1744 elif 'xrd2d' in dettype: 

1745 self.has_xrd2d = 'counts' in detgrp 

1746 elif 'xrd1d' in dettype: 

1747 self.has_xrd1d = 'counts' in detgrp 

1748 elif det == 'xrd': ## compatible with old version 

1749 try: 

1750 detgrp['data1d'] 

1751 self.has_xrd1d = True 

1752 except: 

1753 pass 

1754 try: 

1755 detgrp['data2D'] 

1756 self.has_xrd2d = True 

1757 except: 

1758 pass 

1759 

1760 def print_flags(self): 

1761 print(' HAS XRF, XRD1D, XRD2D: %s, %s, %s' % (self.has_xrf, 

1762 self.has_xrd1d, 

1763 self.has_xrd2d)) 

1764 

1765 def resize_arrays(self, nrow, force_shrink=True): 

1766 "resize all arrays for new nrow size" 

1767 if not self.check_hostid(): 

1768 raise GSEXRM_Exception(NOT_OWNER % self.filename) 

1769 if not self.write_access: 

1770 raise GSEXRM_Exception(READ_ONLY % self.filename) 

1771 if version_ge(self.version, '2.0.0'): 

1772 

1773 g = self.xrmmap['positions/pos'] 

1774 old, npts, nx = g.shape 

1775 if nrow < old and not force_shrink: 

1776 return 

1777 g.resize((nrow, npts, nx)) 

1778 

1779 for g in self.xrmmap.values(): 

1780 type_attr = bytes2str(g.attrs.get('type', '')) 

1781 if type_attr.find('det') > -1: 

1782 if type_attr.startswith('scalar'): 

1783 for aname in g.keys(): 

1784 oldnrow, npts = g[aname].shape 

1785 g[aname].resize((nrow, npts)) 

1786 elif type_attr.startswith('mca'): 

1787 oldnrow, npts, nchan = g['counts'].shape 

1788 g['counts'].resize((nrow, npts, nchan)) 

1789 for aname in ('livetime', 'realtime', 

1790 'inpcounts', 'outcounts', 'dtfactor'): 

1791 g[aname].resize((nrow, npts)) 

1792 elif type_attr.startswith('virtual mca'): 

1793 oldnrow, npts, nchan = g['counts'].shape 

1794 g['counts'].resize((nrow, npts, nchan)) 

1795 for aname in ('livetime', 'realtime', 

1796 'inpcounts', 'outcounts', 'dtfactor'): 

1797 if aname in g: 

1798 g[aname].resize((nrow, npts)) 

1799 

1800 elif type_attr.startswith('xrd2d'): 

1801 oldnrow, npts, xpixx, xpixy = g['counts'].shape 

1802 g['counts'].resize((nrow, npts, xpixx, xpixy)) 

1803 elif type_attr.startswith('xrd1d'): 

1804 oldnrow, npts, qstps = g['counts'].shape 

1805 g['counts'].resize((nrow, npts, qstps)) 

1806 

1807 if self.azwdgs > 1: 

1808 for g in self.xrmmap['work/xrdwedge'].values(): 

1809 g['counts'].resize((nrow, npts, qstps)) 

1810 

1811 if version_ge(self.version, '2.1.0'): 

1812 for bname in ('det_raw', 'det_cor', 'sum_raw', 'sum_cor'): 

1813 g = self.xrmmap['roimap'][bname] 

1814 old, npts, nx = g.shape 

1815 g.resize((nrow, npts, nx)) 

1816 else: 

1817 for g in self.xrmmap['roimap'].values(): # loop through detectors in roimap 

1818 for h in g.values(): # loop through rois in roimap 

1819 for aname in ('raw','cor'): 

1820 oldnrow, npts = h[aname].shape 

1821 h[aname].resize((nrow, npts)) 

1822 

1823 else: ## old file format method 

1824 

1825 realmca_groups = [] 

1826 virtmca_groups = [] 

1827 for g in self.xrmmap.values(): 

1828 # include both real and virtual mca detectors! 

1829 type_attr = bytes2str(g.attrs.get('type', '')) 

1830 if type_attr.find('det') > -1 or type_attr.find('mca') > -1: 

1831 if type_attr.startswith('mca'): 

1832 realmca_groups.append(g) 

1833 elif type_attr.startswith('virtual mca'): 

1834 virtmca_groups.append(g) 

1835 elif type_attr.startswith('xrd2d'): 

1836 oldnrow, npts, xpixx, xpixy = g['counts'].shape 

1837 g['counts'].resize((nrow, npts, xpixx, xpixy)) 

1838 elif type_attr.startswith('xrd1d'): 

1839 oldnrow, npts, qstps = g['counts'].shape 

1840 g['counts'].resize((nrow, npts, qstps)) 

1841 

1842 if self.azwdgs > 1: 

1843 for g in self.xrmmap['work/xrdwedge'].values(): 

1844 g['counts'].resize((nrow, npts, qstps)) 

1845 

1846 oldnrow, npts, nchan = realmca_groups[0]['counts'].shape 

1847 for g in realmca_groups: 

1848 g['counts'].resize((nrow, npts, nchan)) 

1849 for aname in ('livetime', 'realtime', 

1850 'inpcounts', 'outcounts', 'dtfactor'): 

1851 g[aname].resize((nrow, npts)) 

1852 

1853 for g in virtmca_groups: 

1854 g['counts'].resize((nrow, npts, nchan)) 

1855 

1856 g = self.xrmmap['positions/pos'] 

1857 old, npts, nx = g.shape 

1858 g.resize((nrow, npts, nx)) 

1859 

1860 for bname in ('det_raw', 'det_cor', 'sum_raw', 'sum_cor'): 

1861 g = self.xrmmap['roimap'][bname] 

1862 old, npts, nx = g.shape 

1863 g.resize((nrow, npts, nx)) 

1864 

1865 self.h5root.flush() 

1866 

1867 def add_work_array(self, data, name, parent='work', 

1868 dtype='virtual detector', **kws): 

1869 ''' 

1870 add an array to the work group of processed arrays 

1871 ''' 

1872 workgroup = ensure_subgroup(parent, self.xrmmap, dtype=dtype) 

1873 if name is None: 

1874 name = 'array_%3.3i' % (1+len(workgroup)) 

1875 if name in workgroup: 

1876 raise ValueError("array name '%s' exists in '%s" % (name, parent)) 

1877 ds = workgroup.create_dataset(name, data=data) 

1878 for key, val in kws.items(): 

1879 ds.attrs[key] = val 

1880 self.h5root.flush() 

1881 

1882 def add_work_arrays(self, arraydict, parent='work'): 

1883 if parent in self.xrmmap: 

1884 del self.xrmmap[parent] 

1885 self.h5root.flush() 

1886 

1887 workgroup = ensure_subgroup(parent, self.xrmmap) 

1888 workgroup.attrs['orderby'] = 'order' 

1889 count = 0 

1890 for key, data in arraydict.items(): 

1891 count += 1 

1892 name = fix_varname(key) 

1893 if name in workgroup: 

1894 del workgroup[name] 

1895 ds = workgroup.create_dataset(name, data=data) 

1896 ds.attrs['order'] = count 

1897 self.h5root.flush() 

1898 

1899 def del_work_array(self, name, parent='work'): 

1900 ''' 

1901 delete an array to the work group of processed arrays 

1902 ''' 

1903 workgroup = ensure_subgroup(parent, self.xrmmap) 

1904 name = h5str(name) 

1905 if name in workgroup: 

1906 del workgroup[name] 

1907 self.h5root.flush() 

1908 

1909 def get_work_array(self, name, parent='work'): 

1910 ''' 

1911 get an array from the work group of processed arrays by index or name 

1912 ''' 

1913 workgroup = ensure_subgroup(parent, self.xrmmap) 

1914 dat = None 

1915 name = h5str(name) 

1916 if name in workgroup: 

1917 dat = workgroup[name] 

1918 return dat 

1919 

1920 def work_array_names(self, parent='work'): 

1921 ''' 

1922 return list of work array descriptions 

1923 ''' 

1924 workgroup = ensure_subgroup(parent, self.xrmmap) 

1925 return [h5str(g) for g in workgroup.keys()] 

1926 

1927 def add_area(self, amask, name=None, desc=None): 

1928 '''add a selected area, with optional name 

1929 the area is encoded as a boolean array the same size as the map 

1930 

1931 ''' 

1932 if not self.check_hostid(): 

1933 raise GSEXRM_Exception(NOT_OWNER % self.filename) 

1934 if not self.write_access: 

1935 raise GSEXRM_Exception(READ_ONLY % self.filename) 

1936 

1937 area_grp = ensure_subgroup('areas', self.xrmmap, dtype='areas') 

1938 if name is None: 

1939 name = 'area_001' 

1940 if len(area_grp) > 0: 

1941 count = len(area_grp) 

1942 while name in area_grp and count < 9999: 

1943 name = 'area_%3.3i' % (count) 

1944 count += 1 

1945 ds = area_grp.create_dataset(name, data=amask) 

1946 if desc is None: 

1947 desc = name 

1948 ds.attrs['description'] = desc 

1949 self.h5root.flush() 

1950 return name 

1951 

1952 def export_areas(self, filename=None): 

1953 '''export areas to datafile ''' 

1954 if filename is None: 

1955 file_str = '%s_Areas.npz' 

1956 filename = file_str % self.filename 

1957 

1958 areas = ensure_subgroup('areas', self.xrmmap, dtype='areas') 

1959 kwargs = {key: val[:] for key, val in areas.items()} 

1960 np.savez(filename, **kwargs) 

1961 return filename 

1962 

1963 def import_areas(self, filename, overwrite=False): 

1964 '''import areas from datafile exported by export_areas()''' 

1965 fname = os.path.split(filename)[1] 

1966 if fname.endswith('.h5_Areas.npz'): 

1967 fname = fname.replace('.h5_Areas.npz', '') 

1968 

1969 areas = ensure_subgroup('areas', self.xrmmap, dtype='areas') 

1970 

1971 npzdat = np.load(filename) 

1972 for aname in npzdat.files: 

1973 desc = name = aname 

1974 if name in areas and not overwrite: 

1975 name = '%s_%s' % (name, fname) 

1976 desc = '%s from %s' % (aname, fname) 

1977 self.add_area(npzdat[aname], name=name, desc=desc) 

1978 

1979 def get_area(self, name=None): 

1980 ''' 

1981 get area group by name or description 

1982 ''' 

1983 area_grp = ensure_subgroup('areas', self.xrmmap, dtype='areas') 

1984 if name is not None and name in area_grp: 

1985 return area_grp[name] 

1986 else: 

1987 for aname in area_grp: 

1988 if name == bytes2str(area_grp[aname].attrs.get('description','')): 

1989 return area_grp[aname] 

1990 return None 

1991 

1992 def get_area_stats(self, name=None, desc=None): 

1993 '''return statistics for all raw detector counts/sec values 

1994 

1995 for each raw detector returns 

1996 name, length, mean, standard_deviation, 

1997 median, mode, minimum, maximum, 

1998 gmean, hmean, skew, kurtosis 

1999 

2000 ''' 

2001 area = self.get_area(name=name, desc=desc) 

2002 if area is None: 

2003 return None 

2004 

2005 if 'roistats' in area.attrs: 

2006 return json.loads(area.attrs.get('roistats','')) 

2007 

2008 amask = area[()] 

2009 

2010 roidata = [] 

2011 d_addrs = [d.lower() for d in self.xrmmap['roimap/det_address']] 

2012 d_names = [d for d in self.xrmmap['roimap/det_name']] 

2013 # count times 

2014 ctime = [1.e-6*self.xrmmap['roimap/det_raw'][:,:,0][amask]] 

2015 for i in range(self.xrmmap.attrs.get('N_Detectors',0)): 

2016 tname = 'det%i/realtime' % (i+1) 

2017 ctime.append(1.e-6*self.xrmmap[tname][()][amask]) 

2018 

2019 for idet, dname in enumerate(d_names): 

2020 daddr = d_addrs[idet] 

2021 det = 0 

2022 if 'mca' in daddr: 

2023 det = 1 

2024 words = daddr.split('mca') 

2025 if len(words) > 1: 

2026 det = int(words[1].split('.')[0]) 

2027 if idet == 0: 

2028 d = ctime[0] 

2029 else: 

2030 d = self.xrmmap['roimap/det_raw'][:,:,idet][amask]/ctime[det] 

2031 

2032 try: 

2033 hmean, gmean = stats.gmean(d), stats.hmean(d) 

2034 skew, kurtosis = stats.skew(d), stats.kurtosis(d) 

2035 except ValueError: 

2036 hmean, gmean, skew, kurtosis = 0, 0, 0, 0 

2037 mode = stats.mode(d) 

2038 roidata.append((dname, len(d), d.mean(), d.std(), np.median(d), 

2039 stats.mode(d), d.min(), d.max(), 

2040 gmean, hmean, skew, kurtosis)) 

2041 

2042 if 'roistats' not in area.attrs: 

2043 area.attrs['roistats'] = json.dumps(roidata) 

2044 self.h5root.flush() 

2045 

2046 return roidata 

2047 

2048 def get_translation_axis(self, hotcols=None): 

2049 if hotcols is None: 

2050 hotcols = self.hotcols 

2051 posnames = [bytes2str(n.lower()) for n in self.xrmmap['positions/name']] 

2052 if 'x' in posnames: 

2053 x = self.get_pos('x', mean=True) 

2054 elif 'fine x' in posnames: 

2055 x = self.get_pos('fine x', mean=True) 

2056 else: 

2057 x = self.get_pos(0, mean=True) 

2058 

2059 if hotcols and x is not None: 

2060 if len(x) == self.xrmmap[self.get_detname()]['counts'].shape[1]: 

2061 x = x[1:-1] 

2062 

2063 return x 

2064 

2065 def get_rotation_axis(self, axis=None, hotcols=None): 

2066 if hotcols is None: 

2067 hotcols = self.hotcols 

2068 posnames = [bytes2str(n.lower()) for n in self.xrmmap['positions/name']] 

2069 omega = None 

2070 if axis is not None: 

2071 if axis in posnames or type(axis) == int: 

2072 omega = self.get_pos(axis, mean=True) 

2073 else: 

2074 omega = None 

2075 for pname in posnames: 

2076 for aname in ('theta', 'phi', 'omega', 'chi'): 

2077 if aname in pname: 

2078 omega = self.get_pos(pname, mean=True) 

2079 # print(f"using positoner '{pname:s}' as rotation axis ") 

2080 break 

2081 if omega is not None: 

2082 break 

2083 if hotcols and omega is not None: 

2084 if len(omega) == self.xrmmap[self.get_detname()]['counts'].shape[1]: 

2085 omega = omega[1:-1] 

2086 return omega 

2087 

2088 def get_tomography_center(self): 

2089 tomogrp = ensure_subgroup('tomo', self.xrmmap) 

2090 try: 

2091 return tomogrp['center'][()] 

2092 except: 

2093 self.set_tomography_center() 

2094 

2095 return tomogrp['center'][()] 

2096 

2097 def set_tomography_center(self,center=None): 

2098 if center is None: 

2099 center = len(self.get_translation_axis())/2. 

2100 

2101 tomogrp = ensure_subgroup('tomo', self.xrmmap) 

2102 try: 

2103 del tomogrp['center'] 

2104 except: 

2105 pass 

2106 tomogrp.create_dataset('center', data=center) 

2107 

2108 self.h5root.flush() 

2109 

2110 def get_sinogram(self, roi_name, det=None, trim_sino=False, 

2111 hotcols=None, dtcorrect=None, **kws): 

2112 '''extract roi map for a pre-defined roi by name 

2113 

2114 Parameters 

2115 --------- 

2116 roiname : str ROI name 

2117 det : str detector name 

2118 dtcorrect : None or bool [None] deadtime correction 

2119 hotcols : None or bool [None] suppress hot columns 

2120 

2121 Returns 

2122 ------- 

2123 sinogram for ROI data 

2124 sinogram_order (needed for knowing shape of sinogram) 

2125 

2126 Notes 

2127 ----- 

2128 if dtcorrect or hotcols is None, they are taken from 

2129 self.dtcorrect and self.hotcols 

2130 ''' 

2131 if hotcols is None: 

2132 hotcols = self.hotcols 

2133 if dtcorrect is None: 

2134 dtcorrect = self.dtcorrect 

2135 

2136 sino = self.get_roimap(roi_name, det=det, hotcols=hotcols, **kws) 

2137 x = self.get_translation_axis(hotcols=hotcols) 

2138 omega = self.get_rotation_axis(hotcols=hotcols) 

2139 

2140 if omega is None: 

2141 print('\n** Cannot compute tomography: no rotation axis specified in map. **') 

2142 return 

2143 

2144 if trim_sino: 

2145 sino, x, omega = trim_sinogram(sino, x, omega) 

2146 

2147 return reshape_sinogram(sino, x, omega) 

2148 

2149 def get_tomograph(self, sino, omega=None, center=None, hotcols=None, **kws): 

2150 ''' 

2151 returns tomo_center, tomo 

2152 ''' 

2153 if hotcols is None: 

2154 hotcols = self.hotcols 

2155 if center is None: 

2156 center = self.get_tomography_center() 

2157 if omega is None: 

2158 omega = self.get_rotation_axis(hotcols=hotcols) 

2159 if omega is None: 

2160 print('\n** Cannot compute tomography: no rotation axis specified in map. **') 

2161 return 

2162 

2163 center, recon = tomo_reconstruction(sino, omega, center=center, **kws) 

2164 self.set_tomography_center(center=center) 

2165 return recon 

2166 

2167 def save_tomograph(self, datapath, algorithm='gridrec', 

2168 filter_name='shepp', num_iter=1, dtcorrect=None, 

2169 hotcols=None, **kws): 

2170 ''' 

2171 saves group for tomograph for selected detector 

2172 ''' 

2173 if hotcols is None: 

2174 hotcols = self.hotcols 

2175 if dtcorrect is None: 

2176 dtcorrect = self.dtcorrect 

2177 

2178 ## check to make sure the selected detector exists for reconstructions 

2179 detlist = self.get_datapath_list(remove=None) 

2180 if datapath not in detlist: 

2181 print("Detector '%s' not found in data." % datapath) 

2182 print('Known detectors: %s' % detlist) 

2183 return 

2184 datagroup = self.xrmmap[datapath] 

2185 

2186 ## check to make sure there is data to perform tomographic reconstruction 

2187 center = self.get_tomography_center() 

2188 

2189 x = self.get_translation_axis(hotcols=hotcols) 

2190 omega = self.get_rotation_axis(hotcols=hotcols) 

2191 

2192 if omega is None: 

2193 print('\n** Cannot compute tomography: no rotation axis specified in map. **') 

2194 return 

2195 

2196 ## define detector path 

2197 detgroup = datagroup 

2198 while isinstance(detgroup,h5py.Dataset): 

2199 detgroup = detgroup.parent 

2200 detpath = detgroup.name 

2201 

2202 ## create path for saving data 

2203 tpath = datapath.replace('/xrmmap','/tomo') 

2204 tpath = tpath.replace('/scalars','') 

2205 if tpath.endswith('raw'): 

2206 tpath = tpath.replace('_raw','') 

2207 dtcorrect = False 

2208 elif tpath.endswith('counts'): 

2209 tpath = os.path.split(tpath)[0] 

2210 

2211 ## build path for saving data in tomo-group 

2212 grp = self.xrmmap 

2213 for kpath in tpath.split('/'): 

2214 if len(kpath) > 0: 

2215 grp = ensure_subgroup(kpath ,grp) 

2216 tomogrp = grp 

2217 

2218 ## define sino group from datapath 

2219 if 'scalars' in datapath or 'xrd' in datapath: 

2220 sino = datagroup[()] 

2221 elif dtcorrect: 

2222 if 'sum' in datapath: 

2223 sino = np.zeros(np.shape(np.einsum('jki->ijk', datagroup[()]))) 

2224 for i in range(self.nmca): 

2225 idatapath = datapath.replace('sum', str(i+1)) 

2226 idatagroup = self.xrmmap[idatapath] 

2227 idetpath = detpath.replace('sum', str(i+1)) 

2228 idetgroup = self.xrmmap[idetpath] 

2229 sino += np.einsum('jki->ijk', idatagroup[()]) * idetgroup['dtfactor'][()] 

2230 

2231 else: 

2232 sino = np.einsum('jki->ijk', datagroup[()]) * detgroup['dtfactor'][()] 

2233 else: 

2234 sino = datagroup[()] 

2235 

2236 sino,order = reshape_sinogram(sino, x, omega) 

2237 

2238 center, tomo = tomo_reconstruction(sino, algorithm=algorithm, 

2239 filter_name=filter_name, 

2240 num_iter=num_iter, omega=omega, 

2241 center=center, sinogram_order=order) 

2242 

2243 tomogrp.attrs['tomo_alg'] = '-'.join([str(t) for t in (algorithm, filter_name)]) 

2244 tomogrp.attrs['center'] = '%0.2f pixels' % (center) 

2245 

2246 try: 

2247 tomogrp.create_dataset('counts', data=np.swapaxes(tomo,0,2), **self.compress_args) 

2248 except: 

2249 del tomogrp['counts'] 

2250 tomogrp.create_dataset('counts', data=np.swapaxes(tomo,0,2), **self.compress_args) 

2251 

2252 for data_tag in ('energy','q'): 

2253 if data_tag in detgroup.keys(): 

2254 try: 

2255 tomogrp.create_dataset(data_tag, data=detgroup[data_tag]) 

2256 del tomogrp[data_tag] 

2257 except: 

2258 del tomogrp[data_tag] 

2259 tomogrp.create_dataset(data_tag, data=detgroup[data_tag]) 

2260 

2261 for key, val in dict(detgroup.attrs).items(): 

2262 tomogrp.attrs[key] = val 

2263 

2264 self.h5root.flush() 

2265 

2266 def take_ownership(self): 

2267 "claim ownership of file" 

2268 if self.xrmmap is None or not self.write_access: 

2269 return 

2270 self.xrmmap.attrs['Process_Machine'] = get_machineid() 

2271 self.xrmmap.attrs['Process_ID'] = os.getpid() 

2272 self.h5root.flush() 

2273 

2274 def release_ownership(self): 

2275 if not self.write_access: 

2276 return 

2277 self.xrmmap.attrs['Process_Machine'] = '' 

2278 self.xrmmap.attrs['Process_ID'] = 0 

2279 self.xrmmap.attrs['Last_Row'] = self.last_row 

2280 

2281 def check_ownership(self, take_ownership=True): 

2282 return self.check_hostid(take_ownership=take_ownership) 

2283 

2284 def check_hostid(self, take_ownership=True): 

2285 '''checks host and id of file: 

2286 returns True if this process the owner of the file 

2287 

2288 By default, this takes ownership if it can. 

2289 ''' 

2290 if self.xrmmap is None: 

2291 return False 

2292 if not self.write_access: 

2293 return True 

2294 

2295 attrs = self.xrmmap.attrs 

2296 self.folder = attrs['Map_Folder'] 

2297 file_mach = attrs['Process_Machine'] 

2298 file_pid = attrs['Process_ID'] 

2299 if len(file_mach) < 1 or file_pid < 1: 

2300 if take_ownership: 

2301 self.take_ownership() 

2302 return True 

2303 return (file_mach == get_machineid() and file_pid == os.getpid()) 

2304 

2305 def folder_has_newdata(self): 

2306 if self.folder is not None and isGSEXRM_MapFolder(self.folder): 

2307 self.read_master() 

2308 return (self.last_row < len(self.rowdata)-1) 

2309 return False 

2310 

2311 def read_master(self): 

2312 "reads master file for toplevel scan info" 

2313 if self.folder is None or not isGSEXRM_MapFolder(self.folder): 

2314 return 

2315 self.masterfile = os.path.join(nativepath(self.folder), self.MasterFile) 

2316 header, rows, mtime = [], [], -1 

2317 if self.scandb is not None: 

2318 # check that this map folder is the one currently running from scandb: 

2319 try: 

2320 db_folder = toppath(self.scandb.get_info('map_folder')) 

2321 except: 

2322 db_folder = None 

2323 disk_folder = toppath(os.path.abspath(self.folder)) 

2324 

2325 if db_folder == disk_folder: # this is the current map 

2326 mastertext = self.scandb.get_slewscanstatus() 

2327 mtime = time.time() 

2328 header, rows = [], [] 

2329 for srow in mastertext: 

2330 line = str(srow.text.strip()) 

2331 if line.startswith('#'): 

2332 header.append(line) 

2333 else: 

2334 rows.append(line.split()) 

2335 

2336 if len(header) < 1 or mtime < 0: # this is *not* the map that is currently being collected: 

2337 # if file the master file is not new, the current row data is OK: 

2338 

2339 try: 

2340 header, rows = readMasterFile(self.masterfile) 

2341 except IOError: 

2342 raise GSEXRM_Exception("cannot read Master file from '%s'" % 

2343 self.masterfile) 

2344 

2345 mtime = os.stat(self.masterfile).st_mtime 

2346 if mtime < (self.master_modtime+1.0) and len(self.rowdata) > 1: 

2347 # print("READ MASTER not a new masterfile ", len(self.rowdata), len(rows)) 

2348 return len(self.rowdata) 

2349 

2350 self.master_modtime = mtime 

2351 

2352 self.notes['end_time'] = isotime(mtime) 

2353 self.master_header = header 

2354 # carefully read rows to avoid repeated rows due to bad collection 

2355 self.rowdata = [] 

2356 _yl, _xl, _s1 = None, None, None 

2357 for row in rows: 

2358 yval, xrff, sisf = row[0], row[1], row[2] 

2359 il = len(self.rowdata)-1 

2360 if il > -1: 

2361 _yl, _xl, _s1 = (self.rowdata[il][0], 

2362 self.rowdata[il][1], 

2363 self.rowdata[il][2]) 

2364 # skip repeated rows in master file 

2365 if yval != _yl and (xrff != _xl or sisf != _s1): 

2366 self.rowdata.append(row) 

2367 

2368 self.scan_version = 1.00 

2369 self.nrows_expected = None 

2370 self.start_time = time.ctime() 

2371 for line in header: 

2372 words = line.split('=') 

2373 if 'scan.starttime' in words[0].lower(): 

2374 self.start_time = words[1].strip() 

2375 self.notes['scan_start_time'] = self.start_time 

2376 elif 'scan.version' in words[0].lower(): 

2377 self.scan_version = words[1].strip() 

2378 elif 'scan.nrows_expected' in words[0].lower(): 

2379 self.nrows_expected = int(words[1].strip()) 

2380 self.scan_version = float(self.scan_version) 

2381 

2382 # print("read_master scan version = ", self.scan_version) 

2383 self.stop_time = time.ctime(self.master_modtime) 

2384 try: 

2385 last_file = os.path.join(self.folder,rows[-1][2]) 

2386 self.stop_time = time.ctime(os.stat(last_file).st_ctime) 

2387 except: 

2388 pass 

2389 self.notes['scan_end_time'] = self.stop_time 

2390 self.notes['scan_version'] = self.scan_version 

2391 

2392 if self.scan_version < 1.35 and (self.has_xrd2d or self.has_xrd1d): 

2393 xrd_files = [fn for fn in os.listdir(self.folder) if fn.endswith('nc')] 

2394 for i, addxrd in enumerate(xrd_files): 

2395 self.rowdata[i].insert(4, addxrd) 

2396 

2397 cfile = FastMapConfig() 

2398 cfile.Read(os.path.join(self.folder, self.ScanFile)) 

2399 mapconf = self.mapconf = cfile.config 

2400 

2401 if self.filename is None: 

2402 self.filename = mapconf['scan']['filename'] 

2403 if not self.filename.endswith('.h5'): 

2404 self.filename = "%s.h5" % self.filename 

2405 

2406 slow_pos = mapconf['slow_positioners'] 

2407 fast_pos = mapconf['fast_positioners'] 

2408 

2409 scanconf = mapconf['scan'] 

2410 self.dimension = scanconf['dimension'] 

2411 start = mapconf['scan']['start1'] 

2412 stop = mapconf['scan']['stop1'] 

2413 step = mapconf['scan']['step1'] 

2414 span = abs(stop-start) 

2415 self.npts = int(abs(abs(step)*1.01 + span)/abs(step)) 

2416 

2417 pos1 = scanconf['pos1'] 

2418 self.pos_addr = [pos1] 

2419 self.pos_desc = [slow_pos[pos1]] 

2420 # note: XPS gathering file now saving ONLY data for the fast axis 

2421 

2422 if self.dimension > 1: 

2423 yaddr = scanconf['pos2'] 

2424 self.pos_addr.append(yaddr) 

2425 self.pos_desc.append(slow_pos[yaddr]) 

2426 

2427 return len(rows) 

2428 

2429 def get_detname(self, det=None): 

2430 "return XRMMAP group for a detector" 

2431 

2432 mcastr = 'mca' if version_ge(self.version, '2.0.0') else 'det' 

2433 detname = '%ssum' % mcastr 

2434 

2435 if isinstance(det, str): 

2436 for d in self.get_detector_list(): 

2437 if det.lower() == d.lower(): 

2438 detname = d 

2439 elif isinstance(det, int): 

2440 if det in range(1, self.nmca+1): 

2441 detname = '%s%i' % (mcastr, det) 

2442 

2443 return detname 

2444 

2445 def get_detgroup(self, det=None): 

2446 "return XRMMAP group for a detector" 

2447 return self.xrmmap[self.get_detname(det)] 

2448 

2449 def get_energy(self, det=None): 

2450 '''return energy array for a detector''' 

2451 try: 

2452 group = self.xrmmap[det] 

2453 except: 

2454 group = self.get_detgroup(det) 

2455 return group['energy'][()] 

2456 

2457 def get_shape(self): 

2458 '''returns NY, NX shape of array data''' 

2459 ny, nx, npos = self.xrmmap['positions/pos'].shape 

2460 return ny, nx 

2461 

2462 def get_envvar(self, name=None): 

2463 """get environment value by name""" 

2464 if self.envvar is None: 

2465 self.envvar = {} 

2466 env_names = [h5str(a) for a in self.xrmmap['config/environ/name']] 

2467 env_vals = [h5str(a) for a in self.xrmmap['config/environ/value']] 

2468 for name, val in zip(env_names, env_vals): 

2469 name = h5str(name) 

2470 val = h5str(val) 

2471 try: 

2472 fval = float(val) 

2473 except: 

2474 fval = val 

2475 self.envvar[name] = fval 

2476 if name is not None: 

2477 if name in self.envvar: 

2478 return self.envvar[name] 

2479 elif name.lower() in self.envvar: 

2480 return self.envvar[name.lower()] 

2481 

2482 def get_incident_energy(self): 

2483 """ special case of get_envvar""" 

2484 env_names = [h5str(a) for a in self.xrmmap['config/environ/name']] 

2485 env_vals = [h5str(a) for a in self.xrmmap['config/environ/value']] 

2486 for name, val in zip(env_names, env_vals): 

2487 name = name.lower().replace('.', ' ') 

2488 if name.startswith('mono energy'): 

2489 return float(val) 

2490 return DEFAULT_XRAY_ENERGY 

2491 

2492 def get_counts_rect(self, ymin, ymax, xmin, xmax, mapdat=None, 

2493 det=None, dtcorrect=None): 

2494 '''return counts for a map rectangle, optionally 

2495 applying area mask and deadtime correction 

2496 

2497 Parameters 

2498 --------- 

2499 ymin : int low y index 

2500 ymax : int high y index 

2501 xmin : int low x index 

2502 xmax : int high x index 

2503 mapdat : optional, None or map data 

2504 det : optional, None or int index of detector 

2505 dtcorrect : optional, bool [None] dead-time correct data 

2506 

2507 Returns 

2508 ------- 

2509 ndarray for XRF counts in rectangle 

2510 

2511 Does *not* check for errors! 

2512 

2513 Note: if mapdat is None, the map data is taken from the 'det' parameter 

2514 ''' 

2515 if dtcorrect is None: 

2516 dtcorrect = self.dtcorrect 

2517 if mapdat is None: 

2518 mapdat = self.get_detgroup(det) 

2519 

2520 if 'counts' not in mapdat: 

2521 mapdat = self.get_detgroup(None) 

2522 

2523 nx, ny = (xmax-xmin, ymax-ymin) 

2524 sx = slice(xmin, xmax) 

2525 sy = slice(ymin, ymax) 

2526 

2527 if len(mapdat['counts'].shape) == 4: 

2528 counts = mapdat['counts'][sy, sx, :, :] 

2529 else: 

2530 counts = mapdat['counts'][sy, sx, :] 

2531 

2532 if dtcorrect and 'dtfactor' in mapdat: 

2533 counts = counts*mapdat['dtfactor'][sy, sx].reshape(ny, nx, 1) 

2534 return counts 

2535 

2536 def get_mca_area(self, areaname, det=None, dtcorrect=None): 

2537 '''return XRF spectra as MCA() instance for 

2538 spectra summed over a pre-defined area 

2539 

2540 Parameters 

2541 --------- 

2542 areaname : str name of area 

2543 dtcorrect : optional, bool [None] dead-time correct data 

2544 

2545 Returns 

2546 ------- 

2547 MCA object for XRF counts in area 

2548 

2549 ''' 

2550 try: 

2551 area = self.get_area(areaname)[()] 

2552 except: 

2553 raise GSEXRM_Exception("Could not find area '%s'" % areaname) 

2554 if dtcorrect is None: 

2555 dtcorrect = self.dtcorrect 

2556 npixels = area.sum() 

2557 if npixels < 1: 

2558 return None 

2559 

2560 dgroup = self.get_detname(det) 

2561 

2562 # first get data for bounding rectangle 

2563 _ay, _ax = np.where(area) 

2564 ymin, ymax, xmin, xmax = _ay.min(), _ay.max()+1, _ax.min(), _ax.max()+1 

2565 opts = {'dtcorrect': dtcorrect, 'det': det} 

2566 counts = self.get_counts_rect(ymin, ymax, xmin, xmax, **opts) 

2567 ltime, rtime = self.get_livereal_rect(ymin, ymax, xmin, xmax, **opts) 

2568 ltime = ltime[area[ymin:ymax, xmin:xmax]].sum() 

2569 rtime = rtime[area[ymin:ymax, xmin:xmax]].sum() 

2570 counts = counts[area[ymin:ymax, xmin:xmax]] 

2571 while(len(counts.shape) > 1): 

2572 counts = counts.sum(axis=0) 

2573 return self._getmca(dgroup, counts, areaname, npixels=npixels, 

2574 real_time=rtime, live_time=ltime) 

2575 

2576 def get_mca_rect(self, ymin, ymax, xmin, xmax, det=None, dtcorrect=None): 

2577 '''return mca counts for a map rectangle, optionally 

2578 

2579 Parameters 

2580 --------- 

2581 ymin : int low y index 

2582 ymax : int high y index 

2583 xmin : int low x index 

2584 xmax : int high x index 

2585 det : optional, None or int index of detector 

2586 dtcorrect : optional, bool [None] dead-time correct data 

2587 

2588 Returns 

2589 ------- 

2590 MCA object for XRF counts in rectangle 

2591 

2592 ''' 

2593 if dtcorrect is None: 

2594 dtcorrect = self.dtcorrect 

2595 dgroup = self.get_detname(det) 

2596 mapdat = self.get_detgroup(det) 

2597 if 'counts' not in mapdat: 

2598 mapdat = self.get_detgroup(None) 

2599 dgroup = self.get_detname(None) 

2600 counts = self.get_counts_rect(ymin, ymax, xmin, xmax, mapdat=mapdat, 

2601 det=det, dtcorrect=dtcorrect) 

2602 name = 'rect(y=[%i:%i], x==[%i:%i])' % (ymin, ymax, xmin, xmax) 

2603 npix = (ymax-ymin+1)*(xmax-xmin+1) 

2604 ltime, rtime = self.get_livereal_rect(ymin, ymax, xmin, xmax, det=det, 

2605 dtcorrect=dtcorrect) 

2606 counts = counts.sum(axis=0).sum(axis=0) 

2607 return self._getmca(dgroup, counts, name, npixels=npix, 

2608 real_time=rtime.sum(), live_time=ltime.sum()) 

2609 

2610 

2611 def get_livereal_rect(self, ymin, ymax, xmin, xmax, det=None, **kws): 

2612 '''return livetime, realtime for a map rectangle, optionally 

2613 applying area mask and deadtime correction 

2614 

2615 Parameters 

2616 --------- 

2617 ymin : int low y index 

2618 ymax : int high y index 

2619 xmin : int low x index 

2620 xmax : int high x index 

2621 det : optional, None or int index of detector 

2622 

2623 Returns 

2624 ------- 

2625 realtime, livetime in seconds 

2626 

2627 ''' 

2628 tshape = self.get_shape() 

2629 dmap = self.get_detgroup(det) 

2630 

2631 if ymax < 0: ymax += tshape[0] 

2632 if xmax < 0: xmax += tshape[1] 

2633 nx, ny = (xmax-xmin, ymax-ymin) 

2634 sx = slice(xmin, xmax) 

2635 sy = slice(ymin, ymax) 

2636 

2637 if 'livetime' in dmap: 

2638 livetime = 1.e-6*dmap['livetime'][sy, sx] 

2639 realtime = 1.e-6*dmap['realtime'][sy, sx] 

2640 else: 

2641 livetime = self.pixeltime * np.ones((ny, nx)) 

2642 realtime = self.pixeltime * np.ones((ny, nx)) 

2643 

2644 return livetime, realtime 

2645 

2646 def _getmca(self, dgroup, counts, name, npixels=None, **kws): 

2647 '''return an MCA object for a detector group 

2648 (map is one of the 'mca1', ... 'mcasum') 

2649 with specified counts array and a name 

2650 

2651 

2652 Parameters 

2653 --------- 

2654 det : detector object (one of det1, det2, ..., detsum) 

2655 counts : ndarray array of counts 

2656 name : name for MCA 

2657 

2658 Returns 

2659 ------- 

2660 MCA object 

2661 

2662 ''' 

2663 map = self.xrmmap[dgroup] 

2664 cal = map['energy'].attrs 

2665 _mca = MCA(counts=counts, offset=cal['cal_offset'], 

2666 slope=cal['cal_slope'], **kws) 

2667 if self.incident_energy is None: 

2668 self.incident_energy = self.get_incident_energy() 

2669 

2670 _mca.incident_energy = 0.001*self.incident_energy 

2671 _mca.energy = map['energy'][()] 

2672 env_names = [h5str(a) for a in self.xrmmap['config/environ/name']] 

2673 env_addrs = [h5str(a) for a in self.xrmmap['config/environ/address']] 

2674 env_vals = [h5str(a) for a in self.xrmmap['config/environ/value']] 

2675 for desc, val, addr in zip(env_names, env_vals, env_addrs): 

2676 _mca.add_environ(desc=desc, val=val, addr=addr) 

2677 

2678 if npixels is not None: 

2679 _mca.npixels=npixels 

2680 

2681 if version_ge(self.version, '2.0.0'): 

2682 for roi in self.xrmmap['roimap'][dgroup]: 

2683 emin, emax = self.xrmmap['roimap'][dgroup][roi]['limits'][:] 

2684 Eaxis = map['energy'][:] 

2685 

2686 imin = (np.abs(Eaxis-emin)).argmin() 

2687 imax = (np.abs(Eaxis-emax)).argmin() 

2688 _mca.add_roi(roi, left=imin, right=imax) 

2689 else: 

2690 # a workaround for poor practice -- some '1.3.0' files 

2691 # were built with 'roi_names', some with 'roi_name' 

2692 roiname = 'roi_name' 

2693 if roiname not in map: roiname = 'roi_names' 

2694 

2695 roinames = [h5str(a) for a in map[roiname]] 

2696 roilim0 = [lim[0] for lim in map['roi_limits']] 

2697 roilim1 = [lim[1] for lim in map['roi_limits']] 

2698 for roi, lim0, lim1 in zip(roinames, roilim0, roilim1): 

2699 _mca.add_roi(roi, left=lim0, right=lim1) 

2700 

2701 _mca.areaname = _mca.title = name 

2702 path, fname = os.path.split(self.filename) 

2703 _mca.filename = fix_filename(fname) 

2704 fmt = "Data from File '%s', detector '%s', area '%s'" 

2705 _mca.info = fmt % (self.filename, dgroup, name) 

2706 

2707 return _mca 

2708 

2709 def get_xrd1d_area(self, areaname, callback=None, **kws): 

2710 '''return 1D XRD pattern for a pre-defined area 

2711 

2712 Parameters 

2713 --------- 

2714 areaname : str name of area 

2715 

2716 Returns 

2717 ------- 

2718 diffraction pattern for given area 

2719 

2720 ''' 

2721 try: 

2722 area = self.get_area(areaname)[()] 

2723 except: 

2724 raise GSEXRM_Exception("Could not find area '%s'" % areaname) 

2725 return 

2726 npix = area.sum() 

2727 if npix < 1: 

2728 return None 

2729 

2730 stps, xpix, ypix, qdat = 0, 0, 0, None 

2731 sy, sx = [slice(min(_a), max(_a)+1) for _a in np.where(area)] 

2732 xmin, xmax, ymin, ymax = sx.start, sx.stop, sy.start, sy.stop 

2733 nx, ny = (xmax-xmin), (ymax-ymin) 

2734 

2735 xrdgroup = 'xrd1d' 

2736 mapdat = self.xrmmap[xrdgroup] 

2737 counts = self.get_counts_rect(ymin, ymax, xmin, xmax, 

2738 mapdat=mapdat, dtcorrect=False) 

2739 counts = counts[area[ymin:ymax, xmin:xmax]] 

2740 

2741 name = '%s: %s' % (xrdgroup, areaname) 

2742 kws['energy'] = energy = 0.001 * self.get_incident_energy() 

2743 kws['wavelength'] = lambda_from_E(energy, E_units='keV') 

2744 

2745 counts = counts.sum(axis=0) 

2746 xrd = XRD(data1D=counts, steps=len(counts), name=name, **kws) 

2747 if xrdgroup != 'xrd1d': 

2748 xpix, ypix = counts.shape 

2749 xrd = XRD(data2D=counts, xpixels=xpix, ypixels=ypix, name=name, **kws) 

2750 

2751 path, fname = os.path.split(self.filename) 

2752 xrd.filename = fname 

2753 xrd.areaname = xrd.title = name 

2754 xrd.mapname = mapdat.name 

2755 fmt = "Data from File '%s', detector '%s', area '%s'" 

2756 xrd.info = fmt % (self.filename, mapdat.name, name) 

2757 xrd.q = mapdat['q'][()] 

2758 return xrd 

2759 

2760 def get_xrd2d_rect(self, xmin, xmax, ymin, ymax, **kws): 

2761 tshape = self.get_shape() 

2762 

2763 if ymax < 0: ymax += tshape[0] 

2764 if xmax < 0: xmax += tshape[1] 

2765 nx, ny = (xmax-xmin, ymax-ymin) 

2766 sx = slice(xmin, xmax) 

2767 sy = slice(ymin, ymax) 

2768 xrdgroup = 'xrd2d' 

2769 xrdgrp = ensure_subgroup('xrd2d', self.xrmmap, dtype='2DXRD') 

2770 

2771 xmin, xmax, ymin, ymax = sx.start, sx.stop, sy.start, sy.stop 

2772 

2773 xrd_file = os.path.join(self.folder, self.rowdata[0][4]) 

2774 if os.path.exists(xrd_file): 

2775 print("Reading XRD Patterns for rows %d to %d" %(ymin, ymax-1)) 

2776 data = None 

2777 for yrow in range(ymin, ymax+1): 

2778 xrd_file = os.path.join(self.folder, self.rowdata[yrow][4]) 

2779 print(f"read XRD for row {yrow:d}: {xrd_file:s}") 

2780 h5file = h5py.File(xrd_file, 'r') 

2781 rowdat = h5file['entry/data/data'][1:,:,:] 

2782 h5file.close() 

2783 if (yrow % 2) == 1: 

2784 rowdat = rowdat[::-1, :, :] 

2785 rowdat = rowdat[sx,:,:].sum(axis=0) 

2786 if data is None: 

2787 data = rowdat*1.0 

2788 else: 

2789 data += rowdat 

2790 

2791 aname = f'X[{xmin:d}:{xmax-1:d}]/Y[{ymin:d}:{ymax-1:d}]' 

2792 bname = f'X{xmin:03d}Y{ymin:03d}' 

2793 name = f'{xrdgroup:s} {aname:s}' 

2794 energy = 0.001 * self.get_incident_energy() 

2795 kws = {'energy': energy, 

2796 'wavelength': lambda_from_E(energy, E_units='keV')} 

2797 

2798 xrd = XRD(data2D=data, name=name, **kws) 

2799 # print("made xrd ", xrd, kws) 

2800 xrd.filename = self.fname 

2801 xrd.areaname = xrd.title = aname 

2802 xrd.info = f"Data from File '{self.filename}', XRD2D '{aname}'" 

2803 xrd.ponifile = self.xrdcalfile 

2804 return xrd 

2805 

2806 def get_xrd2d_area(self, areaname, callback=None, **kws): 

2807 '''return 2D XRD pattern for a pre-defined area 

2808 

2809 Parameters 

2810 --------- 

2811 areaname : str name of area 

2812 

2813 Returns 

2814 ------- 

2815 diffraction pattern for given area 

2816 

2817 Notes 

2818 ------ 

2819 slow because it really reads from the raw XRD h5 files 

2820 ''' 

2821 try: 

2822 area = self.get_area(areaname)[()] 

2823 except: 

2824 raise GSEXRM_Exception("Could not find area '%s'" % areaname) 

2825 return 

2826 npix = area.sum() 

2827 if npix < 1: 

2828 return None 

2829 

2830 xrdgroup = 'xrd2d' 

2831 xrdgrp = ensure_subgroup('xrd2d', self.xrmmap, dtype='2DXRD') 

2832 

2833 stps, xpix, ypix, qdat = 0, 0, 0, None 

2834 sy, sx = [slice(min(_a), max(_a)+1) for _a in np.where(area)] 

2835 xmin, xmax, ymin, ymax = sx.start, sx.stop, sy.start, sy.stop 

2836 nx, ny = (xmax-xmin), (ymax-ymin) 

2837 xrd_file = os.path.join(self.folder, self.rowdata[0][4]) 

2838 if os.path.exists(xrd_file): 

2839 print("Reading XRD Patterns for rows %d to %d" %(ymin+1, ymax)) 

2840 data = None 

2841 for yrow in range(ymin, ymax+1): 

2842 xrd_file = os.path.join(self.folder, self.rowdata[yrow][4]) 

2843 print(f"read XRD for row {yrow:d}: {xrd_file:s}") 

2844 h5file = h5py.File(xrd_file, 'r') 

2845 rowdat = h5file['entry/data/data'][1:,:,:] 

2846 h5file.close() 

2847 if (yrow % 2) == 1: 

2848 rowdat = rowdat[::-1, :, :] 

2849 rowdat = rowdat[np.where(area[yrow])[0], :, :].sum(axis=0) 

2850 if data is None: 

2851 data = rowdat * 1.0 

2852 else: 

2853 data += rowdat * 1.0 

2854 

2855 name = '%s: %s' % (xrdgroup, areaname) 

2856 kws = {} 

2857 kws['energy'] = energy = 0.001 * self.get_incident_energy() 

2858 kws['wavelength'] = lambda_from_E(energy, E_units='keV') 

2859 # print("MAKE XRD ", data.shape, data.dtype, data.min(), 

2860 # data.max(), data.mean()) 

2861 xrd = XRD(data2D=data, name=name, **kws) 

2862 # print("made xrd ", xrd, kws) 

2863 path, fname = os.path.split(self.filename) 

2864 xrd.filename = fname 

2865 xrd.areaname = xrd.title = areaname 

2866 fmt = "Data from File '%s', XRD 2d, area '%s'" 

2867 xrd.info = fmt % (self.filename, areaname) 

2868 xrd.ponifile = self.xrdcalfile 

2869 return xrd 

2870 

2871 

2872 def get_pos(self, name, mean=True): 

2873 '''return position by name (matching 'roimap/pos_name' if 

2874 name is a string, or using name as an index if it is an integer 

2875 

2876 Parameters 

2877 --------- 

2878 name : str ROI name 

2879 mean : optional, bool [True] return mean x-value 

2880 

2881 with mean=True, and a positioner in the first two position, 

2882 returns a 1-d array of mean x-values 

2883 

2884 with mean=False, and a positioner in the first two position, 

2885 returns a 2-d array of x values for each pixel 

2886 ''' 

2887 index = -1 

2888 if isinstance(name, int): 

2889 index = name 

2890 else: 

2891 for ix, nam in enumerate(self.xrmmap['positions/name']): 

2892 if bytes2str(nam.lower()) == name.lower(): 

2893 index = ix 

2894 break 

2895 if index == -1: 

2896 raise GSEXRM_Exception("Could not find position %s" % repr(name)) 

2897 pos = self.xrmmap['positions/pos'][:, :, index] 

2898 if index in (0, 1) and mean: 

2899 pos = pos.sum(axis=index)/pos.shape[index] 

2900 return pos 

2901 

2902 

2903 def build_xrd_roimap(self, xrd='1d'): 

2904 detname = None 

2905 xrdtype = 'xrd%s detector' % xrd 

2906 

2907 roigroup = ensure_subgroup('roimap', self.xrmmap, dtype='roi maps') 

2908 for det, grp in self.xrmmap.items(): 

2909 if bytes2str(grp.attrs.get('type', '')).startswith(xrdtype): 

2910 detname = det 

2911 ds = ensure_subgroup(det, roigroup) 

2912 ds.attrs['type'] = xrdtype 

2913 return roigroup, detname 

2914 

2915 def add_xrd2droi(self, xyrange, roiname, unit='pixels'): 

2916 

2917 if version_ge(self.version, '2.0.0'): 

2918 if not self.has_xrd2d: 

2919 return 

2920 

2921 roigroup, detname = self.build_xrd_roimap(xrd='2d') 

2922 xrmdet = self.xrmmap[detname] 

2923 

2924 if roiname in roigroup[detname]: 

2925 raise ValueError("Name '%s' exists in 'roimap/%s' arrays." % (roiname,detname)) 

2926 

2927 xyrange = [int(x) for x in xyrange] 

2928 xmin,xmax,ymin,ymax = xyrange 

2929 

2930 xrd2d_counts = xrmdet['counts'][:,:,slice(xmin,xmax),slice(ymin,ymax)] 

2931 xrd2d_counts = xrd2d_counts.sum(axis=2).sum(axis=2) 

2932 if abs(xmax-xmin) > 0 and abs(ymax-ymin) > 0: 

2933 xrd2d_cor = xrd2d_counts/(abs(xmax-xmin)*abs(ymax-ymin)) 

2934 else: 

2935 xrd2d_cor = xrd2d_counts 

2936 

2937 self.save_roi(roiname,detname,xrd2d_counts,xrd2d_cor,xyrange,'area','pixels') 

2938 self.get_roi_list(detname, force=True) 

2939 

2940 else: 

2941 print('Only compatible with newest hdf5 mapfile version.') 

2942 

2943 

2944 def read_xrd1d_ROIFile(self,filename,verbose=False): 

2945 

2946 roidat = readROIFile(filename,xrd=True) 

2947 print('\nReading 1D-XRD ROI file: %s' % filename) 

2948 for iroi, label, xunit, xrange in roidat: 

2949 print(' Adding ROI: %s' % label) 

2950 self.add_xrd1droi(label, xrange, unit=xunit) 

2951 print('Finished.\n') 

2952 

2953 def add_xrd1droi(self, roiname, xrange, unit='q', subtract_bkg=False): 

2954 if not version_ge(self.version, '2.0.0'): 

2955 print('Only compatible with newest hdf5 mapfile version.') 

2956 return 

2957 

2958 if not self.has_xrd1d: 

2959 print('No 1D-XRD data in file') 

2960 return 

2961 

2962 if self.incident_energy is None: 

2963 self.incident_energy = self.get_incident_energy() 

2964 xrange = np.array(xrange) 

2965 if unit.startswith('2th'): ## 2th to 1/A 

2966 qrange = q_from_twth(xrange, 

2967 lambda_from_E(self.incident_energy, E_units='eV')) 

2968 elif unit == 'd': ## A to 1/A 

2969 qrange = q_from_d(xrange) 

2970 else: 

2971 qrange = xrange 

2972 

2973 roigroup, detname = self.build_xrd_roimap(xrd='1d') 

2974 

2975 if roiname in roigroup[detname]: 

2976 raise ValueError("Name '%s' exists in 'roimap/%s' arrays." % (roiname, detname)) 

2977 

2978 counts = self.xrmmap[detname]['counts'] 

2979 q = self.xrmmap[detname]['q'][:] 

2980 

2981 imin = (np.abs(q-qrange[0])).argmin() 

2982 imax = (np.abs(q-qrange[1])).argmin()+1 

2983 

2984 xrd1d_sum = xrd1d_cor = counts[:, :, imin:imax].sum(axis=2) 

2985 if subtract_bkg and imax > imin: 

2986 ibkglo = max(0, imin-3) 

2987 ibkghi = min(len(q), imax+3) 

2988 bkglo = counts[:,:,ibkglo:imin].sum(axis=2)/(imin-ibkglo) 

2989 bkghi = counts[:,:,imax::ibkghi].sum(axis=2)/(ibkghi-imax) 

2990 xrd1d_cor -= (imax-imin)* (bkglo + bkghi)/2.0 

2991 

2992 self.save_roi(roiname, detname, xrd1d_sum, xrd1d_cor, 

2993 qrange,'q', '1/A') 

2994 self.get_roi_list(detname, force=True) 

2995 

2996 

2997 def del_all_xrd1droi(self): 

2998 

2999 ''' delete all 1D-XRD ROI''' 

3000 

3001 roigrp_xrd1d = ensure_subgroup('xrd1d', self.xrmmap['roimap']) 

3002 

3003 for roiname in roigrp_xrd1d.keys(): 

3004 self.del_xrd1droi(roiname) 

3005 

3006 def del_xrd1droi(self, roiname): 

3007 

3008 ''' delete a 1D-XRD ROI''' 

3009 

3010 roigrp_xrd1d = ensure_subgroup('xrd1d', self.xrmmap['roimap']) 

3011 

3012 if roiname not in roigrp_xrd1d.keys(): 

3013 print("No ROI named '%s' found to delete" % roiname) 

3014 return 

3015 

3016 roiname = h5str(roiname) 

3017 if roiname in roigrp_xrd1d: 

3018 del roigrp_xrd1d[roiname] 

3019 self.h5root.flush() 

3020 

3021 

3022 def save_roi(self, roiname, det, raw, cor, drange, dtype, units): 

3023 ds = ensure_subgroup(roiname, self.xrmmap['roimap'][det]) 

3024 ds.create_dataset('raw', data=raw ) 

3025 ds.create_dataset('cor', data=cor ) 

3026 ds.create_dataset('limits', data=drange ) 

3027 ds['limits'].attrs['type'] = dtype 

3028 ds['limits'].attrs['units'] = units 

3029 

3030 self.h5root.flush() 

3031 

3032 def build_mca_roimap(self): 

3033 det_list = [] 

3034 sumdet = None 

3035 roigroup = ensure_subgroup('roimap', self.xrmmap, dtype='roi map') 

3036 for det, grp in zip(self.xrmmap.keys(),self.xrmmap.values()): 

3037 if bytes2str(grp.attrs.get('type', '')).startswith('mca det'): 

3038 det_list += [det] 

3039 ds = ensure_subgroup(det, roigroup) 

3040 ds.attrs['type'] = 'mca detector' 

3041 if (bytes2str(grp.attrs.get('type', '')).startswith('virtual mca') 

3042 and 'sum' in bytes2str(grp.attrs.get('desc', ''))): 

3043 sumdet = det 

3044 ds = ensure_subgroup(det,roigroup) 

3045 ds.attrs['type'] = 'virtual mca detector' 

3046 

3047 return roigroup, det_list, sumdet 

3048 

3049 def add_xrfroi(self, roiname, Erange, unit='keV'): 

3050 if not self.has_xrf: 

3051 return 

3052 

3053 if unit == 'eV': 

3054 Erange[:] = [x/1000. for x in Erange] ## eV to keV 

3055 

3056 roigroup, det_list, sumdet = self.build_mca_roimap() 

3057 if sumdet not in det_list: 

3058 det_list.append(sumdet) 

3059 

3060 for det in det_list: 

3061 if roiname in self.xrmmap['roimap'][det]: 

3062 print(f"ROI '{roiname:s}' exists for detector '{det:s}'. Delete before adding") 

3063 continue 

3064 mapdat = self.xrmmap[det] 

3065 if unit.startswith('chan'): 

3066 emin, emax = Erange 

3067 else: 

3068 en = mapdat['energy'][:] 

3069 emin = (np.abs(en-Erange[0])).argmin() 

3070 emax = (np.abs(en-Erange[1])).argmin()+1 

3071 raw = mapdat['counts'][:, :, emin:emax].sum(axis=2) 

3072 cor = raw * mapdat['dtfactor'] 

3073 self.save_roi(roiname, det, raw, cor, Erange, 'energy', unit) 

3074 self.get_roi_list('mcasum', force=True) 

3075 

3076 def del_xrfroi(self, roiname): 

3077 roigroup, det_list, sumdet = self.build_mca_roimap() 

3078 if sumdet not in det_list: 

3079 det_list.append(sumdet) 

3080 

3081 for det in det_list: 

3082 if roiname in self.xrmmap['roimap'][det]: 

3083 del self.xrmmap['roimap'][det][roiname] 

3084 self.get_roi_list('mcasum', force=True) 

3085 self.h5root.flush() 

3086 

3087 

3088 def check_roi(self, roiname, det=None, version=None): 

3089 if version is None: 

3090 version = self.version 

3091 

3092 if (type(det) is str and det.isdigit()) or type(det) is int: 

3093 det = int(det) 

3094 detname = 'mca%i' % det 

3095 else: 

3096 detname = det 

3097 

3098 if roiname is not None: roiname = roiname.lower() 

3099 # print("Check ROI ", roiname, detname, version) 

3100 

3101 if version_ge(version, '2.0.0'): 

3102 for d in self.get_detector_list(): 

3103 if detname.lower() == d.lower(): 

3104 for rname in self.xrmmap[d]: 

3105 if roiname.lower() == rname.lower(): 

3106 return rname, d 

3107 

3108 if detname is not None: 

3109 detname = detname.replace('det', 'mca') 

3110 

3111 if detname is None: 

3112 detname = 'roimap/mcasum' 

3113 elif not detname.startswith('roimap'): 

3114 detname = 'roimap/%s' % detname 

3115 

3116 try: 

3117 roi_list = [r for r in self.xrmmap[detname]] 

3118 if roiname is None: 

3119 return roi_list, detname 

3120 if roiname not in roi_list: 

3121 for roi in roi_list: 

3122 if roi.lower().startswith(roiname): 

3123 roiname = roi 

3124 except: 

3125 ## provide summed output counts if fail 

3126 detname = 'roimap/mcasum' 

3127 roiname = 'outputcounts' 

3128 

3129 else: 

3130 if detname is not None: 

3131 detname = detname.replace('mca','det') 

3132 

3133 sum_roi = [h5str(r).lower() for r in self.xrmmap['roimap/sum_name']] 

3134 det_roi = [h5str(r).lower() for r in self.xrmmap['roimap/det_name']] 

3135 

3136 if roiname not in sum_roi: 

3137 if roiname is None: 

3138 return np.arange(len(sum_roi)),'roimap/sum_' 

3139 else: 

3140 for roi in sum_roi: 

3141 if roi.startswith(roiname): 

3142 roiname = roi 

3143 

3144 if detname in ['det%d' % (i+1) for i in range(self.nmca)]: 

3145 idet = int(''.join([i for i in detname if i.isdigit()])) 

3146 detname = 'roimap/det_' 

3147 

3148 if roiname not in det_roi: 

3149 roiname = '%s (mca%i)' % (roiname,idet) 

3150 roiname = det_roi.index(roiname) 

3151 

3152 else: 

3153 detname = 'roimap/sum_' 

3154 try: 

3155 roiname = sum_roi.index(roiname) 

3156 except: 

3157 roiname = sum_roi.index('outputcounts') 

3158 

3159 return roiname, detname 

3160 

3161 

3162 def get_roimap(self, roiname, det=None, hotcols=None, zigzag=None, 

3163 dtcorrect=None, minval=None, maxval=None): 

3164 '''extract roi map for a pre-defined roi by name 

3165 Parameters 

3166 --------- 

3167 roiname : str ROI name 

3168 det : str detector name 

3169 dtcorrect : optional, bool [None] dead-time correct data 

3170 hotcols : optional, bool [None] suppress hot columns 

3171 minval: float, trim to minimum value 

3172 maxval: float, trim to maximum value 

3173 

3174 Returns 

3175 ------- 

3176 ndarray for ROI data 

3177 ''' 

3178 if hotcols is None: 

3179 hotcols = self.hotcols 

3180 if zigzag is None: 

3181 zigzag = self.zigzag 

3182 if dtcorrect is None: 

3183 dtcorrect = self.dtcorrect 

3184 

3185 nrow, ncol, npos = self.xrmmap['positions']['pos'].shape 

3186 out = np.zeros((nrow, ncol)) 

3187 

3188 det = self.get_detname(det) 

3189 dtcorrect = dtcorrect and ('mca' in det or 'det' in det) 

3190 

3191 if roiname == '1' or roiname == 1: 

3192 out = np.ones((nrow, ncol)) 

3193 if hotcols: 

3194 out = out[1:-1] 

3195 return out 

3196 

3197 roi, detaddr = self.check_roi(roiname, det) 

3198 ext = '' 

3199 if detaddr.startswith('roimap'): 

3200 ext = 'raw' 

3201 if dtcorrect: 

3202 ext = 'cor' 

3203 

3204 

3205 # print("GetROIMAP roiname=%s|roi=%s|det=%s" % (roiname, roi, det)) 

3206 # print("detaddr=%s|ext=%s|version=%s" % (detaddr, ext, self.version)) 

3207 if version_ge(self.version, '2.0.0'): 

3208 if detaddr.startswith('roimap'): 

3209 roi_ext = '%s/' + ext 

3210 else: 

3211 roi_ext = '%s_' + ext if ext == 'raw' else '%s' 

3212 roiaddr = roi_ext % roi 

3213 # print("looking for detattr, roiaddr ", detaddr, roiaddr) 

3214 try: 

3215 out = self.xrmmap[detaddr][roiaddr][:] 

3216 except (KeyError, OSError): 

3217 _roiname, _roic = roiaddr.split('/') 

3218 try: 

3219 del self.xrmmap[detaddr][_roiname] 

3220 except: 

3221 pass 

3222 rgrp = self.xrmmap[detaddr].create_group(_roiname) 

3223 for aname,dtype in (('raw', np.uint32), 

3224 ('cor', np.float32)): 

3225 rgrp.create_dataset(aname, (1, ncol), dtype, 

3226 chunks=(1, ncol), 

3227 maxshape=(None, ncol), **self.compress_args) 

3228 lmtgrp = rgrp.create_dataset('limits', data=[0., 0.], **self.compress_args) 

3229 lmtgrp.attrs['type'] = 'energy' 

3230 lmtgrp.attrs['units'] = 'keV' 

3231 

3232 out = np.zeros([1, ncol]) 

3233 # print("found roi data ", out.shape, nrow, ncol) 

3234 if version_ge(self.version, '2.1.0') and out.shape != (nrow, ncol): 

3235 _roi, _detaddr = self.check_roi(roiname, det, version='1.0.0') 

3236 detname = '%s%s' % (_detaddr, ext) 

3237 out = self.xrmmap[detname][:, :, _roi] 

3238 # print("from v1, got roi map ", _roi, detname, out.shape) 

3239 if self.write_access: 

3240 self.xrmmap[detaddr][roiaddr].resize((nrow, ncol)) 

3241 self.xrmmap[detaddr][roiaddr][:, :] = out 

3242 

3243 else: # version1 

3244 if det in EXTRA_DETGROUPS: 

3245 detname = "%s/%s" % (det, roiname) 

3246 out = self.xrmmap[detname][:,:] 

3247 else: 

3248 detname = '%s%s' % (detaddr, ext) 

3249 out = self.xrmmap[detname][:, :, roi] 

3250 

3251 if zigzag is not None and zigzag != 0: 

3252 out = remove_zigzag(out, zigzag) 

3253 elif hotcols: 

3254 out = out[:, 1:-1] 

3255 if minval is not None: 

3256 out[np.where(out<minval)] = minval 

3257 if maxval is not None: 

3258 out[np.where(out>maxval)] = maxval 

3259 return out 

3260 

3261 

3262 def get_mca_erange(self, det=None, dtcorrect=None, 

3263 emin=None, emax=None, by_energy=True): 

3264 '''extract map for an ROI set here, by energy range: 

3265 

3266 not implemented 

3267 ''' 

3268 pass 

3269 

3270 def get_rgbmap(self, rroi, groi, broi, det=None, rdet=None, gdet=None, bdet=None, 

3271 hotcols=None, dtcorrect=None, scale_each=True, scales=None): 

3272 '''return a (NxMx3) array for Red, Green, Blue from named 

3273 ROIs (using get_roimap). 

3274 

3275 Parameters 

3276 ---------- 

3277 rroi : str name of ROI for red channel 

3278 groi : str name of ROI for green channel 

3279 broi : str name of ROI for blue channel 

3280 det : optional, None or int [None] index for detector 

3281 dtcorrect : optional, bool [None] dead-time correct data 

3282 hotcols : optional, bool [None] suppress hot columns 

3283 scale_each : optional, bool [True] 

3284 scale each map separately to span the full color range. 

3285 scales : optional, None or 3 element tuple [None] 

3286 multiplicative scale for each map. 

3287 

3288 By default (scales_each=True, scales=None), each map is scaled by 

3289 1.0/map.max() -- that is 1 of the max value for that map. 

3290 

3291 If scales_each=False, each map is scaled by the same value 

3292 (1/max intensity of all maps) 

3293 

3294 ''' 

3295 if hotcols is None: 

3296 hotcols = self.hotcols 

3297 if dtcorrect is None: 

3298 dtcorrect = self.dtcorrect 

3299 if det is not None: 

3300 rdet = gdet = bdet = det 

3301 

3302 kws = dict(hotcols=hotcols, dtcorrect=dtcorrect) 

3303 rmap = self.get_roimap(rroi, det=rdet, **kws) 

3304 gmap = self.get_roimap(groi, det=gdet, **kws) 

3305 bmap = self.get_roimap(broi, det=bdet, **kws) 

3306 

3307 if scales is None or len(scales) != 3: 

3308 scales = (1./rmap.max(), 1./gmap.max(), 1./bmap.max()) 

3309 if scale_each: 

3310 rmap *= scales[0] 

3311 gmap *= scales[1] 

3312 bmap *= scales[2] 

3313 else: 

3314 scale = min(scales[0], scales[1], scales[2]) 

3315 rmap *= scale 

3316 bmap *= scale 

3317 gmap *= scale 

3318 

3319 return np.array([rmap, gmap, bmap]).swapaxes(0, 2).swapaxes(0, 1) 

3320 

3321 def add_roi(self, name, high, low, address='', det=1, 

3322 overwrite=False, **kws): 

3323 '''add named ROI to an XRMMap file. 

3324 These settings will be propogated through the 

3325 ROI maps and all detectors. 

3326 

3327 ''' 

3328 # data structures affected: 

3329 # config/rois/address 

3330 # config/rois/name 

3331 # config/rois/limits 

3332 # roimap/det_address 

3333 # roimap/det_name 

3334 # roimap/det_raw 

3335 # roimap/det_cor 

3336 # roimap/sum_list 

3337 # roimap/sum_name 

3338 # roimap/sum_raw 

3339 # roimap/sum_cor 

3340 # det{I}/roi_address for I = 1, N_detectors (xrmmap attribute) 

3341 # det{I}/roi_name for I = 1, N_detectors (xrmmap attribute) 

3342 # det{I}/roi_limits for I = 1, N_detectors (xrmmap attribute) 

3343 # detsum/roi_address for I = 1, N_detectors (xrmmap attribute) 

3344 # detsum/roi_name for I = 1, N_detectors (xrmmap attribute) 

3345 # detsum/roi_limits for I = 1, N_detectors (xrmmap attribute) 

3346 

3347 '' 

3348 roi_names = [i.lower().strip() for i in self.xrmmap['config/rois/name']] 

3349 if name.lower().strip() in roi_names: 

3350 if overwrite: 

3351 self.del_roi(name) 

3352 else: 

3353 print("An ROI named '%s' exists, use overwrite=True to overwrite" % name) 

3354 return 

3355 

3356 def del_roi(self, name): 

3357 ''' delete an ROI''' 

3358 roi_names = [i.lower().strip() for i in self.xrmmap['config/rois/name']] 

3359 if name.lower().strip() not in roi_names: 

3360 print("No ROI named '%s' found to delete" % name) 

3361 return 

3362 iroi = roi_names.index(name.lower().strip()) 

3363 roi_names = [i in self.xrmmap['config/rois/name']] 

3364 roi_names.pop(iroi) 

3365 

3366 

3367def read_xrmmap(filename, root=None, **kws): 

3368 '''read GSE XRF FastMap data from HDF5 file or raw map folder''' 

3369 key = 'filename' 

3370 if os.path.isdir(filename): 

3371 key = 'folder' 

3372 kws.update({key: filename, 'root': root}) 

3373 

3374 return GSEXRM_MapFile(**kws) 

3375 

3376def process_mapfolder(path, take_ownership=False, **kws): 

3377 """process a single map folder 

3378 with optional keywords passed to GSEXRM_MapFile 

3379 """ 

3380 try: 

3381 kws['xrdcal'] = kws.pop('poni') 

3382 except: 

3383 pass 

3384 if os.path.isdir(path) and isGSEXRM_MapFolder(path): 

3385 print( '\n build map for: %s' % path) 

3386 try: 

3387 g = GSEXRM_MapFile(folder=path, **kws) 

3388 except: 

3389 print( 'Could not create MapFile') 

3390 print( sys.exc_info() ) 

3391 return 

3392 try: 

3393 if take_ownership: 

3394 g.take_ownership() 

3395 if g.check_ownership(): 

3396 g.process() 

3397 else: 

3398 print( 'Skipping file %s: not owner' % path) 

3399 except KeyboardInterrupt: 

3400 sys.exit() 

3401 except: 

3402 print( 'Could not convert %s' % path) 

3403 print( sys.exc_info() ) 

3404 return 

3405 finally: 

3406 g.close() 

3407 

3408def process_mapfolders(folders, ncpus=None, take_ownership=False, **kws): 

3409 """process a list of map folders 

3410 with optional keywords passed to GSEXRM_MapFile 

3411 """ 

3412 try: 

3413 kws['xrdcal'] = kws.pop('poni') 

3414 except: 

3415 pass 

3416 if ncpus is None: 

3417 ncpus = max(1, mp.cpu_count()-1) 

3418 if ncpus == 0: 

3419 for path in folders: 

3420 process_mapfolder(path, **kws) 

3421 else: 

3422 pool = mp.Pool(ncpus) 

3423 kws['take_ownership'] = take_ownership 

3424 myfunc = partial(process_mapfolder, **kws) 

3425 pool.map(myfunc, folders)