Coverage for csv_manager/DC_CSVManager.py: 7%

201 statements  

« prev     ^ index     » next       coverage.py v7.0.4, created at 2023-01-10 09:27 -0600

1""" 

2Copyright 1999 Illinois Institute of Technology 

3 

4Permission is hereby granted, free of charge, to any person obtaining 

5a copy of this software and associated documentation files (the 

6"Software"), to deal in the Software without restriction, including 

7without limitation the rights to use, copy, modify, merge, publish, 

8distribute, sublicense, and/or sell copies of the Software, and to 

9permit persons to whom the Software is furnished to do so, subject to 

10the following conditions: 

11 

12The above copyright notice and this permission notice shall be 

13included in all copies or substantial portions of the Software. 

14 

15THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 

16EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 

17MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 

18IN NO EVENT SHALL ILLINOIS INSTITUTE OF TECHNOLOGY BE LIABLE FOR ANY 

19CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 

20TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 

21SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 

22 

23Except as contained in this notice, the name of Illinois Institute 

24of Technology shall not be used in advertising or otherwise to promote 

25the sale, use or other dealings in this Software without prior written 

26authorization from Illinois Institute of Technology. 

27""" 

28 

29from os import makedirs 

30import csv 

31try: 

32 from ..utils.file_manager import * 

33except: # for coverage 

34 from utils.file_manager import * 

35 

36class DC_CSVManager(): 

37 """ 

38 A class taking care of writing summary file 

39 """ 

40 def __init__(self, dir_path, nFrames, fix_ranges=[]): 

41 self.nFrames = nFrames # number of frames that average 

42 

43 sum_path = fullPath(dir_path, "dc_results") 

44 

45 if not exists(sum_path): 

46 makedirs(sum_path) 

47 

48 if self.nFrames > 0: 

49 # if number of frames that average is specified, summary fill will be summary_nf.csv while n is number of frames 

50 self.filename = fullPath(sum_path, "summary_"+str(nFrames)+"f") 

51 else: 

52 # otherwise, summary file will be summary_m.csv 

53 self.filename = fullPath(sum_path, "summary_m") 

54 

55 fix_range_names = [fr[0] for fr in fix_ranges] 

56 if len(fix_range_names) > 0: 

57 for n in fix_range_names: 

58 self.filename += ("_" + str(n)) 

59 self.filename += ".csv" 

60 

61 self.allpeaks_data = {} # Collect meridian peaks data 

62 self.offmer_data = {} # Collect off-meridian peaks data 

63 self.grp_to_key = {} # Map group number with key 

64 self.gen_data = {} # general data of Diffraction Centroid. Now, there's only integrated width 

65 self.headers = [] # header of csv file 

66 self.keyToImages = {} # Map key to image names 

67 self.loadSummary() # load exist summary file before adding new data 

68 

69 def getRow(self, key): 

70 """ 

71 Return index of key in header. 

72 """ 

73 return self.headers.index(key) 

74 

75 def loadSummary(self): 

76 """ 

77 Load exist summary file, and encapsulate them 

78 """ 

79 self.allpeaks_data = {} 

80 self.offmer_data = {} 

81 self.gen_data = {} 

82 self.keyToImages = {} 

83 self.grp_to_key = {} 

84 currentGroup = -1 

85 header = [] 

86 # Files are selected by auto-grouping 

87 if exists(self.filename): 

88 with open(self.filename, "r") as csvfile: 

89 reader = csv.reader(csvfile) 

90 peak_data = [] 

91 images = [] 

92 g_data = {} 

93 off_mer = {} 

94 

95 for row in reader: 

96 if len(row) == 0: 

97 continue 

98 

99 if row[0] == "group": 

100 header = row[:] 

101 continue 

102 

103 if len(row[0]) > 0: 

104 if len(images) > 0: 

105 k = "...".join([images[0], images[-1]]) 

106 self.allpeaks_data[k] = peak_data 

107 self.offmer_data[k] = off_mer 

108 self.gen_data[k] = g_data 

109 self.keyToImages[k] = images 

110 self.grp_to_key[int(currentGroup)] = k 

111 peak_data = [] 

112 images = [] 

113 g_data = {} 

114 off_mer = {} 

115 

116 if row[0] != "": 

117 currentGroup = row[0] 

118 images.append(row[1]) 

119 g_data["integration area width"] = row[2] 

120 for i in range(3, len(row) - 3): 

121 if "centroid" in header[i]: 

122 if "average" in header[i]: 

123 continue 

124 head = header[i] 

125 head = head.split(" ") 

126 if "_" in head[0] and head[1] == "59": 

127 quadrant = head[0] 

128 off_mer[quadrant] = { 

129 "centroid": [row[header.index(quadrant + " 59 " +"centroid")], 

130 row[header.index(quadrant + " 51 " +"centroid")]], 

131 "baseline": [row[header.index(quadrant + " 59 " + "baseline")], 

132 row[header.index(quadrant + " 51 " + "baseline")]], 

133 "intensity": [row[header.index(quadrant + " 59 " + "intensity")], 

134 row[header.index(quadrant + " 51 " + "intensity")]], 

135 } 

136 else: 

137 side = head[0] 

138 name = head[1] 

139 new_peak = { 

140 "side": side, 

141 "name": name, 

142 "centroid": row[i], 

143 "baseline": row[header.index(side + " " + name + " " + "baseline")], 

144 "intensity": row[header.index(side + " " + name + " " + "intensity")] 

145 } 

146 peak_data.append(new_peak) 

147 else: 

148 images.append(row[1]) 

149 

150 if len(images) > 0: 

151 k = "...".join([images[0], images[-1]]) 

152 self.allpeaks_data[k] = peak_data 

153 self.offmer_data[k] = off_mer 

154 self.gen_data[k] = g_data 

155 self.keyToImages[k] = images 

156 self.grp_to_key[int(currentGroup)] = k 

157 

158 def writeNewData(self, info): 

159 """ 

160 Add new data (DiffractionCentroids info) to encapsulated member, and rewrite summary 

161 :param info: DiffractionCentroids info (dict) 

162 """ 

163 fileList = info["filelist"] 

164 k = "...".join([fileList[0], fileList[-1]]) 

165 if "summary_m" in self.filename: 

166 grpnum = len(self.grp_to_key.keys()) + 1 

167 for (group, key) in self.grp_to_key.items(): 

168 if key == k: 

169 grpnum = group 

170 self.grp_to_key[grpnum] = k 

171 else: 

172 self.grp_to_key[info["grp_num"]] = k 

173 self.keyToImages[k] = fileList 

174 

175 if k in self.allpeaks_data: 

176 del self.allpeaks_data[k] 

177 

178 if k in self.offmer_data: 

179 del self.offmer_data[k] 

180 

181 data = [] 

182 for side in ["top", "bottom"]: 

183 centroids = info[side + "_centroids"] 

184 peaks = info[side + "_peaks"] 

185 baselines = info[side + "_baselines"] 

186 areas = info[side + "_areas"] 

187 names = info[side + "_names"] 

188 reject_state = info["reject"][side] 

189 for i in range(len(peaks)): 

190 peak_dict = {} 

191 peak_dict["side"] = side 

192 peak_dict["name"] = names[i] 

193 if names[i] in reject_state: 

194 peak_dict["centroid"] = "_"+str(centroids[i]) 

195 peak_dict["peak"] = "_"+str(peaks[i]) 

196 peak_dict["intensity"] = "_"+str(areas[i]) 

197 peak_dict["baseline"] = "_"+str(baselines[i]) 

198 else: 

199 peak_dict["centroid"] = str(centroids[i]) 

200 peak_dict["peak"] = str(peaks[i]) 

201 peak_dict["intensity"] = str(areas[i]) 

202 peak_dict["baseline"] = str(baselines[i]) 

203 

204 data.append(peak_dict) 

205 

206 off_mer_data = {} 

207 if "off_mer_baselines" in info: 

208 for q in ["top_left", "top_right", "bottom_left", "bottom_right"]: 

209 dat = { 

210 "baseline" : info["off_mer_baselines"][q], 

211 "centroid" : info["off_mer_peak_info"][q]["centroids"], 

212 "intensity" : info["off_mer_peak_info"][q]["areas"] 

213 } 

214 off_mer_data[q] = dat 

215 

216 self.allpeaks_data[k] = data 

217 self.offmer_data[k] = off_mer_data 

218 l,r = info["int_area"] 

219 self.gen_data[k] = {"integration area width":abs(r-l)} 

220 self.rewriteSummary() 

221 

222 def getOffMerHeaders(self): 

223 """ 

224 Get Off-meridian header for csv file 

225 """ 

226 header = [] 

227 for t in ["centroid", "baseline", "intensity"]: 

228 for p in ["59", "51"]: 

229 for q in ["top_left", "top_right", "bottom_left", "bottom_right"]: 

230 header.append(q + " " + p + " "+t) 

231 header.append(p + " average "+t) 

232 return header 

233 

234 def toInt(self, s): 

235 """ 

236 Convert to int. 

237 """ 

238 try: 

239 result = int(s) 

240 except Exception: 

241 result = s 

242 return result 

243 

244 def rewriteSummary(self): 

245 """ 

246 Re-write summary file from encapsulated data 

247 :return: 

248 """ 

249 # Files are selected by auto-grouping 

250 with open(self.filename, "w") as csvfile: 

251 writer = csv.writer(csvfile, delimiter=",") 

252 grpKey = sorted(self.grp_to_key.items(), key=lambda gk: int(gk[0])) 

253 _, k1 = grpKey[0] 

254 peak_data = self.allpeaks_data[k1] 

255 

256 ### write header for all group ### 

257 header = ["group", "filename", "integration area width"] 

258 top_peaks = sorted([d for d in peak_data if d["side"] == "top"], key=lambda d: self.toInt(d["name"])) 

259 bottom_peaks = sorted([d for d in peak_data if d["side"] == "bottom"], key=lambda d: self.toInt(d["name"])) 

260 nTop = len(top_peaks) 

261 nBottom = len(bottom_peaks) 

262 min_nPeaks = min(nTop, nBottom) 

263 

264 for i in range(min_nPeaks): 

265 peak_header = ["top " + top_peaks[i]["name"] + " centroid", 

266 "bottom " + bottom_peaks[i]["name"] + " centroid", 

267 "average " + bottom_peaks[i]["name"] + " centroid", 

268 "top " + top_peaks[i]["name"] + " baseline", 

269 "bottom " + bottom_peaks[i]["name"] + " baseline", 

270 "average " + bottom_peaks[i]["name"] + " baseline", 

271 "top " + top_peaks[i]["name"] + " intensity", 

272 "bottom " + bottom_peaks[i]["name"] + " intensity", 

273 "average " + bottom_peaks[i]["name"] + " intensity"] 

274 header.extend(peak_header) 

275 header.extend(self.getOffMerHeaders()) 

276 

277 writer.writerow(header) 

278 self.headers = header 

279 

280 ### Write data for each group ### 

281 for (g, k) in grpKey: 

282 off_mer = self.offmer_data[k] 

283 peak_data = self.allpeaks_data[k] 

284 g_data = self.gen_data[k] 

285 filelist = self.keyToImages[k] 

286 top_peaks = sorted([d for d in peak_data if d["side"] == "top"], key=lambda d: d["name"]) 

287 bottom_peaks = sorted([d for d in peak_data if d["side"] == "bottom"], key=lambda d: d["name"]) 

288 

289 for i in range(len(filelist)): 

290 if i != 0: 

291 write_row = ["", str(filelist[i])] 

292 else: 

293 write_row = [str(g), str(filelist[i]), g_data["integration area width"]] 

294 for j in range(min_nPeaks): 

295 peak_detail = [top_peaks[j]["centroid"], bottom_peaks[j]["centroid"], 

296 self.average(top_peaks[j]["centroid"], bottom_peaks[j]["centroid"]), 

297 top_peaks[j]["baseline"], bottom_peaks[j]["baseline"], 

298 self.average(top_peaks[j]["baseline"], bottom_peaks[j]["baseline"]), 

299 top_peaks[j]["intensity"], bottom_peaks[j]["intensity"], 

300 self.average(top_peaks[j]["intensity"], bottom_peaks[j]["intensity"])] 

301 write_row.extend(peak_detail) 

302 

303 if len(off_mer.keys()) > 0: 

304 for h in ["centroid", "baseline", "intensity"]: 

305 for p in [0,1]: 

306 sum_val = 0. 

307 for q in ["top_left", "top_right", "bottom_left", "bottom_right"]: 

308 write_row.append(off_mer[q][h][p]) 

309 sum_val += float(off_mer[q][h][p]) 

310 write_row.append(sum_val/4.) 

311 

312 writer.writerow(write_row) 

313 

314 def average(self, s1, s2): 

315 """ 

316 Find average between 2 float string. Remove '_' if necessary. 

317 """ 

318 rejected = False 

319 if "_" in s1: 

320 rejected = True 

321 s1 = s1.lstrip("_") 

322 if "_" in s2: 

323 rejected = True 

324 s2 = s2.lstrip("_") 

325 

326 result = str(1.*(float(s1)+float(s2))/2.) 

327 if rejected : 

328 result = "_" + result 

329 return result