Coverage for csv_manager/EQ_CSVManager.py: 65%

180 statements  

« prev     ^ index     » next       coverage.py v7.0.4, created at 2023-01-10 09:27 -0600

1""" 

2Copyright 1999 Illinois Institute of Technology 

3 

4Permission is hereby granted, free of charge, to any person obtaining 

5a copy of this software and associated documentation files (the 

6"Software"), to deal in the Software without restriction, including 

7without limitation the rights to use, copy, modify, merge, publish, 

8distribute, sublicense, and/or sell copies of the Software, and to 

9permit persons to whom the Software is furnished to do so, subject to 

10the following conditions: 

11 

12The above copyright notice and this permission notice shall be 

13included in all copies or substantial portions of the Software. 

14 

15THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 

16EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 

17MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 

18IN NO EVENT SHALL ILLINOIS INSTITUTE OF TECHNOLOGY BE LIABLE FOR ANY 

19CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 

20TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 

21SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 

22 

23Except as contained in this notice, the name of Illinois Institute 

24of Technology shall not be used in advertising or otherwise to promote 

25the sale, use or other dealings in this Software without prior written 

26authorization from Illinois Institute of Technology. 

27""" 

28 

29from os.path import exists 

30from os import makedirs 

31import pandas as pd 

32try: 

33 from ..utils.file_manager import fullPath 

34except: # for coverage 

35 from utils.file_manager import fullPath 

36 

37class EQ_CSVManager: 

38 """ 

39 A class taking care of writing results including csv file and failedcases file. 

40 It creates 2 files, summary and summary2, with a different way to display the results for each. 

41 """ 

42 def __init__(self, dir_path): 

43 """ 

44 init with directory path 

45 :param dir_path: 

46 """ 

47 result_path = fullPath(dir_path, "eq_results") 

48 self.dataframe = None 

49 self.dataframe2 = None 

50 if not exists(result_path): 

51 makedirs(result_path) 

52 self.filename = fullPath(result_path, 'summary.csv') 

53 self.colnames = ["Filename","Side","Distance From Center","Area","Sigma D","Sigma S","Sigma C", 

54 "gamma","Z line","Sigma Z","Iz","gamma Z", "Extra peak","Sigma Z EP","Iz EP","gamma Z EP", 

55 "Model", "CenterX", "S10", "d10", 

56 "I11/I10", "Average I11/I10 per fiber", "Fitting error","comment"] 

57 self.filename2 = fullPath(result_path, 'summary2.csv') 

58 self.colnames2 = ['Filename', 'left peak 0', 'right peak 0', 'left peak 1', 'right peak 1', 

59 'left Sigma D', 'right Sigma D', 'left Sigma S', 'right Sigma S', 'left Sigma C', 

60 'right Sigma C', 'left gamma', 'right gamma', 'left Z line', 'right Z line', 

61 'left Sigma Z', 'right Sigma Z', 'left Iz', 'right Iz', 'left gamma Z', 

62 'right gamma Z', 'left Extra peak', 'right Extra peak', 

63 'left Sigma Z EP', 'right Sigma Z EP', 'left Iz EP', 'right Iz EP', 'left gamma Z EP', 

64 'right gamma Z EP', 'Model', 'CenterX', 'S10', 'd10', 'left I11/I10', 'right I11/I10', 

65 'Average I11/I10 per fiber', 'Fitting error', 'comment'] 

66 self.loadFailedCases(dir_path) 

67 self.loadSummary() 

68 self.loadSummary2() 

69 

70 def loadFailedCases(self, direc): 

71 """ 

72 Load failed cases file from the directory and keep them in self.failedcases 

73 :param direc: input directory (str) 

74 :return: - 

75 """ 

76 self.failedcasesfile = fullPath(direc, "failedcases.txt") 

77 self.failedcases = set() 

78 if exists(self.failedcasesfile): 

79 for line in open(self.failedcasesfile, 'r'): 

80 name = line.rstrip('\n') 

81 self.failedcases.add(name) 

82 

83 def loadSummary(self): 

84 """ 

85 Load summary.csv file and keep data in self.dataframe 

86 :return: 

87 """ 

88 if not exists(self.filename): 

89 self.dataframe = pd.DataFrame(columns = self.colnames) 

90 else: 

91 self.dataframe = pd.read_csv(self.filename) 

92 

93 def loadSummary2(self): 

94 """ 

95 Load summary.csv file and keep data in self.dataframe2 

96 :return: 

97 """ 

98 if not exists(self.filename2): 

99 self.dataframe2 = pd.DataFrame(columns = self.colnames2) 

100 else: 

101 self.dataframe2 = pd.read_csv(self.filename2) 

102 

103 def writeNewData(self, bioImg): 

104 """ 

105 Add new data to dataframe, then re-write summary.csv and failed cases file 

106 :param bioImg: EquatorImage object with results in its info dict 

107 :return: - 

108 """ 

109 file_name = bioImg.filename 

110 info = bioImg.info 

111 self.removeData(file_name) 

112 new_datas = [] 

113 

114 # If image is rejected 

115 if "reject" in info and info["reject"]: 

116 data = {} 

117 for k in self.dataframe.columns: 

118 data[k] = '-' 

119 data['Filename'] = file_name 

120 data['comment'] = "REJECTED" 

121 new_datas = [data] 

122 else: 

123 failed = False 

124 # Get all needed infos 

125 if 'peaks' not in info: 

126 data = {} 

127 for k in self.dataframe.columns: 

128 data[k] = '-' 

129 data['Filename'] = file_name 

130 data['comment'] = "No effective peaks detected" 

131 new_datas = [data] 

132 failed = True 

133 else: 

134 if 'fit_results' in info: 

135 fit_results = info['fit_results'] 

136 all_S = fit_results['all_S'] 

137 for side in ['left', 'right']: 

138 areas = fit_results[side+'_areas'] 

139 

140 for i in range(len(areas)): 

141 data = { 

142 "Filename": file_name, 

143 "Side": side, 

144 "Distance From Center": all_S[i], 

145 "Area" : areas[i] 

146 } 

147 if i == 0: 

148 first_row = data 

149 new_datas.append(data) 

150 

151 # Write data to the first row of each side 

152 first_row.update({ 

153 "Sigma C": abs(fit_results[side+'_sigmac']), 

154 "Sigma D": abs(fit_results[side+'_sigmad']), 

155 "Sigma S": abs(fit_results[side+'_sigmas']), 

156 "I11/I10": fit_results[side+'_ratio'], 

157 }) 

158 if fit_results["model"] == 'Voigt': 

159 first_row["gamma"] = fit_results[side+'_gamma'] 

160 if fit_results["isSkeletal"]: 

161 first_row["Z line"] = fit_results[side+'_zline'] 

162 first_row["Sigma Z"] = fit_results[side+'_sigmaz'] 

163 first_row["Iz"] = fit_results[side+'_intz'] 

164 if fit_results["model"] == 'Voigt': 

165 first_row["gamma Z"] = fit_results[side+'_gammaz'] 

166 if fit_results["isExtraPeak"]: 

167 first_row["Extra peak"] = fit_results[side+'_zline_EP'] 

168 first_row["Sigma Z EP"] = fit_results[side+'_sigmaz_EP'] 

169 first_row["Iz EP"] = fit_results[side+'_intz_EP'] 

170 if fit_results["model"] == 'Voigt': 

171 first_row["gamma Z EP"] = fit_results[side+'_gammaz_EP'] 

172 

173 

174 # Write general information to the first row of image 

175 first_row = new_datas[0] 

176 

177 if fit_results["fiterror"] > 0.2: 

178 first_row['comment'] = 'High Fitting Error' 

179 failed = True 

180 

181 first_row['S10'] = fit_results['S10'] 

182 first_row['Model'] = fit_results['model'] 

183 first_row['Average I11/I10 per fiber'] = fit_results['avg_ratio'] 

184 first_row['Model'] = fit_results['model'] 

185 first_row['Fitting error'] = fit_results['fiterror'] 

186 first_row['CenterX'] = round(fit_results['centerX'], 2) 

187 

188 if 'd10' in fit_results.keys(): 

189 first_row['d10'] = fit_results['d10'] 

190 else: 

191 first_row['d10'] = '-' 

192 else: 

193 data = {} 

194 for k in self.dataframe.columns: 

195 data[k] = '-' 

196 data['Filename'] = file_name 

197 data['comment'] = "Model cannot be fit" 

198 new_datas = [data] 

199 failed = True 

200 

201 if failed: 

202 self.failedcases.add(file_name) 

203 elif file_name in self.failedcases: 

204 self.failedcases.remove(file_name) 

205 

206 self.dataframe = pd.concat([self.dataframe, pd.DataFrame.from_records(new_datas)]) 

207 # self.dataframe = self.dataframe.append(new_datas, ignore_index=True) # Future warning deprecated 

208 self.dataframe.reset_index() 

209 self.dataframe.to_csv(self.filename, index=False, columns=self.colnames) # Write to csv file 

210 

211 # Write all failed cases to failed cases file 

212 f = open(self.failedcasesfile, 'w') 

213 f.write("\n".join(list(self.failedcases))) 

214 f.close() 

215 

216 def writeNewData2(self, bioImg): 

217 """ 

218 Add new data to dataframe2, then re-write summary.csv and failed cases file 

219 :param bioImg: EquatorImage object with results in its info dict 

220 :return: - 

221 """ 

222 file_name = bioImg.filename 

223 info = bioImg.info 

224 self.removeData2(file_name) 

225 data = {} 

226 

227 # If image is rejected 

228 if "reject" in info and info["reject"]: 

229 for k in self.dataframe2.columns: 

230 data[k] = '-' 

231 data['Filename'] = file_name 

232 data['comment'] = "REJECTED" 

233 else: 

234 failed = False 

235 # Get all needed infos 

236 if 'peaks' not in info: 

237 for k in self.dataframe2.columns: 

238 data[k] = '-' 

239 data['Filename'] = file_name 

240 data['comment'] = "No effective peaks detected" 

241 failed = True 

242 else: 

243 if 'fit_results' in info: 

244 fit_results = info['fit_results'] 

245 all_S = fit_results['all_S'] 

246 data['Filename'] = file_name 

247 for side in ['left', 'right']: 

248 areas = fit_results[side+'_areas'] 

249 for i in range(len(areas)): 

250 data[f'{side} peak {i}'] = all_S[i] 

251 

252 data.update({ 

253 side+" Sigma C": abs(fit_results[side+'_sigmac']), 

254 side+" Sigma D": abs(fit_results[side+'_sigmad']), 

255 side+" Sigma S": abs(fit_results[side+'_sigmas']), 

256 side+" I11/I10": fit_results[side+'_ratio'], 

257 }) 

258 if fit_results["model"] == 'Voigt': 

259 data[side+" gamma"] = fit_results[side+'_gamma'] 

260 if fit_results["isSkeletal"]: 

261 data[side+" Z line"] = fit_results[side+'_zline'] 

262 data[side+" Sigma Z"] = fit_results[side+'_sigmaz'] 

263 data[side+" Iz"] = fit_results[side+'_intz'] 

264 if fit_results["model"] == 'Voigt': 

265 data[side+" gamma Z"] = fit_results[side+'_gammaz'] 

266 if fit_results["isExtraPeak"]: 

267 data[side+" Extra peak"] = fit_results[side+'_zline_EP'] 

268 data[side+" Sigma Z EP"] = fit_results[side+'_sigmaz_EP'] 

269 data[side+" Iz EP"] = fit_results[side+'_intz_EP'] 

270 if fit_results["model"] == 'Voigt': 

271 data[side+" gamma Z EP"] = fit_results[side+'_gammaz_EP'] 

272 

273 if fit_results["fiterror"] > 0.2: 

274 data['comment'] = 'High Fitting Error' 

275 failed = True 

276 

277 data['S10'] = fit_results['S10'] 

278 data['Model'] = fit_results['model'] 

279 data['Average I11/I10 per fiber'] = fit_results['avg_ratio'] 

280 data['Model'] = fit_results['model'] 

281 data['Fitting error'] = fit_results['fiterror'] 

282 data['CenterX'] = round(fit_results['centerX'], 2) 

283 

284 if 'd10' in fit_results.keys(): 

285 data['d10'] = fit_results['d10'] 

286 else: 

287 data['d10'] = '-' 

288 else: 

289 for k in self.dataframe2.columns: 

290 data[k] = '-' 

291 data['Filename'] = file_name 

292 data['comment'] = "Model cannot be fit" 

293 failed = True 

294 

295 if failed: 

296 self.failedcases.add(file_name) 

297 elif file_name in self.failedcases: 

298 self.failedcases.remove(file_name) 

299 

300 self.dataframe2 = pd.concat([self.dataframe2, pd.DataFrame.from_records([data])]) 

301 # self.dataframe2 = self.dataframe2.append(data, ignore_index=True) # Future warning deprecated 

302 self.dataframe2.reset_index() 

303 self.dataframe2.to_csv(self.filename2, index=False, columns=self.colnames2) # Write to csv file 

304 

305 def removeData(self, file_name): 

306 """ 

307 Remove data from dataframe 

308 :param file_name: (str) 

309 :return: 

310 """ 

311 self.dataframe = self.dataframe[self.dataframe["Filename"] != file_name] 

312 

313 def removeData2(self, file_name): 

314 """ 

315 Remove data from dataframe2 

316 :param file_name: (str) 

317 :return: 

318 """ 

319 self.dataframe2 = self.dataframe2[self.dataframe2["Filename"] != file_name]