Coverage for csv_manager/EQ_CSVManager.py: 65%
180 statements
« prev ^ index » next coverage.py v7.0.4, created at 2023-01-10 09:27 -0600
« prev ^ index » next coverage.py v7.0.4, created at 2023-01-10 09:27 -0600
1"""
2Copyright 1999 Illinois Institute of Technology
4Permission is hereby granted, free of charge, to any person obtaining
5a copy of this software and associated documentation files (the
6"Software"), to deal in the Software without restriction, including
7without limitation the rights to use, copy, modify, merge, publish,
8distribute, sublicense, and/or sell copies of the Software, and to
9permit persons to whom the Software is furnished to do so, subject to
10the following conditions:
12The above copyright notice and this permission notice shall be
13included in all copies or substantial portions of the Software.
15THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18IN NO EVENT SHALL ILLINOIS INSTITUTE OF TECHNOLOGY BE LIABLE FOR ANY
19CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23Except as contained in this notice, the name of Illinois Institute
24of Technology shall not be used in advertising or otherwise to promote
25the sale, use or other dealings in this Software without prior written
26authorization from Illinois Institute of Technology.
27"""
29from os.path import exists
30from os import makedirs
31import pandas as pd
32try:
33 from ..utils.file_manager import fullPath
34except: # for coverage
35 from utils.file_manager import fullPath
37class EQ_CSVManager:
38 """
39 A class taking care of writing results including csv file and failedcases file.
40 It creates 2 files, summary and summary2, with a different way to display the results for each.
41 """
42 def __init__(self, dir_path):
43 """
44 init with directory path
45 :param dir_path:
46 """
47 result_path = fullPath(dir_path, "eq_results")
48 self.dataframe = None
49 self.dataframe2 = None
50 if not exists(result_path):
51 makedirs(result_path)
52 self.filename = fullPath(result_path, 'summary.csv')
53 self.colnames = ["Filename","Side","Distance From Center","Area","Sigma D","Sigma S","Sigma C",
54 "gamma","Z line","Sigma Z","Iz","gamma Z", "Extra peak","Sigma Z EP","Iz EP","gamma Z EP",
55 "Model", "CenterX", "S10", "d10",
56 "I11/I10", "Average I11/I10 per fiber", "Fitting error","comment"]
57 self.filename2 = fullPath(result_path, 'summary2.csv')
58 self.colnames2 = ['Filename', 'left peak 0', 'right peak 0', 'left peak 1', 'right peak 1',
59 'left Sigma D', 'right Sigma D', 'left Sigma S', 'right Sigma S', 'left Sigma C',
60 'right Sigma C', 'left gamma', 'right gamma', 'left Z line', 'right Z line',
61 'left Sigma Z', 'right Sigma Z', 'left Iz', 'right Iz', 'left gamma Z',
62 'right gamma Z', 'left Extra peak', 'right Extra peak',
63 'left Sigma Z EP', 'right Sigma Z EP', 'left Iz EP', 'right Iz EP', 'left gamma Z EP',
64 'right gamma Z EP', 'Model', 'CenterX', 'S10', 'd10', 'left I11/I10', 'right I11/I10',
65 'Average I11/I10 per fiber', 'Fitting error', 'comment']
66 self.loadFailedCases(dir_path)
67 self.loadSummary()
68 self.loadSummary2()
70 def loadFailedCases(self, direc):
71 """
72 Load failed cases file from the directory and keep them in self.failedcases
73 :param direc: input directory (str)
74 :return: -
75 """
76 self.failedcasesfile = fullPath(direc, "failedcases.txt")
77 self.failedcases = set()
78 if exists(self.failedcasesfile):
79 for line in open(self.failedcasesfile, 'r'):
80 name = line.rstrip('\n')
81 self.failedcases.add(name)
83 def loadSummary(self):
84 """
85 Load summary.csv file and keep data in self.dataframe
86 :return:
87 """
88 if not exists(self.filename):
89 self.dataframe = pd.DataFrame(columns = self.colnames)
90 else:
91 self.dataframe = pd.read_csv(self.filename)
93 def loadSummary2(self):
94 """
95 Load summary.csv file and keep data in self.dataframe2
96 :return:
97 """
98 if not exists(self.filename2):
99 self.dataframe2 = pd.DataFrame(columns = self.colnames2)
100 else:
101 self.dataframe2 = pd.read_csv(self.filename2)
103 def writeNewData(self, bioImg):
104 """
105 Add new data to dataframe, then re-write summary.csv and failed cases file
106 :param bioImg: EquatorImage object with results in its info dict
107 :return: -
108 """
109 file_name = bioImg.filename
110 info = bioImg.info
111 self.removeData(file_name)
112 new_datas = []
114 # If image is rejected
115 if "reject" in info and info["reject"]:
116 data = {}
117 for k in self.dataframe.columns:
118 data[k] = '-'
119 data['Filename'] = file_name
120 data['comment'] = "REJECTED"
121 new_datas = [data]
122 else:
123 failed = False
124 # Get all needed infos
125 if 'peaks' not in info:
126 data = {}
127 for k in self.dataframe.columns:
128 data[k] = '-'
129 data['Filename'] = file_name
130 data['comment'] = "No effective peaks detected"
131 new_datas = [data]
132 failed = True
133 else:
134 if 'fit_results' in info:
135 fit_results = info['fit_results']
136 all_S = fit_results['all_S']
137 for side in ['left', 'right']:
138 areas = fit_results[side+'_areas']
140 for i in range(len(areas)):
141 data = {
142 "Filename": file_name,
143 "Side": side,
144 "Distance From Center": all_S[i],
145 "Area" : areas[i]
146 }
147 if i == 0:
148 first_row = data
149 new_datas.append(data)
151 # Write data to the first row of each side
152 first_row.update({
153 "Sigma C": abs(fit_results[side+'_sigmac']),
154 "Sigma D": abs(fit_results[side+'_sigmad']),
155 "Sigma S": abs(fit_results[side+'_sigmas']),
156 "I11/I10": fit_results[side+'_ratio'],
157 })
158 if fit_results["model"] == 'Voigt':
159 first_row["gamma"] = fit_results[side+'_gamma']
160 if fit_results["isSkeletal"]:
161 first_row["Z line"] = fit_results[side+'_zline']
162 first_row["Sigma Z"] = fit_results[side+'_sigmaz']
163 first_row["Iz"] = fit_results[side+'_intz']
164 if fit_results["model"] == 'Voigt':
165 first_row["gamma Z"] = fit_results[side+'_gammaz']
166 if fit_results["isExtraPeak"]:
167 first_row["Extra peak"] = fit_results[side+'_zline_EP']
168 first_row["Sigma Z EP"] = fit_results[side+'_sigmaz_EP']
169 first_row["Iz EP"] = fit_results[side+'_intz_EP']
170 if fit_results["model"] == 'Voigt':
171 first_row["gamma Z EP"] = fit_results[side+'_gammaz_EP']
174 # Write general information to the first row of image
175 first_row = new_datas[0]
177 if fit_results["fiterror"] > 0.2:
178 first_row['comment'] = 'High Fitting Error'
179 failed = True
181 first_row['S10'] = fit_results['S10']
182 first_row['Model'] = fit_results['model']
183 first_row['Average I11/I10 per fiber'] = fit_results['avg_ratio']
184 first_row['Model'] = fit_results['model']
185 first_row['Fitting error'] = fit_results['fiterror']
186 first_row['CenterX'] = round(fit_results['centerX'], 2)
188 if 'd10' in fit_results.keys():
189 first_row['d10'] = fit_results['d10']
190 else:
191 first_row['d10'] = '-'
192 else:
193 data = {}
194 for k in self.dataframe.columns:
195 data[k] = '-'
196 data['Filename'] = file_name
197 data['comment'] = "Model cannot be fit"
198 new_datas = [data]
199 failed = True
201 if failed:
202 self.failedcases.add(file_name)
203 elif file_name in self.failedcases:
204 self.failedcases.remove(file_name)
206 self.dataframe = pd.concat([self.dataframe, pd.DataFrame.from_records(new_datas)])
207 # self.dataframe = self.dataframe.append(new_datas, ignore_index=True) # Future warning deprecated
208 self.dataframe.reset_index()
209 self.dataframe.to_csv(self.filename, index=False, columns=self.colnames) # Write to csv file
211 # Write all failed cases to failed cases file
212 f = open(self.failedcasesfile, 'w')
213 f.write("\n".join(list(self.failedcases)))
214 f.close()
216 def writeNewData2(self, bioImg):
217 """
218 Add new data to dataframe2, then re-write summary.csv and failed cases file
219 :param bioImg: EquatorImage object with results in its info dict
220 :return: -
221 """
222 file_name = bioImg.filename
223 info = bioImg.info
224 self.removeData2(file_name)
225 data = {}
227 # If image is rejected
228 if "reject" in info and info["reject"]:
229 for k in self.dataframe2.columns:
230 data[k] = '-'
231 data['Filename'] = file_name
232 data['comment'] = "REJECTED"
233 else:
234 failed = False
235 # Get all needed infos
236 if 'peaks' not in info:
237 for k in self.dataframe2.columns:
238 data[k] = '-'
239 data['Filename'] = file_name
240 data['comment'] = "No effective peaks detected"
241 failed = True
242 else:
243 if 'fit_results' in info:
244 fit_results = info['fit_results']
245 all_S = fit_results['all_S']
246 data['Filename'] = file_name
247 for side in ['left', 'right']:
248 areas = fit_results[side+'_areas']
249 for i in range(len(areas)):
250 data[f'{side} peak {i}'] = all_S[i]
252 data.update({
253 side+" Sigma C": abs(fit_results[side+'_sigmac']),
254 side+" Sigma D": abs(fit_results[side+'_sigmad']),
255 side+" Sigma S": abs(fit_results[side+'_sigmas']),
256 side+" I11/I10": fit_results[side+'_ratio'],
257 })
258 if fit_results["model"] == 'Voigt':
259 data[side+" gamma"] = fit_results[side+'_gamma']
260 if fit_results["isSkeletal"]:
261 data[side+" Z line"] = fit_results[side+'_zline']
262 data[side+" Sigma Z"] = fit_results[side+'_sigmaz']
263 data[side+" Iz"] = fit_results[side+'_intz']
264 if fit_results["model"] == 'Voigt':
265 data[side+" gamma Z"] = fit_results[side+'_gammaz']
266 if fit_results["isExtraPeak"]:
267 data[side+" Extra peak"] = fit_results[side+'_zline_EP']
268 data[side+" Sigma Z EP"] = fit_results[side+'_sigmaz_EP']
269 data[side+" Iz EP"] = fit_results[side+'_intz_EP']
270 if fit_results["model"] == 'Voigt':
271 data[side+" gamma Z EP"] = fit_results[side+'_gammaz_EP']
273 if fit_results["fiterror"] > 0.2:
274 data['comment'] = 'High Fitting Error'
275 failed = True
277 data['S10'] = fit_results['S10']
278 data['Model'] = fit_results['model']
279 data['Average I11/I10 per fiber'] = fit_results['avg_ratio']
280 data['Model'] = fit_results['model']
281 data['Fitting error'] = fit_results['fiterror']
282 data['CenterX'] = round(fit_results['centerX'], 2)
284 if 'd10' in fit_results.keys():
285 data['d10'] = fit_results['d10']
286 else:
287 data['d10'] = '-'
288 else:
289 for k in self.dataframe2.columns:
290 data[k] = '-'
291 data['Filename'] = file_name
292 data['comment'] = "Model cannot be fit"
293 failed = True
295 if failed:
296 self.failedcases.add(file_name)
297 elif file_name in self.failedcases:
298 self.failedcases.remove(file_name)
300 self.dataframe2 = pd.concat([self.dataframe2, pd.DataFrame.from_records([data])])
301 # self.dataframe2 = self.dataframe2.append(data, ignore_index=True) # Future warning deprecated
302 self.dataframe2.reset_index()
303 self.dataframe2.to_csv(self.filename2, index=False, columns=self.colnames2) # Write to csv file
305 def removeData(self, file_name):
306 """
307 Remove data from dataframe
308 :param file_name: (str)
309 :return:
310 """
311 self.dataframe = self.dataframe[self.dataframe["Filename"] != file_name]
313 def removeData2(self, file_name):
314 """
315 Remove data from dataframe2
316 :param file_name: (str)
317 :return:
318 """
319 self.dataframe2 = self.dataframe2[self.dataframe2["Filename"] != file_name]