Coverage for /Users/Newville/Codes/xraylarch/larch/io/csvfiles.py: 16%

101 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-09 10:08 -0600

1#!/usr/bin/env python 

2""" 

3Code to write and read CVS files 

4 

5""" 

6import sys 

7import os 

8import time 

9import json 

10import platform 

11import csv 

12 

13import numpy as np 

14from dateutil.parser import parse as dateparse 

15from larch import Group 

16from larch.math import interp 

17from larch.utils import bytes2str, fix_varname, gformat 

18 

19maketrans = str.maketrans 

20 

21def groups2csv(grouplist, filename, delim=',', 

22 x='energy', y='norm', individual=False): 

23 """save data from a list of groups to a CSV file 

24 

25 Arguments 

26 --------- 

27 grouplist list of groups to save arrays from 

28 filname name of output file 

29 x name of group member to use for `x` 

30 y name of group member to use for `y` 

31 individual toggle saving individual x/y in separate files 

32 

33 """ 

34 delim = delim.strip() + ' ' 

35 

36 def get_label(grp): 

37 'get label for group' 

38 for attr in ('filename', 'label', 'name', 'file', '__name__'): 

39 o = getattr(grp, attr, None) 

40 if o is not None: 

41 return o 

42 return repr(o) 

43 

44 def save_group(g, delim=', ', x='energy', y='norm'): 

45 label = get_label(g) 

46 _x = getattr(g, x) 

47 _y = getattr(g, y) 

48 _n = len(_x) 

49 labels = [x, label] 

50 outarr = np.array([_x, _y]) 

51 buff = [f"#saved {time.ctime()}", 

52 f"#saving x array={x}, y array={y}", 

53 f"#{label}: {g.filename}", 

54 "#------------------------------------------", 

55 "# %s" % delim.join(labels)] 

56 for i in range(_n): 

57 buff.append(delim.join([gformat(_x[i]), gformat(_y[i])])) 

58 buff.append('') 

59 fnout = f"{label}.csv" 

60 with open(fnout, 'w', encoding=sys.getdefaultencoding()) as fh: 

61 fh.write("\n".join(buff)) 

62 print(f"Wrote group to {fnout}") 

63 

64 if individual is True: 

65 for g in grouplist: 

66 save_group(g, delim=delim, x=x, y=y) 

67 return 

68 

69 ngroups = len(grouplist) 

70 x0 = getattr(grouplist[0], x) 

71 npts = len(x0) 

72 columns = [x0, getattr(grouplist[0], y)] 

73 labels = [x, get_label(grouplist[0]) ] 

74 

75 buff = ["# %d files saved %s" % (len(grouplist), time.ctime()), 

76 "# saving x array='%s', y array='%s'" % (x, y), 

77 "# %s: %s" % (labels[1], grouplist[0].filename)] 

78 

79 for g in grouplist[1:]: 

80 label = get_label(g) 

81 buff.append("# %s: %s" % (label, g.filename)) 

82 labels.append(label) 

83 _x = getattr(g, x) 

84 _y = getattr(g, y) 

85 

86 if ((len(_x) != npts) or (abs(_x -x0)).sum() > 1.0): 

87 columns.append(interp(_x, _y, x0)) 

88 else: 

89 columns.append(_y) 

90 

91 buff.append("#------------------------------------------") 

92 buff.append("# %s" % delim.join(labels)) 

93 for i in range(npts): 

94 buff.append(delim.join([gformat(s[i]) for s in columns])) 

95 

96 buff.append('') 

97 with open(filename, 'w', encoding=sys.getdefaultencoding()) as fh: 

98 fh.write("\n".join(buff)) 

99 

100 print("Wrote %i groups to %s" % (len(columns)-1, filename)) 

101 

102 

103def str2float(word, allow_times=True): 

104 """convert a work to a float 

105 

106 Arguments 

107 --------- 

108 word str, word to be converted 

109 allow_times bool, whether to support time stamps [True] 

110 

111 Returns 

112 ------- 

113 either a float or text 

114 

115 Notes 

116 ----- 

117 The `allow_times` will try to support common date-time strings 

118 using the dateutil module, returning a numerical value as the 

119 Unix timestamp, using 

120 time.mktime(dateutil.parser.parse(word).timetuple()) 

121 """ 

122 mktime = time.mktime 

123 val = word 

124 try: 

125 val = float(word) 

126 except ValueError: 

127 try: 

128 val = mktime(dateparse(word).timetuple()) 

129 except ValueError: 

130 pass 

131 return val 

132 

133def read_csv(filename): 

134 """read CSV file, return group with data as columns""" 

135 csvfile = open(filename, 'r') 

136 dialect = csv.Sniffer().sniff(csvfile.read(), [',',';', '\t']) 

137 csvfile.seek(0) 

138 

139 data = None 

140 isfloat = None 

141 for row in csv.reader(csvfile, dialect): 

142 if data is None: 

143 ncols = len(row) 

144 data = [[] for i in range(ncols)] 

145 isfloat =[None]*ncols 

146 for i, word in enumerate(row): 

147 data[i].append(str2float(word)) 

148 if isfloat[i] is None: 

149 try: 

150 _ = float(word) 

151 isfloat[i] = True 

152 except ValueError: 

153 isfloat[i] = False 

154 

155 out = Group(filename=filename, data=data) 

156 for icol in range(ncols): 

157 cname = 'col_%2.2d' % (icol+1) 

158 val = data[icol] 

159 if isfloat[icol]: 

160 val = np.array(val) 

161 setattr(out, cname, val) 

162 

163 return out