Coverage for /usr/lib/python3/dist-packages/h5py/_debian_h5py_serial/_hl/filters.py: 45%

1# This file is part of h5py, a Python interface to the HDF5 library. 

2# 

3# http://www.h5py.org 

4# 

5# Copyright 2008-2013 Andrew Collette and contributors 

6# 

7# License: Standard 3-clause BSD; see "license.txt" for full license terms 

8# and contributor agreement. 

9 

10""" 

11 Implements support for HDF5 compression filters via the high-level 

12 interface. The following types of filter are available: 

13 

14 "gzip" 

15 Standard DEFLATE-based compression, at integer levels from 0 to 9. 

16 Built-in to all public versions of HDF5. Use this if you want a 

17 decent-to-good ratio, good portability, and don't mind waiting. 

18 

19 "lzf" 

20 Custom compression filter for h5py. This filter is much, much faster 

21 than gzip (roughly 10x in compression vs. gzip level 4, and 3x faster 

22 in decompressing), but at the cost of a worse compression ratio. Use 

23 this if you want cheap compression and portability is not a concern. 

24 

25 "szip" 

26 Access to the HDF5 SZIP encoder. SZIP is a non-mainstream compression 

27 format used in space science on integer and float datasets. SZIP is 

28 subject to license requirements, which means the encoder is not 

29 guaranteed to be always available. However, it is also much faster 

30 than gzip. 

31 

32 The following constants in this module are also useful: 

33 

34 decode 

35 Tuple of available filter names for decoding 

36 

37 encode 

38 Tuple of available filter names for encoding 

39""" 

40from collections.abc import Mapping 

41import operator 

42 

43import numpy as np 

44from .base import product 

45from .compat import filename_encode 

46from .. import h5z, h5p, h5d, h5f 

47 

48 

49_COMP_FILTERS = {'gzip': h5z.FILTER_DEFLATE, 

50 'szip': h5z.FILTER_SZIP, 

51 'lzf': h5z.FILTER_LZF, 

52 'shuffle': h5z.FILTER_SHUFFLE, 

53 'fletcher32': h5z.FILTER_FLETCHER32, 

54 'scaleoffset': h5z.FILTER_SCALEOFFSET } 

55 

56DEFAULT_GZIP = 4 

57DEFAULT_SZIP = ('nn', 8) 

58 

59def _gen_filter_tuples(): 

60 """ Bootstrap function to figure out what filters are available. """ 

61 dec = [] 

62 enc = [] 

63 for name, code in _COMP_FILTERS.items(): 

64 if h5z.filter_avail(code): 

65 info = h5z.get_filter_info(code) 

66 if info & h5z.FILTER_CONFIG_ENCODE_ENABLED: 

67 enc.append(name) 

68 if info & h5z.FILTER_CONFIG_DECODE_ENABLED: 

69 dec.append(name) 

70 

71 return tuple(dec), tuple(enc) 

72 

73decode, encode = _gen_filter_tuples() 

74 

75def _external_entry(entry): 

76 """ Check for and return a well-formed entry tuple for 

77 a call to h5p.set_external. """ 

78 # We require only an iterable entry but also want to guard against 

79 # raising a confusing exception from unpacking below a str or bytes that 

80 # was mistakenly passed as an entry. We go further than that and accept 

81 # only a tuple, which allows simpler documentation and exception 

82 # messages. 

83 if not isinstance(entry, tuple): 

84 raise TypeError( 

85 "Each external entry must be a tuple of (name, offset, size)") 

86 name, offset, size = entry # raise ValueError without three elements 

87 name = filename_encode(name) 

88 offset = operator.index(offset) 

89 size = operator.index(size) 

90 return (name, offset, size) 

91 

92def _normalize_external(external): 

93 """ Normalize external into a well-formed list of tuples and return. """ 

94 if external is None: 

95 return [] 

96 try: 

97 # Accept a solitary name---a str, bytes, or os.PathLike acceptable to 

98 # filename_encode. 

99 return [_external_entry((external, 0, h5f.UNLIMITED))] 

100 except TypeError: 

101 pass 

102 # Check and rebuild each entry to be well-formed. 

103 return [_external_entry(entry) for entry in external] 

104 

105class FilterRefBase(Mapping): 

106 """Base class for referring to an HDF5 and describing its options 

107 

108 Your subclass must define filter_id, and may define a filter_options tuple. 

109 """ 

110 filter_id = None 

111 filter_options = () 

112 

113 # Mapping interface supports using instances as **kwargs for compatibility 

114 # with older versions of h5py 

115 @property 

116 def _kwargs(self): 

117 return { 

118 'compression': self.filter_id, 

119 'compression_opts': self.filter_options 

120 } 

121 

122 def __hash__(self): 

123 return hash((self.filter_id, self.filter_options)) 

124 

125 def __eq__(self, other): 

126 return ( 

127 isinstance(other, FilterRefBase) 

128 and self.filter_id == other.filter_id 

129 and self.filter_options == other.filter_options 

130 ) 

131 

132 def __len__(self): 

133 return len(self._kwargs) 

134 

135 def __iter__(self): 

136 return iter(self._kwargs) 

137 

138 def __getitem__(self, item): 

139 return self._kwargs[item] 

140 

141class Gzip(FilterRefBase): 

142 filter_id = h5z.FILTER_DEFLATE 

143 

144 def __init__(self, level=DEFAULT_GZIP): 

145 self.filter_options = (level,) 

146 

147def fill_dcpl(plist, shape, dtype, chunks, compression, compression_opts, 

148 shuffle, fletcher32, maxshape, scaleoffset, external, 

149 allow_unknown_filter=False): 

150 """ Generate a dataset creation property list. 

151 

152 Undocumented and subject to change without warning. 

153 """ 

154 

155 if shape is None or shape == (): 

156 shapetype = 'Empty' if shape is None else 'Scalar' 

157 if any((chunks, compression, compression_opts, shuffle, fletcher32, 

158 scaleoffset is not None)): 

159 raise TypeError( 

160 f"{shapetype} datasets don't support chunk/filter options" 

161 ) 

162 if maxshape and maxshape != (): 

163 raise TypeError(f"{shapetype} datasets cannot be extended") 

164 return h5p.create(h5p.DATASET_CREATE) 

165 

166 def rq_tuple(tpl, name): 

167 """ Check if chunks/maxshape match dataset rank """ 

168 if tpl in (None, True): 

169 return 

170 try: 

171 tpl = tuple(tpl) 

172 except TypeError: 

173 raise TypeError('"%s" argument must be None or a sequence object' % name) 

174 if len(tpl) != len(shape): 

175 raise ValueError('"%s" must have same rank as dataset shape' % name) 

176 

177 rq_tuple(chunks, 'chunks') 

178 rq_tuple(maxshape, 'maxshape') 

179 

180 if compression is not None: 

181 if isinstance(compression, FilterRefBase): 

182 compression_opts = compression.filter_options 

183 compression = compression.filter_id 

184 

185 if compression not in encode and not isinstance(compression, int): 

186 raise ValueError('Compression filter "%s" is unavailable' % compression) 

187 

188 if compression == 'gzip': 

189 if compression_opts is None: 

190 gzip_level = DEFAULT_GZIP 

191 elif compression_opts in range(10): 

192 gzip_level = compression_opts 

193 else: 

194 raise ValueError("GZIP setting must be an integer from 0-9, not %r" % compression_opts) 

195 

196 elif compression == 'lzf': 

197 if compression_opts is not None: 

198 raise ValueError("LZF compression filter accepts no options") 

199 

200 elif compression == 'szip': 

201 if compression_opts is None: 

202 compression_opts = DEFAULT_SZIP 

203 

204 err = "SZIP options must be a 2-tuple ('ec'|'nn', even integer 0-32)" 

205 try: 

206 szmethod, szpix = compression_opts 

207 except TypeError: 

208 raise TypeError(err) 

209 if szmethod not in ('ec', 'nn'): 

210 raise ValueError(err) 

211 if not (0<szpix<=32 and szpix%2 == 0): 

212 raise ValueError(err) 

213 

214 elif compression_opts is not None: 

215 # Can't specify just compression_opts by itself. 

216 raise TypeError("Compression method must be specified") 

217 

218 if scaleoffset is not None: 

219 # scaleoffset must be an integer when it is not None or False, 

220 # except for integral data, for which scaleoffset == True is 

221 # permissible (will use SO_INT_MINBITS_DEFAULT) 

222 

223 if scaleoffset < 0: 

224 raise ValueError('scale factor must be >= 0') 

225 

226 if dtype.kind == 'f': 

227 if scaleoffset is True: 

228 raise ValueError('integer scaleoffset must be provided for ' 

229 'floating point types') 

230 elif dtype.kind in ('u', 'i'): 

231 if scaleoffset is True: 

232 scaleoffset = h5z.SO_INT_MINBITS_DEFAULT 

233 else: 

234 raise TypeError('scale/offset filter only supported for integer ' 

235 'and floating-point types') 

236 

237 # Scale/offset following fletcher32 in the filter chain will (almost?) 

238 # always triggers a read error, as most scale/offset settings are 

239 # lossy. Since fletcher32 must come first (see comment below) we 

240 # simply prohibit the combination of fletcher32 and scale/offset. 

241 if fletcher32: 

242 raise ValueError('fletcher32 cannot be used with potentially lossy' 

243 ' scale/offset filter') 

244 

245 external = _normalize_external(external) 

246 # End argument validation 

247 

248 if (chunks is True) or \ 

249 (chunks is None and any((shuffle, fletcher32, compression, maxshape, 

250 scaleoffset is not None))): 

251 chunks = guess_chunk(shape, maxshape, dtype.itemsize) 

252 

253 if maxshape is True: 

254 maxshape = (None,)*len(shape) 

255 

256 if chunks is not None: 

257 plist.set_chunk(chunks) 

258 plist.set_fill_time(h5d.FILL_TIME_ALLOC) # prevent resize glitch 

259 

260 # scale-offset must come before shuffle and compression 

261 if scaleoffset is not None: 

262 if dtype.kind in ('u', 'i'): 

263 plist.set_scaleoffset(h5z.SO_INT, scaleoffset) 

264 else: # dtype.kind == 'f' 

265 plist.set_scaleoffset(h5z.SO_FLOAT_DSCALE, scaleoffset) 

266 

267 for item in external: 

268 plist.set_external(*item) 

269 

270 if shuffle: 

271 plist.set_shuffle() 

272 

273 if compression == 'gzip': 

274 plist.set_deflate(gzip_level) 

275 elif compression == 'lzf': 

276 plist.set_filter(h5z.FILTER_LZF, h5z.FLAG_OPTIONAL) 

277 elif compression == 'szip': 

278 opts = {'ec': h5z.SZIP_EC_OPTION_MASK, 'nn': h5z.SZIP_NN_OPTION_MASK} 

279 plist.set_szip(opts[szmethod], szpix) 

280 elif isinstance(compression, int): 

281 if not allow_unknown_filter and not h5z.filter_avail(compression): 

282 raise ValueError("Unknown compression filter number: %s" % compression) 

283 

284 plist.set_filter(compression, h5z.FLAG_OPTIONAL, compression_opts) 

285 

286 # `fletcher32` must come after `compression`, otherwise, if `compression` 

287 # is "szip" and the data is 64bit, the fletcher32 checksum will be wrong 

288 # (see GitHub issue #953). 

289 if fletcher32: 

290 plist.set_fletcher32() 

291 

292 return plist 

293 

294def get_filters(plist): 

295 """ Extract a dictionary of active filters from a DCPL, along with 

296 their settings. 

297 

298 Undocumented and subject to change without warning. 

299 """ 

300 

301 filters = {h5z.FILTER_DEFLATE: 'gzip', h5z.FILTER_SZIP: 'szip', 

302 h5z.FILTER_SHUFFLE: 'shuffle', h5z.FILTER_FLETCHER32: 'fletcher32', 

303 h5z.FILTER_LZF: 'lzf', h5z.FILTER_SCALEOFFSET: 'scaleoffset'} 

304 

305 pipeline = {} 

306 

307 nfilters = plist.get_nfilters() 

308 

309 for i in range(nfilters): 

310 

311 code, _, vals, _ = plist.get_filter(i) 

312 

313 if code == h5z.FILTER_DEFLATE: 

314 vals = vals[0] # gzip level 

315 

316 elif code == h5z.FILTER_SZIP: 

317 mask, pixels = vals[0:2] 

318 if mask & h5z.SZIP_EC_OPTION_MASK: 

319 mask = 'ec' 

320 elif mask & h5z.SZIP_NN_OPTION_MASK: 

321 mask = 'nn' 

322 else: 

323 raise TypeError("Unknown SZIP configuration") 

324 vals = (mask, pixels) 

325 elif code == h5z.FILTER_LZF: 

326 vals = None 

327 else: 

328 if len(vals) == 0: 

329 vals = None 

330 

331 pipeline[filters.get(code, str(code))] = vals 

332 

333 return pipeline 

334 

335CHUNK_BASE = 16*1024 # Multiplier by which chunks are adjusted 

336CHUNK_MIN = 8*1024 # Soft lower limit (8k) 

337CHUNK_MAX = 1024*1024 # Hard upper limit (1M) 

338 

339def guess_chunk(shape, maxshape, typesize): 

340 """ Guess an appropriate chunk layout for a dataset, given its shape and 

341 the size of each element in bytes. Will allocate chunks only as large 

342 as MAX_SIZE. Chunks are generally close to some power-of-2 fraction of 

343 each axis, slightly favoring bigger values for the last index. 

344 

345 Undocumented and subject to change without warning. 

346 """ 

347 # pylint: disable=unused-argument 

348 

349 # For unlimited dimensions we have to guess 1024 

350 shape = tuple((x if x!=0 else 1024) for i, x in enumerate(shape)) 

351 

352 ndims = len(shape) 

353 if ndims == 0: 

354 raise ValueError("Chunks not allowed for scalar datasets.") 

355 

356 chunks = np.array(shape, dtype='=f8') 

357 if not np.all(np.isfinite(chunks)): 

358 raise ValueError("Illegal value in chunk tuple") 

359 

360 # Determine the optimal chunk size in bytes using a PyTables expression. 

361 # This is kept as a float. 

362 dset_size = product(chunks)*typesize 

363 target_size = CHUNK_BASE * (2**np.log10(dset_size/(1024.*1024))) 

364 

365 if target_size > CHUNK_MAX: 

366 target_size = CHUNK_MAX 

367 elif target_size < CHUNK_MIN: 

368 target_size = CHUNK_MIN 

369 

370 idx = 0 

371 while True: 

372 # Repeatedly loop over the axes, dividing them by 2. Stop when: 

373 # 1a. We're smaller than the target chunk size, OR 

374 # 1b. We're within 50% of the target chunk size, AND 

375 # 2. The chunk is smaller than the maximum chunk size 

376 

377 chunk_bytes = product(chunks)*typesize 

378 

379 if (chunk_bytes < target_size or \ 

380 abs(chunk_bytes-target_size)/target_size < 0.5) and \ 

381 chunk_bytes < CHUNK_MAX: 

382 break 

383 

384 if product(chunks) == 1: 

385 break # Element size larger than CHUNK_MAX 

386 

387 chunks[idx%ndims] = np.ceil(chunks[idx%ndims] / 2.0) 

388 idx += 1 

389 

390 return tuple(int(x) for x in chunks)