Coverage for src/hdmf/backends/io.py: 98%

88 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-04 02:57 +0000

1from abc import ABCMeta, abstractmethod 

2import os 

3from pathlib import Path 

4 

5from ..build import BuildManager, GroupBuilder 

6from ..container import Container, HERDManager 

7from .errors import UnsupportedOperation 

8from ..utils import docval, getargs, popargs 

9from warnings import warn 

10 

11 

12class HDMFIO(metaclass=ABCMeta): 

13 

14 @staticmethod 

15 @abstractmethod 

16 def can_read(path): 

17 """Determines whether a given path is readable by this HDMFIO class""" 

18 pass 

19 

20 @docval({'name': 'manager', 'type': BuildManager, 

21 'doc': 'the BuildManager to use for I/O', 'default': None}, 

22 {"name": "source", "type": (str, Path), 

23 "doc": "the source of container being built i.e. file path", 'default': None}, 

24 {'name': 'herd_path', 'type': str, 

25 'doc': 'The path to read/write the HERD file', 'default': None},) 

26 def __init__(self, **kwargs): 

27 manager, source, herd_path = getargs('manager', 'source', 'herd_path', kwargs) 

28 if isinstance(source, Path): 28 ↛ 29line 28 didn't jump to line 29, because the condition on line 28 was never true

29 source = source.resolve() 

30 elif (isinstance(source, str) and 

31 not (source.lower().startswith("http://") or 

32 source.lower().startswith("https://") or 

33 source.lower().startswith("s3://"))): 

34 source = os.path.abspath(source) 

35 

36 self.__manager = manager 

37 self.__built = dict() 

38 self.__source = source 

39 self.herd_path = herd_path 

40 self.herd = None 

41 self.open() 

42 

43 @property 

44 def manager(self): 

45 '''The BuildManager this instance is using''' 

46 return self.__manager 

47 

48 @property 

49 def source(self): 

50 '''The source of the container being read/written i.e. file path''' 

51 return self.__source 

52 

53 @docval(returns='the Container object that was read in', rtype=Container) 

54 def read(self, **kwargs): 

55 """Read a container from the IO source.""" 

56 f_builder = self.read_builder() 

57 if all(len(v) == 0 for v in f_builder.values()): 

58 # TODO also check that the keys are appropriate. print a better error message 

59 raise UnsupportedOperation('Cannot build data. There are no values.') 

60 container = self.__manager.construct(f_builder) 

61 container.read_io = self 

62 if self.herd_path is not None: 

63 from hdmf.common import HERD 

64 try: 

65 self.herd = HERD.from_zip(path=self.herd_path) 

66 if isinstance(container, HERDManager): 66 ↛ 75line 66 didn't jump to line 75, because the condition on line 66 was never false

67 container.link_resources(herd=self.herd) 

68 except FileNotFoundError: 

69 msg = "File not found at {}. HERD not added.".format(self.herd_path) 

70 warn(msg) 

71 except ValueError: 

72 msg = "Check HERD separately for alterations. HERD not added." 

73 warn(msg) 

74 

75 return container 

76 

77 @docval({'name': 'container', 'type': Container, 'doc': 'the Container object to write'}, 

78 {'name': 'herd', 'type': 'HERD', 

79 'doc': 'A HERD object to populate with references.', 

80 'default': None}, allow_extra=True) 

81 def write(self, **kwargs): 

82 container = popargs('container', kwargs) 

83 herd = popargs('herd', kwargs) 

84 

85 """Optional: Write HERD.""" 

86 if self.herd_path is not None: 

87 # If HERD is not provided, create a new one, else extend existing one 

88 if herd is None: 

89 from hdmf.common import HERD 

90 herd = HERD(type_map=self.manager.type_map) 

91 

92 # add_ref_term_set to search for and resolve the TermSetWrapper 

93 herd.add_ref_term_set(container) # container would be the NWBFile 

94 # write HERD 

95 herd.to_zip(path=self.herd_path) 

96 

97 """Write a container to the IO source.""" 

98 f_builder = self.__manager.build(container, source=self.__source, root=True) 

99 self.write_builder(f_builder, **kwargs) 

100 

101 @docval({'name': 'src_io', 'type': 'HDMFIO', 'doc': 'the HDMFIO object for reading the data to export'}, 

102 {'name': 'container', 'type': Container, 

103 'doc': ('the Container object to export. If None, then the entire contents of the HDMFIO object will be ' 

104 'exported'), 

105 'default': None}, 

106 {'name': 'write_args', 'type': dict, 'doc': 'arguments to pass to :py:meth:`write_builder`', 

107 'default': dict()}, 

108 {'name': 'clear_cache', 'type': bool, 'doc': 'whether to clear the build manager cache', 

109 'default': False}) 

110 def export(self, **kwargs): 

111 """Export from one backend to the backend represented by this class. 

112 

113 If `container` is provided, then the build manager of `src_io` is used to build the container, and the resulting 

114 builder will be exported to the new backend. So if `container` is provided, `src_io` must have a non-None 

115 manager property. If `container` is None, then the contents of `src_io` will be read and exported to the new 

116 backend. 

117 

118 The provided container must be the root of the hierarchy of the source used to read the container (i.e., you 

119 cannot read a file and export a part of that file. 

120 

121 Arguments can be passed in for the `write_builder` method using `write_args`. Some arguments may not be 

122 supported during export. 

123 

124 Example usage: 

125 

126 .. code-block:: python 

127 

128 old_io = HDF5IO('old.nwb', 'r') 

129 with HDF5IO('new_copy.nwb', 'w') as new_io: 

130 new_io.export(old_io) 

131 

132 NOTE: When implementing export support on custom backends. Export does not update the Builder.source 

133 on the Builders. As such, when writing LinkBuilders we need to determine if LinkBuilder.source 

134 and LinkBuilder.builder.source are the same, and if so the link should be internal to the 

135 current file (even if the Builder.source points to a different location). 

136 """ 

137 src_io, container, write_args, clear_cache = getargs('src_io', 'container', 'write_args', 'clear_cache', kwargs) 

138 if container is None and clear_cache: 

139 # clear all containers and builders from cache so that they can all get rebuilt with export=True. 

140 # constructing the container is not efficient but there is no elegant way to trigger a 

141 # rebuild of src_io with new source. 

142 container = src_io.read() 

143 if container is not None: 

144 # check that manager exists, container was built from manager, and container is root of hierarchy 

145 if src_io.manager is None: 

146 raise ValueError('When a container is provided, src_io must have a non-None manager (BuildManager) ' 

147 'property.') 

148 old_bldr = src_io.manager.get_builder(container) 

149 if old_bldr is None: 

150 raise ValueError('The provided container must have been read by the provided src_io.') 

151 if old_bldr.parent is not None: 

152 raise ValueError('The provided container must be the root of the hierarchy of the ' 

153 'source used to read the container.') 

154 

155 # NOTE in HDF5IO, clear_cache is set to True when link_data is False 

156 if clear_cache: 

157 # clear all containers and builders from cache so that they can all get rebuilt with export=True 

158 src_io.manager.clear_cache() 

159 else: 

160 # clear only cached containers and builders where the container was modified 

161 src_io.manager.purge_outdated() 

162 bldr = src_io.manager.build(container, source=self.__source, root=True, export=True) 

163 else: 

164 bldr = src_io.read_builder() 

165 self.write_builder(builder=bldr, **write_args) 

166 

167 @abstractmethod 

168 @docval(returns='a GroupBuilder representing the read data', rtype='GroupBuilder') 

169 def read_builder(self): 

170 ''' Read data and return the GroupBuilder representing it ''' 

171 pass 

172 

173 @abstractmethod 

174 @docval({'name': 'builder', 'type': GroupBuilder, 'doc': 'the GroupBuilder object representing the Container'}, 

175 allow_extra=True) 

176 def write_builder(self, **kwargs): 

177 ''' Write a GroupBuilder representing an Container object ''' 

178 pass 

179 

180 @abstractmethod 

181 def open(self): 

182 ''' Open this HDMFIO object for writing of the builder ''' 

183 pass 

184 

185 @abstractmethod 

186 def close(self): 

187 ''' Close this HDMFIO object to further reading/writing''' 

188 pass 

189 

190 def __enter__(self): 

191 return self 

192 

193 def __exit__(self, type, value, traceback): 

194 self.close() 

195 

196 def __del__(self): 

197 self.close()