Coverage for src/hdmf/backends/io.py: 98%
85 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-07-25 05:02 +0000
« prev ^ index » next coverage.py v7.2.5, created at 2023-07-25 05:02 +0000
1from abc import ABCMeta, abstractmethod
2import os
3from pathlib import Path
5from ..build import BuildManager, GroupBuilder
6from ..container import Container, ExternalResourcesManager
7from .errors import UnsupportedOperation
8from ..utils import docval, getargs, popargs
9from warnings import warn
12class HDMFIO(metaclass=ABCMeta):
14 @staticmethod
15 @abstractmethod
16 def can_read(path):
17 """Determines whether a given path is readable by this HDMFIO class"""
18 pass
20 @docval({'name': 'manager', 'type': BuildManager,
21 'doc': 'the BuildManager to use for I/O', 'default': None},
22 {"name": "source", "type": (str, Path),
23 "doc": "the source of container being built i.e. file path", 'default': None},
24 {'name': 'external_resources_path', 'type': str,
25 'doc': 'The path to the ExternalResources', 'default': None},)
26 def __init__(self, **kwargs):
27 manager, source, external_resources_path = getargs('manager', 'source', 'external_resources_path', kwargs)
28 if isinstance(source, Path): 28 ↛ 29line 28 didn't jump to line 29, because the condition on line 28 was never true
29 source = source.resolve()
30 elif (isinstance(source, str) and
31 not (source.lower().startswith("http://") or
32 source.lower().startswith("https://") or
33 source.lower().startswith("s3://"))):
34 source = os.path.abspath(source)
36 self.__manager = manager
37 self.__built = dict()
38 self.__source = source
39 self.external_resources_path = external_resources_path
40 self.external_resources = None
41 self.open()
43 @property
44 def manager(self):
45 '''The BuildManager this instance is using'''
46 return self.__manager
48 @property
49 def source(self):
50 '''The source of the container being read/written i.e. file path'''
51 return self.__source
53 @docval(returns='the Container object that was read in', rtype=Container)
54 def read(self, **kwargs):
55 """Read a container from the IO source."""
56 f_builder = self.read_builder()
57 if all(len(v) == 0 for v in f_builder.values()):
58 # TODO also check that the keys are appropriate. print a better error message
59 raise UnsupportedOperation('Cannot build data. There are no values.')
60 container = self.__manager.construct(f_builder)
61 container.read_io = self
62 if self.external_resources_path is not None:
63 from hdmf.common import ExternalResources
64 try:
65 self.external_resources = ExternalResources.from_norm_tsv(path=self.external_resources_path)
66 if isinstance(container, ExternalResourcesManager): 66 ↛ 75line 66 didn't jump to line 75, because the condition on line 66 was never false
67 container.link_resources(external_resources=self.external_resources)
68 except FileNotFoundError:
69 msg = "File not found at {}. ExternalResources not added.".format(self.external_resources_path)
70 warn(msg)
71 except ValueError:
72 msg = "Check ExternalResources separately for alterations. ExternalResources not added."
73 warn(msg)
75 return container
77 @docval({'name': 'container', 'type': Container, 'doc': 'the Container object to write'},
78 allow_extra=True)
79 def write(self, **kwargs):
80 """Write a container to the IO source."""
81 container = popargs('container', kwargs)
82 f_builder = self.__manager.build(container, source=self.__source, root=True)
83 self.write_builder(f_builder, **kwargs)
85 if self.external_resources_path is not None:
86 external_resources = container.get_linked_resources()
87 if external_resources is not None:
88 external_resources.to_norm_tsv(path=self.external_resources_path)
89 else:
90 msg = "Could not find linked ExternalResources. Container was still written to IO source."
91 warn(msg)
93 @docval({'name': 'src_io', 'type': 'HDMFIO', 'doc': 'the HDMFIO object for reading the data to export'},
94 {'name': 'container', 'type': Container,
95 'doc': ('the Container object to export. If None, then the entire contents of the HDMFIO object will be '
96 'exported'),
97 'default': None},
98 {'name': 'write_args', 'type': dict, 'doc': 'arguments to pass to :py:meth:`write_builder`',
99 'default': dict()},
100 {'name': 'clear_cache', 'type': bool, 'doc': 'whether to clear the build manager cache',
101 'default': False})
102 def export(self, **kwargs):
103 """Export from one backend to the backend represented by this class.
105 If `container` is provided, then the build manager of `src_io` is used to build the container, and the resulting
106 builder will be exported to the new backend. So if `container` is provided, `src_io` must have a non-None
107 manager property. If `container` is None, then the contents of `src_io` will be read and exported to the new
108 backend.
110 The provided container must be the root of the hierarchy of the source used to read the container (i.e., you
111 cannot read a file and export a part of that file.
113 Arguments can be passed in for the `write_builder` method using `write_args`. Some arguments may not be
114 supported during export.
116 Example usage:
118 .. code-block:: python
120 old_io = HDF5IO('old.nwb', 'r')
121 with HDF5IO('new_copy.nwb', 'w') as new_io:
122 new_io.export(old_io)
124 NOTE: When implementing export support on custom backends. Export does not update the Builder.source
125 on the Builders. As such, when writing LinkBuilders we need to determine if LinkBuilder.source
126 and LinkBuilder.builder.source are the same, and if so the link should be internal to the
127 current file (even if the Builder.source points to a different location).
128 """
129 src_io, container, write_args, clear_cache = getargs('src_io', 'container', 'write_args', 'clear_cache', kwargs)
130 if container is None and clear_cache:
131 # clear all containers and builders from cache so that they can all get rebuilt with export=True.
132 # constructing the container is not efficient but there is no elegant way to trigger a
133 # rebuild of src_io with new source.
134 container = src_io.read()
135 if container is not None:
136 # check that manager exists, container was built from manager, and container is root of hierarchy
137 if src_io.manager is None:
138 raise ValueError('When a container is provided, src_io must have a non-None manager (BuildManager) '
139 'property.')
140 old_bldr = src_io.manager.get_builder(container)
141 if old_bldr is None:
142 raise ValueError('The provided container must have been read by the provided src_io.')
143 if old_bldr.parent is not None:
144 raise ValueError('The provided container must be the root of the hierarchy of the '
145 'source used to read the container.')
147 # NOTE in HDF5IO, clear_cache is set to True when link_data is False
148 if clear_cache:
149 # clear all containers and builders from cache so that they can all get rebuilt with export=True
150 src_io.manager.clear_cache()
151 else:
152 # clear only cached containers and builders where the container was modified
153 src_io.manager.purge_outdated()
154 bldr = src_io.manager.build(container, source=self.__source, root=True, export=True)
155 else:
156 bldr = src_io.read_builder()
157 self.write_builder(builder=bldr, **write_args)
159 @abstractmethod
160 @docval(returns='a GroupBuilder representing the read data', rtype='GroupBuilder')
161 def read_builder(self):
162 ''' Read data and return the GroupBuilder representing it '''
163 pass
165 @abstractmethod
166 @docval({'name': 'builder', 'type': GroupBuilder, 'doc': 'the GroupBuilder object representing the Container'},
167 allow_extra=True)
168 def write_builder(self, **kwargs):
169 ''' Write a GroupBuilder representing an Container object '''
170 pass
172 @abstractmethod
173 def open(self):
174 ''' Open this HDMFIO object for writing of the builder '''
175 pass
177 @abstractmethod
178 def close(self):
179 ''' Close this HDMFIO object to further reading/writing'''
180 pass
182 def __enter__(self):
183 return self
185 def __exit__(self, type, value, traceback):
186 self.close()
188 def __del__(self):
189 self.close()