Module catmaid_publish.volumes
Expand source code
import json
from collections import defaultdict
from collections.abc import Sequence
from functools import lru_cache
from pathlib import Path
from typing import Iterable, Optional, Union
import meshio
import navis
import networkx as nx
import numpy as np
import pandas as pd
import pymaid
from catmaid_publish.constants import CACHE_SIZE
from .utils import copy_cache, descendants, entity_graph, fill_in_dict
class AnnotatedVolume(navis.Volume):
def __init__(
self,
vertices: Union[list, np.ndarray],
faces: Union[list, np.ndarray] = None,
name: Optional[str] = None,
color: Union[str, Sequence[Union[int, float]]] = (0.85, 0.85, 0.85, 0.2),
id: Optional[int] = None,
annotations: Optional[set[str]] = None,
**kwargs,
):
super().__init__(vertices, faces, name, color, id, **kwargs)
self.annotations = set() if not annotations else set(annotations)
def get_volume_id(vol: navis.Volume):
"""Depends on some implementation details in both navis and trimesh."""
return vol._kwargs["volume_id"]
def get_volumes(
annotated: list[str],
names: Optional[list[str]],
rename: dict[str, str],
ann_renames: dict[str, str],
) -> tuple[dict[str, tuple[int, meshio.Mesh, set[str]]], dict[str, str]]:
"""
Returns 2-tuple:
(
{vol_renamed: (vol_id, mesh, renamed_annotations)},
{orig_name: out_name}
)
"""
name_to_anns = defaultdict(set)
g = entity_graph()
for v, d in g.nodes(data=True):
if d["type"] != "volume":
continue
for parent in g.predecessors(v):
parent_d = g.nodes[parent]
if parent_d["type"] != "annotation":
continue
name_to_anns[d["name"]].add(parent_d["name"])
if names is not None:
name_set = set(rename)
if annotated:
ann_set = set(annotated)
roots = [
d["id"]
for _, d in g.nodes(data=True)
if d["type"] == "annotation" and d["name"] in ann_set
]
for vol_id in descendants(
g, roots, select_fn=lambda _, d: d["type"] == "volume"
):
name_set.add(g.nodes[vol_id]["name"])
name_set.update(names)
names = sorted(name_set)
volumes: dict[str, navis.Volume] = pymaid.get_volume(names)
rename = fill_in_dict(rename, volumes.keys())
out = {
rename[name]: (
get_volume_id(vol),
meshio.Mesh(vol.vertices, [("triangle", vol.faces)]),
{ann_renames[a] for a in name_to_anns[name] if a in ann_renames},
)
for name, vol in volumes.items()
}
return out, rename
def write_volumes(dpath: Path, volumes: dict[str, tuple[int, meshio.Mesh, set[str]]]):
if not volumes:
return
dpath.mkdir(parents=True, exist_ok=True)
ann_data = defaultdict(set)
with open(dpath / "names.tsv", "w") as f:
f.write("filename\tvolume_name\n")
for name, (vol_id, mesh, anns) in sorted(volumes.items()):
fname = str(vol_id) + ".stl"
f.write(f"{fname}\t{name}\n")
mesh.write(dpath / fname)
for ann in anns:
ann_data[ann].add(name)
with open(dpath / "annotations.json", "w") as f:
json.dump({k: sorted(v) for k, v in ann_data.items()}, f)
def df_to_dict(df: pd.DataFrame, keys, values):
return dict(zip(df[keys], df[values]))
class VolumeReader:
"""Class for reading exported volume data."""
def __init__(self, dpath: Path) -> None:
"""
Parameters
----------
dpath : Path
Path to directory in which the volume data is saved.
"""
self.dpath = dpath
self._names_df = None
@property
def names_df(self) -> pd.DataFrame:
"""Dataframe representing ``names.tsv``.
Returns
-------
pd.DataFrame
Columns ``filename``, ``volume_name``
"""
if self._names_df is None:
self._names_df = pd.read_csv(
self.dpath / "names.tsv",
sep="\t",
)
return self._names_df
@lru_cache
def _dict(self, keys, values):
return df_to_dict(self.names_df, keys, values)
@copy_cache(maxsize=None)
def _get_annotations(self) -> dict[str, set[str]]:
"""Map annotation names to volume names.
Returns
-------
dict[str, set[str]]
"""
d = json.loads((self.dpath / "annotations.json").read_text())
return {k: set(v) for k, v in d.items()}
def get_annotation_graph(self) -> nx.DiGraph:
"""Get graph of annotations to volumes.
Returns
-------
networkx.DiGraph
"""
g = nx.DiGraph()
for k, vs in self._get_annotations().items():
g.add_node(k, type="annotation")
for v in vs:
if v not in g.nodes:
g.add_node(v, type="volume")
g.add_edge(k, v, meta_annotation=False)
return g
def _annotations_for_volume(self, name: str):
d = self._get_annotations()
return {a for a, names in d.items() if name in names}
@copy_cache(maxsize=CACHE_SIZE)
def _read_vol(
self, fpath: Path, name: Optional[str], volume_id: Optional[int]
) -> AnnotatedVolume:
vol = AnnotatedVolume.from_file(fpath)
if name is not None:
d = self._dict("filename", "volume_name")
name = d[fpath.name]
vol.name = name
vol.annotations.update(self._annotations_for_volume(name))
if volume_id is None:
volume_id = int(fpath.stem)
vol.id = volume_id
return vol
def get_by_id(self, volume_id: int) -> AnnotatedVolume:
"""Read a volume with a given (arbitrary) ID.
Parameters
----------
volume_id : int
Returns
-------
AnnotatedVolume
"""
return self._read_vol(
self.dpath / f"{volume_id}.stl",
None,
volume_id,
)
def get_by_name(self, volume_name: str) -> AnnotatedVolume:
"""Read a volume with a given name.
Parameters
----------
volume_name : str
Returns
-------
AnnotatedVolume
"""
d = self._dict("volume_name", "filename")
fname = d[volume_name]
path = self.dpath / fname
return self._read_vol(path, volume_name, None)
def get_by_annotation(self, annotation: str) -> Iterable[AnnotatedVolume]:
"""Lazily iterate through all volumes with the given annotation.
Parameters
----------
annotation : str
Annotation name.
Yields
------
Iterable[AnnotatedVolume]
"""
d = self._get_annotations()
for vol_name in d[annotation]:
yield self.get_by_name(vol_name)
def get_all(self) -> Iterable[AnnotatedVolume]:
"""Lazily iterate through all available volumes.
Iteration is in the order used by ``names.tsv``.
Yields
------
AnnotatedVolume
"""
for fname, name in self._dict("filename", "volume_name").items():
fpath = self.dpath / fname
yield self._read_vol(fpath, name, None)
README = """
# Volumes
Volumes are regions of interest represented by 3D triangular meshes.
Data in this directory can be parsed into `AnnotatedVolume`s
(a subclass of `navis.Volume` which simply adds an attribute `annotations: set[str]`)
using `catmaid_publish.VolumeReader`.
## Files
### `names.tsv`
A tab separated value file with columns
`filename`, `volume_name`.
This maps the name of the volume to the name of the file in which the mesh is stored.
### `*.stl`
Files representing the volume, in ASCII STL format, named with an arbitrary ID.
### `annotations.json`
A JSON file mapping annotation names to an array of the names of volumes it annotates.
""".lstrip()
Functions
def df_to_dict(df: pandas.core.frame.DataFrame, keys, values)
-
Expand source code
def df_to_dict(df: pd.DataFrame, keys, values): return dict(zip(df[keys], df[values]))
def get_volume_id(vol: navis.core.volumes.Volume)
-
Depends on some implementation details in both navis and trimesh.
Expand source code
def get_volume_id(vol: navis.Volume): """Depends on some implementation details in both navis and trimesh.""" return vol._kwargs["volume_id"]
def get_volumes(annotated: list[str], names: Optional[list[str]], rename: dict[str, str], ann_renames: dict[str, str]) ‑> tuple[dict[str, tuple[int, meshio._mesh.Mesh, set[str]]], dict[str, str]]
-
Returns 2-tuple:
( {vol_renamed: (vol_id, mesh, renamed_annotations)}, {orig_name: out_name} )
Expand source code
def get_volumes( annotated: list[str], names: Optional[list[str]], rename: dict[str, str], ann_renames: dict[str, str], ) -> tuple[dict[str, tuple[int, meshio.Mesh, set[str]]], dict[str, str]]: """ Returns 2-tuple: ( {vol_renamed: (vol_id, mesh, renamed_annotations)}, {orig_name: out_name} ) """ name_to_anns = defaultdict(set) g = entity_graph() for v, d in g.nodes(data=True): if d["type"] != "volume": continue for parent in g.predecessors(v): parent_d = g.nodes[parent] if parent_d["type"] != "annotation": continue name_to_anns[d["name"]].add(parent_d["name"]) if names is not None: name_set = set(rename) if annotated: ann_set = set(annotated) roots = [ d["id"] for _, d in g.nodes(data=True) if d["type"] == "annotation" and d["name"] in ann_set ] for vol_id in descendants( g, roots, select_fn=lambda _, d: d["type"] == "volume" ): name_set.add(g.nodes[vol_id]["name"]) name_set.update(names) names = sorted(name_set) volumes: dict[str, navis.Volume] = pymaid.get_volume(names) rename = fill_in_dict(rename, volumes.keys()) out = { rename[name]: ( get_volume_id(vol), meshio.Mesh(vol.vertices, [("triangle", vol.faces)]), {ann_renames[a] for a in name_to_anns[name] if a in ann_renames}, ) for name, vol in volumes.items() } return out, rename
def write_volumes(dpath: pathlib.Path, volumes: dict[str, tuple[int, meshio._mesh.Mesh, set[str]]])
-
Expand source code
def write_volumes(dpath: Path, volumes: dict[str, tuple[int, meshio.Mesh, set[str]]]): if not volumes: return dpath.mkdir(parents=True, exist_ok=True) ann_data = defaultdict(set) with open(dpath / "names.tsv", "w") as f: f.write("filename\tvolume_name\n") for name, (vol_id, mesh, anns) in sorted(volumes.items()): fname = str(vol_id) + ".stl" f.write(f"{fname}\t{name}\n") mesh.write(dpath / fname) for ann in anns: ann_data[ann].add(name) with open(dpath / "annotations.json", "w") as f: json.dump({k: sorted(v) for k, v in ann_data.items()}, f)
Classes
class AnnotatedVolume (vertices: Union[list, numpy.ndarray], faces: Union[list, numpy.ndarray] = None, name: Optional[str] = None, color: Union[str, collections.abc.Sequence[Union[int, float]]] = (0.85, 0.85, 0.85, 0.2), id: Optional[int] = None, annotations: Optional[set[str]] = None, **kwargs)
-
Mesh consisting of vertices and faces.
Subclass of
trimesh.Trimesh
with a few additional methods.Parameters
vertices
:list | array | mesh-like
(N, 3)
vertices coordinates or an object that has.vertices
and.faces
attributes in which casefaces
parameter will be ignored.faces
:list | array
(M, 3)
array of indexed triangle faces.name
:str
, optional- A name for the volume.
color
:tuple
, optional- RGB(A) color.
id
:int
, optional- If not provided, neuron will be assigned a random UUID as
.id
. **kwargs
- Keyword arguments passed through to
trimesh.Trimesh
See Also
:func:
~navis.example_volumeLoads example volume(s).
A Trimesh object contains a triangular 3D mesh.
Parameters
vertices
:(n, 3) float
- Array of vertex locations
faces
:(m, 3)
or(m, 4) int
- Array of triangular or quad faces (triangulated on load)
face_normals
:(m, 3) float
- Array of normal vectors corresponding to faces
vertex_normals
:(n, 3) float
- Array of normal vectors for vertices
metadata
:dict
- Any metadata about the mesh
process
:bool
- if True, Nan and Inf values will be removed
- immediately and vertices will be merged
validate
:bool
- If True, degenerate and duplicate faces will be
- removed immediately, and some functions will alter
- the mesh to ensure consistent results.
use_embree
:bool
- If True try to use pyembree raytracer.
- If pyembree is not available it will automatically fall
- back to a much slower rtree/numpy implementation
initial_cache
:dict
- A way to pass things to the cache in case expensive
- things were calculated before creating the mesh object.
visual
:ColorVisuals
orTextureVisuals
Assigned to self.visual
Expand source code
class AnnotatedVolume(navis.Volume): def __init__( self, vertices: Union[list, np.ndarray], faces: Union[list, np.ndarray] = None, name: Optional[str] = None, color: Union[str, Sequence[Union[int, float]]] = (0.85, 0.85, 0.85, 0.2), id: Optional[int] = None, annotations: Optional[set[str]] = None, **kwargs, ): super().__init__(vertices, faces, name, color, id, **kwargs) self.annotations = set() if not annotations else set(annotations)
Ancestors
- navis.core.volumes.Volume
- trimesh.base.Trimesh
- trimesh.parent.Geometry3D
- trimesh.parent.Geometry
- abc.ABC
class VolumeReader (dpath: pathlib.Path)
-
Class for reading exported volume data.
Parameters
dpath
:Path
- Path to directory in which the volume data is saved.
Expand source code
class VolumeReader: """Class for reading exported volume data.""" def __init__(self, dpath: Path) -> None: """ Parameters ---------- dpath : Path Path to directory in which the volume data is saved. """ self.dpath = dpath self._names_df = None @property def names_df(self) -> pd.DataFrame: """Dataframe representing ``names.tsv``. Returns ------- pd.DataFrame Columns ``filename``, ``volume_name`` """ if self._names_df is None: self._names_df = pd.read_csv( self.dpath / "names.tsv", sep="\t", ) return self._names_df @lru_cache def _dict(self, keys, values): return df_to_dict(self.names_df, keys, values) @copy_cache(maxsize=None) def _get_annotations(self) -> dict[str, set[str]]: """Map annotation names to volume names. Returns ------- dict[str, set[str]] """ d = json.loads((self.dpath / "annotations.json").read_text()) return {k: set(v) for k, v in d.items()} def get_annotation_graph(self) -> nx.DiGraph: """Get graph of annotations to volumes. Returns ------- networkx.DiGraph """ g = nx.DiGraph() for k, vs in self._get_annotations().items(): g.add_node(k, type="annotation") for v in vs: if v not in g.nodes: g.add_node(v, type="volume") g.add_edge(k, v, meta_annotation=False) return g def _annotations_for_volume(self, name: str): d = self._get_annotations() return {a for a, names in d.items() if name in names} @copy_cache(maxsize=CACHE_SIZE) def _read_vol( self, fpath: Path, name: Optional[str], volume_id: Optional[int] ) -> AnnotatedVolume: vol = AnnotatedVolume.from_file(fpath) if name is not None: d = self._dict("filename", "volume_name") name = d[fpath.name] vol.name = name vol.annotations.update(self._annotations_for_volume(name)) if volume_id is None: volume_id = int(fpath.stem) vol.id = volume_id return vol def get_by_id(self, volume_id: int) -> AnnotatedVolume: """Read a volume with a given (arbitrary) ID. Parameters ---------- volume_id : int Returns ------- AnnotatedVolume """ return self._read_vol( self.dpath / f"{volume_id}.stl", None, volume_id, ) def get_by_name(self, volume_name: str) -> AnnotatedVolume: """Read a volume with a given name. Parameters ---------- volume_name : str Returns ------- AnnotatedVolume """ d = self._dict("volume_name", "filename") fname = d[volume_name] path = self.dpath / fname return self._read_vol(path, volume_name, None) def get_by_annotation(self, annotation: str) -> Iterable[AnnotatedVolume]: """Lazily iterate through all volumes with the given annotation. Parameters ---------- annotation : str Annotation name. Yields ------ Iterable[AnnotatedVolume] """ d = self._get_annotations() for vol_name in d[annotation]: yield self.get_by_name(vol_name) def get_all(self) -> Iterable[AnnotatedVolume]: """Lazily iterate through all available volumes. Iteration is in the order used by ``names.tsv``. Yields ------ AnnotatedVolume """ for fname, name in self._dict("filename", "volume_name").items(): fpath = self.dpath / fname yield self._read_vol(fpath, name, None)
Instance variables
var names_df : pandas.core.frame.DataFrame
-
Dataframe representing
names.tsv
.Returns
pd.DataFrame
- Columns
filename
,volume_name
Expand source code
@property def names_df(self) -> pd.DataFrame: """Dataframe representing ``names.tsv``. Returns ------- pd.DataFrame Columns ``filename``, ``volume_name`` """ if self._names_df is None: self._names_df = pd.read_csv( self.dpath / "names.tsv", sep="\t", ) return self._names_df
Methods
def get_all(self) ‑> Iterable[AnnotatedVolume]
-
Lazily iterate through all available volumes.
Iteration is in the order used by
names.tsv
.Yields
Expand source code
def get_all(self) -> Iterable[AnnotatedVolume]: """Lazily iterate through all available volumes. Iteration is in the order used by ``names.tsv``. Yields ------ AnnotatedVolume """ for fname, name in self._dict("filename", "volume_name").items(): fpath = self.dpath / fname yield self._read_vol(fpath, name, None)
def get_annotation_graph(self) ‑> networkx.classes.digraph.DiGraph
-
Get graph of annotations to volumes.
Returns
networkx.DiGraph
Expand source code
def get_annotation_graph(self) -> nx.DiGraph: """Get graph of annotations to volumes. Returns ------- networkx.DiGraph """ g = nx.DiGraph() for k, vs in self._get_annotations().items(): g.add_node(k, type="annotation") for v in vs: if v not in g.nodes: g.add_node(v, type="volume") g.add_edge(k, v, meta_annotation=False) return g
def get_by_annotation(self, annotation: str) ‑> Iterable[AnnotatedVolume]
-
Lazily iterate through all volumes with the given annotation.
Parameters
annotation
:str
- Annotation name.
Yields
Iterable[AnnotatedVolume]
Expand source code
def get_by_annotation(self, annotation: str) -> Iterable[AnnotatedVolume]: """Lazily iterate through all volumes with the given annotation. Parameters ---------- annotation : str Annotation name. Yields ------ Iterable[AnnotatedVolume] """ d = self._get_annotations() for vol_name in d[annotation]: yield self.get_by_name(vol_name)
def get_by_id(self, volume_id: int) ‑> AnnotatedVolume
-
Expand source code
def get_by_id(self, volume_id: int) -> AnnotatedVolume: """Read a volume with a given (arbitrary) ID. Parameters ---------- volume_id : int Returns ------- AnnotatedVolume """ return self._read_vol( self.dpath / f"{volume_id}.stl", None, volume_id, )
def get_by_name(self, volume_name: str) ‑> AnnotatedVolume
-
Expand source code
def get_by_name(self, volume_name: str) -> AnnotatedVolume: """Read a volume with a given name. Parameters ---------- volume_name : str Returns ------- AnnotatedVolume """ d = self._dict("volume_name", "filename") fname = d[volume_name] path = self.dpath / fname return self._read_vol(path, volume_name, None)