"""This module contains special GrammarTypes used for large data in output files"""
import pyparsing as pp
import re
import io
import math
from numbers import Integral
import numpy as np
import copy
import os
import warnings
from typing import Union
from functools import partial
from .grammar_type import GrammarType, compare_numpy_values
from ..dependencies import DependentValue
from ..decorators import add_to_signature, cached_property
from ..grammar import SkipToRegex, Forward
[docs]
def _shape_lines(shape, *, items_per_line):
size = math.prod(shape)
if any(x < 0 for x in shape):
raise ValueError("Invalid shape")
return (size + items_per_line - 1) // items_per_line
[docs]
class RestOfTheFile(GrammarType):
"""Match anything up to the end of the file"""
datatype = str
datatype_name = "string"
_grammar = pp.Regex(".*$", re.M | re.S).set_parse_action(lambda x: x[0])
_grammar.skipWhitespace = False
[docs]
def grammar_name(self):
return "<the rest of the file>"
[docs]
class Prefixed(GrammarType):
"""This value consists from a few lines, each prefixed with a given prefix"""
[docs]
@add_to_signature(GrammarType.__init__, prepend=True)
def __init__(self, data_prefix, allow_empty=True, *args, **kwargs):
self.data_prefix = data_prefix
self.allow_empty = allow_empty
super().__init__(*args, **kwargs)
[docs]
@cached_property
def _grammar(self):
pref = re.escape(self.data_prefix)
out = f"({pref}[^\n]*)(\n{pref}[^\n]*)*"
if self.allow_empty:
out = f"({out})?"
return pp.Regex(out)
[docs]
def _string(self, value):
return re.replace("^|\n", f"{self.data_prefix}\\1", value)
[docs]
class RawData(GrammarType):
"""Match anything up to the end of the file or to the given delimiter"""
[docs]
@add_to_signature(GrammarType.__init__)
def __init__(
self,
*,
lines=None,
indented=False,
line_length=None,
ends_with: Union[str, re.Pattern] = None,
ends_with_str=None,
include_ends_with=False,
**kwargs,
):
"""
Parameters
----------
lines
Number of lines to read. A string denotes the exact path of an
option containing the count. A callable derives the count from the
options named by its parameters; :class:`DependentValue` supports
explicit paths such as ``..N``.
indented
If there are <n> spaces before data, pass n to this arg.
If the file has the following structure:
.. code-block:: text
.......................................
....rest of the splitted line.....
....rest of the splitted line.
....... The second line................
..... the rest of the ............
............ second line ...
............
Pass a tuple with two integers into this argument.
The first number of tuple is the max. number of characters on a line,
longer lines will be splitted.
The second number is the number of spaces placed on the begining of the
new lines created by splitting the old.
line_length
Wrap the lines longer than a given number
ends_with
The data ends with a given string.
ends_with_str
If ends_with is regex, print this on end of the data
include_ends_with
Include the ending delimiter to the data.
``**kwargs``
Any other arguments are passed to the :meth:`GrammarType constructor<GrammarType.__init__>`
"""
self.ends_with = ends_with
self.ends_with_str = self.ends_with if ends_with_str is None else ends_with_str
self.indented = " " * indented if isinstance(indented, int) else indented
self.lines = lines
self.line_length = line_length
self.include_ends_with = include_ends_with
self._line_dependency = None
if lines is not None and not isinstance(lines, Integral):
self._line_dependency = DependentValue.create(lines)
self.forward = None
super().__init__(**kwargs)
[docs]
def _n_lines_grammar(self, lines):
"""return a grammar for n lines of text"""
lines = int(lines)
if lines < 0:
raise ValueError("The number of lines cannot be negative")
if lines == 0:
return pp.Empty().set_parse_action(lambda: [""])
out = pp.Regex(f"([^\n]*\n){{{lines - 1}}}[^\n]*(?=\n|$)", re.S)
out.leave_whitespace()
return out
[docs]
def _grammar(self, param_name=False):
if self.lines is not None:
if isinstance(self.lines, Integral):
out = self._n_lines_grammar(self.lines)
else:
if self.forward is None or not self._line_dependency.hooks:
missing = ", ".join(self._line_dependency.paths)
raise KeyError(
f"No line-count dependencies found: {missing}"
)
out = self.forward
elif self.ends_with:
if isinstance(self.ends_with, re.Pattern):
out = SkipToRegex(self.ends_with, include_pattern=self.include_ends_with)
else:
out = pp.SkipTo(pp.Suppress(self.ends_with), include=self.include_ends_with)
out.set_parse_action(lambda x: x[0])
else:
out = RestOfTheFile._grammar.copy()
def parse(v):
v = v[0]
if self.indented:
if isinstance(self.indented, tuple):
v = v.replace("\n" + " " * self.indented[1], "")
else:
v = re.sub(f"(^|\n){self.indented}", r"\1", v)
if self.line_length:
v = re.sub(
f"([^\n]{{{self.line_length}}}[^{self.written_delimiter}\n]*)\n", f"\\1{self.written_delimiter}", v
)
return v
if self.indented or self.line_length:
out.add_parse_action(parse)
return out
[docs]
def _string(self, val):
out = str(val)
if self.line_length:
out = re.sub(
f"([^\n]{{{self.line_length}}}[^{self.written_delimiter}\n]*){self.written_delimiter}", "\\1\n", out
)
indented = self.indented
if indented:
if isinstance(indented, tuple):
first = indented[0]
nexts = first - indented[1]
prefix = " " * indented[1]
def g():
for i in out.split("\n"):
yield i[:first]
s = first
ln = len(i)
while s < ln:
e = s + nexts
yield prefix + i[s:e]
s = e
out = "\n".join(g())
else:
out = re.sub("(^|\n(?!$))", r"\1" + indented, out)
if self.ends_with_str and self.include_ends_with:
out += self.ends_with_str
return out
[docs]
def added_to_container(self, container):
if self._line_dependency is not None:
self.forward = Forward()
self._line_dependency.bind(container, self._set_number_of_lines)
super().added_to_container(container)
[docs]
def _set_number_of_lines(self, lines):
self.forward << self._n_lines_grammar(lines)
def __del__(self):
# Construction can fail before RawData.__init__ initializes this field.
try:
dependency = self._line_dependency
except AttributeError:
return
if dependency is not None:
dependency.unbind()
[docs]
def copy(self):
out = super().copy()
if self._line_dependency is not None:
out._line_dependency = self._line_dependency.copy()
out.forward = None
return out
[docs]
def convert(self, val):
return str(val)
[docs]
class NumpyArray(RawData):
"""Match anything up to the end of the file or to the given delimiter, as numpy array"""
array_access = True
[docs]
@add_to_signature(RawData.__init__, prepend=True, old_as_keyword=True)
def __init__(
self,
dtype=None,
*,
delimiter=None,
written_delimiter=None,
shape=None,
written_shape=None,
items_per_line=None,
item_format="% .18e",
dtypes=None,
trivial_parsing=False,
no_newline_at_end=True,
**kwargs,
):
"""
Parameters
----------
delimiter
None - default behavior.
int - the number will take given fixed number of chars
written_delimiter
Delimiter used only for writing. By default, use ``delimiter`` or
one space when the parsing delimiter is not specified.
shape
Resize to given shape after read. A string, callable or
:class:`DependentValue` derives the shape from other options.
written_shape
Resize to given shape before writing
items_per_line
Flatten the array and wrap its output after this many scalar items.
If ``shape`` is dynamic, its value also determines the number of
input lines.
item_format
Output format of the array (just for writing).
trivial_parsing
Use numpy.fromstring for simple whitespace-delimited numeric data.
Requires one explicit integer or floating-point dtype.
dtype
Type of the resulting data. Pass ``'line'`` to get array of whole lines
dtypes
More dtypes can be given. Then, the first, that match the data, is used
**kwargs
Any other arguments are passed to the :meth:`GrammarType constructor<GrammarType.__init__>`
"""
self.delimiter = delimiter
self.written_delimiter = (
("" if isinstance(delimiter, int) else delimiter or " ")
if written_delimiter is None
else written_delimiter
)
self.written_shape = written_shape
self.items_per_line = items_per_line
self.item_format = item_format
self.shape = shape
self._shape_dependency = (
DependentValue.create(shape)
if isinstance(shape, (str, DependentValue)) or callable(shape)
else None
)
self._parsed_shape = None
self.no_newline_at_end = no_newline_at_end
if dtypes is None:
if dtype == "line":
dtypes = dtype
else:
dtypes = [dtype]
else:
if dtype is not None:
raise ValueError("Use either dtype or dtypes, but not both")
self.dtypes = dtypes
self.trivial_parsing = trivial_parsing
if trivial_parsing:
if trivial_parsing is not True:
if len(self.dtypes) != 1 or self.dtypes[0] is not None:
raise ValueError(
"Specify either the dtype in dtypes or trivial_parsing, not both"
)
self.dtypes = [trivial_parsing]
if len(self.dtypes) != 1:
raise ValueError(
"trivial_parsing requires exactly one numeric dtype"
)
try:
dtype_kind = (
np.dtype(self.dtypes[0]).kind
if self.dtypes[0] is not None
else None
)
except (TypeError, ValueError):
dtype_kind = None
if dtype_kind not in ("i", "u", "f"):
raise ValueError(
"trivial_parsing requires an explicit integer or floating-point dtype"
)
if self.delimiter is not None and (
not isinstance(self.delimiter, str)
or len(self.delimiter) != 1
or not self.delimiter.isspace()
):
raise ValueError(
"trivial_parsing supports whitespace-delimited data only"
)
lines = kwargs.pop("lines", None)
if items_per_line is not None:
if not isinstance(items_per_line, int) or items_per_line <= 0:
raise ValueError("items_per_line has to be a positive integer")
if lines is None:
if self._shape_dependency is not None:
lines = self._shape_dependency.mapped(
partial(_shape_lines, items_per_line=items_per_line)
)
elif shape is not None and -1 not in shape:
lines = _shape_lines(shape, items_per_line=items_per_line)
super().__init__(lines=lines, **kwargs)
[docs]
def _validate(self, value, why="set"):
return isinstance(value, np.ndarray)
[docs]
def validate(
self,
value,
param_name="<Unknown>",
why="set",
option=None,
):
"""Validate the array and, when possible, its configured shape."""
super().validate(
value,
param_name=param_name,
why=why,
option=option,
)
if option is None:
return True
expected = self.expected_shape(option)
if expected is None:
return True
actual = value.shape
matches = len(actual) == len(expected) and all(
wanted == -1 or current == wanted
for current, wanted in zip(actual, expected)
)
if not matches:
self._valueError(
value,
f"array has shape {actual}, expected {expected}",
option._get_path,
)
return True
[docs]
def convert(self, value):
return np.asarray(value)
[docs]
def _string(self, value):
value = np.asarray(value)
if self.written_shape:
value = value.reshape(self.written_shape)
if self.items_per_line is not None:
values = value.reshape(-1)
lines = []
for start in range(0, len(values), self.items_per_line):
chunk = values[start : start + self.items_per_line]
lines.append(
self.written_delimiter.join(
self.item_format % item for item in chunk
)
)
output = "\n".join(lines)
if lines and not self.no_newline_at_end:
output += "\n"
return super()._string(output)
out = io.StringIO()
np.savetxt(out, value, delimiter=self.written_delimiter, fmt=self.item_format)
if self.no_newline_at_end and out.tell():
out.seek(out.tell() - 1, os.SEEK_SET)
out.truncate()
out = out.getvalue()
return super()._string(out)
is_the_same_value = staticmethod(compare_numpy_values)
[docs]
def _grammar(self, param_name=False):
grammar = super()._grammar(param_name)
def parse(v):
v = v[0]
if self.dtypes == "line":
v = np.array([i.rstrip() for i in v.split("\n")], dtype=object)
else:
if self.items_per_line is not None:
if isinstance(self.delimiter, int):
shape = (
self._parsed_shape
if self._shape_dependency is not None
else self.shape
)
if shape and all(i >= 0 for i in shape):
remaining = math.prod(shape)
lines = []
for line in v.splitlines():
count = min(self.items_per_line, remaining)
# Pyparsing may skip padding before the first
# fixed-width field of a value. Restore it
# from the known record width before parsing.
lines.append(line.rjust(count * self.delimiter))
remaining -= count
v = "".join(lines)
else:
v = v.replace("\n", "")
else:
v = v.replace("\n", " ")
if self.trivial_parsing:
dtype = self.dtypes[0]
inferred_shape = None
if (
self.items_per_line is None
and self.shape is None
and self._shape_dependency is None
):
# genfromtxt returns a 2-D array for multiple rows, but
# fromstring always returns a flat array. Preserve that
# public behavior while rejecting ragged numeric rows.
rows = [line for line in v.splitlines() if line.strip()]
if rows:
columns = len(rows[0].split())
if any(len(line.split()) != columns for line in rows[1:]):
raise ValueError(
"Inconsistent number of values per line in numeric array"
)
inferred_shape = (len(rows), columns)
try:
with warnings.catch_warnings():
# NumPy versions before 2.0 warn rather than fail
# when fromstring stops at non-numeric input.
warnings.simplefilter("error", DeprecationWarning)
v = np.fromstring(
v, dtype=dtype, sep=self.delimiter or " "
)
except DeprecationWarning as exc:
raise ValueError(
"Could not parse all numeric array data"
) from exc
if inferred_shape is not None:
expected_size = math.prod(inferred_shape)
if v.size != expected_size:
raise ValueError(
f"Parsed {v.size} values, expected {expected_size}"
)
if inferred_shape[0] > 1:
v = v.reshape(inferred_shape)
else:
last_error = None
for dt in self.dtypes:
try:
v = np.genfromtxt(
io.StringIO(v),
delimiter=self.delimiter,
dtype=dt,
)
break
except Exception as exc:
last_error = exc
else:
if not last_error:
raise ValueError("No dtype specified")
raise last_error
shape = (
self._parsed_shape
if self._shape_dependency is not None
else self.shape
)
if shape:
v = v.reshape(tuple(shape))
return v
grammar.add_parse_action(parse)
return grammar
[docs]
def copy_value(self, value):
return copy.deepcopy(value)
[docs]
def added_to_container(self, container):
if self._shape_dependency is not None:
self._shape_dependency.bind(container, self._set_parsed_shape)
super().added_to_container(container)
def __del__(self):
dependency = getattr(self, "_shape_dependency", None)
if dependency is not None:
dependency.unbind()
super().__del__()
[docs]
def _set_parsed_shape(self, shape):
self._parsed_shape = tuple(int(item) for item in shape)
[docs]
def expected_shape(self, item):
"""Return the shape expected for a runtime option."""
if self._shape_dependency is not None:
shape = self._shape_dependency.runtime_value(item)
else:
shape = self.shape
return None if shape is None else tuple(int(value) for value in shape)
[docs]
def copy(self):
out = super().copy()
if self._shape_dependency is not None:
out._shape_dependency = self._shape_dependency.copy()
out._parsed_shape = None
return out