Source code for ase2sprkkr.common.container_definitions

from .configuration_definitions import RealItemDefinition, BaseDefinition, Validator
from .grammar import as_delimiter, delimitedList
from .misc import dict_first_item
from .repeated_configuration_containers import RepeatedConfigurationContainer
from .configuration_containers import Section
from .repetition import Repeated
from .decorators import add_to_signature, cache
from .parsing_results import dict_from_parsed


import pyparsing as pp
from typing import Iterable, Optional, Union

from .warnings import DataValidityError
import re
import inspect
from io import StringIO


[docs] class ContainerDefinition(RealItemDefinition): """Base class for a definition (of contained data, format, etc) of either a whole configuration file (e.g. :class:`InputParameters<ase2sprkkr.input_parameters.input_parameters.InputParameters>` or e.g. :class:`Potential<ase2sprkkr.potentials.potentials.Potential>`) or its :class:`Section<ase2sprkkr.common.configuration_containers.Section>`. """ force_order = False """ Force order of its members """ delimiter = pp.Empty().set_name("") is_option = False """Container definitions create sections or configuration roots, not options.""" value_name_format = None """ The (print) format, how the name is written """ write_last_delimiter = True item_type = "section"
[docs] @staticmethod def _dict_from_named_values(args, items=None): """auxiliary method that creates dictionary from the arguments""" items = items or {} for value in args: items[value.name] = value return items
dir_common_attributes = True """ In dir listing, show the common 'object' attributes """
[docs] def __init__( self, name, members=[], written_name=None, info=None, description=None, is_optional=None, is_hidden=False, is_expert=False, experimental=False, has_hidden_members=False, name_in_grammar=None, force_order=None, name_regex=False, result_class=None, is_repeated=False, repeated_delimiter=None, repeated_count=None, repeated_with_name=False, write_condition=None, warning_condition=None, validators: Optional[Union[Validator, Iterable[Validator]]] = (), ): """ Definition of container (e.g. section of an input file). For the rest of the parameters see the :class:`RealItemDefinition`. Parameters ---------- has_hidden_members: bool If true, this section is not intended for a direct editing force_order: bool If True, the items has to retain the order, if False, the items can be in the input file in any order. validators: callable or iterable of callables Semantic validators passed to :class:`RealItemDefinition`. """ super().__init__( name=name, written_name=written_name, is_optional=is_optional, is_hidden=is_hidden, is_expert=is_expert, experimental=experimental, name_in_grammar=name_in_grammar, info=info, description=description, name_regex=name_regex, result_class=result_class, is_repeated=is_repeated, repeated_delimiter=repeated_delimiter, repeated_count=repeated_count, repeated_with_name=repeated_with_name, write_condition=write_condition, warning_condition=warning_condition, validators=validators, ) if not isinstance(members, dict): members = self._dict_from_named_values(members) if self.value_name_format: for i in members.values(): i.value_name_format = self.value_name_format self._members = members self.has_hidden_members = has_hidden_members if force_order is not None: self.force_order = force_order if isinstance(self.delimiter, str): self.delimiter = as_delimiter(self.delimiter)
default_repeated_type = Repeated.LIST_SECTION configuration_type_name = "SECTION" """ Name of the container type in the runtime documentation """
[docs] def added_to_container(self, container): """Attach this container and propagate the callback to its children.""" super().added_to_container(container) for member in self._members.values(): member.added_to_container(self)
[docs] def allow_duplication(self): return self.is_repeated
def __repr__(self): return f"<{self.configuration_type_name} {self.name}>"
[docs] def data_description(self, verbose: Union[bool, str, int] = False, show_hidden: bool = False, prefix: str = ""): """ Return the runtime documentation for the configuration described by this object. Parameters ---------- verbose If ``False``, only one line with the section name and basic info is returned. If ``True``, the items contained in the section are listed. If ``'all'``, add also detailed info about all descendants. If an ``int`` is given, print detailed informations about n levels. I.e. ``1`` is the same as ``True`` show_hidden If False, do not show hidden members. prefix The string, with with each line will begin (commonly the spaces for the indentation). """ def container_name(): out = self.configuration_type_name if out: out += " " out += self.name return out out = f"{prefix}{container_name()}" flags = [] if self.force_order: flags.append("fixed-order") if self.is_hidden: flags.append("hidden") if self.is_optional: flags.append("optional") if self.is_expert: flags.append("expert") if self.experimental: flags.append("experimental") if self.is_repeated: flags.append("repeated") if flags: flags = ", ".join(flags) out += f" ({flags})" if verbose: if isinstance(verbose, int): verbose -= 1 else: verbose = verbose if verbose == "all" else False add = self.additional_data_description(verbose, show_hidden, prefix) if self.info_in_data_description: info = self.info(False) if info: add = prefix + info + "\n\n" + add + "\n" if add: out += " contains:" under = prefix + "-" * len(out) + "\n" out = f"{prefix}{out}\n{under}{add}" return out
[docs] def additional_data_description(self, verbose: Union[bool, str, int] = False, show_hidden=False, prefix: str = ""): """ Return the description (documentation for runtime) of the items in the container. Parameters ---------- verbose If ``True``, include detailed description of the children. If ``'all'``, include even detailed description. If ``int`` is given, print detailed informations up to n levels. show_hidden If False, do not show hidden members. prefix The string, with with each line will begin (commonly the spaces for the indentation). """ cprefix = prefix + self._description_indentation out = [] def write(i): s = i.data_description(verbose, show_hidden, cprefix) if not i.info_in_data_description: if "\n" not in s: info = i.info(False) if info: s = s + (" " * (max(40 - len(s), 0) + 2)) + info else: ccprefix = cprefix + i._description_indentation s += "\n\n" s += ccprefix + i.info(False).replace("\n", "\n" + ccprefix) s += "\n" out.append(s) expert = False for i in self: if i.is_hidden and not show_hidden: continue if not i.is_expert: write(i) else: expert = True if expert: out.append(f"{cprefix}\n{cprefix}Expert options:") out.append(cprefix + "--------------") cprefix += self._description_indentation for i in self: if i.is_expert: if i.is_hidden and not show_hidden: continue write(i) return "\n".join(out)
def __iter__(self): return iter(self._members.values())
[docs] def members(self): return self._members.values()
[docs] def names(self): return self._members.keys()
def __getitem__(self, key): return self._members[key] def __setitem__(self, key, value): self._members[key] = value def __contains__(self, key): return key in self._members
[docs] def get_member(self, name): """Return the definition at the given explicit dotted path. A leading ``..`` selects the parent container, so ``..N`` resolves ``N`` in the direct parent and ``....N`` resolves it two levels up. """ if name.startswith(".."): if self.container is None: raise KeyError(f"No parent of {self}") return self.container.get_member(name[2:]) if "." in name: section_name, child_name = name.split(".", 1) return self._members[section_name].get_member(child_name) return self._members[name]
[docs] def remove(self, name): del self._members[name] return self
[docs] def copy(self, add=[], remove=[], defaults={}, **kwargs): """Get the args to copy the container. The method is just repeated to add named argumets. Parameters ---------- """ arguments = self._get_init_args_for_copy(add, remove, defaults, **kwargs) return self.__class__(**arguments)
[docs] def _get_init_args_for_copy(self, add=[], remove=[], defaults={}, **kwargs): """Get the args to copy the container.""" members = dict(self._members) for i in remove: del members[i] added = self._dict_from_named_values(add) # Replacements are already supplied by the caller. Copy only members # that survive, retaining the original order of replaced members. members = { name: added[name] if name in added else member.copy() for name, member in members.items() } members.update(added) for i, v in defaults.items(): members[i].default_value = members[i].type.convert(v) out = super()._get_init_args_for_copy(**kwargs) out["members"] = members return out
[docs] def copy_member(self, name, **kwargs) -> BaseDefinition: """Copy a member, allowing to redefine its properties. Returns ------- new_member: BaseDefinition The newly created member """ out = self._members[name].copy(**kwargs) self._members[name] = out return out
[docs] def _grammar_of_values( self, allow_dangerous: bool = False, delimiter=None, *, allow_incomplete=False ): if self.custom_class: custom_value = self.custom_member_grammar(self.excluded_names_condition()) else: custom_value = None delimiter = delimiter or self.delimiter def repeated_grammars(): """If the item can be repeated, do it here - we don't know, whether there is a fixed order in any way (e.g. the item is followed by the items without name in grammar) """ for i in self._members.values(): g = i._grammar and i._grammar( allow_dangerous, allow_incomplete=allow_incomplete ) if not g: continue if i.is_repeated and i.repeated_with_name: g = i.repeated_count.grammar(g, i.repeated_delimiter if i.repeated_delimiter is not None else delimiter) yield i, g def grammars(): """This function iterates over the items of the container, joining all the without name_in_grammar with the previous ones.""" it = iter(repeated_grammars()) head_item, grammar_chain = next(it) for item, grammar in it: if item.is_independent_on_the_predecessor: yield head_item, grammar_chain head_item, grammar_chain = item, grammar else: add = delimiter + grammar if item.condition and self.force_order: add = item.condition.prepare_grammar(item, add) # Conditional optionality is checked on the runtime tree. if allow_incomplete or item.is_optional: add = pp.Optional(add) grammar_chain = grammar_chain + add yield head_item, grammar_chain if self.force_order: init = pp.Empty().set_name("<INIT>") def set_loc(loc, toks): init.location = loc init.set_parse_action(set_loc) first = pp.Empty().add_condition(lambda loc, toks: loc == init.location).set_name("<FIRST>") if custom_value: cvs = pp.ZeroOrMore(custom_value + delimiter).set_name("<custom...>") after = delimiter + cvs else: after = pp.Forward() << delimiter after.add_condition(lambda loc, toks: loc != init.location) inter_cvs = (first | after).set_name("<?DELIM>") inter = first | delimiter.copy().add_condition(lambda loc, toks: loc != init.location) def sequence(): for head, g in repeated_grammars(): if head.is_independent_on_the_predecessor: delim = inter_cvs else: delim = inter g = delim + g if head.condition: g = head.condition.prepare_grammar(head, g) # A callback cannot be resolved during grammar construction. if allow_incomplete or head.is_optional: g = pp.Optional(g) yield g values = pp.And([i for i in sequence()]) if custom_value: if not self._first_section_has_to_be_first(): values = cvs + values values += pp.ZeroOrMore(delimiter + custom_value) values = init + values else: it = grammars() # store the first fixed "chain of sections" first = self._first_section_has_to_be_first() and next(it)[1] # the rest has any order values = pp.MatchFirst([i for head, i in it]) if custom_value: values |= custom_value values = delimitedList(values, delimiter) if first: values = first + pp.Optional(delimiter + values) def parse_values(tokens): parsed = dict_from_parsed(tokens.asList(), allow_incomplete=allow_incomplete) # Ordered grammars enforce presence themselves in strict mode. # In incomplete mode they, too, must report omitted members. if not self.force_order or allow_incomplete: for member in self._members.values(): # Conditional presence must be checked against the complete # runtime configuration, not an individual parsed section. if ( member.is_stored and not member.is_optional and member.condition is None and member.name not in parsed and member.name not in parsed.checks ): message = f"Missing non-optional input item {member.get_path()}" if allow_incomplete: DataValidityError.warn(message) else: raise pp.ParseException(message) return parsed values.set_parse_action(parse_values) if self.write_last_delimiter: values = values + pp.Optional(delimiter) if self.is_repeated and not self.repeated_with_name: values = self.repeated_count.grammar(values, self.repeated_delimiter) #values = pp.Group(values, aslist=True).set_name(f"<{self.name}[]>") values = pp.Group(values, aslist=True).set_name(f"<{self.name}[]>") return values
[docs] def _allow_duplicates_of(self, name): """Can a given element (identified by name) have more values in the parsed results? (However, not all definitions have to specify allow_duplicates, just the ones that have a value). For the others, this function raises an error. """ return self[name].allow_duplication()
[docs] def _create_grammar(self, allow_dangerous=False, *, allow_incomplete=False): delimiter = self.delimiter values = self._grammar_of_values( allow_dangerous, delimiter, allow_incomplete=allow_incomplete ) out = self._tuple_with_my_name(values, delimiter) out.set_name(self.name) return out
[docs] @classmethod @cache def delimited_custom_value_grammar(cls): """Return the grammar for the custom child with delimiter. The delimiter can delimite it either from the previous child or from the section name.""" return cls.child_class.delimiter + cls.custom_value_grammar()
custom_name_characters = pp.alphanums + "_-()" """ Which characters can appears in an unknown child (value/section) name """
[docs] @classmethod def custom_member_grammar(cls, name_condition=None): """Grammar for the custom - unknown - child""" name = pp.Word(cls.custom_name_characters).set_parse_action(lambda x: x[0].strip()) if name_condition: name.add_condition(name_condition) out = (name + cls.delimited_custom_value_grammar()).set_parse_action(lambda x: tuple(x)) out.set_name(cls.custom_value_name) return out
[docs] def all_member_names(self): for i in self: yield from i.all_names_in_grammar()
[docs] def excluded_names_condition(self): """Add the condition to the element, that its value is not any of given names""" names = set((_ending_numbers.sub("", i).upper() for i in self.all_member_names())) if not names: return def cond(x): striped = _ending_numbers.sub("", x[0]).upper() return striped not in names return cond
[docs] def _first_section_has_to_be_first(self): """Has/ve the first child(s) in an unordered sequence fixed position?""" return not dict_first_item(self._members).is_independent_on_the_predecessor
[docs] def parse_file(self, file, return_value_only=True, allow_dangerous=False, allow_incomplete=False): """Parse a file, optionally warning about missing required members.""" grammar = self.grammar(allow_dangerous, allow_incomplete=allow_incomplete) out = grammar.parse_file(file, parse_all=True) return self.parse_return(out, return_value_only)
[docs] def parse(self, string, whole_string=True, return_value_only=True, allow_dangerous=False, allow_incomplete=False): """Parse a string, optionally warning about missing required members.""" grammar = self.grammar(allow_dangerous, allow_incomplete=allow_incomplete) out = grammar.parse_string(string, parse_all=True) return self.parse_return(out, return_value_only)
[docs] def parse_return(self, val, return_value_only: bool = True): """Clean up the parsed values (unpack then from unnecessary containers) Parameters ---------- return_value_only Return only value, not name - value tuple """ val = val[0] if return_value_only: val = val[1] return val
[docs] async def parse_from_stream( self, stream, up_to, start=None, whole_string=True, return_value_only=True, allow_dangerous=False ): """ Parse string readed from asyncio stream. The stream is readed up to the given delimiter """ result = await stream.readuntil(up_to) result = result[: -len(up_to)].decode("utf8") if start: result = start + result return self.parse(result, whole_string)
[docs] def read_from_file(self, file, allow_dangerous=False, *, allow_incomplete=False, report_invalid=None, **kwargs): """Read a configuration file into an object. ``allow_incomplete`` relaxes presence requirements, not value syntax. """ out = self.result_class(definition=self, **kwargs) out.read_from_file( file, allow_dangerous=allow_dangerous, allow_incomplete=allow_incomplete, report_invalid=report_invalid, ) return out
[docs] def read_from_dict(self, values, **kwargs): out = self.result_class(definition=self, **kwargs) out.set(values, unknown="add") return out
[docs] def read_from_string(self, string, allow_dangerous=False, **kwargs): return self.read_from_file( StringIO(string), allow_dangerous=allow_dangerous, **kwargs )
repeated_class = RepeatedConfigurationContainer """ Class for the repeated sections """
[docs] def create_object(self, container=None, repeated: bool = True): """Create an instance (section) container: BaseConfigurationContainer To which container the created object will belong repeated: Has meaning only for a is_repeated section. Then, if it is True, a Container for repeated values of the section is returned. Otherwise, the container just for one instance of a section is returned. """ self._finalize() if repeated and self.is_repeated: return self.repeated_class(self, container) return super().create_object(container)
[docs] def _save_to_file(self, file, value, always=False, name_in_grammar=None, delimiter="") -> bool: """Save the content of the container to the file (according to the definition) Parameters ---------- file: file File object (open for writing), where the data should be written always: Do not consider conditions Returns ------- something_have_been_written If any value have been written return True, otherwise return False. """ if not always: if not self.write_condition(value._container) or not self.allowed(value._container): return if self.is_expert: if not value.is_changed(): return False else: if not value.has_any_value(): return False if name_in_grammar is None: name_in_grammar = self.name_in_grammar if delimiter: file.write(str(delimiter)) if name_in_grammar: file.write(self.formated_name) file.write(str(self.delimiter)) members = iter(value) if self.write_last_delimiter: for o in members: if o._save_to_file(file, always): file.write(str(self.delimiter)) else: delimiter = "" for o in members: if o._save_to_file(file, always, delimiter=delimiter): delimiter = self.delimiter return True
_ending_numbers = re.compile("[0-9]*$")
[docs] class SectionDefinition(ContainerDefinition): """Base class for definition of the sections in Pot or InputParameters files. It just redefine a few properties/methods to values/behavior typical for the sections """ result_class = Section @property def values(self): return self._members custom_value_name = "CUSTOM_VALUE" """ Just the name that appears in the grammar, when it is printed."""
[docs] @classmethod @cache def delimited_custom_value_grammar(cls): gt = cls.custom_class.grammar_type # here the child (Value) class delimiter should be used out = cls.child_class.delimiter + gt.grammar() optional, df, _ = gt.missing_value() if optional: out = out | pp.Empty().set_parse_action(lambda x: df) return out
[docs] def _generic_info(self): return f"Configuration section {self.name}"
[docs] def accept_value(self, value) -> bool: if isinstance(value, dict): return True return self.is_repeated and isinstance(value, Iterable)
[docs] class ConfigurationRootDefinition(ContainerDefinition): """From this class, the definition of the format of a whole configuration file should be derived.""" write_last_delimiter = False """ Do not print additional newline after the last section """ name_in_grammar = False """ The configuration files has commonly no "name" of its content, they just contains their content. However, in some cases the name_in_grammar could be used for some kind of prefix in the file, however, it is better to use a fixed value for this purpose. """ item_type = "configuration"
[docs] @classmethod def definition_from_dict(cls, name, defs=None): """ Create instance of the definition from a dictionary, creating the sections (and values) definitions recursively. """ def gen(i): section = defs[i] if not isinstance(defs, SectionDefinition): section = cls.child_class(i, section) return section if defs is None: defs = name name = cls.__name__ return cls((gen(i) for i in defs))
[docs] @add_to_signature(ContainerDefinition.__init__, prepend=True) def __init__(self, name, members=[], **kwargs): if not members and not isinstance(name, str): members = name name = self.__class__.__name__ super().__init__(name, members, **kwargs) self._finalize()
@property def sections(self): return self._members custom_value_name = "CUSTOM_SECTION" """ Just the name that appears in the grammar, when it is printed."""
[docs] def _tuple_with_my_name(self, expr, delimiter=None): """Do not create tuple (name, value) for the root class.""" return expr
[docs] def parse_return(self, val, return_value_only=True): """Clean up the parsed values (unpack then from unnecessary containers) There is no name in the parsed results (see how ConfigurationRootDefinition._tuple_with_my_name is redefined). """ val = val[0] return val
[docs] def _create_grammar(self, allow_dangerous=False, *, allow_incomplete=False): """Returns the grammar to parse the configuration file. This method just tweaks the grammar (generated by the common container implementation) to ignore comments, so the comments would be ignored just once. """ out = super()._create_grammar(allow_dangerous, allow_incomplete=allow_incomplete) out = self.add_ignored(out) return out
[docs] def add_ignored(self, grammar): grammar = pp.Suppress(pp.Regex(r"(\s*\n)*")) + grammar grammar = grammar.ignore("#" + pp.restOfLine + pp.LineEnd()) return grammar
[docs] def _generic_info(self): return "Configuration"