Source code for ase2sprkkr.common.process_output_reader

"""Parsers and stream helpers for calculation output."""

import asyncio
import functools
import os
import numpy as np
from .async_utils import run_coro_sync
from .decorators import maybeclassmethod
from .file_utils import filename_from_file


[docs] class ProcessOutputParser: """Parse asynchronous stdout/stderr streams. Process creation, cancellation, file writing and raw-output delivery belong to the execution backend. Descendants only interpret the supplied streams. """
[docs] def __init__(self, print_info=False): self.print_info = print_info
[docs] def result(self, output, error, return_code): """Postprocess parsed stdout/stderr after execution completes.""" if return_code != 0: exc = ValueError(f"The process ended with return value {return_code}") exc.output = output exc.error = error raise exc return output, error, return_code
[docs] async def read_error(self, stderr, *args): while True: line = await stderr.readline() if not line: return
[docs] async def read_output(self, stdout, *args): raise NotImplementedError("Please, redefine ProcessOutputParser.read_output coroutine")
[docs] @maybeclassmethod def read_from_file(self, cls, output, error=None, read_args=[], return_code=0): """Synchronous wrapper for reading output/error from files.""" if not self: self = cls() return run_coro_sync(self.read_output_file(output, error, read_args, return_code))
[docs] async def read_output_file(self, output, error=None, read_args=[], return_code=0): """Read output and error files through the same async parser API.""" async def read_output_file(): with AsyncioFileReader(output) as air: return await self.read_output(air, *read_args) async def read_error_file(): if not error: return None with AsyncioFileReader(error) as air: return await self.read_error(air, *read_args) out_result, err_result = await asyncio.gather(read_output_file(), read_error_file()) return self.result(out_result, err_result, return_code)
[docs] class AsyncioFileReader: """File reader that mimics asyncio StreamReader. It is synchronous internally, but exposes the async methods used by output parsers so the same parser can consume a subprocess stream or a saved file. """
[docs] def __init__(self, filename, buffersize=8192): self._owns_file = isinstance(filename, (str, bytes, os.PathLike)) self.file = None self.file = open(os.fspath(filename), "rb") if self._owns_file else filename self.buffersize = buffersize
def __fspath__(self): filename = filename_from_file(self.file, None) if filename is None: raise TypeError("The underlying stream has no filesystem path") return filename
[docs] def close(self): file, self.file = self.file, None if file is not None and self._owns_file: file.close()
def __del__(self): self.close() def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() async def __aiter__(self): while (line := self.file.readline()) != b"": yield line def __anext__(self): return self.readline()
[docs] async def readline(self): return self.file.readline()
[docs] @staticmethod @functools.lru_cache(maxsize=128) def separator_shifts(sep): shifts = np.ones(len(sep) + 1, dtype=int) shift = 1 for pos in range(len(sep)): while shift <= pos and sep[pos] != sep[pos - shift]: shift += shifts[pos - shift] shifts[pos + 1] = shift return shifts
[docs] async def readuntil(self, sep=b"\n"): lsep = len(sep) if lsep == 0: return b"" shifts = self.separator_shifts(sep) out = b"" buffer = bytearray(self.buffersize + lsep) bufEnd = 0 bufPos = 0 def result(): if not out: return bytes(buffer[:bufPos]) return out + bytes(buffer[:bufPos]) startPos = 0 matchLen = 0 while True: if bufPos == bufEnd: if bufEnd >= self.buffersize: out += buffer[:bufEnd] bufEnd = bufPos = 0 n = lsep - matchLen data = self.file.read(n) ldata = len(data) buffer[bufPos : bufPos + ldata] = data bufEnd += ldata if len(data) < n: raise asyncio.IncompleteReadError(result(), sep) c = buffer[bufPos] bufPos += 1 while matchLen >= 0 and sep[matchLen] != c: startPos += shifts[matchLen] matchLen -= shifts[matchLen] matchLen += 1 if matchLen == lsep: return result()
[docs] async def readline(stdout): line = await stdout.readline() if not line: raise EOFError() return line.decode("utf8")
[docs] async def readline_until(stdout, cond, can_end=True): while True: line = await stdout.readline() if not line: if can_end: return None raise EOFError() if cond(line): return line.decode("utf8")