#!/usr/bin/python3
"""hugpy drive relay: the only kernel reader for hugpy's model drives; it writes the event log.

fanotify notification-class group with FAN_REPORT_DFID_NAME_TARGET and FAN_MARK_FILESYSTEM on
each filesystem holding a declared root. Modify-class events only (modify, close_write, attrib,
moved_from, moved_to, create, delete, delete_self; never access/open), filtered to paths under
the roots (minus excluded state dirs), coalesced, and appended to an append-only, hash-chained
log (format and chain rules: relay_log.py beside this file). Group commit: one write() per
batch, fdatasync in a background thread at most every --sync-ms (or per --sync-bytes), and on
heartbeat. The unix socket is only a wake ping ({"type":"hello"|"wake","seq":N}).

It never reads file contents. Root or CAP_SYS_ADMIN (filesystem marks) + CAP_DAC_READ_SEARCH
(open_by_handle_at) + CAP_LINUX_IMMUTABLE (chattr +a/+i on segments; optional).

    hugpy-drive-relay run --root DIR [--root DIR ...] [--state DIR] [--socket PATH]
    hugpy-drive-relay register NAME --user USER [--group hugpy] [--start head|genesis]
    hugpy-drive-relay verify [--state DIR]
"""
import argparse
import ctypes
import ctypes.util
import datetime
import errno
import grp
import json
import os
import pwd
import select
import signal
import socket
import struct
import sys
import threading
import time
from collections import OrderedDict

sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
import relay_log as rl  # noqa: E402

RELAY_VERSION = '2.0-proto'

FAN_CLOEXEC, FAN_NONBLOCK, FAN_UNLIMITED_QUEUE = 0x1, 0x2, 0x10
FAN_REPORT_FID, FAN_REPORT_DIR_FID, FAN_REPORT_NAME, FAN_REPORT_TARGET_FID = 0x200, 0x400, 0x800, 0x1000
FAN_REPORT_DFID_NAME_TARGET = FAN_REPORT_FID | FAN_REPORT_DIR_FID | FAN_REPORT_NAME | FAN_REPORT_TARGET_FID
FAN_MARK_ADD, FAN_MARK_FILESYSTEM = 0x1, 0x100
FAN_MODIFY, FAN_ATTRIB, FAN_CLOSE_WRITE = 0x2, 0x4, 0x8
FAN_MOVED_FROM, FAN_MOVED_TO, FAN_CREATE, FAN_DELETE, FAN_DELETE_SELF = 0x40, 0x80, 0x100, 0x200, 0x400
FAN_Q_OVERFLOW, FAN_ONDIR = 0x4000, 0x40000000
BITS = ((FAN_MODIFY, 'modify'), (FAN_CLOSE_WRITE, 'close_write'), (FAN_ATTRIB, 'attrib'),
        (FAN_MOVED_FROM, 'moved_from'), (FAN_MOVED_TO, 'moved_to'), (FAN_CREATE, 'create'),
        (FAN_DELETE, 'delete'), (FAN_DELETE_SELF, 'delete_self'))
assert tuple(n for _, n in BITS) == rl.MASK_NAMES
WANTED = sum(bit for bit, _ in BITS)
NAMESPACE = FAN_MOVED_FROM | FAN_MOVED_TO | FAN_CREATE | FAN_DELETE | FAN_DELETE_SELF
MOVES = FAN_MOVED_FROM | FAN_MOVED_TO
INFO_FID, INFO_DFID_NAME, INFO_DFID = 1, 2, 3
META = struct.Struct('=IBBHQii')
INFO = struct.Struct('=BBH')
U32 = struct.Struct('=I')
AT_FDCWD, AT_EMPTY_PATH = -100, 0x1000
STATX_MASK = 0x7ff | 0x800           # STATX_BASIC_STATS | STATX_BTIME
CLOCK_REALTIME, TFD_TIMER_ABSTIME, TFD_TIMER_CANCEL_ON_SET = 0, 1, 2
CLOCK_BACK_TOL_NS = 10_000_000       # realtime-minus-monotonic may only change by a clock step
CLOCK_FWD_TOL_NS = 1_000_000_000
READ_BUFFER = 1 << 18
DIR_CACHE, FH_CACHE = 16384, 65536
DEFAULT_EXCLUDES = ('/var/lib/hugpy/discovery',)

libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True)
libc.fanotify_init.argtypes = [ctypes.c_uint, ctypes.c_uint]
libc.fanotify_mark.argtypes = [ctypes.c_int, ctypes.c_uint, ctypes.c_uint64, ctypes.c_int, ctypes.c_char_p]
libc.open_by_handle_at.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int]
libc.name_to_handle_at.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p,
                                   ctypes.POINTER(ctypes.c_int), ctypes.c_int]
libc.statx.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_uint, ctypes.c_char_p]
libc.statfs.argtypes = [ctypes.c_char_p, ctypes.c_char_p]


class timespec(ctypes.Structure):
    _fields_ = [('tv_sec', ctypes.c_long), ('tv_nsec', ctypes.c_long)]


class itimerspec(ctypes.Structure):
    _fields_ = [('it_interval', timespec), ('it_value', timespec)]


libc.timerfd_create.argtypes = [ctypes.c_int, ctypes.c_int]
libc.timerfd_settime.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.POINTER(itimerspec), ctypes.c_void_p]


def fail(what):
    error = ctypes.get_errno()
    raise OSError(error, f'{what}: {os.strerror(error)}')


def fsid_of(path):
    """Raw kernel fsid (8 bytes, exactly as fanotify reports it) via statfs(2)."""
    buf = ctypes.create_string_buffer(120)
    if libc.statfs(os.fsencode(path), buf) < 0:
        fail(f'statfs {path}')
    return buf.raw[56:64]


def fsid_hex(key):
    val0, val1 = struct.unpack('=II', key)
    return f'{val0:08x}{val1:08x}'


def handle_str(handle):
    size, kind = struct.unpack_from('=Ii', handle)
    return f'{kind}:{handle[8:8 + size].hex()}'


def name_to_handle(path):
    buf = ctypes.create_string_buffer(8 + 128)
    struct.pack_into('=Ii', buf, 0, 128, 0)
    mount_id = ctypes.c_int()
    if libc.name_to_handle_at(AT_FDCWD, os.fsencode(path), buf, ctypes.byref(mount_id), 0) < 0:
        return None, None
    size = U32.unpack_from(buf.raw)[0]
    return buf.raw[:8 + size], mount_id.value


_STX = ctypes.create_string_buffer(256)


def statx_fd(fd):
    if libc.statx(fd, b'', AT_EMPTY_PATH, STATX_MASK, _STX) < 0:
        return None
    raw = _STX.raw
    mask = U32.unpack_from(raw, 0)[0]
    nlink, uid, gid = struct.unpack_from('=III', raw, 16)
    mode = struct.unpack_from('=H', raw, 28)[0]
    ino, size = struct.unpack_from('=QQ', raw, 32)

    def ts(offset):
        sec, nsec = struct.unpack_from('=qI', raw, offset)
        return sec * 1_000_000_000 + nsec
    major, minor = struct.unpack_from('=II', raw, 136)
    return dict(size=size, mtime_ns=ts(112), ctime_ns=ts(96), btime_ns=ts(80) if mask & 0x800 else None,
                ino=ino, dev=os.makedev(major, minor), uid=uid, gid=gid, mode=mode, nlink=nlink)


def mount_point(path):
    dev = os.stat(path).st_dev
    while path != '/':
        parent = os.path.dirname(path)
        if os.stat(parent).st_dev != dev:
            return path
        path = parent
    return '/'


def anchor(entry):
    """Hand a chain head (a checkpoint entry: seq, h, head_seq, head_h) to the off-host ledger.

    Board t4696; deliberately unimplemented. When implemented it must queue to its own thread
    and never block, slow or fail the event loop; the ledger stores (boot, seq, h, rt_ns) and
    a verifier passes those back as relay_log.verify(anchors={seq: h})."""
    return None


class Filesystem:
    __slots__ = ('key', 'fsid', 'mount_fd', 'dev', 'mount', 'roots')

    def __init__(self, key, root):
        self.key, self.fsid = key, fsid_hex(key)
        self.mount_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC)
        st = os.fstat(self.mount_fd)
        self.dev, self.mount, self.roots = st.st_dev, mount_point(root), [root]


class Syncer(threading.Thread):
    """fdatasync off the event loop; kicks while one is running coalesce into one more."""

    def __init__(self, writer):
        super().__init__(daemon=True, name='relay-sync')
        self.writer = writer
        self.cv = threading.Condition()
        self.want = False
        self.count = 0
        self.total_ns = 0
        self.max_ns = 0

    def kick(self):
        with self.cv:
            self.want = True
            self.cv.notify()

    def run(self):
        while True:
            with self.cv:
                while not self.want:
                    self.cv.wait()
                self.want = False
            start = time.monotonic_ns()
            try:
                if self.writer.sync():
                    took = time.monotonic_ns() - start
                    self.count += 1
                    self.total_ns += took
                    self.max_ns = max(self.max_ns, took)
            except OSError as exc:
                print(f'relay: fdatasync failed: {exc}', file=sys.stderr, flush=True)


class Relay:
    def __init__(self, args):
        self.args = args
        self.pid = os.getpid()
        self.state = os.path.realpath(args.state)
        self.roots = sorted({os.path.realpath(r) for r in args.root})
        socket_dir = os.path.realpath(os.path.dirname(args.socket))
        self.excludes = sorted({os.path.realpath(x) for x in (*DEFAULT_EXCLUDES, *args.exclude, self.state, socket_dir)})
        self.root_prefixes = [r if r.endswith('/') else r + '/' for r in self.roots]
        self.exclude_prefixes = [x if x.endswith('/') else x + '/' for x in self.excludes]
        self.writer = rl.LogWriter(self.state, args.segment_bytes, append_only=not args.no_chattr)
        self.syncer = Syncer(self.writer)
        self.filesystems = {}
        self.pending = OrderedDict()
        self.dir_cache = OrderedDict()
        self.fh_paths = OrderedDict()
        self.clients = {}
        self.stats = dict(raw=0, reads=0, events=0, entries=0, bytes=0, self_dropped=0, out_of_scope=0,
                          unresolved=0, unresolved_self=0, overflows=0, wakes=0, interim=0, compactions=0, alerts=0)
        self.stopping = False
        self.heartbeat_now = False
        self.compact_now = False
        self.compacting = False
        self.compaction_results = None
        self.unsynced = 0
        self.unsynced_since = None
        self.last_kick = 0.0
        self.last_wake = 0.0
        self.wake_due = None
        self.wake_seq = 0
        self.clock = None
        self.root_ids = {}
        self.started = time.monotonic()

    # --- setup --------------------------------------------------------------------------

    def setup(self):
        a = self.args
        flags = FAN_CLOEXEC | FAN_NONBLOCK | FAN_REPORT_DFID_NAME_TARGET
        if a.unlimited_queue:
            flags |= FAN_UNLIMITED_QUEUE
        self.fan = libc.fanotify_init(flags, os.O_RDONLY | os.O_LARGEFILE | os.O_CLOEXEC)
        if self.fan < 0:
            fail('fanotify_init')
        mask = WANTED | FAN_ONDIR
        for root in self.roots:
            key = fsid_of(root)
            if key in self.filesystems:
                self.filesystems[key].roots.append(root)
                continue
            if libc.fanotify_mark(self.fan, FAN_MARK_ADD | FAN_MARK_FILESYSTEM, mask, AT_FDCWD, os.fsencode(root)) < 0:
                fail(f'fanotify_mark {root}')
            self.filesystems[key] = Filesystem(key, root)
            st = os.stat(root)
            self.root_ids[root] = (st.st_dev, st.st_ino)
        self._cleanup_compaction_leftovers()
        consumers = os.path.join(self.state, 'consumers')
        if not os.path.isdir(consumers):
            # 0750, no setgid: RestrictSUIDSGID= forbids it here. `register` (root) makes each
            # consumers/<name>/ 2750 <user>:hugpy so cursor files stay group-hugpy readable.
            os.mkdir(consumers, 0o750)
        append_only, flag_note = self._probe_flags()
        self.writer.append_only = append_only
        last = self.writer.open()
        boot = self.writer.boot
        cursors = rl.registered(self.state)
        floor = max((c['seq'] for c in cursors.values() if c), default=0)
        now_rt, now_mono = time.time_ns(), time.monotonic_ns()
        if last is None:
            reason = 'first-start'
        else:
            reason = 'restart' if last['type'] == 'stop' else 'unclean-restart'
            if last['boot'] != boot:
                reason += '+reboot'
        if self.writer.torn_bytes:
            reason += '+torn-tail'
        seq = None
        if floor > self.writer.seq:
            reason += '+log-behind-cursor'
            seq = floor + 1
        self.append('gap', seq=seq, prev_seq=self.writer.seq, prev_rt_ns=last and last['rt_ns'],
                    prev_boot=last and last['boot'], reason=reason)
        if last is not None and last['boot'] == boot:
            self._check_clock(now_rt, now_mono, last['rt_ns'] - last['mono_ns'], 'restart')
        marks = []
        for fs in self.filesystems.values():
            for root in fs.roots:
                handle, mount_id = name_to_handle(root)
                marks.append(dict(fsid=fs.fsid, root=root, mount=fs.mount, dev=fs.dev, mount_id=mount_id,
                                  fh=handle_str(handle) if handle else None))
        self.append('start', version=RELAY_VERSION, pid=self.pid, uid=os.geteuid(), roots=self.roots,
                    excludes=self.excludes, marks=marks, append_only=append_only, append_only_note=flag_note,
                    config=dict(segment_bytes=a.segment_bytes, heartbeat_s=a.heartbeat_s, sync_ms=a.sync_ms,
                                sync_bytes=a.sync_bytes, idle_ms=a.idle_ms, max_hold_s=a.max_hold_s,
                                wake_ms=a.wake_ms, checkpoint_s=a.checkpoint_s,
                                unlimited_queue=bool(a.unlimited_queue)))
        self.commit(force_sync=True)
        self.syncer.start()
        self._open_socket()
        self._open_timerfd()
        self.sig_r, self.sig_w = socket.socketpair()
        self.sig_r.setblocking(False)
        self.sig_w.setblocking(False)
        signal.set_wakeup_fd(self.sig_w.fileno())
        for sig, action in ((signal.SIGTERM, 'stop'), (signal.SIGINT, 'stop'),
                            (signal.SIGUSR1, 'heartbeat'), (signal.SIGUSR2, 'compact')):
            signal.signal(sig, self._signal(action))
        now = time.monotonic()
        self.next_heartbeat = now + a.heartbeat_s
        self.next_checkpoint = now + a.checkpoint_s
        self.next_compact_check = now + a.compact_check_s
        self.next_hold_scan = now + 1.0
        print(f'relay: seq {self.writer.seq}, {len(self.filesystems)} filesystem(s), roots {self.roots}, '
              f'append-only {append_only} ({flag_note}), gap reason {reason}', file=sys.stderr, flush=True)

    def _signal(self, action):
        def handler(_sig, _frame):
            if action == 'stop':
                self.stopping = True
            elif action == 'heartbeat':
                self.heartbeat_now = True
            else:
                self.compact_now = True
        return handler

    def _probe_flags(self):
        if self.args.no_chattr:
            return False, 'disabled'
        probe = os.path.join(self.writer.log_dir, f'.flagprobe.{self.pid}')
        fd = os.open(probe, os.O_WRONLY | os.O_CREAT | os.O_APPEND | os.O_CLOEXEC, 0o600)
        try:
            rl.set_flags(fd, add=rl.FS_APPEND_FL)
            rl.set_flags(fd, remove=rl.FS_APPEND_FL)
            return True, 'ok'
        except OSError as exc:
            return False, exc.strerror
        finally:
            os.close(fd)
            os.unlink(probe)

    def _cleanup_compaction_leftovers(self):
        log_dir = self.writer.log_dir
        for name in os.listdir(log_dir):
            if name.startswith('.') and name.endswith('.tmp'):
                os.unlink(os.path.join(log_dir, name))
        for name in os.listdir(log_dir):
            if name.endswith(rl.CSEG):
                full = os.path.join(log_dir, name[:-len(rl.CSEG)] + rl.SEG)
                if os.path.exists(full):  # rename happened after the record was durable
                    try:
                        rl.path_flags(full, remove=rl.FS_APPEND_FL | rl.FS_IMMUTABLE_FL)
                    except OSError:
                        pass
                    os.unlink(full)

    def _open_socket(self):
        path = self.args.socket
        os.makedirs(os.path.dirname(path), exist_ok=True)
        try:
            os.unlink(path)
        except FileNotFoundError:
            pass
        self.server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM | socket.SOCK_CLOEXEC)
        self.server.bind(path)
        os.chmod(path, 0o660)
        if self.args.socket_group:
            try:
                os.chown(path, -1, grp.getgrnam(self.args.socket_group).gr_gid)
            except (KeyError, PermissionError) as exc:
                print(f'relay: socket group {self.args.socket_group}: {exc}', file=sys.stderr, flush=True)
        self.server.listen(16)
        self.server.setblocking(False)

    def _open_timerfd(self):
        """A realtime timer cancelled by any clock_settime: immediate clock-step detection."""
        self.tfd = libc.timerfd_create(CLOCK_REALTIME, os.O_CLOEXEC | os.O_NONBLOCK)
        if self.tfd < 0:
            self.tfd = None
            return
        self._arm_timerfd()

    def _arm_timerfd(self):
        spec = itimerspec()
        spec.it_value.tv_sec = int(time.time()) + 10 * 365 * 86400
        if libc.timerfd_settime(self.tfd, TFD_TIMER_ABSTIME | TFD_TIMER_CANCEL_ON_SET, ctypes.byref(spec), None) < 0:
            os.close(self.tfd)
            self.tfd = None

    # --- appending ----------------------------------------------------------------------

    def _check_clock(self, rt, mono, previous_offset, where):
        offset = rt - mono
        step = offset - previous_offset
        if step < -CLOCK_BACK_TOL_NS:
            signal_name = 'realtime-backwards'
        elif step > CLOCK_FWD_TOL_NS:
            signal_name = 'realtime-forward-jump'
        else:
            return
        self.stats['alerts'] += 1
        self.writer.append('alert', rt_ns=rt, mono_ns=mono, signal=signal_name,
                           detail=dict(step_ns=step, where=where, prev_offset_ns=previous_offset,
                                       offset_ns=offset))

    def append(self, type_, **fields):
        rt, mono = time.time_ns(), time.monotonic_ns()
        offset = rt - mono
        if self.clock is not None:
            self._check_clock(rt, mono, self.clock, 'running')
        self.clock = offset
        entry = self.writer.append(type_, rt_ns=rt, mono_ns=mono, **fields)
        self.stats['entries'] += 1
        return entry

    def commit(self, force_sync=False):
        written = self.writer.flush()
        now = time.monotonic()
        if written:
            self.stats['bytes'] += written
            self.unsynced += written
            if self.unsynced_since is None:
                self.unsynced_since = now
            self.wake_seq = self.writer.seq
            if self.wake_due is None:
                self.wake_due = max(now, self.last_wake + self.args.wake_ms / 1000)
        if self.writer.segment is not None and not self.writer.sealed \
                and self.writer.segment_size >= self.args.segment_bytes:
            self.rotate()
            now = time.monotonic()
        if force_sync:
            if self.unsynced:
                self.writer.sync()
                self.syncer.count += 1
            self.unsynced, self.unsynced_since = 0, None
        elif self.unsynced and (now - self.unsynced_since >= self.args.sync_ms / 1000 or (
                self.unsynced >= self.args.sync_bytes and now - self.last_kick >= self.args.sync_min_gap_ms / 1000)):
            self.syncer.kick()
            self.last_kick = now
            self.unsynced, self.unsynced_since = 0, None
        if self.wake_due is not None and now >= self.wake_due:
            self.wake(now)

    def rotate(self):
        entry = self.writer.seal()
        self.unsynced, self.unsynced_since = 0, None
        if entry is not None:
            anchor(entry)
            self.compact_now = True

    def wake(self, now):
        line = (rl.canonical(dict(type='wake', seq=self.writer.seq)) + '\n').encode()
        for fd, client in list(self.clients.items()):
            try:
                client.send(line)
            except BlockingIOError:
                pass  # it has unread wakes; it will look at the log anyway
            except OSError:
                self.drop(fd)
        self.stats['wakes'] += 1
        self.last_wake = now
        self.wake_due = None

    # --- kernel events ------------------------------------------------------------------

    def read_events(self):
        for _ in range(16):
            try:
                data = os.read(self.fan, READ_BUFFER)
            except BlockingIOError:
                return
            self.stats['reads'] += 1
            self.process(data)
            self.commit()
            if len(data) < READ_BUFFER // 2:
                return

    def process(self, data):
        batch = OrderedDict()
        now = time.monotonic()
        offset, size = 0, len(data)
        while offset + META.size <= size:
            length, _ver, _res, header, mask, efd, pid = META.unpack_from(data, offset)
            if length < META.size:
                break
            end = offset + length
            if efd >= 0:
                os.close(efd)
            self.stats['raw'] += 1
            if mask & FAN_Q_OVERFLOW:
                self.emit_batch(batch)
                self.stats['overflows'] += 1
                self.append('overflow', fsids=sorted(fs.fsid for fs in self.filesystems.values()))
                print('relay: fanotify queue overflow', file=sys.stderr, flush=True)
                offset = end
                continue
            if pid == self.pid:
                self.stats['self_dropped'] += 1
                offset = end
                continue
            dfid = name = fid = key = None
            info = offset + header
            while info + INFO.size <= end:
                kind, _pad, info_len = INFO.unpack_from(data, info)
                if info_len == 0:
                    break
                if kind in (INFO_FID, INFO_DFID_NAME, INFO_DFID):
                    key = data[info + 4:info + 12]
                    hbytes = U32.unpack_from(data, info + 12)[0]
                    handle = data[info + 12:info + 20 + hbytes]
                    if kind == INFO_FID:
                        fid = handle
                    else:
                        dfid = handle
                        if kind == INFO_DFID_NAME:
                            name = data[info + 20 + hbytes:info + info_len].split(b'\0', 1)[0]
                info += info_len
            offset = end
            bits = mask & WANTED
            fs = self.filesystems.get(key)
            if bits and fs is not None:
                self.event(fs, bits, bool(mask & FAN_ONDIR), dfid, name, fid, pid, batch, now)
        self.emit_batch(batch)

    def dir_path(self, fs, handle):
        cache_key = (fs.key, handle)
        path = self.dir_cache.get(cache_key)
        if path is not None:
            self.dir_cache.move_to_end(cache_key)
            return path
        fd = libc.open_by_handle_at(fs.mount_fd, handle, os.O_PATH | os.O_CLOEXEC)
        if fd < 0:
            return self.fh_paths.get(cache_key)
        try:
            path = os.readlink(f'/proc/self/fd/{fd}')
        finally:
            os.close(fd)
        if path.endswith(' (deleted)'):
            return path[:-len(' (deleted)')]
        self.dir_cache[cache_key] = path
        if len(self.dir_cache) > DIR_CACHE:
            self.dir_cache.popitem(last=False)
        return path

    def in_scope(self, path, is_dir, bits):
        for prefix in self.exclude_prefixes:
            if path.startswith(prefix) or path + '/' == prefix:
                return False
        for prefix in self.root_prefixes:
            if path.startswith(prefix) or path + '/' == prefix:
                return True
        if is_dir and bits & NAMESPACE:  # an ancestor of a root was moved or removed
            probe = path.rstrip('/') + '/'
            return any(root.startswith(probe) for root in self.roots)
        return False

    def event(self, fs, bits, is_dir, dfid, name, fid, pid, batch, now):
        self.stats['events'] += 1
        if is_dir and bits & MOVES:
            self.dir_cache.clear()  # every path below a moved directory changed
            self.fh_paths.clear()
        obj = fid if fid is not None else (dfid if name in (None, b'.') else None)
        if dfid is not None and name not in (None, b'.'):
            directory = self.dir_path(fs, dfid)
            base = os.fsdecode(name)
            path = (directory.rstrip('/') + '/' + base) if directory else None
        elif dfid is not None:
            path = self.dir_path(fs, dfid)
            directory, base = (os.path.dirname(path), os.path.basename(path)) if path else (None, None)
        else:
            path = self.fh_paths.get((fs.key, obj)) if obj is not None else None
            directory, base = (os.path.dirname(path), os.path.basename(path)) if path else (None, None)
        if path is not None:
            if not self.in_scope(path, is_dir, bits):
                self.stats['out_of_scope'] += 1
                return
            if obj is not None:
                self.fh_paths[(fs.key, obj)] = path
                if len(self.fh_paths) > FH_CACHE:
                    self.fh_paths.popitem(last=False)
        else:
            if dfid is None and not bits & (FAN_MODIFY | FAN_CLOSE_WRITE):
                # Inode-only record (attrib/delete_self without parent) of an object never seen
                # in scope: its unlink or rename-over is reported, with a path, by the parent.
                self.stats['unresolved_self'] += 1
                return
            self.stats['unresolved'] += 1
        if path is None or not bits & NAMESPACE:
            # Coalesced per object until close_write, 1 s idle or --max-hold-s.
            if path is None:
                key = ('unresolved', fs.key, dfid if dfid is not None else obj)
            else:
                key = (fs.key, obj if obj is not None else dfid)
            rec = self.pending.get(key)
            if rec is None:
                rec = self.pending[key] = self.record(fs, obj, dfid, path, directory, base, is_dir, now)
            else:
                self.pending.move_to_end(key)
            self.merge(rec, bits, pid, path, directory, base, now)
            if bits & FAN_CLOSE_WRITE and path is not None:
                del self.pending[key]
                batch[('final', key, id(rec))] = rec
            return
        pending = self.pending.pop((fs.key, obj), None) if obj is not None else None
        if pending is not None:
            batch[('final', (fs.key, obj), id(pending))] = pending
        bkey = (fs.key, obj, path)
        rec = batch.get(bkey)
        if rec is None:
            rec = batch[bkey] = self.record(fs, obj, dfid, path, directory, base, is_dir, now)
        self.merge(rec, bits, pid, path, directory, base, now)

    @staticmethod
    def record(fs, obj, dfid, path, directory, base, is_dir, now):
        return dict(fs=fs, obj=obj, dfid=dfid, bits=0, count=0, path=path, dir=directory, name=base,
                    is_dir=is_dir, pid=None, pids=set(), first=now, last=now)

    @staticmethod
    def merge(rec, bits, pid, path, directory, base, now):
        rec['bits'] |= bits
        rec['count'] += 1
        rec['pid'] = pid
        if len(rec['pids']) < 16:
            rec['pids'].add(pid)
        if path is not None:
            rec['path'], rec['dir'], rec['name'] = path, directory, base
        rec['last'] = now

    def statx(self, rec):
        if rec['obj'] is None or rec['path'] is None:
            return None
        fd = libc.open_by_handle_at(rec['fs'].mount_fd, rec['obj'], os.O_PATH | os.O_CLOEXEC)
        if fd < 0:
            return None
        try:
            st = statx_fd(fd)
        finally:
            os.close(fd)
        if st is None or st['nlink'] == 0:
            return None
        return st

    def emit(self, rec):
        fields = dict(mask=[n for bit, n in BITS if rec['bits'] & bit], count=rec['count'],
                      fsid=rec['fs'].fsid, path=rec['path'], dir=rec['dir'], name=rec['name'],
                      is_dir=rec['is_dir'], statx=self.statx(rec), pid=rec['pid'])
        if rec['path'] is None and rec['dfid'] is not None:
            handle = rec['dfid']  # unresolvable: the parent directory's handle (statx is null)
        else:
            handle = rec['obj'] if rec['obj'] is not None else rec['dfid']
        fields['fh'] = handle_str(handle)
        if len(rec['pids']) > 1:
            fields['pids'] = sorted(rec['pids'])
        self.append('event', **fields)

    def emit_batch(self, batch):
        for rec in batch.values():
            self.emit(rec)
        batch.clear()

    def flush_pending(self, now, force=False):
        idle = self.args.idle_ms / 1000
        while self.pending:
            key, rec = next(iter(self.pending.items()))
            if not force and rec['last'] + idle > now:
                break
            del self.pending[key]
            self.emit(rec)
        if self.pending and (force or now >= self.next_hold_scan):
            hold = self.args.max_hold_s
            for key, rec in list(self.pending.items()):
                if force or rec['first'] + hold <= now:
                    del self.pending[key]
                    self.stats['interim'] += 1
                    self.emit(rec)
            self.next_hold_scan = now + 1.0

    # --- clients ------------------------------------------------------------------------

    def accept(self, poller):
        while True:
            try:
                client, _ = self.server.accept()
            except BlockingIOError:
                return
            client.setblocking(False)
            try:
                client.send((rl.canonical(dict(type='hello', seq=self.writer.seq)) + '\n').encode())
            except OSError:
                client.close()
                continue
            self.clients[client.fileno()] = client
            poller.register(client.fileno(), select.POLLIN | select.POLLHUP | select.POLLERR)
            self.poller = poller

    def drop(self, fd):
        client = self.clients.pop(fd, None)
        if client is not None:
            try:
                self.poller.unregister(fd)
            except (KeyError, ValueError):
                pass
            client.close()

    def client_ready(self, fd):
        client = self.clients.get(fd)
        if client is None:
            return
        try:
            if client.recv(4096):
                return  # clients have nothing to say; ignore
        except BlockingIOError:
            return
        except OSError:
            pass
        self.drop(fd)

    # --- periodic -----------------------------------------------------------------------

    def snapshot(self):
        out = dict(self.stats)
        out.update(fsyncs=self.syncer.count, fsync_max_us=self.syncer.max_ns // 1000,
                   fsync_mean_us=self.syncer.total_ns // max(1, self.syncer.count) // 1000,
                   pending=len(self.pending), clients=len(self.clients),
                   segment=self.writer.segment.stem if self.writer.segment else None,
                   segment_bytes=self.writer.segment_size, uptime_s=int(time.monotonic() - self.started),
                   flag_error=self.writer.flag_error)
        try:
            with open('/proc/self/stat') as handle:
                fields = handle.read().rsplit(')', 1)[1].split()
            out['cpu_ticks'] = int(fields[11]) + int(fields[12])
            with open('/proc/self/status') as handle:
                for line in handle:
                    if line.startswith(('VmRSS:', 'VmHWM:')):
                        out[line.split(':')[0].lower() + '_kb'] = int(line.split()[1])
        except OSError:
            pass
        return out

    def heartbeat(self):
        self.append('heartbeat', stats=self.snapshot())
        for root, ident in self.root_ids.items():
            try:
                st = os.stat(root)
                now_ident = (st.st_dev, st.st_ino)
            except OSError:
                now_ident = None
            if now_ident != ident:
                self.stats['alerts'] += 1
                self.append('alert', signal='root-moved', detail=dict(root=root, was=list(ident),
                                                                      now=list(now_ident) if now_ident else None))
                self.root_ids[root] = now_ident
        if self.writer.segment_rt_ns and not self.writer.sealed:
            day = datetime.datetime.fromtimestamp(self.writer.segment_rt_ns / 1e9, datetime.timezone.utc).date()
            if day != datetime.datetime.now(datetime.timezone.utc).date():
                self.commit()
                self.rotate()
        self.commit(force_sync=True)

    def checkpoint(self):
        if self.writer.segment is None or self.writer.sealed:
            return
        entry = self.append('checkpoint', reason='periodic', segment=self.writer.segment.stem,
                            head_seq=self.writer.seq, head_h=self.writer.h)
        self.commit(force_sync=True)
        anchor(entry)

    def compaction_check(self):
        if self.compacting:
            return
        active = self.writer.segment.first_seq if self.writer.segment and not self.writer.sealed else None
        found, keep = rl.compactable(self.state, active)
        if not found:
            return
        self.compacting = True

        def work():
            results = []
            for segment in found:
                try:
                    before = os.path.getsize(segment.path)
                    lines, record = rl.plan_compaction(segment.path, keep)
                    tmp = rl.write_compacted(segment, lines)
                    results.append((segment, tmp, record, before, os.path.getsize(tmp)))
                except (OSError, rl.ChainError) as exc:
                    print(f'relay: compaction of {segment} skipped: {exc}', file=sys.stderr, flush=True)
            self.compaction_results = results
            try:
                self.sig_w.send(b'c')
            except OSError:
                pass
        threading.Thread(target=work, daemon=True, name='relay-compact').start()

    def finish_compactions(self):
        results, self.compaction_results = self.compaction_results, None
        for segment, tmp, record, before, after in results:
            self.append('compaction', **record)
            self.commit(force_sync=True)
            rl.finish_compaction(segment, tmp, immutable=self.writer.append_only)
            self.stats['compactions'] += 1
            print(f'relay: compacted {segment}: {before} -> {after} bytes, kept {record["kept"]}, '
                  f'dropped {record["dropped"]}', file=sys.stderr, flush=True)
        self.compacting = False

    def timeout_ms(self, now):
        deadlines = [self.next_heartbeat, self.next_checkpoint, self.next_compact_check]
        if self.pending:
            first = next(iter(self.pending.values()))
            deadlines += [first['last'] + self.args.idle_ms / 1000, self.next_hold_scan]
        if self.unsynced_since is not None:
            deadlines.append(self.unsynced_since + self.args.sync_ms / 1000)
        if self.wake_due is not None:
            deadlines.append(self.wake_due)
        return max(0, int((min(deadlines) - now) * 1000) + 1)

    def tick(self):
        now = time.monotonic()
        if self.pending:
            self.flush_pending(now)
        if self.compaction_results is not None:
            self.finish_compactions()
        if self.heartbeat_now or now >= self.next_heartbeat:
            self.heartbeat_now = False
            self.next_heartbeat = now + self.args.heartbeat_s
            self.heartbeat()
        if now >= self.next_checkpoint:
            self.next_checkpoint = now + self.args.checkpoint_s
            self.checkpoint()
        if self.compact_now or now >= self.next_compact_check:
            self.compact_now = False
            self.next_compact_check = now + self.args.compact_check_s
            self.compaction_check()
        self.commit()

    def clock_set(self):
        try:
            os.read(self.tfd, 8)
        except OSError as exc:
            if exc.errno != errno.ECANCELED:
                return
        self.append('heartbeat', stats=dict(self.snapshot(), clock_set=True))
        self._arm_timerfd()
        self.commit(force_sync=True)

    def run(self):
        self.setup()
        poller = self.poller = select.poll()
        poller.register(self.fan, select.POLLIN)
        poller.register(self.server.fileno(), select.POLLIN)
        poller.register(self.sig_r.fileno(), select.POLLIN)
        if self.tfd is not None:
            poller.register(self.tfd, select.POLLIN)
        while not self.stopping:
            for fd, _flags in poller.poll(self.timeout_ms(time.monotonic())):
                if fd == self.fan:
                    self.read_events()
                elif fd == self.server.fileno():
                    self.accept(poller)
                elif fd == self.sig_r.fileno():
                    try:
                        while self.sig_r.recv(4096):
                            pass
                    except BlockingIOError:
                        pass
                elif fd == self.tfd:
                    self.clock_set()
                else:
                    self.client_ready(fd)
            self.tick()
        self.shutdown()

    def shutdown(self):
        self.read_events()
        self.flush_pending(time.monotonic(), force=True)
        self.append('stop', reason='signal', stats=self.snapshot())
        self.commit()
        self.writer.close()
        try:
            os.unlink(self.args.socket)
        except OSError:
            pass
        print(f'relay: stopped at seq {self.writer.seq}', file=sys.stderr, flush=True)


def main():
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    sub = parser.add_subparsers(dest='command', required=True)
    run = sub.add_parser('run', help='observe and write the log')
    run.add_argument('--root', action='append', required=True, help='declared root (repeat)')
    run.add_argument('--state', default=rl.DEFAULT_STATE)
    run.add_argument('--socket', default=rl.DEFAULT_SOCKET)
    run.add_argument('--socket-group', default='hugpy')
    run.add_argument('--exclude', action='append', default=[])
    run.add_argument('--segment-bytes', type=int, default=64 << 20)
    run.add_argument('--heartbeat-s', type=float, default=60.0)
    run.add_argument('--checkpoint-s', type=float, default=900.0)
    run.add_argument('--compact-check-s', type=float, default=3600.0)
    run.add_argument('--sync-ms', type=float, default=100.0)
    run.add_argument('--sync-bytes', type=int, default=64 << 10)
    run.add_argument('--sync-min-gap-ms', type=float, default=20.0)
    run.add_argument('--idle-ms', type=float, default=1000.0)
    run.add_argument('--max-hold-s', type=float, default=60.0)
    run.add_argument('--wake-ms', type=float, default=2.0)
    run.add_argument('--unlimited-queue', action='store_true')
    run.add_argument('--no-chattr', action='store_true')
    reg = sub.add_parser('register', help='create consumers/NAME owned by USER (root)')
    reg.add_argument('name')
    reg.add_argument('--user', required=True)
    reg.add_argument('--group', default='hugpy')
    reg.add_argument('--start', choices=('head', 'genesis'), default='head')
    reg.add_argument('--state', default=rl.DEFAULT_STATE)
    ver = sub.add_parser('verify', help='verify the chain, including compacted segments')
    ver.add_argument('--state', default=rl.DEFAULT_STATE)
    args = parser.parse_args()
    if args.command == 'run':
        Relay(args).run()
    elif args.command == 'register':
        user = pwd.getpwnam(args.user)
        print(json.dumps(rl.register(args.name, user.pw_uid, grp.getgrnam(args.group).gr_gid,
                                     args.state, args.start)))
    else:
        result = rl.verify(args.state)
        print(json.dumps(result))
        sys.exit(0 if result['ok'] else 1)


if __name__ == '__main__':
    main()
