Source code for parlay.items.threaded_item

from twisted.internet import defer
from parlay.items.base import MSG_TYPES, MSG_STATUS
from parlay.protocols.utils import message_id_generator
from twisted.python.failure import Failure
from base import BaseItem
from parlay.server.broker import Broker, run_in_broker
import sys
import json
import logging


# a list of Item proxy classes for Scripts
ITEM_PROXIES = {}


DEFAULT_TIMEOUT = 120


# list of deferreds to cancel when cleaning up
CLEANUP_DEFERRED = set()


[docs]def cleanup(): for d in CLEANUP_DEFERRED: if not d.called: d.cancel()
# cleanup our deferreds Broker.call_on_stop(cleanup)
[docs]class ListenerStatus(object): """ Enum object for keeping or removing listeners """ KEEP_LISTENER = False REMOVE_LISTENER = True
[docs]class ThreadedItem(BaseItem): """Base object for all Parlay scripts""" # a list of functions that will be alerted when a new script instance is created stop_reactor_on_close = True def __init__(self, item_id, name, reactor=None, adapter=None): BaseItem.__init__(self, item_id, name, adapter=adapter) self._reactor = self._adapter.reactor if reactor is None else reactor self._msg_listeners = [] self._system_errors = [] self._system_events = [] self._timer = None self._auto_update_discovery = True #: If True auto update discovery with broadcast discovery messages self.discovery = {} #: The current discovery information to pull from self._message_id_generator = message_id_generator(65535, 100) # Add this listener so it will be first in the list to pickup errors, warnings and events. self.add_listener(self._system_listener) self.add_listener(self._discovery_request_listener) self._adapter.subscribe(self._discovery_broadcast_listener, type='DISCOVERY_BROADCAST') # we need to overrite publish so we can register our callback for broker type messages
[docs] def publish(self, msg): self._adapter.publish(msg, self._runListeners)
def _discovery_broadcast_listener(self, msg): """ Listen for discovery broadcast listeners and update our discovery accordingly """ if self._auto_update_discovery and msg['CONTENTS'].get("status", "") == "ok": self.discovery = msg['CONTENTS'].get('discovery', self.discovery) return ListenerStatus.KEEP_LISTENER def _system_listener(self, msg): """ This should be the first listener in the list. It will store any non-response errors and events :param msg:Msg from the broker :return: False so that it is never removed """ # If it is not a response and it is an error, save it in the list if msg['TOPICS'].get('MSG_TYPE', "") != 'RESPONSE': status = msg['TOPICS'].get('MSG_STATUS', "") if status == 'ERROR': self._system_errors.append(msg) elif status == 'WARNING' or status == 'INFO': self._system_events.append(msg) return ListenerStatus.KEEP_LISTENER def _discovery_request_listener(self, msg): """ Respond to a get_protocol_discovery message with an empty get_protocol_discovery_response message. :param msg: incoming message :return: False so that it is never removed from the listener list """ if msg['TOPICS'].get('type', "") == 'get_protocol_discovery': msg = {'TOPICS': {'type': 'get_protocol_discovery_response'}, 'CONTENTS': {"CHILDREN": [self.get_discovery()]}} self.publish(msg) return ListenerStatus.KEEP_LISTENER
[docs] def open(self, protocol, **params): """ :param protocol: protocol being used :param params: other parameters :return: """ msg = {"TOPICS": {'type': 'broker', 'request': 'open_protocol'}, "CONTENTS": {'protocol_name': protocol, 'params': params}} self._reactor.maybeCallFromThread(self.publish, msg) def wait_for_response(): result = defer.Deferred() def listener(_msg): if _msg['TOPICS'].get('response', "") == 'open_protocol_response': if _msg['CONTENTS']['STATUS'] == 'ok': result.callback(_msg['CONTENTS']['STATUS']) else: result.errback(Failure(RuntimeError(_msg['CONTENTS']['STATUS']))) return True # we're done here return False # keep waiting self.add_listener(listener) return result return self._reactor.maybeblockingCallFromThread(wait_for_response)
[docs] def close_protocol(self, protocol_id): """ close a protocol by id :param protocol_id: :return: """ msg = {"TOPICS": {'type': 'broker', 'request': 'close_protocol'}, "CONTENTS": {'protocol': protocol_id}} self._reactor.maybeCallFromThread(self.publish, msg) def wait_for_response(): result = defer.Deferred() def listener(_msg): if _msg['TOPICS'].get('response', "") == 'close_protocol_response': if _msg['CONTENTS']['STATUS'] == 'ok': result.callback(_msg['CONTENTS']['STATUS']) else: result.errback(Failure(RuntimeError(_msg['CONTENTS']['STATUS']))) return True # we're done here return False # keep waiting self.add_listener(listener) return result return self._reactor.maybeblockingCallFromThread(wait_for_response)
[docs] def add_listener(self, listener_function): """ Add functions to the listener list """ self._msg_listeners.append(listener_function)
############################################################################################### ################### The functions below are used by the script ###############################
[docs] def make_msg(self, to, command, msg_type=MSG_TYPES.COMMAND, direct=True, response_req=True, _extra_topics=None, **kwargs): """ Prepare a message for the broker to disperse """ msg = {'TOPICS': {}, 'CONTENTS': kwargs} if _extra_topics is not None: msg["TOPICS"] = _extra_topics # we can assume some keyword values msg['TOPICS']['TX_TYPE'] = 'DIRECT' if direct else "BROADCAST" msg['TOPICS']['MSG_TYPE'] = msg_type msg['TOPICS']['RESPONSE_REQ'] = response_req msg['TOPICS']['MSG_ID'] = self._message_id_generator.next() msg['TOPICS']['TO'] = to msg['TOPICS']['FROM'] = self.item_id if command is not None: msg['CONTENTS']['COMMAND'] = command return msg
[docs] def send_parlay_message(self, msg, timeout=DEFAULT_TIMEOUT, wait=None): """ Send a command. This will be sent from the reactor thread. If a response is required, we will wait for it. :param msg The Message to send :param timeout If we require a response and don't get one back int timeout seconds, raise a timeout exception :param wait If set to True, will block until a response, if false will continue without blocking, If set to None, till auto discover based on message RESPONSE_REQ. """ if wait is None: wait = msg['TOPICS'].get('RESPONSE_REQ', False) if wait: # block the thread until we get a response or timeout return self._reactor.maybeblockingCallFromThread(self._send_parlay_message_from_thread, msg=msg, timeout=timeout) else: # send this to the reactor without waiting for a response self._reactor.maybeCallFromThread(self.publish, msg) return None # nothing to wait on, no response
[docs] def discover(self, force=True): """ Run a discovery so that the script knows what items are attached and can get handles to them. :param force: If True, will force a rediscovery, if False will take the last cached discovery """ if not self._reactor.running: raise Exception("You must call parlay.utils.setup() at the beginning of a script!") print "Running discovery..." # block the thread until we get a discovery or error return self._reactor.maybeblockingCallFromThread(self._in_reactor_discover, force)
[docs] def save_discovery(self, path): """ Save the current discovery information to a file so it can be loaded later :param path: The Path to the file to save to (Warning: will be overwritten) """ with open(path, "w") as f: # pretty print in case a human wants to read it json.dump(self.discovery, f, indent=4, sort_keys=True)
[docs] def load_discovery(self, path): """ Load discovery from a file. :param path: The path to the file that has the JSON discovery """ with open(path, 'r') as f: self.discovery = json.load(f)
@run_in_broker @defer.inlineCallbacks
[docs] def get_item_by_id(self, item_id): """ Returns a handler object that can be used to send messages to an item. :param item_id: globally unique id of the item :return: a proxy object for the item """ if not self._reactor.running: raise Exception("You must call parlay.utils.setup() at the beginning of a script!") def find(): g = self._find_item_info(self.discovery, item_id, "ID") item_disc = next(g) return self._proxy_item(item_disc) try: defer.returnValue(find()) except StopIteration: # discover and try again try: yield self.discover(force=False) defer.returnValue(find()) except StopIteration: raise KeyError("Couldn't find item with id " + str(item_id))
@run_in_broker @defer.inlineCallbacks
[docs] def get_item_by_name(self, item_name): """ Returns a handler object that can be used to send messages to an item. :param item_name: globally unique name of the item :return: a proxy object for the item """ if not self._reactor.running: raise Exception("You must call parlay.utils.setup() at the beginning of a script!") def find(): g = self._find_item_info(self.discovery, item_name, "NAME") item_disc = next(g) return self._proxy_item(item_disc) try: defer.returnValue(find()) except StopIteration: # discover and try again try: yield self.discover(force=False) defer.returnValue(find()) except StopIteration: raise KeyError("Couldn't find item with name " + str(item_name))
@run_in_broker @defer.inlineCallbacks
[docs] def get_all_items_with_name(self, item_name): """ Returns a handler object that can be used to send messages to an item. :param item_name: globally unique name of the item :return: a proxy object for the item """ if not self._reactor.running: raise Exception("You must call parlay.utils.setup() at the beginning of a script!") result = [self._proxy_item(x) for x in self._find_item_info(self.discovery, item_name, "NAME")] if len(result) == 0: # retry after discover if it fails yield self.discover(force=False) result = [self._proxy_item(x) for x in self._find_item_info(self.discovery, item_name, "NAME")] defer.returnValue(result)
def _proxy_item(self, item_disc): """ Get an Item Proxy object by the discovery :param item_disc: The item discovery object :type item_disc: dict """ # Do we have a valid item discovery object? if item_disc is None: raise KeyError("Couldn't make Proxy Item with None type") # now that we have the discovery, let's try and construct a proxy out of it templates = [x.strip() for x in item_disc.get("TYPE", "").split("/")] template = None # find and stop at the first valid one for t in templates: if template is None: template = ITEM_PROXIES.get(t, None) # if it's still None, then that means that we couldn't find it if template is None: raise KeyError("Couldn't find template proxy for:" + item_disc.get("TYPE", "")) # we have a good template class! Let's construct it try: result = template(item_disc, self) return result except Exception as e: print "Could not construct proxy Item. Caught Exception :" + str(e) raise def _find_item_info(self, discovery, item_id, key): """ Find the item with a given id recursively (or None if it can't be found) :type: discovery list """ for item in discovery: if item.get(key, None) == item_id: yield item # see if a child found it for child in self._find_item_info(item.get('CHILDREN', []), item_id, key): yield child
[docs] def sleep(self, timeout): """ Sleep for <timeout> seconds. This call is BLOCKING. :param timeout: number of seconds to sleep """ if not self._reactor.running: raise Exception("You must call parlay.utils.setup() at the beginning of a script!") return run_in_broker(lambda: self._sleep(timeout))()
####################### The following must be run from the reactor thread ################### ############################# Do not call directly from script thread ##################### def _send_parlay_message_from_thread(self, msg, timeout): """ Send the command and wait for the callback.This must be called only from the reactor thread. NOTE: caller is blocked. :param msg: message to send :param timeout timeout ins econds """ response = defer.Deferred() timer = None timeout_msg = {'TOPICS': {'MSG_TYPE': 'TIMEOUT'}} def listener(received_msg): # See if this is the response we are waiting for if received_msg['TOPICS'].get('MSG_TYPE', "") == MSG_TYPES.RESPONSE: if received_msg['TOPICS']['TO'] == self.item_id and \ received_msg['TOPICS'].get('MSG_ID', None) == msg['TOPICS']['MSG_ID']: if received_msg['TOPICS'].get('MSG_STATUS', "") == MSG_STATUS.PROGRESS: return False # keep waiting, an ACK means its not finished yet, it just got our msg if timer is not None: # Clear the timer timer.cancel() if received_msg['TOPICS'].get('MSG_STATUS', "") == MSG_STATUS.ERROR: # return error to waiting thread response.errback(Failure(ErrorResponse(received_msg))) else: # send the response back to the waiting thread response.callback(received_msg) return True # remove this listener from the list else: # not our response. Check for a system error. if len(self._system_errors) > 0: if timer is not None: # clear out the timer timer.cancel() # report an error to the waiting thread response.errback(Failure(AsyncSystemError(self._system_errors.pop(0)))) return True # remove this listener return False # not for this listener - don't remove def cb(_msg): # got a timeout or started with an error # remove the listener if listener in self._msg_listeners: self._msg_listeners.remove(listener) # send failure to thread waiting. response.errback(Failure(AsyncSystemError(_msg))) # If we already have a system error, fail if len(self._system_errors) > 0: self._timer = self._reactor.callLater(0, cb, self._system_errors.pop(0)) else: # set a timeout, if requested if timeout > 0: timer = self._reactor.callLater(timeout, cb, timeout_msg) # add our listener to the listener ist self.add_listener(listener) # send the message self.publish(msg) return response def _in_reactor_discover(self, force): """ Discovery called from within the reactor context """ # call this back with the discovery result = defer.Deferred() def discovery_listener(msg): if msg['TOPICS'].get("type", "") != 'broker' and \ msg['TOPICS'].get("response", "") != "get_discovery_response": return False # not the msg we're looking for if msg['CONTENTS'].get("status", "") == "ok": self.discovery = msg['CONTENTS'].get('discovery', {}) result.callback(self.discovery) else: result.errback(Failure(Exception(msg.get("status", "NO STATUS")))) return True # we're done here self.add_listener(discovery_listener) self.publish({"TOPICS": {'type': 'broker', 'request': 'get_discovery'}, "CONTENTS": {'force': force}}) return result def _sleep(self, timeout): """ Support a script delay. The delay will stop early with an error if there is a system error. :param timeout: :return:deferred """ response = defer.Deferred() CLEANUP_DEFERRED.add(response) timer = None def listener(received_msg): # look for system errors while we are waiting if len(self._system_errors) > 0: # cancel out the timer if timer is not None: timer.cancel() # return the error to our waiting thread response.errback(Failure(AsyncSystemError(self.errors.pop(0)))) return True # remove the listener from the list return False # don't remove def cb(msg): # remove ourselves from cleanup list CLEANUP_DEFERRED.remove(response) # remove our listener function if it is in the list. if listener in self._msg_listeners: self._msg_listeners.remove(listener) # if this is the normal timeout, just send the timeout message if msg['TOPICS']['MSG_TYPE'] == 'TIMEOUT': # Timed out - no error response.callback(msg) else: # Error response.errback(Failure(AsyncSystemError(msg))) # check we don't already have an error if len(self._system_errors) > 0: self._timer = self._reactor.callLater(0, cb, self._system_errors.pop(0)) else: timer = self._reactor.callLater(timeout, cb, {'TOPICS': {'MSG_TYPE': 'TIMEOUT'}}) self.add_listener(listener) return response def _runListeners(self, msg): remove_list = [] for i, listener in enumerate(self._msg_listeners): if listener(msg): remove_list.append(i) # Now that we are done running the list, we can remove the ones slated for removal. if len(remove_list) > 0: self._msg_listeners = [x for i, x in enumerate(self._msg_listeners) if i not in remove_list]
[docs]class ErrorResponse(Exception): def __init__(self, error_msg): self.error_msg = error_msg self.description = error_msg['CONTENTS'].get('DESCRIPTION', '') # copy our traceback from the other exception self._traceback = error_msg["CONTENTS"].get('TRACEBACK', None) self.str = "Response Error: {} \n\n\nOriginal Traceback:\n {}".format(self.description, self._traceback) def __str__(self): return self.str
[docs]class AsyncSystemError(Exception): """ This error class is for asynchronous system errors. """ def __init__(self, error_msg): self.error_msg = error_msg self.description = error_msg.get('CONTENTS', {}).get('DESCRIPTION', '') self.code = error_msg.get('CONTENTS', {}).get('ERROR_CODE', 0) def __str__(self): return "Critical Error: {} \nCODE: {} \nERROR_MESSAGE: {} ".format(self.description, self.code, self.error_msg)