Package ivy :: Module ivy
[hide private]
[frames] | no frames]

Source Code for Module ivy.ivy

   1  """ 
   2  Using an IvyServer 
   3  ------------------ 
   4   
   5  The following code is a typical example of use: 
   6   
   7  .. python:: 
   8      from ivy.ivy import IvyServer 
   9       
  10      class MyAgent(IvyServer): 
  11        def __init__(self, name): 
  12          IvyServer.__init__(self,'MyAgent') 
  13          self.name = name 
  14          self.start('127.255.255.255:2010') 
  15          self.bind_msg(self.handle_hello, 'hello .*') 
  16          self.bind_msg(self.handle_button, 'BTN ([a-fA-F0-9]+)') 
  17           
  18        def handle_hello(self, agent): 
  19          print '[Agent %s] GOT hello from %r'%(self.name, agent) 
  20         
  21        def handle_button(self, agent, btn_id): 
  22          print '[Agent %s] GOT BTN button_id=%s from %r'%(self.name, btn_id, agent) 
  23          # let's answer! 
  24          self.send_msg('BTN_ACK %s'%btn_id) 
  25       
  26      a=MyAgent('007') 
  27   
  28   
  29  Implementation details 
  30  ---------------------- 
  31   
  32  An Ivy client is made of several threads: 
  33   
  34    - an `IvyServer` instance 
  35   
  36    - a UDP server, lanched by the Ivy server, listening to incoming UDP 
  37      broadcast messages 
  38   
  39    - `IvyTimer` objects  
  40   
  41  :group Messages types: BYE, ADD_REGEXP, MSG, ERROR, DEL_REGEXP, END_REGEXP, 
  42    END_INIT, START_REGEXP, START_INIT, DIRECT_MSG, DIE 
  43  :group Separators: ARG_START, ARG_END 
  44  :group Misc. constants: DEFAULT_IVYBUS, PROTOCOL_VERSION, IVY_SHOULD_NOT_DIE 
  45    IvyApplicationConnected, IvyApplicationDisconnected, DEFAULT_TTL 
  46  :group Objects and functions related to logging: ivylogger, debug, log, warn, 
  47    error, ivy_loghdlr, ivy_logformatter 
  48   
  49  Copyright (c) 2005-2011 Sebastien Bigaret <sbigaret@users.sourceforge.net> 
  50  """ 
  51   
  52  import logging, os 
  53  ivylogger = logging.getLogger('Ivy') 
  54   
  55  if os.environ.get('IVY_LOG_TRACE'): 
  56    logging.TRACE=logging.DEBUG-1 
  57    logging.addLevelName(logging.TRACE, "TRACE") 
  58    trace = lambda *args, **kw: ivylogger.log(logging.TRACE, *args, **kw) 
  59  else: 
  60    trace = lambda *args, **kw: None 
  61   
  62  debug = ivylogger.debug 
  63  info = log = ivylogger.info 
  64  warn = ivylogger.warning 
  65  error = ivylogger.error 
  66   
  67  ivy_loghdlr = logging.StreamHandler() # stderr by default 
  68  ivy_logformatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') 
  69   
  70  ivy_loghdlr.setFormatter(ivy_logformatter) 
  71  ivylogger.addHandler(ivy_loghdlr) 
  72  #ivylogger.setLevel(logging.DEBUG) 
  73  ivylogger.setLevel(logging.INFO) 
  74   
  75  ## 
  76  DEFAULT_IVYBUS = '127:2010' 
  77  PROTOCOL_VERSION = 3 
  78   
  79  # Message types. Refer to "The Ivy architecture and protocol" for details 
  80  BYE = 0 
  81  ADD_REGEXP = 1 
  82  MSG = 2 
  83  ERROR = 3 
  84  DEL_REGEXP = 4 
  85   
  86  # START_REGEXP and END_REGEXP are the ones declared in ivy.c 
  87  # however we'll use the aliases START_INIT and END_INIT here 
  88  END_REGEXP   = END_INIT   = 5 
  89  START_REGEXP = START_INIT = 6 
  90   
  91  DIRECT_MSG = 7 
  92  DIE = 8 
  93   
  94  # Other constants 
  95  ARG_START = '\002' 
  96  ARG_END = '\003' 
  97   
  98  # for multicast, arbitrary TTL value taken from ivysocket.c:SocketAddMember 
  99  DEFAULT_TTL = 64   
 100   
 101  IvyApplicationConnected = 1 
 102  IvyApplicationDisconnected = 2 
 103   
 104  IvyRegexpAdded = 3 
 105  IvyRegexpRemoved = 4 
 106   
 107  IVY_SHOULD_NOT_DIE = 'Ivy Application Should Not Die' 
 108   
 109   
110 -def void_function(*arg, **kw):
111 "A function that accepts any number of parameters and does nothing" 112 pass
113 114 #
115 -def UDP_init_and_listen(broadcast_addr, port, socket_server):
116 """ 117 Called by an IvyServer at startup; the method is responsible for: 118 119 - sending the initial UDP broadcast message, 120 121 - waiting for incoming UDP broadcast messages being sent by new clients 122 connecting on the bus. When it receives such a message, a connection 123 is established to that client and that connection (a socket) is then 124 passed to the IvyServer instance. 125 126 :Parameters: 127 - `broadcast_addr`: the broadcast address used on the Ivy bus 128 - `port`: the port dedicated to the Ivy bus 129 - `socket_server`: instance of an IvyServer handling communications 130 for our client. 131 """ 132 log('Starting Ivy UDP Server on %r:%r'%(broadcast_addr,port)) 133 import socket 134 s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 135 on=1 136 s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, on) 137 if hasattr(socket, 'SO_REUSEPORT'): 138 s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, on) 139 s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, on) 140 141 s.bind(('',port)) # '' means: INADDR_ANY 142 143 # Multicast 144 if is_multicast(broadcast_addr): 145 debug('Broadcast address is a multicast address') 146 import struct 147 ifaddr = socket.INADDR_ANY 148 mreq=struct.pack('4sl', 149 socket.inet_aton(broadcast_addr), 150 socket.htonl(ifaddr)) 151 s.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) 152 s.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, DEFAULT_TTL) 153 # /Multicast 154 155 s.sendto("%li %s %s %s\n"%(PROTOCOL_VERSION,socket_server.port, 156 socket_server.agent_id, socket_server.agent_name), 157 (broadcast_addr, port)) 158 159 s.settimeout(0.1) 160 while socket_server.isAlive(): 161 try: 162 udp_msg, (ip, ivybus_port) = s.recvfrom(1024) 163 except socket.timeout: 164 continue 165 166 debug('UDP got: %r from: %r', udp_msg, ip) 167 168 appid = appname = None 169 try: 170 udp_msg_l = udp_msg.split(' ') 171 protocol_version, port_number = udp_msg_l[:2] 172 if len(udp_msg_l) > 2: 173 # "new" udp protocol, with id & appname 174 appid = udp_msg_l[2] 175 appname = ' '.join(udp_msg_l[3:]).strip('\n') 176 debug('IP %s has id: %s and name: %s', ip, appid, appname) 177 else: 178 debug('Received message w/o app. id & name from %r', ip) 179 180 port_number = int(port_number) 181 protocol_version = int(protocol_version) 182 183 except (ValueError): # unpack error, invalid literal for int() 184 warn('Received an invalid UDP message (%r) from :', udp_msg) 185 186 if protocol_version != PROTOCOL_VERSION: 187 error('Received a UDP broadcast msg. w/ protocol version:%s , expected: %s', protocol_version, PROTOCOL_VERSION) 188 continue 189 190 if appid == socket_server.agent_id: 191 # this is us! 192 debug('UDP from %r: ignored: we sent that one!', ip) 193 continue 194 195 # build a new socket and delegate its handling to the SocketServer 196 debug('New client connected: %s:%s', ip, port_number) 197 198 new_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 199 new_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, on) 200 trace('New client %s:%s, socket %r', ip, port_number, new_socket) 201 # Since we already have a client's name and id, lets register it 202 # (this was previously done in IvyHandler's __init__() only) 203 # but we want to check that we did not receive more than once a 204 # broadcast coming from the same 205 new_client = socket_server._get_client(ip, port_number, new_socket, 206 appid, appname) 207 if new_client is None: 208 # an agent with that app-id is already registered 209 info('UDP from %s:%s (%s): discarding message, an application w/ id=%s is already registered', ip, port_number, appname, appid) 210 continue 211 212 try: 213 new_socket.connect((ip, port_number)) 214 except: # e.g., timeout on connect 215 from traceback import format_exc 216 info('Client %r: failed to connect to its socket, ignoring it', 217 new_client) 218 debug('Client %r: failed to connect to its socket, got:%s', 219 new_client, format_exc()) 220 socket_server.remove(ip, port_number, 221 trigger_application_callback=False) 222 else: 223 socket_server.process_request(new_socket, (ip, port_number)) 224 log('UDP Server stopped')
225
226 -class IvyProtocolError(Exception): pass
227 -class IvyMalformedMessage(Exception): pass
228 -class IvyIllegalStateError(RuntimeError): pass
229
230 -def decode_msg(msg):
231 """ 232 233 :return: msg_id, numerical_id, parameters 234 :except IvyMalformedMessage: 235 """ 236 try: 237 msg_id, _msg = msg.split(' ', 1) 238 msg_id = int(msg_id) 239 num_id, params = _msg.split(ARG_START, 1) 240 num_id = int(num_id) 241 242 if ARG_END in params: 243 # there is an extra ARG_END after the last parameter and 244 # before the newline 245 params = params[:-1].split(ARG_END) 246 247 except ValueError: 248 raise IvyMalformedMessage 249 return msg_id, num_id, params
250
251 -def encode_message(msg_type, numerical_id, params=''):
252 """ 253 254 params is string -> added as-is 255 params is list -> concatenated, separated by ARG_END 256 """ 257 msg = "%s %s"%(msg_type, numerical_id) + ARG_START 258 if type(params) is type(''): 259 msg += params 260 else: 261 msg += ARG_END.join(params) 262 msg += ARG_END 263 trace('encode_message(params: %s) -> %s'%(repr(params),repr(msg+'\n'))) 264 return msg + '\n'
265 266 267 268 NOT_INITIALIZED = 0 269 INITIALIZATION_IN_PROGRESS=1 270 INITIALIZED=2 271
272 -class IvyClient:
273 """ 274 Represents a client connected to the bus. Every callback methods 275 registered by an agent receive an object of this type as their first 276 parameter, so that they know which agent on the bus is the cause of the 277 event which triggered the callback. 278 279 An IvyClient is responsible for: 280 281 - managing the remote agent's subscriptions, 282 283 - sending messages to the remote agent. 284 285 It is **not** responsible for receiving messages from the client: another 286 object is in charge of that, namely an `IvyHandler` object. 287 288 The local IvyServer creates one IvyClient per agent on the bus. 289 290 MT-safety 291 --------- 292 See the discussion in `regexps`. 293 294 :group Protocol-related methods: start_init, end_init, 295 send_new_subscription, remove_subscription, wave_bye 296 :group Manipulating the remote agent's subscriptions: 297 add_regexp, remove_regexp 298 :group Sending messages: send_msg, send_direct_message, send_die_message 299 300 :ivar regexps: a dictionary mapping subscriptions' ids (as delivered by 301 `add_regexp`) to the corresponding regular expressions. Precisely, it 302 maps ids to tuples being ``(regexp_as_string, compiled_regexp)``. You 303 shouldn't directly access or manipulate this variable, use `add_regexp` 304 and `remove_regexp` instead; however if you really need/want to, you 305 must acquire the `regexp_lock` before accessing/modifying it. 306 307 :ivar regexp_lock: a non-reentrant lock protecting the variable 308 `regexps`. Used by methods `add_regexp`, `remove_regexp` and `send_msg`. 309 :type regexp_lock: `threading.Lock` 310 """ 311 agent_id = None 312 agent_name = None 313 port = None 314 socket = None 315 #struct _client in ivysocket.c
316 - def __init__(self, ip, port, client_socket, 317 agent_id=None, agent_name=None):
318 self.agent_id = agent_id 319 # agent_name will be overridden when start_init() is called 320 # but nevermind, 321 self.agent_name = agent_name 322 self.ip = ip 323 self.port = port 324 self.socket = client_socket 325 self.regexps = {} # maps regexp_id to (regexp_string, compiled_regexp) 326 import socket 327 self.fqdn = socket.getfqdn(ip) 328 self.status = NOT_INITIALIZED 329 self.socket.settimeout(0.1) 330 331 import threading 332 # regexps are modified w/ add_regexp and remove_regexp, called 333 # by the server, while they are accessed by send_message() 334 # called by the corresponding IvyHandler-thread --> needs to be 335 # protected against concurrent access. 336 self.regexps_lock=threading.Lock() # non-reentrant, faster than RLock
337
338 - def start_init(self, agent_name):
339 """ 340 Finalizes the initialization process by setting the client's 341 agent_name. This is a Ivy protocol requirement that an application 342 sends its agent-name only once during the initial handshake (beginning 343 with message of type ``START_INIT`` and ending with a type 344 ``END_INIT``). After this method is called, we expect to receive the 345 initial subscriptions for that client (or none); the initialization 346 process completes after `end_init` is called. 347 348 :except IvyIllegalStateError: if the method has already been called 349 once 350 """ 351 if self.status != NOT_INITIALIZED: 352 raise IvyIllegalStateError 353 self.agent_name = agent_name 354 self.status = INITIALIZATION_IN_PROGRESS 355 debug('Client:%r: Starting initialization', self)
356
357 - def end_init(self):
358 """ 359 Should be called when the initalization process ends. 360 361 :except IvyIllegalStateError: if the method has already been called 362 (and ``self.status`` has already been set to ``INITIALIZED``) 363 """ 364 if self.status is INITIALIZED: 365 raise IvyIllegalStateError 366 debug('Client:%r: Initialization ended', self) 367 self.status = INITIALIZED
368
369 - def add_regexp(self, regexp_id, regexp):
370 """ 371 :except IvyIllegalStateError: if the client has not been fully 372 initialized yet (see `start_init`) 373 """ 374 if self.status not in ( INITIALIZATION_IN_PROGRESS, INITIALIZED ): 375 # initialization has not begun 376 raise IvyIllegalStateError 377 378 # TODO: handle error on compile 379 import re 380 debug('Client:%r: Adding regexp id=%s: %r', self, regexp_id, regexp) 381 self.regexps_lock.acquire() 382 try: 383 self.regexps[regexp_id] = (regexp, re.compile(regexp)) 384 finally: 385 self.regexps_lock.release()
386
387 - def remove_regexp(self, regexp_id):
388 """ 389 Removes a regexp 390 391 :return: the regexp that has been removed 392 :except IvyIllegalStateError: if the client has not been fully 393 initialized yet (see `start_init`) 394 :except KeyError: if no such subscription exists 395 """ 396 if self.status not in ( INITIALIZATION_IN_PROGRESS, INITIALIZED ): 397 # initialization has not begun 398 raise IvyIllegalStateError 399 debug('Client:%r: removing regexp id=%s', self, regexp_id) 400 regexp = None 401 402 self.regexps_lock.acquire() 403 try: 404 regexp = self.regexps.pop(regexp_id)[0] 405 finally: 406 self.regexps_lock.release() 407 return regexp
408
409 - def get_regexps(self):
410 self.regexps_lock.acquire() 411 try: 412 return [ (idx, s[0]) for idx, s in self.regexps.items()] 413 finally: 414 self.regexps_lock.release()
415
416 - def send_msg(self, msg):
417 """ 418 Sends a message to the client. The message is compared to the 419 client's subscriptions and it is sent if one of them matches. 420 421 :return: ``True`` if the message was actually sent to the client, that 422 is: if there is a regexp matching the message in the client's 423 subscriptions; returns ``False`` otherwise. 424 425 """ 426 if self.status is not INITIALIZED: 427 return 428 debug('Client:%r: Searching a subscription matching msg %r', 429 self, msg) 430 # TODO: if 2 regexps match a message, we should be able to tell 431 # TODO: which one is selected (for example, try them in the order 432 # TODO: of their subscriptions) 433 self.regexps_lock.acquire() 434 try: 435 436 for id, (s, r) in self.regexps.items(): 437 captures = r.match(msg) 438 if captures: 439 captures=captures.groups() 440 # The following is needed to reproduce the very same 441 # behaviour #observed w/ the C library 442 # (tested w/ pyhello.py and ivyprobe) 443 if len(captures)==0: 444 captures='' 445 debug('Client:%r: msg being sent: %r (regexp: %r)', 446 self,captures,s) 447 self._send(MSG, id, captures) 448 return True 449 return False 450 451 finally: 452 self.regexps_lock.release()
453
454 - def send_direct_message(self, num_id, msg):
455 """ 456 Sends a direct message 457 458 Note: the message will be encoded by `encode_message` with 459 ``numerical_id=num_id`` and ``params==msg``; this means that if `msg` 460 is not a string but a list or a tuple, the direct message will contain 461 more than one parameter. This is an **extension** of the original Ivy 462 design, supported by python, but if you want to inter-operate with 463 applications using the standard Ivy API the message you send *must* be 464 a string. See in particular in ``ivy.h``:: 465 466 typedef void (*MsgDirectCallback)( IvyClientPtr app, void *user_data, int id, char *msg ) ; 467 468 """ 469 if self.status is INITIALIZED: 470 debug('Client:%r: a direct message being sent: id: %r msg: %r', 471 self, id, msg) 472 self._send(DIRECT_MSG, num_id, msg)
473
474 - def send_die_message(self, num_id=0, msg=""):
475 """ 476 Sends a die message 477 """ 478 if self.status is INITIALIZED: 479 debug('Client:%r: die msg being sent: num_id: %r msg: %r', 480 self, num_id, msg) 481 self._send(DIE, num_id, msg)
482
483 - def send_new_subscription(self, idx, regexp):
484 """ 485 Notifies the remote agent that we (the local agent) subscribe to 486 a new type of messages 487 488 :Parameters: 489 - `idx`: the index/id of the new subscription. It is the 490 responsability of the local agent to make sure that every 491 subscription gets a unique id. 492 - `regexp`: a regular expression. The subscription consists in 493 receiving messages mathcing the regexp. 494 """ 495 self._send(ADD_REGEXP, idx, regexp)
496
497 - def remove_subscription(self, idx):
498 """ 499 Notifies the remote agent that we (the local agent) are not 500 interested in a given subscription. 501 502 :Parameters: 503 - `idx`: the index/id of a subscription previously registered with 504 `send_new_subscription`. 505 """ 506 self._send(DEL_REGEXP, idx)
507
508 - def wave_bye(self, id=0):
509 "Notifies the remote agent that we are about to quit" 510 self._send(BYE, id)
511
512 - def send_error(self, num_id, msg):
513 """ 514 Sends an error message 515 """ 516 self._send(ERROR, num_id, msg)
517
518 - def __eq__(self, client):
519 """ 520 cf. dict[client] or dict[(ip,port)] UNNEEDED FOR THE MOMENT 521 """ 522 if isinstance(client, IvyClient): 523 return self.ip==client.ip and self.port==client.port 524 525 import types 526 if type(client) in (types.TupleType, types.ListType) \ 527 and len(client) == 2: 528 return self.ip == client[0] and self.port == client[1] 529 530 return False
531
532 - def __hash__(self):
533 "``hash((self.ip, self.port))``" 534 return hash((self.ip, self.port))
535
536 - def __repr__(self):
537 "Returns ``'ip:port (agent_name)'``" 538 return "%s:%s (%s)"%(self.ip, self.port, self.agent_name)
539
540 - def __str__(self):
541 "Returns ``'agent_name@FQDN'``" 542 return "%s@%s"%(self.agent_name, self.fqdn)
543
544 - def _send(self, msg_type, *params):
545 """ 546 Internally used to send message to the remote agent through the opened 547 socket `self.socket`. This method catches all exceptions 548 `socket.error` and `socket.timeout` and ignores them, simply logging 549 them at the "info" level. 550 551 The errors that can occur are for example:: 552 553 socket.timeout: timed out 554 socket.error: (104, 'Connection reset by peer') 555 socket.error: (32, 'Broken pipe') 556 557 They can happen after a client disconnects abruptly (because it was 558 killed, because the network is down, etc.). We assume here that if and 559 when an error happens here, a disconnection will be detected shortly 560 afterwards by the server which then removes this agent from the bus. 561 Hence, we ognore the error; please also note that not ignoring the 562 error can have an impact on code, for example, IyServer.send_msg() 563 does not expect that IvyClient.send() fails and if it fails, it is 564 possible that the server does not send the message to all possible 565 subscribers. 566 567 .. note:: ``ivysocket.c:SocketSendRaw()`` also ignores error, simply 568 logging them. 569 570 """ 571 import socket 572 try: 573 self.socket.send(encode_message(msg_type, *params)) 574 except (socket.timeout, socket.error), exc: 575 log('[ignored] Error on socket with %r: %s', self, exc)
576 577 import SocketServer, threading
578 -class IvyServer(SocketServer.ThreadingTCPServer):
579 """ 580 An Ivy server is responsible for receiving and handling the messages 581 that other clients send on an Ivy bus to a given agent. 582 583 An IvyServer has two important attributes: `usesDaemons` and 584 `server_termination`. 585 586 587 :ivar usesDaemons: 588 whether the threads are daemonic or not. Daemonic 589 threads do not prevent python from exiting when the main thread stop, 590 while non-daemonic ones do. Default is False. This attribute should 591 be set through at `__init__()` time and should not be modified 592 afterwards. 593 594 :ivar server_termination: 595 a `threading.Event` object that is set on server shutdown. It can be 596 used either to test whether the server has been stopped 597 (``server_termination.isSet()``) or to wait until it is stopped 598 (``server_termination.wait()``). Application code should not try to set 599 the Event directly, rather it will call `stop()` to terminate the 600 server. 601 602 :ivar port: tells on which port the TCP server awaits connection 603 604 MT-safety 605 --------- 606 All public methods (not starting with an underscore ``_``) are 607 MT-safe 608 609 :group Communication on the ivybus: start, send_msg, send_direct_message, 610 send_ready_message, handle_msg, stop 611 612 :group Inspecting the ivybus: get_clients, _get_client, get_client_with_name 613 614 :group Our own subscriptions: get_subscriptions, bind_msg, unbind_msg, 615 _add_subscription, _remove_subscription, _get_fct_for_subscription 616 617 """ 618 # Impl. note: acquiring/releasing the global lock in methods 619 # requiring it could be done w/ a decorator instead of repeating 620 # the acquire-try-finally-release block each time, but then we 621 # won't be compatible w/ py < 2.4 and I do not want this 622
623 - def __init__(self, agent_name, ready_msg="", 624 app_callback = void_function, 625 die_callback = void_function, 626 usesDaemons=False):
627 """ 628 Builds a new IvyServer. A client only needs to call `start()` on the 629 newly created instances to connect to the corresponding Ivy bus and to 630 start communicating with other applications. 631 632 MT-safety: both functions `app_callback` and `die_callback` must be 633 prepared to be called concurrently 634 635 :Parameters: 636 - `agent_name`: the client's agent name 637 - `ready_msg`: a message to send to clients when they connect 638 - `app_callback`: a function called each time a client connects or 639 disconnects. This function is called with a single parameter 640 indicating which event occured: `IvyApplicationConnected` or 641 `IvyApplicationDisconnected`. 642 - `die_callback`: called when the IvyServer receives a DIE message 643 - `usesDaemons`: see above. 644 645 :see: `bind_msg()`, `start()` 646 """ 647 self._thread = None 648 649 # the empty string is equivalent to INADDR_ANY 650 SocketServer.TCPServer.__init__(self, ('',0),IvyHandler) 651 self.port = self.socket.getsockname()[1] 652 #self.allow_reuse_address=True 653 654 # private, maps (ip,port) to IvyClient! 655 self._clients = {} 656 657 # idx -> (regexp, function), see bind_msg() for details, below 658 self._subscriptions = {} 659 # the next index to use within the _subscriptions map. 660 self._next_subst_idx = 0 661 662 self.agent_name = agent_name 663 self.ready_message = ready_msg 664 665 # app_callback's parameter event=CONNECTED / DISCONNECTED 666 self.app_callback = app_callback 667 self.die_callback = die_callback 668 self.direct_callback = void_function 669 self.regexp_change_callback = void_function 670 671 # the global_lock protects: _clients, _subscriptions 672 # and _next_subst_idx 673 self._global_lock = threading.RLock() 674 675 self.usesDaemons = usesDaemons 676 self.server_termination = threading.Event() 677 678 import time,random 679 self.agent_id=agent_name+time.strftime('%Y%m%d%H%M%S')+"%05i"%random.randint(0,99999)+str(self.port)
680
681 - def serve_forever(self):
682 """ 683 Handle requests (calling `handle_request()`) until doomsday... or 684 until `stop()` is called. 685 686 This method is registered as the target method for the thread. 687 It is also responsible for launching the UDP server in a separate 688 thread, see `UDP_init_and_listen` for details. 689 690 You should not need to call this method, use `start` instead. 691 """ 692 broadcast, port = decode_ivybus(self.ivybus) 693 l=lambda server=self: UDP_init_and_listen(broadcast, port, server) 694 t2 = threading.Thread(target=l) 695 t2.setDaemon(self.usesDaemons) 696 log('Starting UDP listener') 697 t2.start() 698 699 self.socket.settimeout(0.1) 700 while not self.server_termination.isSet(): 701 self.handle_request() 702 log('TCP Ivy Server terminated')
703
704 - def start(self, ivybus=None):
705 """ 706 Binds the server to the ivybus. The server remains connected until 707 `stop` is called, or until it receives and accepts a 'die' message. 708 709 :except IvyIllegalStateError: if the server has already been 710 started 711 """ 712 if self._thread is not None: 713 error('Cannot start: IvyServer already started') 714 raise IvyIllegalStateError('not running') 715 716 self.ivybus = ivybus 717 import socket 718 719 log('Starting IvyServer on port %li', self.port) 720 self.server_termination.clear() 721 self._thread = threading.Thread(target=self.serve_forever) 722 self._thread.setDaemon(self.usesDaemons) 723 self._thread.start()
724
725 - def stop(self):
726 """ 727 Disconnects the server from the ivybus. It also sets the 728 `server_termination` event. 729 730 :except IvyIllegalStateError: if the server is not running started 731 """ 732 if not self.isAlive(): 733 error('Cannot stop: not running') 734 raise IvyIllegalStateError('not running') 735 736 self._global_lock.acquire() 737 try: 738 import socket 739 for client in self._clients.values(): 740 try: 741 client.wave_bye() 742 except socket.error, e: 743 pass 744 finally: 745 self._global_lock.release() 746 self.server_termination.set() 747 self._thread.join() 748 self._thread = None
749
750 - def isAlive(self):
751 if self._thread is None: 752 return False 753 return self._thread.isAlive()
754
755 - def get_clients(self):
756 """ 757 Returns the list of the agent names of all connected clients 758 759 :see: get_client_with_name 760 """ 761 self._global_lock.acquire() 762 try: 763 return [c.agent_name for c in self._clients.values() 764 if c.status == INITIALIZED] 765 finally: 766 self._global_lock.release()
767
768 - def _get_client(self, ip, port, socket=None, 769 agent_id=None,agent_name=None):
770 """ 771 Returns the corresponding client, and create a new one if needed. 772 773 If agent_id is not None, the method checks whether a client with the 774 same id is already registered; if it exists, the method exits by 775 returning None. 776 777 You should not need to call this, use `get_client_with_name` instead 778 """ 779 self._global_lock.acquire() 780 try: 781 # if agent_id is provided, check whether it was already registered 782 if agent_id and agent_id in [c.agent_id for c in self._clients.values()]: 783 return None 784 return self._clients.setdefault( (ip,port), 785 IvyClient(ip, port, socket, 786 agent_id, agent_name) ) 787 finally: 788 self._global_lock.release()
789
790 - def get_client_with_name(self, name):
791 """ 792 Returns the list of the clients registered with a given agent-name 793 794 :see: get_clients 795 """ 796 clients=[] 797 self._global_lock.acquire() 798 try: 799 for client in self._clients.values(): 800 if client.agent_name == name: 801 clients.append(client) 802 return clients 803 finally: 804 self._global_lock.release()
805
806 - def handle_new_client(self, client):
807 """ 808 finalisation de la connection avec le client 809 TODO: peut-etre ajouter un flag (en cours de cnx) sur le client, 810 qui empecherait l'envoi de msg. etc. tant que la cnx. n'est pas 811 confirmee 812 """ 813 self.app_callback(client, IvyApplicationConnected)
814
815 - def handle_die_message(self, msg_id, from_client=None):
816 "" 817 should_die=(self.die_callback(from_client,msg_id) != IVY_SHOULD_NOT_DIE) 818 log("Received a die msg from: %s with id: %s -- should die=%s", 819 from_client or "<unknown>", msg_id, should_die) 820 if should_die: 821 self.stop() 822 return should_die
823
824 - def handle_direct_msg(self, client, num_id, msg):
825 "" 826 log("Received a direct msg from: %s with id: %s -- %s", 827 client or "<unknown>", num_id, msg) 828 self.direct_callback(client, num_id, msg)
829
830 - def handle_regexp_change(self, client, event, id, regexp):
831 """ 832 """ 833 log("Regexp change: %s %s regexp %d: %s", 834 client or "<unknown>", 835 event==ADD_REGEXP and "add" or "remove", 836 id, regexp) 837 if event==ADD_REGEXP: 838 event=IvyRegexpAdded 839 else: 840 event=IvyRegexpRemoved 841 self.regexp_change_callback(client, event, id, regexp)
842
843 - def remove_client(self, ip, port, trigger_application_callback=True):
844 """ 845 Removes a registered client 846 847 This method is responsible for calling ``server.app_callback`` 848 849 :return: the removed client, or None if no such client was found 850 851 .. note:: NO NETWORK CLEANUP IS DONE 852 """ 853 self._global_lock.acquire() 854 try: 855 try: 856 removed_client = self._clients[(ip,port)] 857 except KeyError: 858 debug("Trying to remove a non registered client %s:%s",ip,port) 859 return None 860 debug("Removing client %r", removed_client) 861 del self._clients[removed_client] 862 if trigger_application_callback: 863 self.app_callback(removed_client, IvyApplicationDisconnected) 864 865 return removed_client 866 finally: 867 self._global_lock.release()
868
869 - def send_msg(self, message):
870 """ 871 Examine the message and choose to send a message to the clients 872 that subscribed to such a msg 873 874 :return: the number of clients to which the message was sent 875 """ 876 self._global_lock.acquire() 877 count = 0 878 try: 879 for client in self._clients.values(): 880 if client.send_msg(message): 881 count = count + 1 882 finally: 883 self._global_lock.release() 884 885 return count
886
887 - def send_direct_message(self, agent_name, num_id, msg):
888 self._global_lock.acquire() 889 try: 890 for client in self._clients.values(): 891 # TODO: what if multiple clients w/ the same name?!! 892 if client.agent_name == agent_name: 893 self.client.send_direct_message(num_id, msg) 894 return True 895 return False 896 finally: 897 self._global_lock.release()
898
899 - def send_ready_message(self, client):
900 """ 901 """ 902 if self.ready_message: 903 client.send_msg(self.ready_message)
904
905 - def _add_subscription(self, regexp, fct):
906 """ 907 Registers a new regexp and binds it to the supplied fct. The id 908 assigned to the subscription and returned by method is **unique** 909 to that subscription for the life-time of the server object: even in 910 the case when a subscription is unregistered, its id will _never_ 911 be assigned to another subscription. 912 913 :return: the unique id for that subscription 914 """ 915 # explicit lock here: even if this method is private, it is 916 # responsible for the uniqueness of a subscription's id, so we 917 # prefer to lock it one time too much than taking the risk of 918 # forgetting it (hence, the need for a reentrant lock) 919 self._global_lock.acquire() 920 try: 921 idx = self._next_subst_idx 922 self._next_subst_idx += 1 923 self._subscriptions[idx] = (regexp, fct) 924 return idx 925 finally: 926 self._global_lock.release()
927
928 - def _remove_subscription(self, idx):
929 """ 930 Unregisters the corresponding regexp 931 932 .. warning:: this method is not MT-safe, callers must acquire the 933 global lock 934 935 :return: the regexp that has been removed 936 :except KeyError: if no such subscription can be found 937 """ 938 return self._subscriptions.pop(idx)[0]
939
940 - def _get_fct_for_subscription(self, idx):
941 """ 942 943 .. warning:: this method is not MT-safe, callers must acquire the 944 global lock 945 """ 946 return self._subscriptions[int(idx)][1]
947
948 - def handle_msg(self, client, idx, *params):
949 """ 950 Simply call the function bound to the subscription id `idx` with 951 the supplied parameters. 952 """ 953 self._global_lock.acquire() 954 try: 955 try: 956 return self._get_fct_for_subscription(int(idx))(client, *params) 957 except KeyError: 958 # it is possible that we receive a message for a regexp that 959 # was subscribed then unregistered 960 warn('Asked to handle an unknown subscription: id:%r params: %r' 961 ' --ignoring', idx, params) 962 finally: 963 self._global_lock.release()
964
965 - def get_subscriptions(self):
966 self._global_lock.acquire() 967 try: 968 return [ (idx, s[0]) for idx, s in self._subscriptions.items()] 969 finally: 970 self._global_lock.release()
971
972 - def bind_direct_msg(self, on_direct_msg_fct):
973 """ 974 """ 975 self.direct_callback = on_direct_msg_fct
976
977 - def bind_regexp_change(self, on_regexp_change_callback):
978 """ 979 """ 980 self.regexp_change_callback = on_regexp_change_callback
981
982 - def bind_msg(self, on_msg_fct, regexp):
983 """ 984 Registers a new subscriptions, by binding a regexp to a function, so 985 that this function is called whenever a message matching the regexp 986 is received. 987 988 :Parameters: 989 - `on_msg_fct`: a function accepting as many parameters as there is 990 groups in the regexp. For example: 991 992 - the regexp ``'^hello .*'`` corresponds to a function called w/ no 993 parameter, 994 - ``'^hello (.*)'``: one parameter, 995 - ``'^hello=([^ ]*) from=([^ ]*)'``: two parameters 996 997 - `regexp`: (string) a regular expression 998 999 :return: the binding's id, which can be used to unregister the binding 1000 with `unbind_msg()` 1001 """ 1002 self._global_lock.acquire() 1003 idx = self._add_subscription(regexp, on_msg_fct) 1004 try: 1005 for client in self._clients.values(): 1006 client.send_new_subscription(idx, regexp) 1007 finally: 1008 self._global_lock.release() 1009 return idx
1010
1011 - def unbind_msg(self, id):
1012 """ 1013 Unbinds a subscription 1014 1015 :param id: the binding's id, as returned by `bind_msg()` 1016 1017 :return: the regexp corresponding to the unsubscribed binding 1018 :except KeyError: if no such subscription can be found 1019 """ 1020 self._global_lock.acquire() 1021 regexp = None 1022 try: 1023 regexp = self._remove_subscription(id) # KeyError 1024 # notify others that we have no interest anymore in this regexp 1025 for client in self._clients.values(): 1026 client.remove_subscription(id) 1027 finally: 1028 self._global_lock.release() 1029 return regexp
1030
1031 -class IvyHandler(SocketServer.StreamRequestHandler): #BaseRequestHandler):
1032 """ 1033 An IvyHandler is associated to one IvyClient connected to our server. 1034 1035 It runs into a dedicate thread as long as the remote client is connected 1036 to us. 1037 1038 It is in charge of examining all messages that are received and to 1039 take any appropriate actions. 1040 1041 Implementation note: the IvyServer is accessible in ``self.server`` 1042 """
1043 - def handle(self):
1044 """ 1045 """ 1046 # self.request is the socket object 1047 # self.server is the IvyServer 1048 import time 1049 bufsize=1024 1050 socket = self.request 1051 ip = self.client_address[0] 1052 port = self.client_address[1] 1053 1054 trace('New IvyHandler for %s:%s, socket %r', ip, port, socket) 1055 1056 client = self.server._get_client(ip, port, socket) 1057 debug("Got a request from ip=%s port=%s", ip, port) 1058 1059 # First, send our initial subscriptions 1060 socket.send(encode_message(START_INIT, self.server.port, 1061 self.server.agent_name)) 1062 for idx, subscr in self.server.get_subscriptions(): 1063 socket.send(encode_message(ADD_REGEXP, idx, subscr)) 1064 socket.send(encode_message(END_REGEXP, 0)) 1065 1066 while self.server.isAlive(): 1067 from socket import error as socket_error 1068 from socket import timeout as socket_timeout 1069 try: 1070 msgs = socket.recv(bufsize) 1071 except socket_timeout, e: 1072 #debug('timeout on socket bound to client %r', client) 1073 continue 1074 except socket_error, e: 1075 log('Error on socket with %r: %s', client, e) 1076 self.server.remove_client(ip, port) 1077 break # the server will close the TCP connection 1078 1079 if not msgs: 1080 # client is not connected anymore 1081 log('Lost connection with %r', client) 1082 self.server.remove_client(ip, port) 1083 break # the server will close the TCP connection 1084 1085 # Sometimes the message is not fully read on the first try, 1086 # so we insist to get the final newline 1087 if msgs[-1:] != '\n': 1088 1089 # w/ the following idioms (also replicated a second time below) 1090 # we make sure that we wait until we get a message containing 1091 # the final newline, or if the server is terminated we stop 1092 # handling the request 1093 while self.server.isAlive(): 1094 try: 1095 _msg = socket.recv(bufsize) 1096 break 1097 except socket_timeout: 1098 continue 1099 if not self.server.isAlive(): 1100 break 1101 1102 msgs += _msg 1103 while _msg[-1:] != '\n' and self.server.isAlive(): 1104 1105 while self.server.isAlive(): 1106 try: 1107 _msg = socket.recv(bufsize) 1108 break 1109 except socket_timeout: 1110 continue 1111 1112 msgs += _msg 1113 1114 if not self.server.isAlive(): 1115 break 1116 1117 debug("Got a request from ip=%s port=%s: %r", ip, port, msgs) 1118 1119 msgs = msgs[:-1] 1120 1121 1122 msgs=msgs.split('\n') 1123 for msg in msgs: 1124 keep_connection_alive = self.process_ivymessage(msg, client) 1125 if not keep_connection_alive: 1126 self.server.remove_client(ip, port) 1127 break 1128 log('Closing connection to client %r', client)
1129
1130 - def process_ivymessage(self, msg, client):
1131 """ 1132 Examines the message (after passing it through the `decode_msg()` 1133 filter) and takes the appropriate actions depending on the message 1134 types. Please refer to the document `The Ivy Architecture and 1135 Protocol <http://www.tls.cena.fr/products/ivy/documentation>`_ and to 1136 the python code for further details. 1137 1138 1139 :Parameters: 1140 - `msg`: (should not include a newline at end) 1141 1142 :return: ``False`` if the connection should be terminated, ``True`` 1143 otherwise 1144 1145 """ 1146 # cf. static void Receive() in ivy.c 1147 try: 1148 msg_id, num_id, params = decode_msg(msg) 1149 except IvyMalformedMessage: 1150 warn('Received an incorrect message: %r from: %r', msg, client) 1151 # TODO: send back an error message 1152 return True 1153 1154 debug('Got: msg_id: %r, num_id: %r, params: %r', 1155 msg_id, num_id, params) 1156 1157 err_msg = "" 1158 try: 1159 if msg_id == BYE: 1160 # num_id: not meaningful. No parameter. 1161 log('%s waves bye-bye: disconnecting', client) 1162 return False 1163 1164 elif msg_id == ADD_REGEXP: 1165 # num_id=id for the regexp, one parameter: the regexp 1166 err_msg = 'Client %r was not properly initialized'%client 1167 log('%s sending a new subscription id:%r regexp:%r ', 1168 client, num_id, params) 1169 client.add_regexp(num_id, params) 1170 self.server.handle_regexp_change(client, ADD_REGEXP, 1171 num_id, params) 1172 # TODO: handle errors (e.g. 2 subscriptions w/ the same id) 1173 1174 elif msg_id == DEL_REGEXP: 1175 # num_id=id for the regexp to removed, no parameter 1176 err_msg = 'Client %r was not properly initialized'%client 1177 log('%s removing subscription id:%r', client, num_id) 1178 try: 1179 regexp = client.remove_regexp(num_id) 1180 self.server.handle_regexp_change(client, DEL_REGEXP, 1181 num_id, regexp) 1182 except KeyError: 1183 # TODO: what else? 1184 warn('%s tried to remove a non-registered subscription w/ id:%r', client, num_id) 1185 1186 elif msg_id == MSG: 1187 # num_id: regexp_id, parameters: the substrings captured by 1188 # the regexp 1189 log('From %s: (regexp=%s) %r', client, num_id, params) 1190 self.server.handle_msg(client, num_id, *params) 1191 1192 elif msg_id == ERROR: 1193 # num_id: not meaningful, parameter=error msg 1194 warn('Client %r sent a protocol error: %s', client, params) 1195 # TODO: send BYE and close connection, as in ivy.c? 1196 1197 elif msg_id == START_INIT: 1198 # num_id: tcp port number, parameter: the client's agentname 1199 err_msg = 'Client %r sent the initial subscription more than once'%client 1200 client.start_init(params) 1201 log('%s connected from %r', params, client) 1202 1203 elif msg_id == END_INIT: 1204 # num_id: not meaningful. No parameter. 1205 client.end_init() 1206 # app. callback 1207 self.server.handle_new_client(client) 1208 # send ready message 1209 self.server.send_ready_message(client) 1210 1211 elif msg_id == DIE: 1212 # num_id: not meaningful. No parameter. 1213 self.server.handle_die_message(num_id, client) 1214 1215 elif msg_id == DIRECT_MSG: 1216 # num_id: not meaningful. 1217 log('Client %r sent us a direct msg num_id:%s msg:%r', 1218 client, num_id, params) 1219 self.server.handle_direct_msg(client, num_id, params) 1220 1221 else: 1222 warn('Unhandled msg from %r: %r', client, msg) 1223 1224 except IvyIllegalStateError: 1225 raise IvyProtocolError, err_msg 1226 1227 return True
1228 1229 import threading
1230 -class IvyTimer(threading.Thread):
1231 """ 1232 An IvyTimer object is responsible for calling a function regularly. It is 1233 bound to an IvyServer and stops when its server stops. 1234 1235 Interacting with a timer object 1236 ------------------------------- 1237 1238 - Each timer gets a unique id, stored in the attribute ``id``. Note that 1239 a dead timer's id can be reassigned to another one (a dead timer is a 1240 timer that has been stopped) 1241 1242 - To start a timer, simply call its method ``start()`` 1243 1244 - To modify a timer's delay: simply assign ``timer.delay``, the 1245 modification will be taken into account after the next tick. The delay 1246 should be given in milliseconds. 1247 1248 - to stop a timer, assign ``timer.abort`` to ``True``, the timer will stop 1249 at the next tick (the callback won't be called) 1250 1251 MT-safety 1252 --------- 1253 **Please note:** ``start()`` starts a new thread; if the same function is 1254 given as the callback to different timers, that function should be 1255 prepared to be called concurrently. Specifically, if the callback 1256 accesses shared variables, they should be protected against concurrency 1257 problems (using locks e.g.). 1258 1259 """
1260 - def __init__(self, server, nbticks, delay, callback):
1261 """ 1262 Creates a new timer. After creation, call the timer's ``start()`` 1263 method to activate it. 1264 1265 :Parameters: 1266 - `server`: the `IvyServer` related to this timer --when the server 1267 stops, so does the timer. 1268 - `nbticks`: the number of repetition to make. ``0`` (zero) means: 1269 endless loop 1270 - `delay`: the delay, in milliseconds, between two ticks 1271 - `callback`: a function called at each tick. This function is 1272 called with one parameter, the timer itself 1273 1274 """ 1275 threading.Thread.__init__(self) 1276 self.server = server 1277 self.nbticks = nbticks 1278 self.delay = delay # milliseconds 1279 self.callback = callback 1280 self.abort = False 1281 self.id = id(self) 1282 self.setDaemon(server.usesDaemons)
1283
1284 - def run(self):
1285 import time 1286 ticks = -1 1287 while self.server.isAlive() and not self.abort and ticks<self.nbticks: 1288 1289 if self.nbticks: # 0 means: endless 1290 ticks += 1 1291 self.callback(self) 1292 time.sleep(self.delay/1000.0) 1293 log('IvyTimer %s terminated', id(self))
1294 1295
1296 -def is_multicast(ip):
1297 """ 1298 Tells whether the specified ip is a multicast address or not 1299 1300 :param ip: an IPv4 address in dotted-quad string format, for example 1301 192.168.2.3 1302 """ 1303 return int(ip.split('.')[0]) in range(224,239)
1304
1305 -def decode_ivybus(ivybus=None):
1306 """ 1307 Transforms the supplied string into the corrersponding broadcast address 1308 and port 1309 1310 :param ivybus: if ``None`` or empty, defaults to environment variable 1311 ``IVYBUS`` 1312 1313 :return: a tuple made of (broadcast address, port number). For example: 1314 :: 1315 1316 >>> print decode_ivybus('192.168.12:2010') 1317 ('192.168.12.255', 2010) 1318 1319 """ 1320 if not ivybus: 1321 import os 1322 ivybus = os.getenv('IVYBUS', DEFAULT_IVYBUS) 1323 1324 broadcast, port = ivybus.split(':', 1) 1325 port = int(port) 1326 broadcast = broadcast.strip('.') 1327 broadcast += '.' + '.'.join( ['255',]*(4-len(broadcast.split('.')))) 1328 # if broadcast is multicast it had 4 elements -> previous line added a '.' 1329 broadcast = broadcast.strip('.') 1330 debug('Decoded ivybus %s:%s', broadcast, port) 1331 return broadcast, port
1332 1333 1334 1335 if __name__=='__main__': 1336 s=IvyServer(agent_name='TEST_APP', 1337 ready_msg="[Youkou]") 1338 s.start() 1339 import time 1340
1341 - def dflt_fct(*args):
1342 log("DFLT_FCT: Received: %r", args)
1343 1344 time.sleep(1) 1345 1346 1347 for regexp in ('^test .*', '^test2 (.*)$', 'test3 ([^-]*)-?(.*)', '(.*)'): 1348 s.bind_msg(dflt_fct, regexp) 1349 time.sleep(1) 1350 1351 1352 s.send_msg('glop pas glop -et paf') 1353 s.send_msg('glosp pas glop -et paf') 1354 time.sleep(1000000) 1355