Package wtf :: Package opi :: Module listener
[hide private]
[frames] | no frames]

Source Code for Module wtf.opi.listener

  1  # -*- coding: ascii -*- 
  2  # 
  3  # Copyright 2006-2012 
  4  # Andr\xe9 Malo or his licensors, as applicable 
  5  # 
  6  # Licensed under the Apache License, Version 2.0 (the "License"); 
  7  # you may not use this file except in compliance with the License. 
  8  # You may obtain a copy of the License at 
  9  # 
 10  #     http://www.apache.org/licenses/LICENSE-2.0 
 11  # 
 12  # Unless required by applicable law or agreed to in writing, software 
 13  # distributed under the License is distributed on an "AS IS" BASIS, 
 14  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 15  # See the License for the specific language governing permissions and 
 16  # limitations under the License. 
 17  """ 
 18  Listener Socket 
 19  =============== 
 20   
 21  Here's the abstraction to the socket handling implemented. 
 22   
 23  :Variables: 
 24   - `AF_INET`: INET address family 
 25   - `AF_INET6`: INET6 address family (``None`` if not available) 
 26   - `AF_UNIX`: UNIX address family (``None`` if not available) 
 27   
 28  :Types: 
 29   - `AF_INET`: ``int`` 
 30   - `AF_INET6`: ``int`` 
 31   - `AF_UNIX`: ``int`` 
 32  """ 
 33  __author__ = u"Andr\xe9 Malo" 
 34  __docformat__ = "restructuredtext en" 
 35   
 36  import errno as _errno 
 37  import os as _os 
 38  import re as _re 
 39  import socket as _socket 
 40  import sys as _sys 
 41  import warnings as _warnings 
 42   
 43  from wtf import Error, WtfWarning 
 44  from wtf import osutil as _osutil 
 45  from wtf.config import ConfigurationError 
 46   
 47  AF_INET = _socket.AF_INET 
 48  AF_INET6 = getattr(_socket, "AF_INET6", None) 
 49  AF_UNIX = getattr(_socket, "AF_UNIX", None) 
 50   
 51   
52 -class ShutdownWarning(WtfWarning):
53 """ Socket shutdown failures """
54
55 -class ListenerWarning(WtfWarning):
56 """ Duplicate listener detected """
57
58 -class SocketError(Error):
59 """ Socket error """
60
61 -class SocketTimeout(SocketError):
62 """ Socket timeout """
63
64 -class SocketPollError(SocketError):
65 """ Socket poll error """
66 67
68 -class ListenerSocket(object):
69 """ 70 Abstraction to the listener socket 71 72 This actually can contain more than one actual socket, but provides 73 an interface as it was one. 74 75 :CVariables: 76 - `_TYPES`: Supported socket types and configuration patterns 77 (``(('name', (regex, ...)), ...)``) 78 79 :IVariables: 80 - `_sockets`: List of actual sockets (``[socket, ...]``) 81 82 :Types: 83 - `_TYPES`: ``tuple`` 84 """ 85 _TYPES = ( 86 (u'tcp', ( 87 _re.compile(ur'(?:(?P<ip>[^:]+|\[[^\]]+]|\*):)?(?P<port>\d+)$'), 88 )), 89 (u'unix', ( 90 _re.compile(ur'(?P<path>.+)\((?P<perm>\d+)\)$'), 91 _re.compile(ur'(?P<path>.+)(?P<perm>)$'), 92 )), 93 ) 94 _sockets = None 95
96 - def __init__(self, listen, basedir=None):
97 """ 98 Initialization 99 100 :Parameters: 101 - `listen`: The addresses to listen on, may not be empty 102 - `basedir`: Basedir for relative paths 103 104 :Types: 105 - `listen`: ``iterable`` 106 - `basedir`: ``str`` 107 """ 108 # pylint: disable = R0912 109 110 if not listen: 111 raise ConfigurationError("No listeners configured") 112 113 # The way some OS work require us to follow a two-step-approach here: 114 # First we "prepare" the sockets by determining the details for 115 # every socket. The we reorder them, so we can filter out dupes 116 # or includes. Includes are bindings which are already handled 117 # by another binding, like localhost:1206 is included in *:1206 118 # A special, but related problem is the inclusion of IPv4 in IPv6. 119 msg = "Invalid listen configuration: %s" 120 self._sockets, kinds = [], dict(self._TYPES) 121 for bind in listen: 122 obind, fixed, tocheck = repr(bind), None, self._TYPES 123 if ':' in bind: 124 fixed = bind[:bind.find(':')].lower() 125 if fixed in kinds: 126 tocheck = (fixed, kinds[fixed]) 127 bind = bind[len(fixed) + 1:] 128 else: 129 fixed = None 130 for kind, rexs in tocheck: 131 if bind.startswith(kind + ':'): 132 fixed, bind = bind, bind[len(kind) + 1:] 133 for rex in rexs: 134 match = rex.match(bind) 135 if match: 136 break 137 else: 138 match = None 139 if match is not None: 140 method = getattr(self, "_setup_" + kind) 141 try: 142 method(match.group, basedir) 143 except ConfigurationError, e: 144 stre = str(e) 145 e = _sys.exc_info() 146 try: 147 raise e[0], (msg % obind) + ": " + stre, e[2] 148 finally: 149 del e 150 break 151 else: 152 raise ConfigurationError(msg % obind) 153 154 self.accept = self._finalize_listeners(msg)
155
156 - def _finalize_listeners(self, msg):
157 """ 158 Finalize the listening sockets 159 160 This method actually sets the sockets to the LISTEN state. 161 162 :Parameters: 163 - `msg`: Configuration error message template 164 165 :Types: 166 - `msg`: ``str`` 167 168 :return: Socket acceptor 169 :rtype: ``callable`` 170 171 :Exceptions: 172 - `ConfigurationError`: No listeners available 173 """ 174 if not self._sockets: 175 raise ConfigurationError("No listener sockets") 176 177 memory, toremove = {}, [] 178 for socket in sorted(self._sockets): 179 if socket.key() in memory or socket.anykey() in memory: 180 # Do not issue the warning on any-ipv4/ipv6 inclusion 181 if socket.key() != socket.anykey() or \ 182 memory[socket.key()] == socket.family(): 183 _warnings.warn("Duplicate listen: %s" % (socket.bindspec), 184 category=ListenerWarning) 185 toremove.append(socket) 186 continue 187 _osutil.close_on_exec(socket) 188 socket.setblocking(False) 189 try: 190 socket.bind() 191 socket.listen(_socket.SOMAXCONN) 192 except _socket.error, e: 193 stre = str(e) 194 e = _sys.exc_info() 195 try: 196 raise ConfigurationError, \ 197 (msg % socket.bindspec) + ": " + stre, e[2] 198 finally: 199 del e 200 else: 201 memory[socket.key()] = socket.family() 202 203 while toremove: 204 socket = toremove.pop() 205 self._sockets.remove(socket) 206 try: 207 socket.close() 208 except (_socket.error, OSError), e: 209 _warnings.warn("Socket shutdown problem: %s" % str(e), 210 category=ShutdownWarning) 211 212 return Acceptor(item.realsocket for item in self._sockets)
213
214 - def __del__(self):
215 self.close()
216
217 - def close(self):
218 """ Shutdown the sockets """ 219 sockets, self._sockets = self._sockets, None 220 if sockets is not None: 221 for socket in sockets: 222 try: 223 socket.close() 224 except (_socket.error, OSError), e: 225 _warnings.warn("Socket shutdown problem: %s" % str(e), 226 category=ShutdownWarning)
227
228 - def _setup_tcp(self, bind, basedir=None):
229 """ 230 Setup TCP/IP(v6) socket and append it to the global list 231 232 :Parameters: 233 - `bind`: Bind parameter accessor (``match.group``) 234 - `basedir`: Basedir for relative paths (unused) 235 236 :Types: 237 - `bind`: ``callable`` 238 - `basedir`: ``basestring`` 239 """ 240 # pylint: disable = W0613 241 242 obind = repr(bind(0)) 243 host, port, flags = bind(u'ip'), bind(u'port'), 0 244 port = int(port) 245 if not host or host == u'*': 246 host, flags = None, _socket.AI_PASSIVE 247 elif host.startswith(u'[') and host.endswith(u']'): 248 host = host[1:-1].encode('ascii') # IPv6 notation [xxx:xxx:xxx] 249 else: 250 host = host.encode('idna') 251 try: 252 adi = _socket.getaddrinfo(host, port, 253 _socket.AF_UNSPEC, _socket.SOCK_STREAM, 0, flags) 254 for family, stype, proto, _, bind in adi: 255 if not _socket.has_ipv6 and family == AF_INET6: 256 continue 257 258 try: 259 socket = _socket.socket(family, stype, proto) 260 except _socket.error, e: 261 if e[0] == _errno.EAFNOSUPPORT and host is None and \ 262 family == AF_INET6: 263 # grmpf. 264 # There are systems (e.g. linux) which emit 265 # IPv6 on ANY, even if they don't support it. 266 # Or is it the libc? Who cares anyway... 267 continue 268 raise 269 socket.setsockopt(_socket.SOL_SOCKET, _socket.SO_REUSEADDR, 1) 270 self._sockets.append( 271 InetSocket(socket, obind, host, family, bind) 272 ) 273 except _socket.error: 274 e = _sys.exc_info() 275 try: 276 raise ConfigurationError, e[1], e[2] 277 finally: 278 del e
279
280 - def _setup_unix(self, bind, basedir=None):
281 """ 282 Setup UNIX domain socket 283 284 :Parameters: 285 - `bind`: Bind parameter accessor (``match.group``) 286 - `basedir`: Basedir for relative paths 287 288 :Types: 289 - `bind`: ``callable`` 290 - `basedir`: ``str`` 291 """ 292 if AF_UNIX is None: 293 raise ConfigurationError("UNIX domain sockets are not available") 294 295 obind = repr(bind(0)) 296 if bind(u'perm'): 297 try: 298 socket_perm = int(bind('perm'), 8) 299 except (TypeError, ValueError): 300 raise ConfigurationError("Invalid permission") 301 umask = 0777 & ~socket_perm 302 else: 303 umask = None 304 basedir = basedir or _os.getcwd() 305 if not isinstance(basedir, unicode): 306 basedir = basedir.decode(_sys.getfilesystemencoding()) 307 path = _os.path.normpath(_os.path.join( 308 basedir, bind(u'path') 309 )).encode(_sys.getfilesystemencoding()) 310 socket = _socket.socket(AF_UNIX, _socket.SOCK_STREAM) 311 self._sockets.append(UnixSocket(socket, obind, path, umask))
312 313
314 -class SocketDecorator(object):
315 """ 316 Socket decorating container 317 318 Derive from this container in order to build new concrete containers. 319 These containers are necessary for proper duplicate/warning/error 320 handling, because we need some context for the socket. The socket 321 ordering is also defined in these containers (via `__cmp__`). 322 323 :See: `UnixSocket`, `InetSocket` 324 325 :CVariables: 326 - `_famcomp`: Index for address family comparisons 327 328 :IVariables: 329 - `realsocket`: The actual socket object 330 - `bindspec`: The bind specification from the config 331 332 :Types: 333 - `_famcomp`: ``dict`` 334 - `realsocket`: ``socket.socket`` 335 - `bindspec`: ``str`` 336 """ 337 _famcomp = dict((fam, idx) for idx, fam in enumerate(( 338 AF_UNIX, AF_INET6, AF_INET 339 )) if fam is not None) 340
341 - def __init__(self, socket, bindspec):
342 """ 343 Initialization 344 345 :Parameters: 346 - `socket`: The socket object to decorate 347 - `bindspec`: The bind specification from config 348 349 :Types: 350 - `socket`: ``socket.socket`` 351 - `bindspec`: ``str`` 352 """ 353 self.realsocket = socket 354 self.bindspec = bindspec
355
356 - def __cmp__(self, other):
357 """ 358 Compare 3-way with another object 359 360 Comparison is done by the socket family index. If the other object 361 is not a `SocketDecorator`, the ``id()`` s are compared. 362 363 :Parameters: 364 - `other`: The other object 365 366 :Types: 367 - `other`: `SocketDecorator` 368 369 :return: Comparison result (``-1``, ``0``, ``1``) for `self` being 370 less, equal or greater than/to `other` 371 :rtype: ``int`` 372 373 :Exceptions: 374 - `NotImplementedError`: The socket family of either socket is not 375 in the index 376 """ 377 if not isinstance(other, self.__class__): 378 return cmp(id(self), id(other)) 379 try: 380 return cmp( 381 self._famcomp[self.family()], 382 self._famcomp[other.family()] 383 ) 384 except KeyError: 385 raise NotImplementedError()
386
387 - def __eq__(self, other):
388 """ 389 Compary (2-way) by identity 390 391 :Parameters: 392 - `other`: The other object 393 394 :Types: 395 - `other`: `SocketDecorator` 396 397 :return: Are the objects identical? 398 :rtype: ``bool`` 399 """ 400 return id(self) == id(other)
401
402 - def __repr__(self):
403 """ 404 String representation of the object (suitable for debugging) 405 406 :return: The string representation 407 :rtype: ``str`` 408 """ 409 return "<%s.%s fileno=%s, family=%s, key=%r>" % ( 410 self.__class__.__module__, self.__class__.__name__, 411 self.fileno(), self.family(), self.key(), 412 )
413
414 - def __del__(self):
415 """ Destructor """ 416 self.close()
417
418 - def __getattr__(self, name):
419 """ 420 Delegate all undefined symbol requests to the real socket 421 422 :Parameters: 423 - `name`: The symbol to look up 424 425 :Types: 426 - `name`: ``str`` 427 """ 428 return getattr(self.realsocket, name)
429
430 - def bind(self):
431 """ Bind the socket according to its bindspec """ 432 raise NotImplementedError()
433
434 - def family(self):
435 """ 436 Determine the socket address family 437 438 :return: The family 439 :rtype: ``int`` 440 """ 441 raise NotImplementedError()
442
443 - def key(self):
444 """ 445 Determine the key of the socket, derived from the bindspec 446 447 This key can be considered a normalized version of the bindspec. It 448 has to be hashable. 449 450 :return: The key 451 :rtype: any 452 """ 453 raise NotImplementedError()
454
455 - def anykey(self):
456 """ 457 Determine the key of the socket if bindspec would point to ANY 458 459 :See: `key` 460 461 :return: The key 462 :rtype: any 463 """ 464 raise NotImplementedError()
465 466
467 -class UnixSocket(SocketDecorator):
468 """ 469 Decorator for UNIX domain sockets 470 471 :IVariables: 472 - `_bound`: Was the socket bound to a path? 473 - `_path`: The path to bind to 474 - `_umask`: The umask to be set when binding to the path (maybe ``None``) 475 - `_normpath`: The normalized path (symlinks resolved) (used as key) 476 477 :Types: 478 - `_bound`: ``bool`` 479 - `_path`: ``str`` 480 - `_umask`: ``int`` 481 - `_normpath`: ``str`` 482 """
483 - def __init__(self, socket, bindspec, path, umask):
484 """ 485 Initialization 486 487 :Parameters: 488 - `socket`: The actual socket object 489 - `bindspec`: Binding string from configuration 490 - `path`: Path to bind to 491 - `umask`: Umask to apply when binding to the path 492 493 :Types: 494 - `socket`: ``socket.socket`` 495 - `bindspec`: ``str`` 496 - `path`: ``str`` 497 - `umask`: ``int`` 498 """ 499 super(UnixSocket, self).__init__(socket, bindspec) 500 self._bound, self._path, self._umask = False, path, umask 501 self._normpath = _os.path.normpath(_os.path.realpath(path))
502
503 - def close(self):
504 """ Remove the socket path and close the file handle """ 505 _osutil.unlink_silent(self._path) 506 self.realsocket.close()
507
508 - def bind(self):
509 """ Bind to the socket path """ 510 old_umask = None 511 try: 512 if self._umask is not None: 513 old_umask = _os.umask(self._umask) 514 _osutil.unlink_silent(self._path) 515 self._bound, _ = True, self.realsocket.bind(self._path) 516 finally: 517 if old_umask is not None: 518 _os.umask(old_umask)
519
520 - def family(self):
521 """ Determine the socket family """ 522 return AF_UNIX
523
524 - def key(self):
525 """ Determine the socket key """ 526 return self._normpath
527
528 - def anykey(self):
529 """ Determine ANY key """ 530 return None
531 532
533 -class InetSocket(SocketDecorator):
534 """ 535 Decorator for TCP/IP(v6) sockets 536 537 :IVariables: 538 - `_bind`: bind value from ``getaddrinfo(3)`` 539 - `_host`: Hostname/IP (or ``None`` for ANY) 540 - `_family`: socket family 541 542 :Types: 543 - `_bind`: ``tuple`` 544 - `_host`: ``str`` 545 - `_family`: ``int`` 546 """
547 - def __init__(self, socket, bindspec, host, family, bind):
548 """ 549 Initialization 550 551 :Parameters: 552 - `socket`: Actual socket object 553 - `bindspec`: Bind specification from config 554 - `host`: Hostname/IP or ``None`` 555 - `family`: Socket family 556 - `bind`: bind value from ``getaddrinfo(3)`` 557 558 :Types: 559 - `socket`: ``socket.socket`` 560 - `bindspec`: ``str`` 561 - `host`: ``str`` 562 - `family`: ``int`` 563 - `bind`: ``tuple`` 564 """ 565 super(InetSocket, self).__init__(socket, bindspec) 566 self._bind, self._host, self._family = bind, host, family
567
568 - def __cmp__(self, other):
569 """ 570 Compare (3-way) to a different object 571 572 In addition to the base's ``__cmp__`` method, we compare the host 573 and the rest of the bind value. 574 """ 575 # pylint: disable = W0212 576 return ( 577 super(InetSocket, self).__cmp__(other) or 578 cmp(self._host or '', other._host or '') or 579 cmp(self._bind[1:], other._bind[1:]) 580 )
581
582 - def bind(self):
583 """ Bind the socket according to bindspec """ 584 self.realsocket.bind(self._bind)
585
586 - def family(self):
587 """ Determine the socket family """ 588 return self._family
589
590 - def key(self):
591 """ Determine the socket key """ 592 if self._host is None: 593 return self.anykey() 594 return (self._host, self._family, self._bind[1])
595
596 - def anykey(self):
597 """ Determine the socket ANY key """ 598 return (None, AF_INET, self._bind[1])
599 600
601 -class Acceptor(object):
602 """ Acceptor for multiple connections """ 603 _IGNOREFAIL = set(getattr(_errno, _name, None) for _name in """ 604 EINTR 605 ENOBUFS 606 EPROTO 607 ECONNABORTED 608 ECONNRESET 609 ETIMEDOUT 610 EHOSTUNREACH 611 ENETUNREACH 612 EAGAIN 613 EWOULDBLOCK 614 """.split()) 615 if None in _IGNOREFAIL: 616 _IGNOREFAIL.remove(None) 617
618 - def __init__(self, sockets):
619 """ 620 Initialization 621 622 :Parameters: 623 - `sockets`: List of sockets to poll 624 625 :Types: 626 - `sockets`: ``iterable`` 627 """ 628 import collections, select 629 try: 630 pollset = select.poll 631 except AttributeError: 632 pollset = _SelectAdapter() 633 else: 634 pollset = _PollAdapter() 635 self._fdmap = {} 636 for socket in sockets: 637 fd = socket.fileno() 638 pollset.add(fd) 639 self._fdmap[fd] = socket 640 self._set = pollset 641 self._backlog = collections.deque()
642
643 - def __call__(self, timeout=None):
644 """ 645 Accept a new connection 646 647 :Parameters: 648 - `timeout`: Timeout in seconds 649 650 :Types: 651 - `timeout`: ``float`` 652 653 :return: New socket and the peername 654 :rtype: ``tuple`` 655 656 :Exceptions: 657 - `SocketTimeout`: accept call timed out 658 - `SocketError`: An error occured while accepting the socket 659 """ 660 while True: 661 try: 662 sock, peer = self._accept(timeout) 663 except _socket.error, e: 664 if e[0] in self._IGNOREFAIL: 665 continue 666 e = _sys.exc_info() 667 try: 668 raise SocketError, e[1], e[2] 669 finally: 670 del e 671 _osutil.close_on_exec(sock.fileno()) 672 return sock, peer
673
674 - def _accept(self, timeout=None):
675 """ 676 Accept a connection 677 678 :Parameters: 679 - `timeout`: Timeout in seconds 680 681 :Types: 682 - `timeout`: ``float`` 683 684 :return: The new connection socket and the peername 685 :rtype: ``tuple`` 686 687 :Exceptions: 688 - `SocketTimeout`: accept call timed out 689 - `SocketPollError`: Error with poll call 690 - `socket.error`: Socket error 691 """ 692 backlog = self._backlog 693 if not backlog: 694 pollset, timeout_used = self._set, timeout 695 if timeout_used is None: 696 timeout_used = 1000 697 else: 698 timeout_used = int(timeout_used * 1000) 699 while True: 700 try: 701 ready = pollset.poll(timeout_used) 702 except pollset.error, e: 703 if e[0] == _errno.EINTR: 704 continue 705 e = _sys.exc_info() 706 try: 707 raise SocketPollError, e[1], e[2] 708 finally: 709 del e 710 if ready: 711 break 712 elif timeout is None: 713 continue 714 raise SocketTimeout(timeout) 715 backlog.extendleft(item[0] for item in ready) 716 return self._fdmap[backlog.pop()].accept()
717 718
719 -class _AdapterInterface(object):
720 """ 721 Adapter poll API to select implementation 722 723 :IVariables: 724 - `error`: Exception to catch on poll() 725 726 :Types: 727 - `error`: ``Exception`` 728 """ 729
730 - def __init__(self):
731 """ Initialization """
732
733 - def add(self, fd):
734 """ 735 Register a new file descriptor 736 737 :Parameters: 738 - `fd`: File descriptor to register 739 740 :Types: 741 - `fd`: ``int`` 742 743 :Exceptions: 744 - `ValueError`: Error while creating an integer out of `fd` 745 - `TypeError`: Error while creating an integer out of `fd` 746 """
747
748 - def remove(self, fd):
749 """ 750 Unregister a file descriptor 751 752 :Parameters: 753 - `fd`: File descriptor to unregister 754 755 :Types: 756 - `fd`: ``int`` 757 758 :Exceptions: 759 - `ValueError`: Error while creating an integer out of `fd` 760 - `TypeError`: Error while creating an integer out of `fd` 761 - `KeyError`: The descriptor was not registered before 762 """
763
764 - def poll(self, timeout=None):
765 """ 766 Poll the list of descriptors 767 768 :Parameters: 769 - `timeout`: Poll timeout in milliseconds 770 771 :Types: 772 - `timeout`: ``int`` 773 774 :return: List of (descriptor, event) tuples, event is useless, though 775 :rtype: ``list`` 776 777 :Exceptions: 778 - `self.error`: Select error occured 779 """
780 781
782 -class _SelectAdapter(object):
783 # pylint: disable = C0111 784 __implements__ = [_AdapterInterface] 785
786 - def __init__(self):
787 import select 788 self.error = select.error 789 self._rfds = set()
790
791 - def add(self, fd):
792 self._rfds.add(int(fd))
793
794 - def remove(self, fd):
795 self._rfds.remove(int(fd))
796
797 - def poll(self, timeout=None):
798 import select 799 if timeout is not None: 800 timeout = float(timeout) / 1000.0 801 rfds, _, _ = select.select(self._rfds, (), (), timeout) 802 return [(item, 0) for item in rfds]
803 804
805 -class _PollAdapter(object):
806 # pylint: disable = C0111 807 __implements__ = [_AdapterInterface] 808
809 - def __init__(self):
810 import select 811 self.error = select.error 812 self._pollset = select.poll() 813 self.poll = self._pollset.poll
814
815 - def add(self, fd):
816 import select 817 self._pollset.register(fd, select.POLLIN)
818
819 - def remove(self, fd):
820 self._pollset.unregister(fd)
821
822 - def poll(self, timeout=None): # pylint: disable = E0202
823 return self._pollset.poll(timeout)
824