Package multivac
[hide private]
[frames] | no frames]

Source Code for Package multivac

  1  #! /usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3  """ 
  4  MultiVAC generic framework for building tree-based status applications. 
  5   
  6  This modules provides an implementation of a generic framework, called 
  7  MultiVAC, for use in building applications working with trees of elements, in 
  8  which the status of a node derives from the status of the leaves. 
  9   
 10  Two classes are defined, one for each table in the relational model 
 11  L{Element},  L{Project}. Also, two additional classes are defined for the 
 12  association relationships between the former: L{TagElement} and L{TagProject}. 
 13  """ 
 14   
 15  from version import * 
 16   
 17  import os 
 18  import sys 
 19  import warnings 
 20   
 21  from sqlalchemy import inspect 
 22  from sqlalchemy.engine import create_engine, Engine 
 23  from sqlalchemy.events import event 
 24  from sqlalchemy.ext.associationproxy import association_proxy 
 25  from sqlalchemy.ext.hybrid import hybrid_property 
 26  from sqlalchemy.orm import mapper, relationship, scoped_session, sessionmaker 
 27  from sqlalchemy.orm.collections import attribute_mapped_collection 
 28  from sqlalchemy.orm.exc import NoResultFound 
 29  from sqlalchemy.orm.session import object_session 
 30  from sqlalchemy.orm.util import identity_key 
 31  from sqlalchemy.schema import Column, MetaData, Table 
 32  from sqlalchemy.schema import ForeignKeyConstraint, Index, PrimaryKeyConstraint, UniqueConstraint 
 33  from sqlalchemy.sql import functions 
 34  from sqlalchemy.sql.expression import literal, literal_column 
 35  from sqlalchemy.types import Boolean, DateTime, Integer, String, Text 
 36   
 37  #: Metadata object for this model 
 38  metadata = MetaData() 
 39  #: Name-indexed dict containing the definition of every table 
 40  tables = {} 
 41  #: Name-indexed dict containing the mapper of every table 
 42  mappers = {} 
43 44 -def init(session_maker, **kwargs):
45 """ 46 Initialize the db connection, set up the tables and map the classes. 47 48 @param session_maker: Session generator to bind to the model 49 @type session_maker: sqlalchemy.orm.session.Session factory 50 @param kwargs: Additional settings for the mapper 51 @type kwargs: dict 52 """ 53 54 # Bind engine 55 metadata.bind = session_maker.bind 56 # Setup model 57 setup_tables() 58 setup_mappers(**kwargs) 59 setup_events(session_maker, **kwargs)
60
61 -def setup_tables():
62 """ 63 Define the tables, columns, keys and constraints of the DB. 64 """ 65 66 global tables 67 68 tables['element'] = Table('element', metadata, 69 Column('id', Integer, nullable=False), 70 Column('forced_status', Boolean, nullable=False, default=False), 71 Column('name', String(120), nullable=False), 72 Column('parent_id', Integer, nullable=True), 73 Column('project_id', Integer, nullable=False), 74 Column('status', String(20), nullable=False), 75 PrimaryKeyConstraint('id'), 76 ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'), 77 ForeignKeyConstraint(['project_id', 'parent_id'], ['element.project_id', 'element.id'], ondelete='CASCADE'), 78 UniqueConstraint('project_id', 'parent_id', 'name'), 79 UniqueConstraint('project_id', 'id') 80 ) 81 Index('element_uk_root', tables['element'].c.project_id, tables['element'].c.name, 82 postgresql_where=tables['element'].c.parent_id == None, unique=True 83 ) 84 85 tables['status_history'] = Table('status_history', metadata, 86 Column('id', Integer, nullable=False), 87 Column('element_id', Integer, nullable=False), 88 Column('timestamp', DateTime, nullable=False, default=functions.now()), 89 Column('status', String(20), nullable=False), 90 PrimaryKeyConstraint('id'), 91 ForeignKeyConstraint(['element_id'], ['element.id'], ondelete='CASCADE'), 92 ) 93 Index('status_history_ik_order', tables['status_history'].c.element_id, tables['status_history'].c.timestamp) 94 95 tables['project'] = Table('project', metadata, 96 Column('id', Integer, nullable=False), 97 Column('name', String(20), nullable=False), 98 PrimaryKeyConstraint('id'), 99 UniqueConstraint('name'), 100 ) 101 102 tables['tag_element'] = Table('tag_element', metadata, 103 Column('name', String(20), nullable=False), 104 Column('element_id', Integer, nullable=False), 105 Column('value', Text(), nullable=True), 106 PrimaryKeyConstraint('element_id', 'name'), 107 ForeignKeyConstraint(['element_id'], ['element.id'], ondelete='CASCADE'), 108 ) 109 110 tables['tag_project'] = Table('tag_project', metadata, 111 Column('name', String(20), nullable=False), 112 Column('project_id', Integer, nullable=False), 113 Column('value', Text(), nullable=True), 114 PrimaryKeyConstraint('project_id', 'name'), 115 ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'), 116 )
117
118 -def setup_mappers(**kwargs):
119 """ 120 Define the mapping between tables and classes, and the relationships that link them. 121 122 @kwarg extra_mapping: Mapping between database tables and classes 123 @type extra_mapping: dict 124 @kwarg extra_properties: Dictionary of additional properties for a table 125 @type extra_properties: dict 126 @kwarg extra_extensions: Dictionary of additional extensions for a table 127 @type extra_extensions: dict 128 @kwarg extra_kwargs: Dictionary of additional arguments for a mapper 129 @type extra_kwargs: dict 130 """ 131 132 global mappers 133 134 mapping = { 135 'element' : Element, 136 'status_history' : StatusHistory, 137 'project' : Project, 138 'tag_element' : TagElement, 139 'tag_project' : TagProject, 140 } 141 mapping.update(kwargs.get('extra_mapping', dict())) 142 143 assert issubclass(mapping['element'], Element) 144 assert issubclass(mapping['status_history'], StatusHistory) 145 assert issubclass(mapping['project'], Project) 146 assert issubclass(mapping['tag_element'], TagElement) 147 assert issubclass(mapping['tag_project'], TagProject) 148 149 properties = {} 150 properties['element'] = { 151 '_id' : tables['element'].c.id, 152 '_parent_id' : tables['element'].c.parent_id, 153 '_project_id' : tables['element'].c.project_id, 154 '_status' : tables['element'].c.status, 155 '_children' : relationship(mapping['element'], back_populates='_parent', collection_class=set, 156 primaryjoin = tables['element'].c.parent_id == tables['element'].c.id, 157 cascade='all', passive_deletes=True), 158 '_parent' : relationship(mapping['element'], back_populates='_children', collection_class=set, 159 primaryjoin = tables['element'].c.parent_id == tables['element'].c.id, 160 remote_side = [ tables['element'].c.id ]), 161 '_project' : relationship(mapping['project'], back_populates='elements', collection_class=set), 162 'status_history' : relationship(mapping['status_history'], back_populates='_element', 163 order_by=tables['status_history'].c.id, viewonly=True), 164 'tags' : relationship(mapping['tag_element'], collection_class=attribute_mapped_collection('name'), 165 cascade='all, delete-orphan', passive_deletes=True), 166 } 167 properties['status_history'] = { 168 '_id' : tables['status_history'].c.id, 169 '_element_id' : tables['status_history'].c.element_id, 170 '_timestamp' : tables['status_history'].c.timestamp, 171 '_status' : tables['status_history'].c.status, 172 '_element' : relationship(mapping['element'], back_populates='status_history'), 173 } 174 properties['project'] = { 175 '_id' : tables['project'].c.id, 176 'elements' : relationship(mapping['element'], back_populates='_project', collection_class=set, 177 primaryjoin = tables['element'].c.project_id == tables['project'].c.id, 178 cascade='all', passive_deletes=True), 179 'tags' : relationship(mapping['tag_project'], collection_class=attribute_mapped_collection('name'), 180 cascade='all, delete-orphan', passive_deletes=True), 181 } 182 properties['tag_element'] = {} 183 properties['tag_project'] = {} 184 185 extra_properties = kwargs.get('extra_properties', dict()) 186 for entity in mapping.iterkeys(): 187 properties[entity].update(extra_properties.get(entity, dict())) 188 189 extensions = {} 190 extensions.update(kwargs.get('extra_extensions', dict())) 191 192 options = {} 193 options.update(kwargs.get('extra_kwargs', dict())) 194 195 for name, cls in mapping.iteritems(): 196 mappers[name] = mapper(cls, tables[name], 197 properties=properties.get(name, None), 198 extension=extensions.get(name, None), 199 **options.get(name, {})) 200 201 """ 202 Association proxy to access its tags and retrieve their corresponding value. 203 Example: instance.tag['name'] = 'value' 204 """ 205 Element.tag = association_proxy('tags', 'value', creator=lambda name, value: mapping['tag_element'](name=name, value=value)) 206 Project.tag = association_proxy('tags', 'value', creator=lambda name, value: mapping['tag_project'](name=name, value=value))
207
208 -def setup_events(session_maker, **kwargs):
209 """ 210 Define the events of the model. 211 """ 212 mapping = { 213 'element' : Element, 214 'status_history' : StatusHistory, 215 'project' : Project, 216 'tag_element' : TagElement, 217 'tag_project' : TagProject, 218 } 219 mapping.update(kwargs.get('extra_mapping', dict())) 220 221 event.listen(mapping['element']._children, 'append', mapping['element']._children_added) 222 event.listen(mapping['element']._children, 'remove', mapping['element']._children_removed) 223 event.listen(session_maker, 'before_flush', _session_before_flush) 224 event.listen(Engine, "begin", _sqlite_begin)
225
226 -def _sqlite_begin(conn):
227 if conn.engine.name == 'sqlite': 228 # Foreign keys are NOT enabled by default... WTF! 229 conn.execute("PRAGMA foreign_keys = ON") 230 # Force a single active transaction on a sqlite database. 231 # This is needed to emulate FOR UPDATE locks :( 232 conn.execute("BEGIN EXCLUSIVE")
233
234 -def _session_before_flush(session, flush_context, instances):
235 """ 236 Ensure that when an Element instance is deleted, the children collection of 237 its parent is notified to update and cascade the status change. 238 """ 239 for instance in session.deleted: 240 if isinstance(instance, Element): 241 if instance.parent: 242 instance.parent.children.remove(instance) 243 244 for instance in session.new: 245 if isinstance(instance, Element): 246 session.add(StatusHistory( 247 _element = instance, 248 _status = instance.status 249 )) 250 251 for instance in session.dirty: 252 if isinstance(instance, Element): 253 if inspect(instance).attrs['_status'].history[2]: 254 session.add(StatusHistory( 255 _element = instance, 256 _status = instance.status 257 ))
258
259 -class CTENotSupported(UserWarning):
260 """ 261 User-defined exception to warn the user when calling a method 262 that uses Common Table Expressions (CTE) to perform its work. 263 In most cases, there is an alternative slow path of code. 264 """ 265 pass
266
267 -class ORM_Base(object):
268 """ 269 Base class for mapping the tables. 270 """ 271
272 - def __init__(self, *args, **kwargs):
273 """ 274 Base contructor for all mapped entities. 275 Set the value of attributes based on keyword arguments. 276 277 @param args: Optional arguments to the constructor 278 @type args: tuple 279 @param kwargs: Optional keyword arguments to the constructor 280 @type kwargs: dict 281 """ 282 for name, value in kwargs.iteritems(): 283 setattr(self, name, value)
284
285 -class Element(ORM_Base):
286 """ 287 Mapping class for the table «element». 288 """ 289 290 @hybrid_property
291 - def id(self):
292 """ 293 Read only accessor to prevent setting this field. 294 295 @return: Surrogate primary key 296 @rtype: int 297 """ 298 return self._id
299 300 @hybrid_property
301 - def children(self):
302 """ 303 The collection of child Elements of this instance. 304 305 @return: This instance collection of children Elements 306 @rtype: set<L{Element}> 307 """ 308 return self._children
309 310 @hybrid_property
311 - def parent_id(self):
312 """ 313 Read only accessor to prevent setting this field. 314 315 @return: Foreign key for the parent L{Element} relationship 316 @rtype: int 317 """ 318 return self._parent_id
319 320 @hybrid_property
321 - def parent(self):
322 """ 323 The parent Element of this instance. 324 325 @return: The parent Element 326 @rtype: L{Element} 327 """ 328 return self._parent
329 330 @parent.setter
331 - def parent(self, parent):
332 """ 333 Setter for the related parent Element of this instance. 334 Ensures project coherence between itself and the parent, and proper 335 children collection initialization. Also cascades status changes. 336 337 @param parent: The parent Element to be assigned 338 @type parent: L{Element} 339 """ 340 # Avoid infinite recursion 341 if self.parent == parent: 342 return 343 assert parent == None or isinstance(parent, Element) 344 # Check project coherence (parent - self) 345 if self.parent != None and parent != None: 346 assert parent.project == self.project 347 # Ensure children initialization and check for existence. DO NOT REMOVE 348 if parent != None: 349 assert parent.children != None 350 # Assign new parent 351 self._parent = parent 352 if self.parent != None: 353 self.project = parent.project 354 # Cascade status changes only if it has a parent 355 if self.status and not self.parent.forced_status: 356 self._cascade_status()
357 358 @hybrid_property
359 - def project_id(self):
360 """ 361 Read only accessor to prevent setting this field. 362 363 @return: Foreign key for the parent L{Project} relationship 364 @rtype: int 365 """ 366 return self._project_id
367 368 @hybrid_property
369 - def project(self):
370 """ 371 The related Project of this instance. 372 373 @return: The related Project 374 @rtype: L{Project} 375 """ 376 return self._project
377 378 @project.setter
379 - def project(self, project):
380 """ 381 Setter for the related Project of this instance. 382 Prevents a second assignation. 383 384 @param project: The Project to be assigned 385 @type project: L{Project} 386 """ 387 # Avoid infinite recursion 388 if self.project == project: 389 return 390 assert isinstance(project, Project) 391 # Avoid second assignation 392 if self.project == None: 393 self._project = project 394 else: 395 raise AttributeError('This attribute cannot be modified once it has been assigned.')
396 397 @hybrid_property
398 - def status(self):
399 """ 400 The status of this instance 401 402 @return: status 403 @rtype: str 404 """ 405 return self._status
406 407 @status.setter
408 - def status(self, status):
409 """ 410 Setter for the status of this instance. 411 Ensures the cascade of a status change. 412 413 @param status: The status to be assigned 414 @type status: str 415 """ 416 # Avoid infinite recursion 417 if self.status == status: 418 return 419 else: 420 self._status = status 421 # Cascade status changes 422 if self.parent and not self.parent.forced_status: 423 self._cascade_status()
424
425 - def __init__(self, *args, **kwargs):
426 """ 427 Constructor for Element instances. 428 Ensures that the «forced_status» field is assigned first to cascade 429 status properly. 430 431 @param args: Optional arguments to the constructor 432 @type args: tuple 433 @param kwargs: Optional keyword arguments to the constructor 434 @type kwargs: dict 435 """ 436 # Assign first force_status field 437 if 'forced_status' in kwargs: 438 setattr(self, 'forced_status', kwargs['forced_status']) 439 del kwargs['forced_status'] 440 # Assign the rest of the fields 441 super(Element, self).__init__(*args, **kwargs)
442 443 @classmethod
444 - def _children_added(cls, parent, child, initiator):
445 """ 446 Listener to be executed when an element has to be added to a 447 children collection. 448 Check the added child status and update the parent's one. 449 450 @param parent: The Element that has a new child added 451 @type parent: L{Element} 452 @param child: The Element being added as a child 453 @type child: L{Element} 454 """ 455 if not child.status or parent.forced_status: 456 return 457 if parent.status > child.status: 458 parent.status = child.status 459 elif (parent.status < child.status) and (len(parent.children) == 1): 460 parent.status = child.status
461 462 @classmethod
463 - def _children_removed(cls, parent, child, initiator):
464 """ 465 Listener to be executed when an element has to be removed from a 466 children collection. 467 Check the removed child status and update the parent's one. 468 469 @param parent: The Element that has a child removed 470 @type parent: L{Element} 471 @param child: The Element being removed as a child 472 @type child: L{Element} 473 """ 474 475 if not child.status or parent.forced_status: 476 return 477 new_children = parent.children.difference([child]) 478 if parent.status == child.status and new_children: 479 new_status = min([c.status for c in new_children]) 480 if parent.status != new_status: 481 parent.status = new_status
482
483 - def _cascade_status(self):
484 """ 485 Propagate its status to its parent, in a recursive manner. 486 """ 487 if self.parent.status > self.status: 488 self.parent.status = self.status 489 elif self.parent.status < self.status: 490 new_status = min([c.status for c in self.parent.children]) 491 if self.parent.status != new_status: 492 self.parent.status = new_status
493 494 @property
495 - def ancestors(self):
496 """ 497 Retrieve all the ancestors of this node. 498 If the node has no parents, return an empty list. 499 Else, start retrieving them from the identity map and, when not there, 500 fetch the rest from the database using a CTE. 501 502 @return: A list of all the ancestors of this node, ordered by proximity. 503 @rtype: list<L{Element}> 504 """ 505 cls = self.__class__ 506 session = object_session(self) 507 # Must flush to get the parent_id from the database 508 session.flush() 509 if self._parent_id is None: 510 # End of the recursion. This Element has no parent. 511 return [] 512 else: 513 # Best case available: retrieve the parent from the identity map 514 key = identity_key(cls, self._parent_id) 515 parent = session.identity_map.get(key, None) 516 if parent: 517 # Parent found in identity map, recurse and return. 518 parents = [parent] 519 parents.extend(parent.ancestors) 520 return parents 521 # Parent NOT found in identity map. Must use CTE 522 if session.bind.name != 'postgresql': 523 # If not using PostgreSQL, warn the user and use the non-optimized method. 524 warnings.warn('CTE are only supported on PostgreSQL. Using slower technique for "ancestor" method.', CTENotSupported) 525 parents = [self.parent] 526 parents.extend(self.parent.ancestors) 527 return parents 528 # Use a CTE to retrieve ALL ancestors 529 l0 = literal_column('0').label('level') 530 q_base = session.query(cls, l0).filter_by( 531 id = self._parent_id 532 ).cte(recursive = True) 533 l1 = literal_column('level + 1').label('level') 534 q_rec = session.query(cls, l1).filter( 535 q_base.c.parent_id == cls.id 536 ) 537 q_cte = q_base.union_all(q_rec) 538 return session.query(cls).select_from(q_cte).order_by(q_cte.c.level).all()
539
540 - def __repr__(self):
541 """ 542 Returns a printable representation of this instance. 543 544 @return: A descriptive string containing most of this instance fields 545 @rtype: str 546 """ 547 return u"%s(id=%s, name=%s, parent_id=%s, project_id=%s, status=%s, forced_status=%s)" % ( 548 self.__class__.__name__, 549 repr(self.id), 550 repr(self.name), 551 repr(self.parent_id), 552 repr(self.project_id), 553 repr(self.status), 554 repr(self.forced_status), 555 )
556
557 - def __str__(self):
558 """ 559 Coerces this instance to a string. 560 561 @return: The name field 562 @rtype: str 563 """ 564 return str(self.name)
565
566 -class StatusHistory(ORM_Base):
567 """ 568 Mapping class for the table «status_history». 569 """ 570 571 @hybrid_property
572 - def id(self):
573 """ 574 Read only accessor to prevent setting this field. 575 576 @return: Surrogate primary key 577 @rtype: int 578 """ 579 return self._id
580 581 @hybrid_property
582 - def element_id(self):
583 """ 584 Read only accessor to prevent setting this field. 585 586 @return: Surrogate primary key 587 @rtype: int 588 """ 589 return self._element_id
590 591 @hybrid_property
592 - def element(self):
593 """ 594 The related Element of this instance. 595 596 @return: The related Element 597 @rtype: L{Element} 598 """ 599 return self._element
600 601 @hybrid_property
602 - def timestamp(self):
603 """ 604 Read only accessor to prevent setting this field. 605 606 @return: The time this status was updated. 607 @rtype: datetime 608 """ 609 return self._timestamp
610 611 @hybrid_property
612 - def status(self):
613 """ 614 The status of this instance 615 616 @return: status 617 @rtype: str 618 """ 619 return self._status
620
621 - def __repr__(self):
622 """ 623 Returns a printable representation of this instance. 624 625 @return: A descriptive string containing most of this instance fields 626 @rtype: str 627 """ 628 return u"%s(element_id=%s, timestamp=%s, status=%s)" % ( 629 self.__class__.__name__, 630 repr(self.element_id), 631 repr(self.timestamp), 632 repr(self.status), 633 )
634
635 - def __str__(self):
636 """ 637 Coerces this instance to a string. 638 639 @return: The status field 640 @rtype: str 641 """ 642 return str(self.status)
643
644 -class Project(ORM_Base):
645 """ 646 Mapping class for the table «project». 647 """ 648 649 @hybrid_property
650 - def id(self):
651 """ 652 Read only accessor to prevent setting this field. 653 654 @return: Surrogate primary key 655 @rtype: int 656 """ 657 return self._id
658
659 - def __repr__(self):
660 """ 661 Returns a printable representation of this instance. 662 663 @return: A descriptive string containing most of this instance fields 664 @rtype: str 665 """ 666 return u"%s(id=%s, name=%s)" % ( 667 self.__class__.__name__, 668 repr(self.id), 669 repr(self.name), 670 )
671
672 - def __str__(self):
673 """ 674 Coerces this instance to a string. 675 676 @return: The name field 677 @rtype: str 678 """ 679 return str(self.name)
680
681 -class TagElement(ORM_Base):
682 """ 683 Mapping class for the table «tagelement». 684 """ 685
686 - def __init__(self, *args, **kwargs):
687 """ 688 Constructor for TagElement instances. 689 Ensures that the «name» field is specified. 690 691 @param args: Optional arguments to the constructor 692 @type args: tuple 693 @param kwargs: Optional keyword arguments to the constructor 694 @type kwargs: dict 695 """ 696 assert 'name' in kwargs 697 assert kwargs['name'] is not None 698 super(TagElement, self).__init__(*args, **kwargs)
699
700 - def __repr__(self):
701 """ 702 Returns a printable representation of this instance. 703 704 @return: A descriptive string containing most of this instance fields 705 @rtype: str 706 """ 707 return u"%s(element_id=%s, name=%s, value=%s)" % ( 708 self.__class__.__name__, 709 repr(self.element_id), 710 repr(self.name), 711 repr(self.value), 712 )
713
714 - def __str__(self):
715 """ 716 Coerces this instance to a string. 717 718 @return: The name and value fields 719 @rtype: str 720 """ 721 return str(dict(self.name, self.value))
722
723 -class TagProject(ORM_Base):
724 """ 725 Mapping class for the table «tagproject». 726 """ 727
728 - def __init__(self, *args, **kwargs):
729 """ 730 Constructor for TagProject instances. 731 Ensures that the «name» field is specified. 732 733 @param args: Optional arguments to the constructor 734 @type args: tuple 735 @param kwargs: Optional keyword arguments to the constructor 736 @type kwargs: dict 737 """ 738 assert 'name' in kwargs 739 assert kwargs['name'] is not None 740 super(TagProject, self).__init__(*args, **kwargs)
741
742 - def __repr__(self):
743 """ 744 Returns a printable representation of this instance. 745 746 @return: A descriptive string containing most of this instance fields 747 @rtype: str 748 """ 749 return u"%s(project_id=%s, name=%s, value=%s)" % ( 750 self.__class__.__name__, 751 repr(self.project_id), 752 repr(self.name), 753 repr(self.value), 754 )
755
756 - def __str__(self):
757 """ 758 Coerces this instance to a string. 759 760 @return: The name and value fields 761 @rtype: str 762 """ 763 return str(dict(self.name, self.value))
764