Package intermine :: Package lists :: Module listmanager
[hide private]
[frames] | no frames]

Source Code for Module intermine.lists.listmanager

  1  import weakref 
  2   
  3  # Use core json for 2.6+, simplejson for <=2.5 
  4  try: 
  5      import json 
  6  except ImportError: 
  7      import simplejson as json 
  8   
  9  import urllib 
 10  import codecs 
 11   
 12  from intermine.errors import WebserviceError 
 13  from intermine.lists.list import List 
14 15 -class ListManager(object):
16 """ 17 A Class for Managing List Content and Operations 18 ================================================ 19 20 This class serves as a delegate for the intermine.webservice.Service class, 21 managing list content and operations. 22 23 This class is not meant to be called itself, but rather for its 24 methods to be called by the service object. 25 26 Note that the methods for creating lists can conflict in threaded applications, if 27 two threads are each allocated the same unused list name. You are 28 strongly advised to use locks to synchronise any list creation requests (create_list, 29 or intersect, union, subtract, diff) unless you are choosing your own names each time. 30 """ 31 32 DEFAULT_LIST_NAME = "my_list_" 33 DEFAULT_DESCRIPTION = "List created with Python client library" 34 35 INTERSECTION_PATH = '/lists/intersect/json' 36 UNION_PATH = '/lists/union/json' 37 DIFFERENCE_PATH = '/lists/diff/json' 38 SUBTRACTION_PATH = '/lists/subtract/json' 39
40 - def __init__(self, service):
41 self.service = weakref.proxy(service) 42 self.lists = None 43 self._temp_lists = set()
44
45 - def refresh_lists(self):
46 """Update the list information with the latest details from the server""" 47 self.lists = {} 48 url = self.service.root + self.service.LIST_PATH 49 sock = self.service.opener.open(url) 50 data = sock.read() 51 sock.close() 52 list_info = json.loads(data) 53 if not list_info.get("wasSuccessful"): 54 raise ListServiceError(list_info.get("error")) 55 for l in list_info["lists"]: 56 l = ListManager.safe_dict(l) # Workaround for python 2.6 unicode key issues 57 self.lists[l["name"]] = List(service=self.service, manager=self, **l)
58 59 @staticmethod
60 - def safe_dict(d):
61 """Recursively clone json structure with UTF-8 dictionary keys""" 62 if isinstance(d, dict): 63 return dict([(k.encode('utf-8'), v) for k,v in d.iteritems()]) 64 else: 65 return d
66
67 - def get_list(self, name):
68 """Return a list from the service by name, if it exists""" 69 if self.lists is None: 70 self.refresh_lists() 71 return self.lists.get(name)
72
73 - def l(self, name):
74 """Alias for get_list""" 75 return self.get_list(name)
76
77 - def get_all_lists(self):
78 """Get all the lists on a webservice""" 79 if self.lists is None: 80 self.refresh_lists() 81 return self.lists.values()
82
83 - def get_all_list_names(self):
84 """Get all the names of the lists in a particular webservice""" 85 if self.lists is None: 86 self.refresh_lists() 87 return self.lists.keys()
88
89 - def get_list_count(self):
90 """ 91 Return the number of lists accessible at the given webservice. 92 This number will vary depending on who you are authenticated as. 93 """ 94 return len(self.get_all_list_names())
95
96 - def get_unused_list_name(self):
97 """ 98 Get an unused list name 99 ======================= 100 101 This method returns a new name that does not conflict 102 with any currently existing list name. 103 104 The list name is only guaranteed to be unused at the time 105 of allocation. 106 """ 107 list_names = self.get_all_list_names() 108 counter = 1 109 name = self.DEFAULT_LIST_NAME + str(counter) 110 while name in list_names: 111 counter += 1 112 name = self.DEFAULT_LIST_NAME + str(counter) 113 self._temp_lists.add(name) 114 return name
115
116 - def _get_listable_query(self, queryable):
117 q = queryable.to_query() 118 if not q.views: 119 q.add_view(q.root.name + ".id") 120 else: 121 # Check to see if the class of the selected items is unambiguous 122 up_to_attrs = set((v[0:v.rindex(".")] for v in q.views)) 123 if len(up_to_attrs) == 1: 124 q.select(up_to_attrs.pop() + ".id") 125 return q
126
127 - def _create_list_from_queryable(self, queryable, name, description, tags):
128 q = self._get_listable_query(queryable) 129 uri = q.get_list_upload_uri() 130 params = q.to_query_params() 131 params["listName"] = name 132 params["description"] = description 133 params["tags"] = ";".join(tags) 134 form = urllib.urlencode(params) 135 resp = self.service.opener.open(uri, form) 136 data = resp.read() 137 resp.close() 138 return self.parse_list_upload_response(data)
139
140 - def create_list(self, content, list_type="", name=None, description=None, tags=[]):
141 """ 142 Create a new list in the webservice 143 =================================== 144 145 If no name is given, the list will be considered to be a temporary 146 list, and will be automatically deleted when the program ends. To prevent 147 this happening, give the list a name, either on creation, or by renaming it. 148 149 This method is not thread safe for anonymous lists - it will need synchronisation 150 with locks if you intend to create lists with multiple threads in parallel. 151 152 @rtype: intermine.lists.List 153 """ 154 if description is None: 155 description = self.DEFAULT_DESCRIPTION 156 157 if name is None: 158 name = self.get_unused_list_name() 159 160 try: 161 ids = open(content).read() 162 except (TypeError, IOError): 163 if isinstance(content, basestring): 164 ids = content 165 else: 166 try: 167 return self._create_list_from_queryable(content, name, description, tags) 168 except AttributeError: 169 ids = "\n".join(map(lambda x: '"' + x + '"', iter(content))) 170 171 uri = self.service.root + self.service.LIST_CREATION_PATH 172 query_form = { 173 'name': name, 174 'type': list_type, 175 'description': description, 176 'tags': ";".join(tags) 177 } 178 uri += "?" + urllib.urlencode(query_form) 179 data = self.service.opener.post_plain_text(uri, ids) 180 return self.parse_list_upload_response(data)
181
182 - def parse_list_upload_response(self, response):
183 """ 184 Intepret the response from the webserver to a list request, and return the List it describes 185 """ 186 try: 187 response_data = json.loads(response) 188 except ValueError: 189 raise ListServiceError("Error parsing response: " + response) 190 if not response_data.get("wasSuccessful"): 191 raise ListServiceError(response_data.get("error")) 192 self.refresh_lists() 193 new_list = self.get_list(response_data["listName"]) 194 failed_matches = response_data.get("unmatchedIdentifiers") 195 new_list._add_failed_matches(failed_matches) 196 return new_list
197
198 - def delete_lists(self, lists):
199 """Delete the given lists from the webserver""" 200 all_names = self.get_all_list_names() 201 for l in lists: 202 if isinstance(l, List): 203 name = l.name 204 else: 205 name = str(l) 206 if name not in all_names: 207 continue 208 uri = self.service.root + self.service.LIST_PATH 209 query_form = {'name': name} 210 uri += "?" + urllib.urlencode(query_form) 211 response = self.service.opener.delete(uri) 212 response_data = json.loads(response) 213 if not response_data.get("wasSuccessful"): 214 raise ListServiceError(response_data.get("error")) 215 self.refresh_lists()
216
217 - def remove_tags(self, to_remove_from, tags):
218 """ 219 Add the tags to the given list 220 ============================== 221 222 Returns the current tags of this list. 223 """ 224 uri = self.service.root + self.service.LIST_TAG_PATH 225 form = {"name": to_remove_from.name, "tags": ";".join(tags)} 226 uri += "?" + urllib.urlencode(form) 227 body = self.service.opener.delete(uri) 228 return self._body_to_json(body)["tags"]
229
230 - def add_tags(self, to_tag, tags):
231 """ 232 Add the tags to the given list 233 ============================== 234 235 Returns the current tags of this list. 236 """ 237 uri = self.service.root + self.service.LIST_TAG_PATH 238 form = {"name": to_tag.name, "tags": ";".join(tags)} 239 resp = self.service.opener.open(uri, urllib.urlencode(form)) 240 body = resp.read() 241 resp.close() 242 return self._body_to_json(body)["tags"]
243
244 - def get_tags(self, im_list):
245 """ 246 Get the up-to-date set of tags for a given list 247 =============================================== 248 249 Returns the current tags of this list. 250 """ 251 uri = self.service.root + self.service.LIST_TAG_PATH 252 form = {"name": im_list.name} 253 uri += "?" + urllib.urlencode(form) 254 resp = self.service.opener.open(uri) 255 body = resp.read() 256 resp.close() 257 return self._body_to_json(body)["tags"]
258
259 - def _body_to_json(self, body):
260 try: 261 data = json.loads(body) 262 except ValueError: 263 raise ListServiceError("Error parsing response: " + body) 264 if not data.get("wasSuccessful"): 265 raise ListServiceError(data.get("error")) 266 return data
267
268 - def delete_temporary_lists(self):
269 """Delete all the lists considered temporary (those created without names)""" 270 self.delete_lists(self._temp_lists) 271 self._temp_lists = set()
272
273 - def intersect(self, lists, name=None, description=None, tags=[]):
274 """Calculate the intersection of a given set of lists, and return the list representing the result""" 275 return self._do_operation(self.INTERSECTION_PATH, "Intersection", lists, name, description, tags)
276
277 - def union(self, lists, name=None, description=None, tags=[]):
278 """Calculate the union of a given set of lists, and return the list representing the result""" 279 return self._do_operation(self.UNION_PATH, "Union", lists, name, description, tags)
280
281 - def xor(self, lists, name=None, description=None, tags=[]):
282 """Calculate the symmetric difference of a given set of lists, and return the list representing the result""" 283 return self._do_operation(self.DIFFERENCE_PATH, "Difference", lists, name, description, tags)
284
285 - def subtract(self, lefts, rights, name=None, description=None, tags=[]):
286 """Calculate the subtraction of rights from lefts, and return the list representing the result""" 287 left_names = self.make_list_names(lefts) 288 right_names = self.make_list_names(rights) 289 if description is None: 290 description = "Subtraction of " + ' and '.join(right_names) + " from " + ' and '.join(left_names) 291 if name is None: 292 name = self.get_unused_list_name() 293 uri = self.service.root + self.SUBTRACTION_PATH 294 uri += '?' + urllib.urlencode({ 295 "name": name, 296 "description": description, 297 "references": ';'.join(left_names), 298 "subtract": ';'.join(right_names), 299 "tags": ";".join(tags) 300 }) 301 resp = self.service.opener.open(uri) 302 data = resp.read() 303 resp.close() 304 return self.parse_list_upload_response(data)
305
306 - def _do_operation(self, path, operation, lists, name, description, tags):
307 list_names = self.make_list_names(lists) 308 if description is None: 309 description = operation + " of " + ' and '.join(list_names) 310 if name is None: 311 name = self.get_unused_list_name() 312 uri = self.service.root + path 313 uri += '?' + urllib.urlencode({ 314 "name": name, 315 "lists": ';'.join(list_names), 316 "description": description, 317 "tags": ";".join(tags) 318 }) 319 resp = self.service.opener.open(uri) 320 data = resp.read() 321 resp.close() 322 return self.parse_list_upload_response(data)
323 324
325 - def make_list_names(self, lists):
326 """Turn a list of things into a list of list names""" 327 list_names = [] 328 for l in lists: 329 try: 330 t = l.list_type 331 list_names.append(l.name) 332 except AttributeError: 333 try: 334 m = l.model 335 list_names.append(self.create_list(l).name) 336 except AttributeError: 337 list_names.append(str(l)) 338 339 return list_names
340
341 -class ListServiceError(WebserviceError):
342 """Errors thrown when something goes wrong with list requests""" 343 pass
344