Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import logging 

2import time 

3 

4log = logging.getLogger(__name__) 

5 

6 

7class NeedRegenerationException(Exception): 

8 """An exception that when raised in the 'with' block, 

9 forces the 'has_value' flag to False and incurs a 

10 regeneration of the value. 

11 

12 """ 

13 

14 

15NOT_REGENERATED = object() 

16 

17 

18class Lock(object): 

19 """Dogpile lock class. 

20 

21 Provides an interface around an arbitrary mutex 

22 that allows one thread/process to be elected as 

23 the creator of a new value, while other threads/processes 

24 continue to return the previous version 

25 of that value. 

26 

27 :param mutex: A mutex object that provides ``acquire()`` 

28 and ``release()`` methods. 

29 :param creator: Callable which returns a tuple of the form 

30 (new_value, creation_time). "new_value" should be a newly 

31 generated value representing completed state. "creation_time" 

32 should be a floating point time value which is relative 

33 to Python's ``time.time()`` call, representing the time 

34 at which the value was created. This time value should 

35 be associated with the created value. 

36 :param value_and_created_fn: Callable which returns 

37 a tuple of the form (existing_value, creation_time). This 

38 basically should return what the last local call to the ``creator()`` 

39 callable has returned, i.e. the value and the creation time, 

40 which would be assumed here to be from a cache. If the 

41 value is not available, the :class:`.NeedRegenerationException` 

42 exception should be thrown. 

43 :param expiretime: Expiration time in seconds. Set to 

44 ``None`` for never expires. This timestamp is compared 

45 to the creation_time result and ``time.time()`` to determine if 

46 the value returned by value_and_created_fn is "expired". 

47 :param async_creator: A callable. If specified, this callable will be 

48 passed the mutex as an argument and is responsible for releasing the mutex 

49 after it finishes some asynchronous value creation. The intent is for 

50 this to be used to defer invocation of the creator callable until some 

51 later time. 

52 

53 """ 

54 

55 def __init__( 

56 self, 

57 mutex, 

58 creator, 

59 value_and_created_fn, 

60 expiretime, 

61 async_creator=None, 

62 ): 

63 self.mutex = mutex 

64 self.creator = creator 

65 self.value_and_created_fn = value_and_created_fn 

66 self.expiretime = expiretime 

67 self.async_creator = async_creator 

68 

69 def _is_expired(self, createdtime): 

70 """Return true if the expiration time is reached, or no 

71 value is available.""" 

72 

73 return not self._has_value(createdtime) or ( 

74 self.expiretime is not None 

75 and time.time() - createdtime > self.expiretime 

76 ) 

77 

78 def _has_value(self, createdtime): 

79 """Return true if the creation function has proceeded 

80 at least once.""" 

81 return createdtime > 0 

82 

83 def _enter(self): 

84 value_fn = self.value_and_created_fn 

85 

86 try: 

87 value = value_fn() 

88 value, createdtime = value 

89 except NeedRegenerationException: 

90 log.debug("NeedRegenerationException") 

91 value = NOT_REGENERATED 

92 createdtime = -1 

93 

94 generated = self._enter_create(value, createdtime) 

95 

96 if generated is not NOT_REGENERATED: 

97 generated, createdtime = generated 

98 return generated 

99 elif value is NOT_REGENERATED: 

100 # we called upon the creator, and it said that it 

101 # didn't regenerate. this typically means another 

102 # thread is running the creation function, and that the 

103 # cache should still have a value. However, 

104 # we don't have a value at all, which is unusual since we just 

105 # checked for it, so check again (TODO: is this a real codepath?) 

106 try: 

107 value, createdtime = value_fn() 

108 return value 

109 except NeedRegenerationException: 

110 raise Exception( 

111 "Generation function should " 

112 "have just been called by a concurrent " 

113 "thread." 

114 ) 

115 else: 

116 return value 

117 

118 def _enter_create(self, value, createdtime): 

119 if not self._is_expired(createdtime): 

120 return NOT_REGENERATED 

121 

122 _async = False 

123 

124 if self._has_value(createdtime): 

125 has_value = True 

126 if not self.mutex.acquire(False): 

127 log.debug("creation function in progress elsewhere, returning") 

128 return NOT_REGENERATED 

129 else: 

130 has_value = False 

131 log.debug("no value, waiting for create lock") 

132 self.mutex.acquire() 

133 

134 try: 

135 log.debug("value creation lock %r acquired" % self.mutex) 

136 

137 if not has_value: 

138 # we entered without a value, or at least with "creationtime == 

139 # 0". Run the "getter" function again, to see if another 

140 # thread has already generated the value while we waited on the 

141 # mutex, or if the caller is otherwise telling us there is a 

142 # value already which allows us to use async regeneration. (the 

143 # latter is used by the multi-key routine). 

144 try: 

145 value, createdtime = self.value_and_created_fn() 

146 except NeedRegenerationException: 

147 # nope, nobody created the value, we're it. 

148 # we must create it right now 

149 pass 

150 else: 

151 has_value = True 

152 # caller is telling us there is a value and that we can 

153 # use async creation if it is expired. 

154 if not self._is_expired(createdtime): 

155 # it's not expired, return it 

156 log.debug("Concurrent thread created the value") 

157 return value, createdtime 

158 

159 # otherwise it's expired, call creator again 

160 

161 if has_value and self.async_creator: 

162 # we have a value we can return, safe to use async_creator 

163 log.debug("Passing creation lock to async runner") 

164 

165 # so...run it! 

166 self.async_creator(self.mutex) 

167 _async = True 

168 

169 # and return the expired value for now 

170 return value, createdtime 

171 

172 # it's expired, and it's our turn to create it synchronously, *or*, 

173 # there's no value at all, and we have to create it synchronously 

174 log.debug( 

175 "Calling creation function for %s value", 

176 "not-yet-present" if not has_value else "previously expired", 

177 ) 

178 return self.creator() 

179 finally: 

180 if not _async: 

181 self.mutex.release() 

182 log.debug("Released creation lock") 

183 

184 def __enter__(self): 

185 return self._enter() 

186 

187 def __exit__(self, type_, value, traceback): 

188 pass