图片解析应用
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

323 lines
12 KiB

  1. import threading
  2. import time as mod_time
  3. import uuid
  4. from types import SimpleNamespace, TracebackType
  5. from typing import Optional, Type
  6. from redis.exceptions import LockError, LockNotOwnedError
  7. from redis.typing import Number
  8. class Lock:
  9. """
  10. A shared, distributed Lock. Using Redis for locking allows the Lock
  11. to be shared across processes and/or machines.
  12. It's left to the user to resolve deadlock issues and make sure
  13. multiple clients play nicely together.
  14. """
  15. lua_release = None
  16. lua_extend = None
  17. lua_reacquire = None
  18. # KEYS[1] - lock name
  19. # ARGV[1] - token
  20. # return 1 if the lock was released, otherwise 0
  21. LUA_RELEASE_SCRIPT = """
  22. local token = redis.call('get', KEYS[1])
  23. if not token or token ~= ARGV[1] then
  24. return 0
  25. end
  26. redis.call('del', KEYS[1])
  27. return 1
  28. """
  29. # KEYS[1] - lock name
  30. # ARGV[1] - token
  31. # ARGV[2] - additional milliseconds
  32. # ARGV[3] - "0" if the additional time should be added to the lock's
  33. # existing ttl or "1" if the existing ttl should be replaced
  34. # return 1 if the locks time was extended, otherwise 0
  35. LUA_EXTEND_SCRIPT = """
  36. local token = redis.call('get', KEYS[1])
  37. if not token or token ~= ARGV[1] then
  38. return 0
  39. end
  40. local expiration = redis.call('pttl', KEYS[1])
  41. if not expiration then
  42. expiration = 0
  43. end
  44. if expiration < 0 then
  45. return 0
  46. end
  47. local newttl = ARGV[2]
  48. if ARGV[3] == "0" then
  49. newttl = ARGV[2] + expiration
  50. end
  51. redis.call('pexpire', KEYS[1], newttl)
  52. return 1
  53. """
  54. # KEYS[1] - lock name
  55. # ARGV[1] - token
  56. # ARGV[2] - milliseconds
  57. # return 1 if the locks time was reacquired, otherwise 0
  58. LUA_REACQUIRE_SCRIPT = """
  59. local token = redis.call('get', KEYS[1])
  60. if not token or token ~= ARGV[1] then
  61. return 0
  62. end
  63. redis.call('pexpire', KEYS[1], ARGV[2])
  64. return 1
  65. """
  66. def __init__(
  67. self,
  68. redis,
  69. name: str,
  70. timeout: Optional[Number] = None,
  71. sleep: Number = 0.1,
  72. blocking: bool = True,
  73. blocking_timeout: Optional[Number] = None,
  74. thread_local: bool = True,
  75. ):
  76. """
  77. Create a new Lock instance named ``name`` using the Redis client
  78. supplied by ``redis``.
  79. ``timeout`` indicates a maximum life for the lock in seconds.
  80. By default, it will remain locked until release() is called.
  81. ``timeout`` can be specified as a float or integer, both representing
  82. the number of seconds to wait.
  83. ``sleep`` indicates the amount of time to sleep in seconds per loop
  84. iteration when the lock is in blocking mode and another client is
  85. currently holding the lock.
  86. ``blocking`` indicates whether calling ``acquire`` should block until
  87. the lock has been acquired or to fail immediately, causing ``acquire``
  88. to return False and the lock not being acquired. Defaults to True.
  89. Note this value can be overridden by passing a ``blocking``
  90. argument to ``acquire``.
  91. ``blocking_timeout`` indicates the maximum amount of time in seconds to
  92. spend trying to acquire the lock. A value of ``None`` indicates
  93. continue trying forever. ``blocking_timeout`` can be specified as a
  94. float or integer, both representing the number of seconds to wait.
  95. ``thread_local`` indicates whether the lock token is placed in
  96. thread-local storage. By default, the token is placed in thread local
  97. storage so that a thread only sees its token, not a token set by
  98. another thread. Consider the following timeline:
  99. time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds.
  100. thread-1 sets the token to "abc"
  101. time: 1, thread-2 blocks trying to acquire `my-lock` using the
  102. Lock instance.
  103. time: 5, thread-1 has not yet completed. redis expires the lock
  104. key.
  105. time: 5, thread-2 acquired `my-lock` now that it's available.
  106. thread-2 sets the token to "xyz"
  107. time: 6, thread-1 finishes its work and calls release(). if the
  108. token is *not* stored in thread local storage, then
  109. thread-1 would see the token value as "xyz" and would be
  110. able to successfully release the thread-2's lock.
  111. In some use cases it's necessary to disable thread local storage. For
  112. example, if you have code where one thread acquires a lock and passes
  113. that lock instance to a worker thread to release later. If thread
  114. local storage isn't disabled in this case, the worker thread won't see
  115. the token set by the thread that acquired the lock. Our assumption
  116. is that these cases aren't common and as such default to using
  117. thread local storage.
  118. """
  119. self.redis = redis
  120. self.name = name
  121. self.timeout = timeout
  122. self.sleep = sleep
  123. self.blocking = blocking
  124. self.blocking_timeout = blocking_timeout
  125. self.thread_local = bool(thread_local)
  126. self.local = threading.local() if self.thread_local else SimpleNamespace()
  127. self.local.token = None
  128. self.register_scripts()
  129. def register_scripts(self) -> None:
  130. cls = self.__class__
  131. client = self.redis
  132. if cls.lua_release is None:
  133. cls.lua_release = client.register_script(cls.LUA_RELEASE_SCRIPT)
  134. if cls.lua_extend is None:
  135. cls.lua_extend = client.register_script(cls.LUA_EXTEND_SCRIPT)
  136. if cls.lua_reacquire is None:
  137. cls.lua_reacquire = client.register_script(cls.LUA_REACQUIRE_SCRIPT)
  138. def __enter__(self) -> "Lock":
  139. if self.acquire():
  140. return self
  141. raise LockError(
  142. "Unable to acquire lock within the time specified",
  143. lock_name=self.name,
  144. )
  145. def __exit__(
  146. self,
  147. exc_type: Optional[Type[BaseException]],
  148. exc_value: Optional[BaseException],
  149. traceback: Optional[TracebackType],
  150. ) -> None:
  151. self.release()
  152. def acquire(
  153. self,
  154. sleep: Optional[Number] = None,
  155. blocking: Optional[bool] = None,
  156. blocking_timeout: Optional[Number] = None,
  157. token: Optional[str] = None,
  158. ):
  159. """
  160. Use Redis to hold a shared, distributed lock named ``name``.
  161. Returns True once the lock is acquired.
  162. If ``blocking`` is False, always return immediately. If the lock
  163. was acquired, return True, otherwise return False.
  164. ``blocking_timeout`` specifies the maximum number of seconds to
  165. wait trying to acquire the lock.
  166. ``token`` specifies the token value to be used. If provided, token
  167. must be a bytes object or a string that can be encoded to a bytes
  168. object with the default encoding. If a token isn't specified, a UUID
  169. will be generated.
  170. """
  171. if sleep is None:
  172. sleep = self.sleep
  173. if token is None:
  174. token = uuid.uuid1().hex.encode()
  175. else:
  176. encoder = self.redis.get_encoder()
  177. token = encoder.encode(token)
  178. if blocking is None:
  179. blocking = self.blocking
  180. if blocking_timeout is None:
  181. blocking_timeout = self.blocking_timeout
  182. stop_trying_at = None
  183. if blocking_timeout is not None:
  184. stop_trying_at = mod_time.monotonic() + blocking_timeout
  185. while True:
  186. if self.do_acquire(token):
  187. self.local.token = token
  188. return True
  189. if not blocking:
  190. return False
  191. next_try_at = mod_time.monotonic() + sleep
  192. if stop_trying_at is not None and next_try_at > stop_trying_at:
  193. return False
  194. mod_time.sleep(sleep)
  195. def do_acquire(self, token: str) -> bool:
  196. if self.timeout:
  197. # convert to milliseconds
  198. timeout = int(self.timeout * 1000)
  199. else:
  200. timeout = None
  201. if self.redis.set(self.name, token, nx=True, px=timeout):
  202. return True
  203. return False
  204. def locked(self) -> bool:
  205. """
  206. Returns True if this key is locked by any process, otherwise False.
  207. """
  208. return self.redis.get(self.name) is not None
  209. def owned(self) -> bool:
  210. """
  211. Returns True if this key is locked by this lock, otherwise False.
  212. """
  213. stored_token = self.redis.get(self.name)
  214. # need to always compare bytes to bytes
  215. # TODO: this can be simplified when the context manager is finished
  216. if stored_token and not isinstance(stored_token, bytes):
  217. encoder = self.redis.get_encoder()
  218. stored_token = encoder.encode(stored_token)
  219. return self.local.token is not None and stored_token == self.local.token
  220. def release(self) -> None:
  221. """
  222. Releases the already acquired lock
  223. """
  224. expected_token = self.local.token
  225. if expected_token is None:
  226. raise LockError("Cannot release an unlocked lock", lock_name=self.name)
  227. self.local.token = None
  228. self.do_release(expected_token)
  229. def do_release(self, expected_token: str) -> None:
  230. if not bool(
  231. self.lua_release(keys=[self.name], args=[expected_token], client=self.redis)
  232. ):
  233. raise LockNotOwnedError(
  234. "Cannot release a lock that's no longer owned",
  235. lock_name=self.name,
  236. )
  237. def extend(self, additional_time: int, replace_ttl: bool = False) -> bool:
  238. """
  239. Adds more time to an already acquired lock.
  240. ``additional_time`` can be specified as an integer or a float, both
  241. representing the number of seconds to add.
  242. ``replace_ttl`` if False (the default), add `additional_time` to
  243. the lock's existing ttl. If True, replace the lock's ttl with
  244. `additional_time`.
  245. """
  246. if self.local.token is None:
  247. raise LockError("Cannot extend an unlocked lock", lock_name=self.name)
  248. if self.timeout is None:
  249. raise LockError("Cannot extend a lock with no timeout", lock_name=self.name)
  250. return self.do_extend(additional_time, replace_ttl)
  251. def do_extend(self, additional_time: int, replace_ttl: bool) -> bool:
  252. additional_time = int(additional_time * 1000)
  253. if not bool(
  254. self.lua_extend(
  255. keys=[self.name],
  256. args=[self.local.token, additional_time, "1" if replace_ttl else "0"],
  257. client=self.redis,
  258. )
  259. ):
  260. raise LockNotOwnedError(
  261. "Cannot extend a lock that's no longer owned",
  262. lock_name=self.name,
  263. )
  264. return True
  265. def reacquire(self) -> bool:
  266. """
  267. Resets a TTL of an already acquired lock back to a timeout value.
  268. """
  269. if self.local.token is None:
  270. raise LockError("Cannot reacquire an unlocked lock", lock_name=self.name)
  271. if self.timeout is None:
  272. raise LockError(
  273. "Cannot reacquire a lock with no timeout",
  274. lock_name=self.name,
  275. )
  276. return self.do_reacquire()
  277. def do_reacquire(self) -> bool:
  278. timeout = int(self.timeout * 1000)
  279. if not bool(
  280. self.lua_reacquire(
  281. keys=[self.name], args=[self.local.token, timeout], client=self.redis
  282. )
  283. ):
  284. raise LockNotOwnedError(
  285. "Cannot reacquire a lock that's no longer owned",
  286. lock_name=self.name,
  287. )
  288. return True