codekingpro/portable-devtools
115k
1#ifndef GREENLET_THREAD_STATE_HPP
2#define GREENLET_THREAD_STATE_HPP
3
4#include <cstdlib>
5#include <ctime>
6#include <stdexcept>
7#include <atomic>
8
9#include "greenlet_internal.hpp"
10#include "greenlet_refs.hpp"
11#include "greenlet_thread_support.hpp"
12
13using greenlet::LockGuard;
14using greenlet::refs::BorrowedObject;
15using greenlet::refs::BorrowedGreenlet;
16using greenlet::refs::BorrowedMainGreenlet;
17using greenlet::refs::OwnedMainGreenlet;
18using greenlet::refs::OwnedObject;
19using greenlet::refs::OwnedGreenlet;
20using greenlet::refs::OwnedList;
21using greenlet::refs::PyErrFetchParam;
22using greenlet::refs::PyArgParseParam;
23using greenlet::refs::ImmortalString;
24using greenlet::refs::CreatedModule;
25using greenlet::refs::PyErrPieces;
26using greenlet::refs::NewReference;
27
28
29namespace greenlet {
30/**
31 * Thread-local state of greenlets.
32 *
33 * Each native thread will get exactly one of these objects,
34 * automatically accessed through the best available thread-local
35 * mechanism the compiler supports (``thread_local`` for C++11
36 * compilers or ``__thread``/``declspec(thread)`` for older GCC/clang
37 * or MSVC, respectively.)
38 *
39 * Previously, we kept thread-local state mostly in a bunch of
40 * ``static volatile`` variables in the main greenlet file.. This had
41 * the problem of requiring extra checks, loops, and great care
42 * accessing these variables if we potentially invoked any Python code
43 * that could release the GIL, because the state could change out from
44 * under us. Making the variables thread-local solves this problem.
45 *
46 * When we detected that a greenlet API accessing the current greenlet
47 * was invoked from a different thread than the greenlet belonged to,
48 * we stored a reference to the greenlet in the Python thread
49 * dictionary for the thread the greenlet belonged to. This could lead
50 * to memory leaks if the thread then exited (because of a reference
51 * cycle, as greenlets referred to the thread dictionary, and deleting
52 * non-current greenlets leaked their frame plus perhaps arguments on
53 * the C stack). If a thread exited while still having running
54 * greenlet objects (perhaps that had just switched back to the main
55 * greenlet), and did not invoke one of the greenlet APIs *in that
56 * thread, immediately before it exited, without some other thread
57 * then being invoked*, such a leak was guaranteed.
58 *
59 * This can be partly solved by using compiler thread-local variables
60 * instead of the Python thread dictionary, thus avoiding a cycle.
61 *
62 * To fully solve this problem, we need a reliable way to know that a
63 * thread is done and we should clean up the main greenlet. On POSIX,
64 * we can use the destructor function of ``pthread_key_create``, but
65 * there's nothing similar on Windows; a C++11 thread local object
66 * reliably invokes its destructor when the thread it belongs to exits
67 * (non-C++11 compilers offer ``__thread`` or ``declspec(thread)`` to
68 * create thread-local variables, but they can't hold C++ objects that
69 * invoke destructors; the C++11 version is the most portable solution
70 * I found). When the thread exits, we can drop references and
71 * otherwise manipulate greenlets and frames that we know can no
72 * longer be switched to.
73 *
74 * There are two small wrinkles. The first is that when the thread
75 * exits, it is too late to actually invoke Python APIs: the Python
76 * thread state is gone, and the GIL is released. To solve *this*
77 * problem, our destructor uses ``Py_AddPendingCall`` to transfer the
78 * destruction work to the main thread.
79 *
80 * The second is that once the thread exits, the thread local object
81 * is invalid and we can't even access a pointer to it, so we can't
82 * pass it to ``Py_AddPendingCall``. This is handled by actually using
83 * a second object that's thread local (ThreadStateCreator) and having
84 * it dynamically allocate this object so it can live until the
85 * pending call runs.
86 */
87
88
89
90class ThreadState {
91private:
92 // As of commit 08ad1dd7012b101db953f492e0021fb08634afad
93 // this class needed 56 bytes in o Py_DEBUG build
94 // on 64-bit macOS 11.
95 // Adding the vector takes us up to 80 bytes ()
96
97 /* Strong reference to the main greenlet */
98 OwnedMainGreenlet main_greenlet;
99
100 /* Strong reference to the current greenlet. */
101 OwnedGreenlet current_greenlet;
102
103 /* Strong reference to the trace function, if any. */
104 OwnedObject tracefunc;
105
106 // Use std::allocator (malloc/free) instead of PythonAllocator
107 // (PyMem_Malloc) for the deleteme list. During Py_FinalizeEx on
108 // Python < 3.11, the PyObject_Malloc pool that holds ThreadState
109 // can be disrupted, corrupting any PythonAllocator-backed
110 // containers. Using std::allocator makes this vector independent
111 // of Python's allocator lifecycle.
112 typedef std::vector<PyGreenlet*> deleteme_t;
113 /* A vector of raw PyGreenlet pointers representing things that need
114 deleted when this thread is running. The vector owns the
115 references, but you need to manually INCREF/DECREF as you use
116 them. We don't use a vector<refs::OwnedGreenlet> because we
117 make copy of this vector, and that would become O(n) as all the
118 refcounts are incremented in the copy.
119 */
120 deleteme_t deleteme;
121#ifdef Py_GIL_DISABLED
122 // On free-threaded builds, we need to protect shared access to
123 // the deleteme list by a mutex. It can be written from one thread
124 // while being read in another
125 Mutex deleteme_lock;
126#endif
127
128#ifdef GREENLET_NEEDS_EXCEPTION_STATE_SAVED
129 void* exception_state;
130#endif
131
132#ifdef Py_GIL_DISABLED
133 static std::atomic<std::clock_t> _clocks_used_doing_gc;
134#else
135 static std::clock_t _clocks_used_doing_gc;
136#endif
137 static ImmortalString get_referrers_name;
138
139 G_NO_COPIES_OF_CLS(ThreadState);
140
141
142 // Allocates a main greenlet for the thread state. If this fails,
143 // exits the process. Called only during constructing a ThreadState.
144 MainGreenlet* alloc_main()
145 {
146 PyGreenlet* gmain;
147
148 /* create the main greenlet for this thread */
149 gmain = reinterpret_cast<PyGreenlet*>(PyType_GenericAlloc(&PyGreenlet_Type, 0));
150 if (gmain == NULL) {
151 throw PyFatalError("alloc_main failed to alloc"); //exits the process
152 }
153
154 MainGreenlet* const main = new MainGreenlet(gmain, this);
155
156 assert(Py_REFCNT(gmain) == 1);
157 assert(gmain->pimpl == main);
158 return main;
159 }
160
161
162public:
163 // Allocate ThreadState with malloc/free rather than Python's
164 // object allocator. ThreadState outlives many Python objects and
165 // must remain valid throughout Py_FinalizeEx. On Python < 3.11,
166 // PyObject_Malloc pools can be disrupted during early
167 // finalization, corrupting any C++ objects stored in them.
168 static void* operator new(size_t count)
169 {
170 void* p = std::malloc(count);
171 if (!p) {
172 throw std::bad_alloc();
173 }
174 return p;
175 }
176
177 static void operator delete(void* ptr)
178 {
179 std::free(ptr);
180 }
181
182 static void init()
183 {
184 ThreadState::get_referrers_name = "get_referrers";
185 ThreadState::set_clocks_used_doing_gc(0);
186 }
187
188 ThreadState()
189 {
190
191#ifdef GREENLET_NEEDS_EXCEPTION_STATE_SAVED
192 this->exception_state = slp_get_exception_state();
193#endif
194
195 // XXX: Potentially dangerous, exposing a not fully
196 // constructed object.
197 MainGreenlet* const main = this->alloc_main();
198 this->main_greenlet = OwnedMainGreenlet::consuming(
199 main->self()
200 );
201 assert(this->main_greenlet);
202 this->current_greenlet = main->self();
203 // The main greenlet starts with 1 refs: The returned one. We
204 // then copied it to the current greenlet.
205 assert(this->main_greenlet.REFCNT() == 2);
206 }
207
208 inline void restore_exception_state()
209 {
210#ifdef GREENLET_NEEDS_EXCEPTION_STATE_SAVED
211 // It's probably important this be inlined and only call C
212 // functions to avoid adding an SEH frame.
213 slp_set_exception_state(this->exception_state);
214#endif
215 }
216
217 inline bool has_main_greenlet() const noexcept
218 {
219 return bool(this->main_greenlet);
220 }
221
222 // Called from the ThreadStateCreator when we're in non-standard
223 // threading mode. In that case, there is an object in the Python
224 // thread state dictionary that points to us. The main greenlet
225 // also traverses into us, in which case it's crucial not to
226 // traverse back into the main greenlet.
227 int tp_traverse(visitproc visit, void* arg, bool traverse_main=true)
228 {
229 if (traverse_main) {
230 Py_VISIT(main_greenlet.borrow_o());
231 }
232 if (traverse_main || current_greenlet != main_greenlet) {
233 Py_VISIT(current_greenlet.borrow_o());
234 }
235 Py_VISIT(tracefunc.borrow());
236 return 0;
237 }
238
239 inline BorrowedMainGreenlet borrow_main_greenlet() const noexcept
240 {
241 assert(this->main_greenlet);
242 assert(this->main_greenlet.REFCNT() >= 2);
243 return this->main_greenlet;
244 };
245
246 inline OwnedMainGreenlet get_main_greenlet() const noexcept
247 {
248 return this->main_greenlet;
249 }
250
251 /**
252 * If we have a main greenlet, mark it as dead by setting its
253 * thread_state to null (this part is atomic with respect to other
254 * threads looking at the main greenlet's thread_state).
255 */
256 inline bool mark_main_greenlet_dead() noexcept
257 {
258 PyGreenlet* main_greenlet = this->main_greenlet.borrow();
259 if (!main_greenlet) {
260 return false;
261 }
262 assert(main_greenlet->pimpl->thread_state() == this
263 || main_greenlet->pimpl->thread_state() == nullptr);
264 dynamic_cast<MainGreenlet*>(main_greenlet->pimpl)->thread_state(nullptr);
265 return true;
266 }
267
268 /**
269 * In addition to returning a new reference to the currunt
270 * greenlet, this performs any maintenance needed.
271 */
272 inline OwnedGreenlet get_current()
273 {
274 /* green_dealloc() cannot delete greenlets from other threads, so
275 it stores them in the thread dict; delete them now. */
276 this->clear_deleteme_list();
277 //assert(this->current_greenlet->main_greenlet == this->main_greenlet);
278 //assert(this->main_greenlet->main_greenlet == this->main_greenlet);
279 return this->current_greenlet;
280 }
281
282 /**
283 * As for non-const get_current();
284 */
285 inline BorrowedGreenlet borrow_current()
286 {
287 this->clear_deleteme_list();
288 return this->current_greenlet;
289 }
290
291 /**
292 * Does no maintenance.
293 */
294 inline OwnedGreenlet get_current() const
295 {
296 return this->current_greenlet;
297 }
298
299 template<typename T, refs::TypeChecker TC>
300 inline bool is_current(const refs::PyObjectPointer<T, TC>& obj) const
301 {
302 return this->current_greenlet.borrow_o() == obj.borrow_o();
303 }
304
305 inline void set_current(const OwnedGreenlet& target)
306 {
307 this->current_greenlet = target;
308 }
309
310private:
311 /**
312 * Deref and remove the greenlets from the deleteme list. Must be
313 * holding the GIL.
314 *
315 * If *murder* is true, then we must be called from a different
316 * thread than the one that these greenlets were running in.
317 * In that case, if the greenlet was actually running, we destroy
318 * the frame reference and otherwise make it appear dead before
319 * proceeding; otherwise, we would try (and fail) to raise an
320 * exception in it and wind up right back in this list.
321 */
322 inline void clear_deleteme_list(const bool murder=false)
323 {
324#ifdef Py_GIL_DISABLED
325 LockGuard deleteme_guard(this->deleteme_lock);
326#endif
327 if (this->deleteme.empty()) {
328 return;
329 }
330 // Move the list contents out with swap — a constant-time
331 // pointer exchange that never allocates. The previous
332 // code used a copy (deleteme_t copy = this->deleteme)
333 // which allocated through PythonAllocator / PyMem_Malloc;
334 // that could SIGSEGV during early Py_FinalizeEx on Python
335 // < 3.11 when the allocator is partially torn down.
336 deleteme_t copy;
337 std::swap(copy, this->deleteme);
338
339 // During Py_FinalizeEx cleanup, the GC or atexit handlers
340 // may have already collected objects in this list,
341 // leaving dangling pointers. Attempting Py_DECREF on
342 // freed memory causes a SIGSEGV. g_greenlet_shutting_down
343 // covers the early atexit phase; Py_IsFinalizing() covers
344 // later phases. Thus, we deliberately leak.
345 if (greenlet::IsShuttingDown()) {
346 return;
347 }
348
349 // Preserve any pending exception so that cleanup-triggered
350 // errors don't accidentally swallow an unrelated exception
351 // (e.g. one set by throw() before a switch).
352 PyErrPieces incoming_err;
353
354 for(deleteme_t::iterator it = copy.begin(), end = copy.end();
355 it != end;
356 ++it ) {
357 PyGreenlet* to_del = *it;
358 if (murder) {
359 // Force each greenlet to appear dead; we can't raise an
360 // exception into it anymore anyway.
361 to_del->pimpl->murder_in_place();
362 }
363
364 // The only reference to these greenlets should be in
365 // this list, decreffing them should let them be
366 // deleted again, triggering calls to green_dealloc()
367 // in the correct thread (if we're not murdering).
368 // This may run arbitrary Python code and switch
369 // threads or greenlets!
370 Py_DECREF(to_del);
371 if (PyErr_Occurred()) {
372 PyErr_WriteUnraisable(nullptr);
373 PyErr_Clear();
374 }
375 }
376 // Not worried about C++ exception safety here in terms of
377 // making sure we restore the error. Either we'll catch it
378 // above and establish the error from that exception
379 // (which, yes, might overwrite something from before we
380 // entered, but we're in an undefined situation at that
381 // point) or we won't catch it at all and will crash the
382 // process.
383 //
384 // As for Python exception safety, there's no chance we're
385 // overwriting an exception (from the loop) with no
386 // exception (captured NULLs before we entered the loop),
387 // because there CAN'T BE any exception from the loop ---
388 // we clear them. So we're either restoring a pre-existing
389 // exception, or leaving the exception unset (by restoring
390 // NULL).
391 incoming_err.PyErrRestore();
392 }
393
394public:
395
396 /**
397 * Returns a new reference, or a false object.
398 */
399 inline OwnedObject get_tracefunc() const
400 {
401 return tracefunc;
402 };
403
404
405 inline void set_tracefunc(BorrowedObject tracefunc)
406 {
407 assert(tracefunc);
408 if (tracefunc == BorrowedObject(Py_None)) {
409 this->tracefunc.CLEAR();
410 }
411 else {
412 this->tracefunc = tracefunc;
413 }
414 }
415
416 /**
417 * Given a reference to a greenlet that some other thread
418 * attempted to delete (has a refcount of 0) store it for later
419 * deletion when the thread this state belongs to is current.
420 */
421 inline void delete_when_thread_running(PyGreenlet* to_del)
422 {
423 Py_INCREF(to_del);
424#ifdef Py_GIL_DISABLED
425 LockGuard deleteme_guard(this->deleteme_lock);
426#endif
427 this->deleteme.push_back(to_del);
428 }
429
430 /**
431 * Set to std::clock_t(-1) to disable.
432 */
433 inline static std::clock_t clocks_used_doing_gc()
434 {
435#ifdef Py_GIL_DISABLED
436 return ThreadState::_clocks_used_doing_gc.load(std::memory_order_relaxed);
437#else
438 return ThreadState::_clocks_used_doing_gc;
439#endif
440 }
441
442 inline static void set_clocks_used_doing_gc(std::clock_t value)
443 {
444#ifdef Py_GIL_DISABLED
445 ThreadState::_clocks_used_doing_gc.store(value, std::memory_order_relaxed);
446#else
447 ThreadState::_clocks_used_doing_gc = value;
448#endif
449 }
450
451 inline static void add_clocks_used_doing_gc(std::clock_t value)
452 {
453#ifdef Py_GIL_DISABLED
454 ThreadState::_clocks_used_doing_gc.fetch_add(value, std::memory_order_relaxed);
455#else
456 ThreadState::_clocks_used_doing_gc += value;
457#endif
458 }
459
460 // Runs in some arbitrary thread that Python is using to invoke
461 // pending callbacks. This may not be the thread that was
462 // running the greenlets.
463 ~ThreadState()
464 {
465 if (!PyInterpreterState_Head()) {
466 // We shouldn't get here (our callers protect us)
467 // but if we do, all we can do is bail early.
468 return;
469 }
470
471 // During interpreter finalization, Python APIs like
472 // PyImport_ImportModule are unsafe (the import machinery may
473 // be partially torn down). On Python < 3.11, perform only the
474 // minimal cleanup that is safe: clear our strong references
475 // so we don't leak, but skip the GC-based leak detection.
476 //
477 // Python 3.11+ restructured interpreter finalization so that
478 // these APIs remain safe during shutdown.
479 if (greenlet::IsShuttingDown()) {
480 this->tracefunc.CLEAR();
481 if (this->current_greenlet) {
482 this->current_greenlet->murder_in_place();
483 this->current_greenlet.CLEAR();
484 }
485 this->main_greenlet.CLEAR();
486 return;
487 }
488
489 // We should not have an "origin" greenlet; that only exists
490 // for the temporary time during a switch, which should not
491 // be in progress as the thread dies.
492 //assert(!this->switching_state.origin);
493
494 this->tracefunc.CLEAR();
495
496 // Forcibly GC as much as we can.
497 this->clear_deleteme_list(true);
498
499 // The pending call did this.
500 assert(this->main_greenlet->thread_state() == nullptr);
501
502 // If the main greenlet is the current greenlet,
503 // then we "fell off the end" and the thread died.
504 // It's possible that there is some other greenlet that
505 // switched to us, leaving a reference to the main greenlet
506 // on the stack, somewhere uncollectible. Try to detect that.
507 if (this->current_greenlet == this->main_greenlet && this->current_greenlet) {
508 assert(
509 this->current_greenlet->is_currently_running_in_some_thread()
510 || this->current_greenlet->was_running_in_dead_thread()
511 );
512 // Drop one reference we hold.
513 this->current_greenlet.CLEAR();
514 assert(!this->current_greenlet);
515 // Only our reference to the main greenlet should be left,
516 // But hold onto the pointer in case we need to do extra cleanup.
517 PyGreenlet* old_main_greenlet = this->main_greenlet.borrow();
518 Py_ssize_t cnt = this->main_greenlet.REFCNT();
519 this->main_greenlet.CLEAR();
520 if (ThreadState::clocks_used_doing_gc() != std::clock_t(-1)
521 && cnt == 2 && Py_REFCNT(old_main_greenlet) == 1) {
522 // Highly likely that the reference is somewhere on
523 // the stack, not reachable by GC. Verify.
524 // XXX: This is O(n) in the total number of objects.
525 // TODO: Add a way to disable this at runtime, and
526 // another way to report on it.
527 std::clock_t begin = std::clock();
528 NewReference gc(PyImport_ImportModule("gc"));
529 if (gc) {
530 OwnedObject get_referrers = gc.PyRequireAttr(ThreadState::get_referrers_name);
531 OwnedList refs(get_referrers.PyCall(old_main_greenlet));
532 if (refs && refs.empty()) {
533 assert(refs.REFCNT() == 1);
534 // We found nothing! So we left a dangling
535 // reference: Probably the last thing some
536 // other greenlet did was call
537 // 'getcurrent().parent.switch()' to switch
538 // back to us. Clean it up. This will be the
539 // case on CPython 3.7 and newer, as they use
540 // an internal calling conversion that avoids
541 // creating method objects and storing them on
542 // the stack.
543 Py_DECREF(old_main_greenlet);
544 }
545 else if (refs
546 && refs.size() == 1
547 && PyCFunction_Check(refs.at(0))
548 && Py_REFCNT(refs.at(0)) == 2) {
549 assert(refs.REFCNT() == 1);
550 // Ok, we found a C method that refers to the
551 // main greenlet, and its only referenced
552 // twice, once in the list we just created,
553 // once from...somewhere else. If we can't
554 // find where else, then this is a leak.
555 // This happens in older versions of CPython
556 // that create a bound method object somewhere
557 // on the stack that we'll never get back to.
558 if (PyCFunction_GetFunction(refs.at(0).borrow()) == (PyCFunction)green_switch) {
559 BorrowedObject function_w = refs.at(0);
560 refs.clear(); // destroy the reference
561 // from the list.
562 // back to one reference. Can *it* be
563 // found?
564 assert(function_w.REFCNT() == 1);
565 refs = get_referrers.PyCall(function_w);
566 if (refs && refs.empty()) {
567 // Nope, it can't be found so it won't
568 // ever be GC'd. Drop it.
569 Py_CLEAR(function_w);
570 }
571 }
572 }
573 std::clock_t end = std::clock();
574 ThreadState::add_clocks_used_doing_gc(end - begin);
575 }
576 }
577 }
578
579 // We need to make sure this greenlet appears to be dead,
580 // because otherwise deallocing it would fail to raise an
581 // exception in it (the thread is dead) and put it back in our
582 // deleteme list.
583 if (this->current_greenlet) {
584 this->current_greenlet->murder_in_place();
585 this->current_greenlet.CLEAR();
586 }
587
588 if (this->main_greenlet) {
589 // Couldn't have been the main greenlet that was running
590 // when the thread exited (because we already cleared this
591 // pointer if it was). This shouldn't be possible?
592
593 // If the main greenlet was current when the thread died (it
594 // should be, right?) then we cleared its self pointer above
595 // when we cleared the current greenlet's main greenlet pointer.
596 // assert(this->main_greenlet->main_greenlet == this->main_greenlet
597 // || !this->main_greenlet->main_greenlet);
598 // // self reference, probably gone
599 // this->main_greenlet->main_greenlet.CLEAR();
600
601 // This will actually go away when the ivar is destructed.
602 this->main_greenlet.CLEAR();
603 }
604
605 if (PyErr_Occurred()) {
606 PyErr_WriteUnraisable(NULL);
607 PyErr_Clear();
608 }
609
610 }
611
612};
613
614ImmortalString ThreadState::get_referrers_name(nullptr);
615#ifdef Py_GIL_DISABLED
616std::atomic<std::clock_t> ThreadState::_clocks_used_doing_gc(0);
617#else
618std::clock_t ThreadState::_clocks_used_doing_gc(0);
619#endif
620
621
622
623
624
625}; // namespace greenlet
626
627#endif
628