From 6d178604b2741d244898a5c486ef2f29a2047847 Mon Sep 17 00:00:00 2001 From: Zekun Li Date: Fri, 22 Sep 2017 18:35:33 -0700 Subject: [PATCH 1/3] bpo-31558: Add gc.freeze() --- Include/internal/mem.h | 1 + Lib/test/test_gc.py | 4 ++++ Modules/gcmodule.c | 42 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/Include/internal/mem.h b/Include/internal/mem.h index f3a8f56e8055bc..a606d76d7873f4 100644 --- a/Include/internal/mem.h +++ b/Include/internal/mem.h @@ -185,6 +185,7 @@ struct _gc_runtime_state { collections, and are awaiting to undergo a full collection for the first time. */ Py_ssize_t long_lived_pending; + struct gc_generation permanent_generation; }; PyAPI_FUNC(void) _PyGC_Initialize(struct _gc_runtime_state *); diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 914efec3d3ef55..edefd364e963b0 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -734,6 +734,10 @@ def test_get_stats(self): self.assertEqual(new[1]["collections"], old[1]["collections"]) self.assertEqual(new[2]["collections"], old[2]["collections"] + 1) + def test_freeze(self): + gc.freeze() + self.assertGreater(gc.get_freeze_stats(), 0) + class GCCallbackTests(unittest.TestCase): def setUp(self): diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 832e27ebe664e2..dce2483a3e353d 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -71,6 +71,10 @@ _PyGC_Initialize(struct _gc_runtime_state *state) state->generations[i] = generations[i]; }; state->generation0 = GEN_HEAD(0); + struct gc_generation permanent_generation = { + {{&permanent_generation.head, &permanent_generation.head, 0}}, 0, 0 + }; + state->permanent_generation = permanent_generation; } /*-------------------------------------------------------------------------- @@ -813,6 +817,8 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, for (i = 0; i < NUM_GENERATIONS; i++) PySys_FormatStderr(" %zd", gc_list_size(GEN_HEAD(i))); + PySys_WriteStderr("\ngc: objects in permanent generation: %d", + gc_list_size(&_PyRuntime.gc.permanent_generation.head)); t1 = _PyTime_GetMonotonicClock(); PySys_WriteStderr("\n"); @@ -1405,6 +1411,36 @@ gc_is_tracked(PyObject *module, PyObject *obj) return result; } +PyDoc_STRVAR(gc_freeze__doc__, +"freeze() -> None\n" +"\n" +"Freeze all current tracked objects and ignore them for future collections.\n" +"This can be used before a fork to make the gc copy-on-write friendly.\n" +"Note: collection before a fork may free pages for future allocation\n" +"which can cause copy-on-write.\n" +); + +static PyObject * +gc_freeze(PyObject *module) +{ + for (int i = 0; i < NUM_GENERATIONS; ++i) { + gc_list_merge(GEN_HEAD(i), &_PyRuntime.gc.permanent_generation.head); + _PyRuntime.gc.generations[i].count = 0; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(gc_get_freeze_stats__doc__, +"get_freeze_stats() -> n\n" +"\n" +"Return the number of objects in permanent generations.\n" +); + +static PyObject * +gc_get_freeze_stats(PyObject *module) { + return Py_BuildValue("i", gc_list_size(&_PyRuntime.gc.permanent_generation.head)); +} + PyDoc_STRVAR(gc__doc__, "This module provides access to the garbage collector for reference cycles.\n" @@ -1422,7 +1458,9 @@ PyDoc_STRVAR(gc__doc__, "get_objects() -- Return a list of all objects tracked by the collector.\n" "is_tracked() -- Returns true if a given object is tracked.\n" "get_referrers() -- Return the list of objects that refer to an object.\n" -"get_referents() -- Return the list of objects that an object refers to.\n"); +"get_referents() -- Return the list of objects that an object refers to.\n" +"freeze() -- Freeze all tracked objects and ignore them for future collections.\n" +"get_freeze_stats() -- Return the number of objects in the permanent generation.\n"); static PyMethodDef GcMethods[] = { GC_ENABLE_METHODDEF @@ -1441,6 +1479,8 @@ static PyMethodDef GcMethods[] = { gc_get_referrers__doc__}, {"get_referents", gc_get_referents, METH_VARARGS, gc_get_referents__doc__}, + {"freeze", gc_freeze, METH_NOARGS, gc_freeze__doc__}, + {"get_freeze_stats", gc_get_freeze_stats, METH_NOARGS, gc_get_freeze_stats__doc__}, {NULL, NULL} /* Sentinel */ }; From fdd12ff84f4d30e259f7517d4fc9fbceca34c372 Mon Sep 17 00:00:00 2001 From: Zekun Li Date: Sun, 1 Oct 2017 00:16:59 -0400 Subject: [PATCH 2/3] address comments --- Include/internal/mem.h | 3 ++- Lib/test/test_gc.py | 2 +- Modules/clinic/gcmodule.c.h | 52 ++++++++++++++++++++++++++++++++++++- Modules/gcmodule.c | 44 +++++++++++++++++-------------- 4 files changed, 78 insertions(+), 23 deletions(-) diff --git a/Include/internal/mem.h b/Include/internal/mem.h index a606d76d7873f4..471cdf45df2766 100644 --- a/Include/internal/mem.h +++ b/Include/internal/mem.h @@ -167,6 +167,8 @@ struct _gc_runtime_state { /* linked lists of container objects */ struct gc_generation generations[NUM_GENERATIONS]; PyGC_Head *generation0; + /* a permanent generation which won't be collected */ + struct gc_generation permanent_generation; struct gc_generation_stats generation_stats[NUM_GENERATIONS]; /* true if we are currently running the collector */ int collecting; @@ -185,7 +187,6 @@ struct _gc_runtime_state { collections, and are awaiting to undergo a full collection for the first time. */ Py_ssize_t long_lived_pending; - struct gc_generation permanent_generation; }; PyAPI_FUNC(void) _PyGC_Initialize(struct _gc_runtime_state *); diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index edefd364e963b0..1d2b32b91dc877 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -736,7 +736,7 @@ def test_get_stats(self): def test_freeze(self): gc.freeze() - self.assertGreater(gc.get_freeze_stats(), 0) + self.assertGreater(gc.get_freeze_count(), 0) class GCCallbackTests(unittest.TestCase): diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 771c57387bb840..6dc9e21e8b9ab8 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -255,4 +255,54 @@ PyDoc_STRVAR(gc_is_tracked__doc__, #define GC_IS_TRACKED_METHODDEF \ {"is_tracked", (PyCFunction)gc_is_tracked, METH_O, gc_is_tracked__doc__}, -/*[clinic end generated code: output=5a58583f00ab018e input=a9049054013a1b77]*/ + +PyDoc_STRVAR(gc_freeze__doc__, +"freeze($module, /)\n" +"--\n" +"\n" +"Freeze all current tracked objects and ignore them for future collections.\n" +"\n" +"This can be used before a fork to make the gc copy-on-write friendly.\n" +"Note: collection before a fork may free pages for future allocation\n" +"which can cause copy-on-write."); + +#define GC_FREEZE_METHODDEF \ + {"freeze", (PyCFunction)gc_freeze, METH_NOARGS, gc_freeze__doc__}, + +static PyObject * +gc_freeze_impl(PyObject *module); + +static PyObject * +gc_freeze(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return gc_freeze_impl(module); +} + +PyDoc_STRVAR(gc_get_freeze_count__doc__, +"get_freeze_count($module, /)\n" +"--\n" +"\n" +"Return the number of objects in permanent generations."); + +#define GC_GET_FREEZE_COUNT_METHODDEF \ + {"get_freeze_count", (PyCFunction)gc_get_freeze_count, METH_NOARGS, gc_get_freeze_count__doc__}, + +static int +gc_get_freeze_count_impl(PyObject *module); + +static PyObject * +gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = gc_get_freeze_count_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} +/*[clinic end generated code: output=004d91aafb1827f0 input=a9049054013a1b77]*/ diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index dce2483a3e353d..2d3879bda6f725 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -817,7 +817,7 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, for (i = 0; i < NUM_GENERATIONS; i++) PySys_FormatStderr(" %zd", gc_list_size(GEN_HEAD(i))); - PySys_WriteStderr("\ngc: objects in permanent generation: %d", + PySys_WriteStderr("\ngc: objects in permanent generation: %zd", gc_list_size(&_PyRuntime.gc.permanent_generation.head)); t1 = _PyTime_GetMonotonicClock(); @@ -1411,17 +1411,19 @@ gc_is_tracked(PyObject *module, PyObject *obj) return result; } -PyDoc_STRVAR(gc_freeze__doc__, -"freeze() -> None\n" -"\n" -"Freeze all current tracked objects and ignore them for future collections.\n" -"This can be used before a fork to make the gc copy-on-write friendly.\n" -"Note: collection before a fork may free pages for future allocation\n" -"which can cause copy-on-write.\n" -); +/*[clinic input] +gc.freeze + +Freeze all current tracked objects and ignore them for future collections. + +This can be used before a fork to make the gc copy-on-write friendly. +Note: collection before a fork may free pages for future allocation +which can cause copy-on-write. +[clinic start generated code]*/ static PyObject * -gc_freeze(PyObject *module) +gc_freeze_impl(PyObject *module) +/*[clinic end generated code: output=502159d9cdc4c139 input=8798c406cd01d7c4]*/ { for (int i = 0; i < NUM_GENERATIONS; ++i) { gc_list_merge(GEN_HEAD(i), &_PyRuntime.gc.permanent_generation.head); @@ -1430,14 +1432,16 @@ gc_freeze(PyObject *module) Py_RETURN_NONE; } -PyDoc_STRVAR(gc_get_freeze_stats__doc__, -"get_freeze_stats() -> n\n" -"\n" -"Return the number of objects in permanent generations.\n" -); +/*[clinic input] +gc.get_freeze_count -> int -static PyObject * -gc_get_freeze_stats(PyObject *module) { +Return the number of objects in permanent generations. +[clinic start generated code]*/ + +static int +gc_get_freeze_count_impl(PyObject *module) +/*[clinic end generated code: output=e4e2ebcc77e5cbf3 input=590241a6a398cfa2]*/ +{ return Py_BuildValue("i", gc_list_size(&_PyRuntime.gc.permanent_generation.head)); } @@ -1460,7 +1464,7 @@ PyDoc_STRVAR(gc__doc__, "get_referrers() -- Return the list of objects that refer to an object.\n" "get_referents() -- Return the list of objects that an object refers to.\n" "freeze() -- Freeze all tracked objects and ignore them for future collections.\n" -"get_freeze_stats() -- Return the number of objects in the permanent generation.\n"); +"get_freeze_count() -- Return the number of objects in the permanent generation.\n"); static PyMethodDef GcMethods[] = { GC_ENABLE_METHODDEF @@ -1479,8 +1483,8 @@ static PyMethodDef GcMethods[] = { gc_get_referrers__doc__}, {"get_referents", gc_get_referents, METH_VARARGS, gc_get_referents__doc__}, - {"freeze", gc_freeze, METH_NOARGS, gc_freeze__doc__}, - {"get_freeze_stats", gc_get_freeze_stats, METH_NOARGS, gc_get_freeze_stats__doc__}, + GC_FREEZE_METHODDEF + GC_GET_FREEZE_COUNT_METHODDEF {NULL, NULL} /* Sentinel */ }; From 03f49152240d81ef2d12b544a698863885106e6b Mon Sep 17 00:00:00 2001 From: Zekun Li Date: Fri, 6 Oct 2017 22:38:46 -0700 Subject: [PATCH 3/3] fix a segfault and address comments --- Doc/library/gc.rst | 27 +++++++++++++++++++++++++++ Lib/test/test_gc.py | 2 ++ Modules/clinic/gcmodule.c.h | 28 ++++++++++++++++++++++++---- Modules/gcmodule.c | 32 +++++++++++++++++++++++++------- 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 87d682445bae23..153d8fb7045623 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -174,6 +174,33 @@ The :mod:`gc` module provides the following functions: .. versionadded:: 3.1 +.. function:: freeze() + + Freeze all the objects tracked by gc - move them to a permanent generation + and ignore all the future collections. This can be used before a POSIX + fork() call to make the gc copy-on-write friendly or to speed up collection. + Also collection before a POSIX fork() call may free pages for future + allocation which can cause copy-on-write too so it's advised to disable gc + in master process and freeze before fork and enable gc in child process. + + .. versionadded:: 3.7 + + +.. function:: unfreeze() + + Unfreeze the objects in the permanent generation, put them back into the + oldest generation. + + .. versionadded:: 3.7 + + +.. function:: get_freeze_count() + + Return the number of objects in the permanent generation. + + .. versionadded:: 3.7 + + The following variables are provided for read-only access (you can mutate the values but should not rebind them): diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 1d2b32b91dc877..904fc7d88c0318 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -737,6 +737,8 @@ def test_get_stats(self): def test_freeze(self): gc.freeze() self.assertGreater(gc.get_freeze_count(), 0) + gc.unfreeze() + self.assertEqual(gc.get_freeze_count(), 0) class GCCallbackTests(unittest.TestCase): diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 6dc9e21e8b9ab8..c31ef65eb06b74 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -262,8 +262,8 @@ PyDoc_STRVAR(gc_freeze__doc__, "\n" "Freeze all current tracked objects and ignore them for future collections.\n" "\n" -"This can be used before a fork to make the gc copy-on-write friendly.\n" -"Note: collection before a fork may free pages for future allocation\n" +"This can be used before a POSIX fork() call to make the gc copy-on-write friendly.\n" +"Note: collection before a POSIX fork() call may free pages for future allocation\n" "which can cause copy-on-write."); #define GC_FREEZE_METHODDEF \ @@ -278,11 +278,31 @@ gc_freeze(PyObject *module, PyObject *Py_UNUSED(ignored)) return gc_freeze_impl(module); } +PyDoc_STRVAR(gc_unfreeze__doc__, +"unfreeze($module, /)\n" +"--\n" +"\n" +"Unfreeze all objects in the permanent generation.\n" +"\n" +"Put all objects in the permanent generation back into oldest generation."); + +#define GC_UNFREEZE_METHODDEF \ + {"unfreeze", (PyCFunction)gc_unfreeze, METH_NOARGS, gc_unfreeze__doc__}, + +static PyObject * +gc_unfreeze_impl(PyObject *module); + +static PyObject * +gc_unfreeze(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return gc_unfreeze_impl(module); +} + PyDoc_STRVAR(gc_get_freeze_count__doc__, "get_freeze_count($module, /)\n" "--\n" "\n" -"Return the number of objects in permanent generations."); +"Return the number of objects in the permanent generation."); #define GC_GET_FREEZE_COUNT_METHODDEF \ {"get_freeze_count", (PyCFunction)gc_get_freeze_count, METH_NOARGS, gc_get_freeze_count__doc__}, @@ -305,4 +325,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=004d91aafb1827f0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4f41ec4588154f2b input=a9049054013a1b77]*/ diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 2d3879bda6f725..6e26c7a68f6037 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -72,7 +72,7 @@ _PyGC_Initialize(struct _gc_runtime_state *state) }; state->generation0 = GEN_HEAD(0); struct gc_generation permanent_generation = { - {{&permanent_generation.head, &permanent_generation.head, 0}}, 0, 0 + {{&state->permanent_generation.head, &state->permanent_generation.head, 0}}, 0, 0 }; state->permanent_generation = permanent_generation; } @@ -1416,14 +1416,14 @@ gc.freeze Freeze all current tracked objects and ignore them for future collections. -This can be used before a fork to make the gc copy-on-write friendly. -Note: collection before a fork may free pages for future allocation +This can be used before a POSIX fork() call to make the gc copy-on-write friendly. +Note: collection before a POSIX fork() call may free pages for future allocation which can cause copy-on-write. [clinic start generated code]*/ static PyObject * gc_freeze_impl(PyObject *module) -/*[clinic end generated code: output=502159d9cdc4c139 input=8798c406cd01d7c4]*/ +/*[clinic end generated code: output=502159d9cdc4c139 input=b602b16ac5febbe5]*/ { for (int i = 0; i < NUM_GENERATIONS; ++i) { gc_list_merge(GEN_HEAD(i), &_PyRuntime.gc.permanent_generation.head); @@ -1432,17 +1432,33 @@ gc_freeze_impl(PyObject *module) Py_RETURN_NONE; } +/*[clinic input] +gc.unfreeze + +Unfreeze all objects in the permanent generation. + +Put all objects in the permanent generation back into oldest generation. +[clinic start generated code]*/ + +static PyObject * +gc_unfreeze_impl(PyObject *module) +/*[clinic end generated code: output=1c15f2043b25e169 input=2dd52b170f4cef6c]*/ +{ + gc_list_merge(&_PyRuntime.gc.permanent_generation.head, GEN_HEAD(NUM_GENERATIONS-1)); + Py_RETURN_NONE; +} + /*[clinic input] gc.get_freeze_count -> int -Return the number of objects in permanent generations. +Return the number of objects in the permanent generation. [clinic start generated code]*/ static int gc_get_freeze_count_impl(PyObject *module) -/*[clinic end generated code: output=e4e2ebcc77e5cbf3 input=590241a6a398cfa2]*/ +/*[clinic end generated code: output=e4e2ebcc77e5cbf3 input=4b759db880a3c6e4]*/ { - return Py_BuildValue("i", gc_list_size(&_PyRuntime.gc.permanent_generation.head)); + return gc_list_size(&_PyRuntime.gc.permanent_generation.head); } @@ -1464,6 +1480,7 @@ PyDoc_STRVAR(gc__doc__, "get_referrers() -- Return the list of objects that refer to an object.\n" "get_referents() -- Return the list of objects that an object refers to.\n" "freeze() -- Freeze all tracked objects and ignore them for future collections.\n" +"unfreeze() -- Unfreeze all objects in the permanent generation.\n" "get_freeze_count() -- Return the number of objects in the permanent generation.\n"); static PyMethodDef GcMethods[] = { @@ -1484,6 +1501,7 @@ static PyMethodDef GcMethods[] = { {"get_referents", gc_get_referents, METH_VARARGS, gc_get_referents__doc__}, GC_FREEZE_METHODDEF + GC_UNFREEZE_METHODDEF GC_GET_FREEZE_COUNT_METHODDEF {NULL, NULL} /* Sentinel */ };