matt/agila_open_tool
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
23f2d29e3202de5b34ba76409a12ed058d54799f
agila_open_tool/tools/v/thirdparty/libgc/gc.c
T

45850 lines
1.3 MiB
Plaintext
Raw Blame History

/*
* Copyright (c) 1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996 by Silicon Graphics. All rights reserved.
* Copyright (c) 1998 by Fergus Henderson. All rights reserved.
* Copyright (c) 2000-2009 by Hewlett-Packard Development Company.
* All rights reserved.
* Copyright (c) 2009-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* This file could be used for the following purposes:
* - get the complete collector as a single link object file (module);
* - enable more compiler optimizations.
*
* Tip: to get the highest level of compiler optimizations, the typical
* compiler options to use (assuming gcc) are:
* `-O3 -march=native -fprofile-generate`
*
* Warning: gcc for Linux (for C++ clients only): use `-fexceptions` both for
* the collector library and the client as otherwise `GC_thread_exit_proc()`
* is not guaranteed to be invoked (see the comments in `pthread_start.c`
* file).
*/
/*
* V note: the bundled libgc must be compiled with `-DALL_INTERIOR_POINTERS=1`
* (set in `vlib/builtin/builtin_d_gcboehm.c.v`). Without it, the conservative
* collector misses interior pointers into GC-allocated buffers — sumtype
* payloads referenced via offset (e.g. `toml.ast_to_any`, `yaml.Any.to_json`)
* get reclaimed mid-walk and `-gc boehm` builds crash with `signal 11`.
* Keep this requirement in sync when re-amalgamating from bdwgc.
*/
#define GC_SINGLE_OBJ_BUILD
#ifndef __cplusplus
/* `static` is desirable here for more efficient linkage. */
/* TODO: Enable this in case of the compilation as C++ code. */
# define GC_INNER STATIC
# define GC_EXTERN GC_INNER
/* Note: `STATIC` macro is defined in `gcconfig.h` file. */
#endif
/* Small files go first... */
/*
* Copyright (c) 2001 by Hewlett-Packard Company. All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1995 by Xerox Corporation. All rights reserved.
* Copyright (c) 1997 by Silicon Graphics. All rights reserved.
* Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* This is mostly an internal header file. Typical clients should
* not use it. Clients that define their own object kinds with
* debugging allocators will probably want to include this, however.
* No attempt is made to keep the namespace clean. This should not be
* included from header files that are frequently included by clients.
*/
#ifndef GC_DBG_MLC_H
#define GC_DBG_MLC_H
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#ifndef GC_PRIVATE_H
#define GC_PRIVATE_H
#ifdef HAVE_CONFIG_H
/* include/config.h. Generated from config.h.in by configure. */
/* include/config.h.in. Generated from configure.ac by autoheader. */
/* Define to recognize all pointers to the interior of objects. */
#define ALL_INTERIOR_POINTERS 1
/* AO load, store and/or test-and-set primitives are implemented in
`libatomic_ops` using locks. */
/* #undef BASE_ATOMIC_OPS_EMULATED */
/* Erroneously cleared dirty bits checking. Use only for debugging of the
incremental collector. */
/* #undef CHECKSUMS */
/* Define to discover thread stack bounds on Darwin without trying to walk the
frames on the stack. */
/* #undef DARWIN_DONT_PARSE_STACK */
/* Define to force debug headers on all objects. */
#define DBG_HDRS_ALL 1
/* Do not use user32.dll import library (Win32). */
/* #undef DONT_USE_USER32_DLL */
/* Define to support pointer mask/shift set at runtime. */
/* #undef DYNAMIC_POINTER_MASK */
/* Define to enable eCos target support. */
/* #undef ECOS */
/* Wine `getenv` may not return `NULL` for missing entry. */
/* #undef EMPTY_GETENV_RESULTS */
/* Define to enable alternative finalization interface. */
#define ENABLE_DISCLAIM 1
/* Define to enable internal debug assertions. */
/* #undef GC_ASSERTIONS */
/* Define to enable atomic uncollectible allocation. */
#define GC_ATOMIC_UNCOLLECTABLE 1
/* Use GCC atomic intrinsics instead of `libatomic_ops` primitives. */
#define GC_BUILTIN_ATOMIC 1
/* Define to build dynamic libraries with only API symbols exposed. */
/* #undef GC_DLL */
/* Skip the initial guess of data root sets. */
/* #undef GC_DONT_REGISTER_MAIN_STATIC_DATA */
/* Define to turn on `GC_suspend_thread` support (Linux only). */
#define GC_ENABLE_SUSPEND_THREAD 1
/* Define to include support for `gcj`. */
#define GC_GCJ_SUPPORT 1
/* Define if backtrace information is supported. */
/* #undef GC_HAVE_BUILTIN_BACKTRACE */
/* Define to use `pthread_sigmask` function if needed. */
/* #undef GC_HAVE_PTHREAD_SIGMASK */
/* Enable Win32 `DllMain`-based approach of threads registering. */
/* #undef GC_INSIDE_DLL */
/* Missing execinfo.h header. */
/* #undef GC_MISSING_EXECINFO_H */
/* Missing `sigsetjmp` function. */
/* #undef GC_NO_SIGSETJMP */
/* Disable threads discovery in the collector. */
/* #undef GC_NO_THREADS_DISCOVERY */
/* Read environment variables from the GC 'env' file. */
/* #undef GC_READ_ENV_FILE */
/* Define and export `GC_wcsdup` function. */
#define GC_REQUIRE_WCSDUP 1
/* Define to support platform-specific threads. */
#define GC_THREADS 1
/* Force the GC to use signals based on `SIGRTMIN+k`. */
/* #undef GC_USESIGRT_SIGNALS */
/* Define to cause the collector to redefine `malloc` and intercepted
`pthreads` routines with their real names while using `dlsym` to refer to
the original routines. */
/* #undef GC_USE_DLOPEN_WRAP */
/* The major version number of this GC release. */
#define GC_VERSION_MAJOR 8
/* The micro version number of this GC release. */
#define GC_VERSION_MICRO 0
/* The minor version number of this GC release. */
#define GC_VERSION_MINOR 3
/* Define to support pthreads-win32 or winpthreads. */
/* #undef GC_WIN32_PTHREADS */
/* Define to install `pthread_atfork` handlers by default. */
#define HANDLE_FORK 1
/* Define to use `dladdr` function. */
#define HAVE_DLADDR 1
/* Define to 1 if you have the <dlfcn.h> header file. */
#define HAVE_DLFCN_H 1
/* Define to 1 if you have the 'dl_iterate_phdr' function. */
/* #undef HAVE_DL_ITERATE_PHDR */
/* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1
/* `libatomic_ops` `AO_or` primitive implementation is lock-free. */
/* #undef HAVE_LOCKFREE_AO_OR */
/* Define to use `pthread_setname_np(const char *)` function. */
#define HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID 1
/* Define to use `pthread_setname_np(pthread_t, const char *)` function. */
/* #undef HAVE_PTHREAD_SETNAME_NP_WITH_TID */
/* Define to use `pthread_setname_np(pthread_t, const char *, void *)`
function. */
/* #undef HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG */
/* Define to use `pthread_set_name_np(pthread_t, const char *)` function. */
/* #undef HAVE_PTHREAD_SET_NAME_NP */
/* Define to 1 if you have the <stdint.h> header file. */
#define HAVE_STDINT_H 1
/* Define to 1 if you have the <stdio.h> header file. */
#define HAVE_STDIO_H 1
/* Define to 1 if you have the <stdlib.h> header file. */
#define HAVE_STDLIB_H 1
/* Define to 1 if you have the <strings.h> header file. */
#define HAVE_STRINGS_H 1
/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1
/* Define to 1 if you have the <sys/stat.h> header file. */
#define HAVE_SYS_STAT_H 1
/* Define to 1 if you have the <sys/types.h> header file. */
#define HAVE_SYS_TYPES_H 1
/* Define to 1 if you have the <unistd.h> header file. */
#define HAVE_UNISTD_H 1
/* Do not define `DYNAMIC_LOADING` even if supported (i.e., build the
collector with disabled tracing of dynamic library data roots). */
/* #undef IGNORE_DYNAMIC_LOADING */
/* Define to make it somewhat safer by default to finalize objects out of
order by specifying a nonstandard finalization mark procedure. */
#define JAVA_FINALIZATION 1
/* Define to save back-pointers in debugging headers. */
#define KEEP_BACK_PTRS 1
/* Define to optimize for large heaps or root sets. */
/* #undef LARGE_CONFIG */
/* Define to the sub-directory where libtool stores uninstalled libraries. */
#define LT_OBJDIR ".libs/"
/* Define to build the collector with the support of the functionality to
print max length of chain through unreachable objects ending in a reachable
one. */
/* #undef MAKE_BACK_GRAPH */
/* Number of sequential garbage collections during those a candidate block for
unmapping should be marked as free. */
#define MUNMAP_THRESHOLD 7
/* Define to not use system clock (cross compiling). */
/* #undef NO_CLOCK */
/* Disable debugging, like `GC_dump` and its callees. */
/* #undef NO_DEBUGGING */
/* Define to make the collector not allocate executable memory by default. */
#define NO_EXECUTE_PERMISSION 1
/* Missing `getcontext` function. */
/* #undef NO_GETCONTEXT */
/* Prohibit installation of `pthread_atfork` handlers. */
/* #undef NO_HANDLE_FORK */
/* Name of package */
#define PACKAGE "gc"
/* Define to the address where bug reports for this package should be sent. */
#define PACKAGE_BUGREPORT "https://github.com/bdwgc/bdwgc/issues"
/* Define to the full name of this package. */
#define PACKAGE_NAME "gc"
/* Define to the full name and version of this package. */
#define PACKAGE_STRING "gc 8.3.0"
/* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME "gc"
/* Define to the home page for this package. */
#define PACKAGE_URL ""
/* Define to the version of this package. */
#define PACKAGE_VERSION "8.3.0"
/* Define to enable parallel marking. */
#define PARALLEL_MARK 1
/* If defined, redirect `free` to this function. */
/* #undef REDIRECT_FREE */
/* If defined, redirect `malloc` to this function. */
/* #undef REDIRECT_MALLOC */
/* If defined, redirect `realloc` to this function. */
/* #undef REDIRECT_REALLOC */
/* The number of caller frames saved when allocating with the debugging API.
*/
/* #undef SAVE_CALL_COUNT */
/* Shorten the headers to minimize object size at the expense of checking for
writes past the end. */
/* #undef SHORT_DBG_HDRS */
/* Define to tune the collector for small heap sizes. */
/* #undef SMALL_CONFIG */
/* Define to 1 if all of the C89 standard headers exist (not just the ones
required in a freestanding environment). This macro is provided for
backward compatibility; new code need not use it. */
#define STDC_HEADERS 1
/* Define to work around a Solaris 5.3 bug (see dyn_load.c). */
/* #undef SUNOS53_SHARED_LIB */
/* Define to enable thread-local allocation optimization. */
/* #undef THREAD_LOCAL_ALLOC */
/* Use Unicode (W) variant of Win32 API instead of ASCII (A) one. */
/* #undef UNICODE */
/* Define to use of compiler-support for thread-local variables. */
/* #undef USE_COMPILER_TLS */
/* Define to use `mmap` instead of `sbrk` to expand the heap. */
#define USE_MMAP 1
/* Define to return memory to OS with `munmap` calls. */
#define USE_MUNMAP 1
/* Use `rwlock` for the allocator lock instead of mutex. */
/* #undef USE_RWLOCK */
/* Define to use Win32 `VirtualAlloc` (instead of `sbrk` or `mmap`) to expand
the heap. */
/* #undef USE_WINALLOC */
/* Define to support tracking `GC_malloc` and friends for heap profiling
tools. */
/* #undef VALGRIND_TRACKING */
/* Version number of package */
#define VERSION "8.3.0"
/* The POSIX feature macro. */
/* #undef _POSIX_C_SOURCE */
/* Indicates the use of `pthreads` (NetBSD). */
/* #undef _PTHREADS */
/* Required define if using POSIX threads. */
#define _REENTRANT 1
/* Define to '__inline__' or '__inline' if that's what the C compiler
calls it, or to nothing if 'inline' is not supported under any name. */
#ifndef __cplusplus
/* #undef inline */
#endif
#endif
#if !defined(GC_BUILD) && !defined(NOT_GCBUILD)
# define GC_BUILD
#endif
#if (defined(__linux__) || defined(__GLIBC__) || defined(__GNU__) \
|| defined(__CYGWIN__) || defined(HAVE_DLADDR) \
|| (defined(__COSMOPOLITAN__) && defined(USE_MUNMAP)) \
|| defined(GC_HAVE_PTHREAD_SIGMASK) \
|| defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID) \
|| defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG) \
|| defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID)) \
&& !defined(_GNU_SOURCE)
/* Cannot test `LINUX`, since this must be defined before other includes. */
# define _GNU_SOURCE 1
#endif
#if defined(__INTERIX) && !defined(_ALL_SOURCE)
# define _ALL_SOURCE 1
#endif
#if (defined(DGUX) && defined(GC_THREADS) || defined(DGUX386_THREADS) \
|| defined(GC_DGUX386_THREADS)) \
&& !defined(_USING_POSIX4A_DRAFT10)
# define _USING_POSIX4A_DRAFT10 1
#endif
#if defined(__MINGW32__) && !defined(__MINGW_EXCPT_DEFINE_PSDK) \
&& defined(__i386__) \
&& defined(GC_EXTERN) /*< defined in `extra/gc.c` file */
/* See the description in `mark.c` file. */
# define __MINGW_EXCPT_DEFINE_PSDK 1
#endif
#if defined(NO_DEBUGGING) && !defined(GC_ASSERTIONS) && !defined(NDEBUG)
/* To turn off assertion checking (in `atomic_ops.h` file). */
# define NDEBUG 1
#endif
#ifndef GC_H
# include "gc/gc.h"
#endif
#include <stdlib.h>
#if !defined(sony_news)
# include <stddef.h>
#endif
#ifdef DGUX
# include <sys/resource.h>
# include <sys/time.h>
#endif
#ifdef BSD_TIME
# include <sys/resource.h>
# include <sys/time.h>
#endif
#ifdef PARALLEL_MARK
# define AO_REQUIRE_CAS
# if !defined(__GNUC__) && !defined(AO_ASSUME_WINDOWS98)
# define AO_ASSUME_WINDOWS98
# endif
#endif
#include "gc/gc_mark.h"
#include "gc/gc_tiny_fl.h"
typedef GC_word word;
#ifndef PTR_T_DEFINED
/*
* A generic pointer to which we can add byte displacements and which
* can be used for address comparisons.
*/
typedef char *ptr_t;
# define PTR_T_DEFINED
#endif
#ifndef SIZE_MAX
# include <limits.h>
#endif
#if defined(SIZE_MAX) && !defined(CPPCHECK)
/*
* A constant representing maximum value for `size_t` type. Note: an extra
* cast is used to workaround some buggy `SIZE_MAX` definitions.
*/
# define GC_SIZE_MAX ((size_t)SIZE_MAX)
#else
# define GC_SIZE_MAX (~(size_t)0)
#endif
#if (GC_GNUC_PREREQ(3, 0) || defined(__clang__)) && !defined(LINT2)
/* Equivalent to `e`, but predict that usually `e` is true (false). */
# define LIKELY(e) __builtin_expect(e, 1 /* `TRUE` */)
# define UNLIKELY(e) __builtin_expect(e, 0 /* `FALSE` */)
#else
# define LIKELY(e) (e)
# define UNLIKELY(e) (e)
#endif /* __GNUC__ */
/*
* Saturated addition of `size_t` values. Used to avoid value wrap around
* on overflow. The arguments should have no side effects.
*/
#define SIZET_SAT_ADD(a, b) \
(LIKELY((a) < GC_SIZE_MAX - (b)) ? (a) + (b) : GC_SIZE_MAX)
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996 by Silicon Graphics. All rights reserved.
* Copyright (c) 2000-2004 Hewlett-Packard Development Company, L.P.
* Copyright (c) 2009-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* This header is private to the collector. It is almost always
* included from `gc_priv.h` file. However it is possible to include
* it by itself if just the configuration macros are needed. In that
* case, a few declarations relying on types declared in `gc_priv.h`
* file will be omitted.
*/
#ifndef GCCONFIG_H
#define GCCONFIG_H
#ifndef GC_H
# ifdef HAVE_CONFIG_H
# endif
#endif
#ifdef CPPCHECK
# undef CLOCKS_PER_SEC
# undef FIXUP_POINTER
# undef POINTER_MASK
# undef POINTER_SHIFT
# undef REDIRECT_REALLOC
# undef _MAX_PATH
#endif
#ifndef PTR_T_DEFINED
typedef char *ptr_t;
# define PTR_T_DEFINED
#endif
#if !defined(sony_news)
# include <stddef.h> /*< for `size_t`, etc. */
#endif
/*
* Note: only wrap our own declarations, and not the included headers;
* in this case, wrap our entire file, but temporarily unwrap/rewrap
* around `#include` entities; macros do not need such wrapping.
*/
#ifdef __cplusplus
# define EXTERN_C_BEGIN extern "C" {
# define EXTERN_C_END } /* extern "C" */
#else
# define EXTERN_C_BEGIN
# define EXTERN_C_END
#endif
EXTERN_C_BEGIN
/* Convenient internal macro to test version of Clang. */
#if defined(__clang__) && defined(__clang_major__)
# define GC_CLANG_PREREQ(major, minor) \
((__clang_major__ << 8) + __clang_minor__ >= ((major) << 8) + (minor))
# define GC_CLANG_PREREQ_FULL(major, minor, patchlevel) \
(GC_CLANG_PREREQ(major, (minor) + 1) \
|| (__clang_major__ == (major) && __clang_minor__ == (minor) \
&& __clang_patchlevel__ >= (patchlevel)))
#else
# define GC_CLANG_PREREQ(major, minor) 0 /*< `FALSE` */
# define GC_CLANG_PREREQ_FULL(major, minor, patchlevel) 0
#endif
/*
* Machine-dependent parameters. Some tuning parameters can be found
* near the top of `gc_priv.h` file.
*/
/* Machine-specific parts contributed by various people. See `README` file. */
#if defined(__ANDROID__) && !defined(HOST_ANDROID)
/* A Linux-based OS. `__ANDROID__` macro is defined by Android NDK gcc. */
# define HOST_ANDROID 1
#endif
#if defined(TIZEN) && !defined(HOST_TIZEN)
/* A Linux-based OS. */
# define HOST_TIZEN 1
#endif
#if defined(__SYMBIAN32__) && !defined(SYMBIAN)
# define SYMBIAN
# ifdef __WINS__
# pragma data_seg(".data2")
# endif
#endif
/* First a unified test for Linux. */
#if (defined(linux) || defined(__linux__) || defined(HOST_ANDROID) \
|| defined(HOST_TIZEN)) \
&& !defined(LINUX) && !defined(__native_client__)
# define LINUX
#endif
/* And one for NetBSD. */
#if defined(__NetBSD__)
# define NETBSD
#endif
/* And one for OpenBSD. */
#if defined(__OpenBSD__)
# define OPENBSD
#endif
/* And one for FreeBSD. */
#if (defined(__FreeBSD__) || defined(__DragonFly__) \
|| defined(__FreeBSD_kernel__)) \
&& !defined(FREEBSD) \
&& !defined(GC_NO_FREEBSD) /*< Orbis compiler defines `__FreeBSD__` */
# define FREEBSD
#endif
#if defined(FREEBSD) || defined(NETBSD) || defined(OPENBSD)
# define ANY_BSD
#endif
#if defined(__COSMOPOLITAN__)
# define COSMO
#endif
#if defined(__EMBOX__)
# define EMBOX
#endif
#if defined(__KOS__)
# define KOS
#endif
#if defined(__QNX__) && !defined(QNX)
# define QNX
#endif
#if defined(__serenity__)
# define SERENITY
#endif
/* And one for Darwin. */
#if defined(macosx) || (defined(__APPLE__) && defined(__MACH__))
# define DARWIN
EXTERN_C_END
# include <TargetConditionals.h>
EXTERN_C_BEGIN
#endif
/* Determine the machine type. */
#if defined(__native_client__)
# define NACL
# if !defined(__portable_native_client__) && !defined(__arm__)
# define I386
# define mach_type_known
# else
/* Here we will rely upon arch-specific defines. */
# endif
#endif
#if defined(__aarch64__) && !defined(ANY_BSD) && !defined(COSMO) \
&& !defined(DARWIN) && !defined(LINUX) && !defined(KOS) \
&& !defined(NN_BUILD_TARGET_PLATFORM_NX) && !defined(QNX) \
&& !defined(SERENITY) && !defined(_WIN32)
# define AARCH64
# define NOSYS
# define mach_type_known
#endif
#if defined(__arm) || defined(__arm__) || defined(__thumb__)
# define ARM32
# if defined(NACL) || defined(SYMBIAN)
# define mach_type_known
# elif !defined(ANY_BSD) && !defined(DARWIN) && !defined(LINUX) \
&& !defined(QNX) && !defined(NN_PLATFORM_CTR) \
&& !defined(SN_TARGET_PSP2) && !defined(_WIN32) && !defined(__CEGCC__) \
&& !defined(GC_NO_NOSYS)
# define NOSYS
# define mach_type_known
# endif
#endif
#if defined(__riscv) && !defined(ANY_BSD) && !defined(LINUX)
# define RISCV
# define NOSYS
# define mach_type_known
#endif
#if defined(vax) || defined(__vax__)
# define VAX
# ifdef ultrix
# define ULTRIX
# else
# define BSD
# endif
# define mach_type_known
#endif
#if defined(NETBSD) && defined(__vax__)
# define VAX
# define mach_type_known
#endif
#if (defined(mips) || defined(__mips) || defined(_mips)) \
&& !defined(__TANDEM) && !defined(ANY_BSD) && !defined(LINUX)
# define MIPS
# if defined(nec_ews) || defined(_nec_ews)
# define EWS4800
# define mach_type_known
# elif defined(ultrix) || defined(__ultrix)
# define ULTRIX
# define mach_type_known
# elif !defined(_WIN32_WCE) && !defined(__CEGCC__) && !defined(__MINGW32CE__)
# define IRIX5 /*< or Irix 6.x */
# define mach_type_known
# endif /* !MSWINCE */
#endif
#if defined(DGUX) && (defined(i386) || defined(__i386__))
# define I386
# ifndef _USING_DGUX
# define _USING_DGUX
# endif
# define mach_type_known
#endif
#if defined(sequent) && (defined(i386) || defined(__i386__))
# define I386
# define SEQUENT
# define mach_type_known
#endif
#if (defined(sun) || defined(__sun)) && (defined(i386) || defined(__i386__))
# define I386
# define SOLARIS
# define mach_type_known
#endif
#if (defined(sun) || defined(__sun)) && defined(__amd64)
# define X86_64
# define SOLARIS
# define mach_type_known
#endif
#if (defined(__OS2__) || defined(__EMX__)) && defined(__32BIT__)
# define I386
# define OS2
# define mach_type_known
#endif
#if (defined(sun) || defined(__sun)) && (defined(sparc) || defined(__sparc))
/* SunOS 5.x */
EXTERN_C_END
# include <errno.h>
EXTERN_C_BEGIN
# define SPARC
# define SOLARIS
# define mach_type_known
#elif defined(sparc) && defined(unix) && !defined(sun) && !defined(linux) \
&& !defined(ANY_BSD)
# define SPARC
# define DRSNX
# define mach_type_known
#endif
#if defined(_IBMR2) /* `&& defined(_AIX)` */
# define POWERPC
# define AIX
# define mach_type_known
#endif
#if defined(_M_XENIX) && defined(_M_SYSV) && defined(_M_I386)
/* TODO: The above test may need refinement. */
# define I386
# if defined(_SCO_ELF)
# define SCO_ELF
# else
# define SCO
# endif
# define mach_type_known
#endif
#if defined(_PA_RISC1_0) || defined(_PA_RISC1_1) || defined(_PA_RISC2_0) \
|| defined(hppa) || defined(__hppa__)
# define HP_PA
# if !defined(LINUX) && !defined(HPUX) && !defined(OPENBSD)
# define HPUX
# endif
# define mach_type_known
#endif
#if defined(__ia64) && (defined(_HPUX_SOURCE) || defined(__HP_aCC))
# define IA64
# ifndef HPUX
# define HPUX
# endif
# define mach_type_known
#endif
#if (defined(__BEOS__) || defined(__HAIKU__)) && defined(_X86_)
# define I386
# define HAIKU
# define mach_type_known
#endif
#if defined(__HAIKU__) && (defined(__amd64__) || defined(__x86_64__))
# define X86_64
# define HAIKU
# define mach_type_known
#endif
#if defined(__alpha) || defined(__alpha__)
# define ALPHA
# if !defined(ANY_BSD) && !defined(LINUX)
# define OSF1 /*< Compaq Tru64 UNIX (Digital UNIX) */
# endif
# define mach_type_known
#endif
#if defined(__rtems__) && (defined(i386) || defined(__i386__))
# define I386
# define RTEMS
# define mach_type_known
#endif
#if defined(NeXT) && defined(mc68000)
# define M68K
# define NEXT
# define mach_type_known
#endif
#if defined(NeXT) && (defined(i386) || defined(__i386__))
# define I386
# define NEXT
# define mach_type_known
#endif
#if defined(bsdi) && (defined(i386) || defined(__i386__))
# define I386
# define BSDI
# define mach_type_known
#endif
#if defined(__386BSD__) && !defined(mach_type_known)
# define I386
# define THREE86BSD
# define mach_type_known
#endif
#if defined(_CX_UX) && defined(_M88K)
# define M88K
# define CX_UX
# define mach_type_known
#endif
#if defined(DGUX) && defined(m88k)
# define M88K
/* `DGUX` macro is already defined. */
# define mach_type_known
#endif
#if defined(_WIN32_WCE) || defined(__CEGCC__) || defined(__MINGW32CE__)
/*
* `SH3`, `SH4`, `MIPS` macros are already defined for the corresponding
* architectures.
*/
# if defined(SH3) || defined(SH4)
# define SH
# endif
# if defined(x86) || defined(__i386__)
# define I386
# endif
# if defined(_M_ARM) || defined(ARM) || defined(_ARM_)
# define ARM32
# endif
# define MSWINCE
# define mach_type_known
#else
# if ((defined(_MSDOS) || defined(_MSC_VER)) && (_M_IX86 >= 300)) \
|| (defined(_WIN32) && !defined(__CYGWIN32__) && !defined(__CYGWIN__) \
&& !defined(__INTERIX) && !defined(SYMBIAN)) \
|| defined(__MINGW32__)
# if defined(__LP64__) || defined(_M_X64)
# define X86_64
# elif defined(_M_ARM)
# define ARM32
# elif defined(_M_ARM64)
# define AARCH64
# else /* _M_IX86 */
# define I386
# endif
# ifdef _XBOX_ONE
# define MSWIN_XBOX1
# else
# ifndef MSWIN32
# define MSWIN32 /*< or Win64 */
# endif
# if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP)
# define MSWINRT_FLAVOR
# endif
# endif
# define mach_type_known
# endif
# if defined(_MSC_VER) && defined(_M_IA64)
# define IA64
/*
* Really Win64, but we do not treat 64-bit variants as a different
* platform.
*/
# define MSWIN32
# endif
#endif /* !_WIN32_WCE && !__CEGCC__ && !__MINGW32CE__ */
#if defined(__DJGPP__)
# define I386
# ifndef DJGPP
/* MSDOS running the DJGPP port of gcc. */
# define DJGPP
# endif
# define mach_type_known
#endif
#if defined(__CYGWIN32__) || defined(__CYGWIN__)
# if defined(__LP64__)
# define X86_64
# else
# define I386
# endif
# define CYGWIN32
# define mach_type_known
#endif /* __CYGWIN__ */
#if defined(__INTERIX)
# define I386
# define INTERIX
# define mach_type_known
#endif
#if defined(_UTS) && !defined(mach_type_known)
# define S370
# define UTS4
# define mach_type_known
#endif
#if defined(__embedded__) && defined(PPC)
# define POWERPC
# define NOSYS
# define mach_type_known
#endif
#if defined(__WATCOMC__) && defined(__386__)
# define I386
# if !defined(OS2) && !defined(MSWIN32) && !defined(DOS4GW)
# if defined(__OS2__)
# define OS2
# elif defined(__WINDOWS_386__) || defined(__NT__)
# define MSWIN32
# else
# define DOS4GW
# endif
# endif
# define mach_type_known
#endif /* __WATCOMC__ && __386__ */
#if defined(__GNU__) && defined(__i386__)
/* The Debian Hurd running on generic PC. */
# define HURD
# define I386
# define mach_type_known
#endif
#if defined(__GNU__) && defined(__x86_64__)
# define HURD
# define X86_64
# define mach_type_known
#endif
#if defined(__TANDEM)
/* Nonstop S-series. */
/* FIXME: Should recognize Integrity series? */
# define MIPS
# define NONSTOP
# define mach_type_known
#endif
#if defined(__tile__) && defined(LINUX)
# ifdef __tilegx__
# define TILEGX
# else
# define TILEPRO
# endif
# define mach_type_known
#endif
#if defined(NN_BUILD_TARGET_PLATFORM_NX)
# define AARCH64
# define NINTENDO_SWITCH
# define mach_type_known
#endif
#if defined(__EMSCRIPTEN__) || defined(EMSCRIPTEN)
# define WEBASSEMBLY
# ifndef EMSCRIPTEN
# define EMSCRIPTEN
# endif
# define mach_type_known
#endif
#if defined(__wasi__)
/* The WebAssembly System Interface (WASI). */
# define WEBASSEMBLY
# define WASI
# define mach_type_known
#endif
#if defined(__aarch64__) \
&& (defined(ANY_BSD) || defined(COSMO) || defined(DARWIN) \
|| defined(LINUX) || defined(KOS) || defined(QNX) \
|| defined(SERENITY))
# define AARCH64
# define mach_type_known
#elif defined(__arc__) && defined(LINUX)
# define ARC
# define mach_type_known
#elif (defined(__arm) || defined(__arm__) || defined(__arm32__) \
|| defined(__ARM__)) \
&& (defined(ANY_BSD) || defined(DARWIN) || defined(LINUX) || defined(QNX) \
|| defined(NN_PLATFORM_CTR) || defined(SN_TARGET_PSP2))
# define ARM32
# define mach_type_known
#elif defined(__avr32__) && defined(LINUX)
# define AVR32
# define mach_type_known
#elif defined(__cris__) && defined(LINUX)
# ifndef CRIS
# define CRIS
# endif
# define mach_type_known
#elif defined(__e2k__) && defined(LINUX)
# define E2K
# define mach_type_known
#elif defined(__hexagon__) && defined(LINUX)
# define HEXAGON
# define mach_type_known
#elif (defined(__i386__) || defined(i386) || defined(__X86__)) \
&& (defined(ANY_BSD) || defined(DARWIN) || defined(EMBOX) \
|| defined(LINUX) || defined(QNX) || defined(SERENITY))
# define I386
# define mach_type_known
#elif (defined(__ia64) || defined(__ia64__)) && defined(LINUX)
# define IA64
# define mach_type_known
#elif defined(__loongarch__) && defined(LINUX)
# define LOONGARCH
# define mach_type_known
#elif defined(__m32r__) && defined(LINUX)
# define M32R
# define mach_type_known
#elif ((defined(__m68k__) || defined(m68k)) \
&& (defined(NETBSD) || defined(OPENBSD))) \
|| (defined(__mc68000__) && defined(LINUX))
# define M68K
# define mach_type_known
#elif (defined(__mips) || defined(_mips) || defined(mips)) \
&& (defined(ANY_BSD) || defined(LINUX))
# define MIPS
# define mach_type_known
#elif (defined(__NIOS2__) || defined(__NIOS2) || defined(__nios2__)) \
&& defined(LINUX)
# define NIOS2 /*< Altera NIOS2 */
# define mach_type_known
#elif defined(__or1k__) && defined(LINUX)
# define OR1K /*< OpenRISC (or1k) */
# define mach_type_known
#elif (defined(__powerpc__) || defined(__powerpc64__) || defined(__ppc__) \
|| defined(__ppc64__) || defined(powerpc) || defined(powerpc64)) \
&& (defined(ANY_BSD) || defined(DARWIN) || defined(LINUX))
# define POWERPC
# define mach_type_known
#elif defined(__riscv) && (defined(ANY_BSD) || defined(LINUX))
# define RISCV
# define mach_type_known
#elif defined(__s390__) && defined(LINUX)
# define S390
# define mach_type_known
#elif defined(__sh__) \
&& (defined(LINUX) || defined(NETBSD) || defined(OPENBSD))
# define SH
# define mach_type_known
#elif (defined(__sparc) || defined(sparc)) \
&& (defined(ANY_BSD) || defined(LINUX))
# define SPARC
# define mach_type_known
#elif defined(__sw_64__) && defined(LINUX)
# define SW_64
# define mach_type_known
#elif (defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) \
|| defined(__X86_64__)) \
&& (defined(ANY_BSD) || defined(COSMO) || defined(DARWIN) \
|| defined(LINUX) || defined(QNX) || defined(SERENITY))
# define X86_64
# define mach_type_known
#endif
/*
* Feel free to add more clauses here. Or manually define the machine
* type here. A machine type is characterized by the architecture.
* Some machine types are further subdivided by OS. Macros such as
* `LINUX`, `FREEBSD`, etc. distinguish them. The distinction in these
* cases is usually the stack starting address.
*/
#if !defined(mach_type_known) && !defined(CPPCHECK)
# error The collector has not been ported to this machine/OS combination
#endif
/*
* The CPU architecture mapping is:
* - `AARCH64`: ARM AArch64 ILP32/64-bit (running `COSMO` environment,
* `DARWIN` (OS X or iOS), `KOS`, `LINUX`, `MSWIN32`, `NETBSD`,
* `NINTENDO_SWITCH`, `NOSYS` environment, `OPENBSD`, `QNX`, `SERENITY`);
* - `ALPHA`: DEC Alpha (running `FREEBSD`, `LINUX`, `NETBSD`, `OPENBSD`,
* `OSF1`);
* - `ARC`: Synopsys ARC (running `LINUX`);
* - `ARM32`: ARMv7 (running `DARWIN` (iOS), `FREEBSD`, `LINUX`, `MSWIN32`,
* `MSWINCE`, `NETBSD`, `NN_PLATFORM_CTR`, `NOSYS` environment,
* `OPENBSD`, `QNX`, `SN_TARGET_PSP2`, `SYMBIAN`);
* - `AVR32`: Atmel RISC (running `LINUX`);
* - `CRIS`: Axis Etrax (running `LINUX`);
* - `E2K`: Elbrus 2000 32/64-bit (running `LINUX`);
* - `HEXAGON`: Qualcomm Hexagon (running `LINUX`);
* - `HP_PA`: HP9000/700 and HP9000/800 32/64-bit (running `HPUX`, `LINUX`,
* `OPENBSD`);
* - `I386`: Intel 486/586/686 (running `BSDI`, `CYGWIN32` environment,
* `DARWIN` (macOS), `DGUX`, `DJGPP` environment, `DOS4GW` environment,
* `EMBOX`, `FREEBSD`, `HAIKU`, `HURD`, `INTERIX`, `LINUX`, `MSWIN32`,
* `MSWINCE`, `NACL` environment, `NETBSD`, `NEXT`, `OPENBSD`, `OS2`,
* `QNX`, `RTEMS`, `SCO`, `SCO_ELF`, `SEQUENT`, `SERENITY`, `SOLARIS`,
* `THREE86BSD`);
* - `IA64`: Intel IPF, e.g. Itanium 32/64-bit (running `HPUX`, `LINUX`,
* `MSWIN32`);
* - `LOONGARCH`: Loongson LoongArch 32/64-bit (running `LINUX`);
* - `M32R`: Renesas M32R (running `LINUX`);
* - `M68K`: Motorola 680x0 (running `LINUX`, `NETBSD`, `NEXT`, `OPENBSD`);
* - `M88K`: Motorola 88xx0 (running `CX_UX`, `DGUX`);
* - `MIPS`: R2000+ 32/64-bit (running `EWS4800`, `FREEBSD`, `IRIX5`,
* `LINUX`, `MSWINCE`, `NETBSD`, `NONSTOP`, `OPENBSD`, `ULTRIX`);
* - `NIOS2`: Altera NIOS2 (running `LINUX`);
* - `OR1K`: OpenRISC/or1k (running `LINUX`);
* - `POWERPC`: IBM/Apple PowerPC 32/64-bit (running `AIX`, `DARWIN`,
* `FREEBSD`, `LINUX`, `NETBSD`, `NOSYS` environment, `OPENBSD`,
* `SN_TARGET_PS3`);
* - `RISCV`: RISC-V 32/64-bit (running `FREEBSD`, `LINUX`, `NETBSD`,
* `NOSYS` environment, `OPENBSD`);
* - `S370`: A 370-like machine (running `UTS4`);
* - `S390`: A 390-like machine 32/64-bit (running `LINUX`);
* - `SH`: Hitachi SuperH (running `LINUX`, `MSWINCE`, `NETBSD`,
* `OPENBSD`);
* - `SH4`: Hitachi SH4 (running `MSWINCE`);
* - `SPARC`: SPARC v7/v8/v9 32/64-bit (running `DRSNX`, `FREEBSD`,
* `LINUX`, `NETBSD`, `OPENBSD`, `SOLARIS`);
* - `SW_64`: Sunway/Shenwei (running `LINUX`);
* - `TILEGX`: Tilera TILE-Gx 32/64-bit (running `LINUX`);
* - `TILEPRO`: Tilera TILEPro (running `LINUX`);
* - `VAX`: DEC VAX (running `BSD`, `ULTRIX`);
* - `WEBASSEMBLY`: WebAssembly/Wasm (running `EMSCRIPTEN` environment,
* `WASI` environment);
* - `X86_64`: AMD x86-64 ILP32/64-bit (running `COSMO` environment,
* `CYGWIN32` environment, `DARWIN` (macOS), `FREEBSD`, `HAIKU`,
* `HURD`, `LINUX`, `MSWIN32`, `MSWIN_XBOX1`, `NETBSD`, `OPENBSD`,
* `PLATFORM_GETMEM` environment, `QNX`, `SERENITY`, `SOLARIS`).
*/
/*
* For each architecture and OS, the following need to be defined:
*
* `CPP_WORDSZ` is a simple integer constant representing the word size
* in bits. We assume byte addressability, where a byte has 8 bits.
* We also assume `CPP_WORDSZ` is either 32 or 64.
* (We care about the length of a pointer address, not hardware
* bus widths. Thus a 64-bit processor with a C compiler that uses
* 32-bit pointers should use `CPP_WORDSZ` of 32, not 64.)
*
* `CPP_PTRSZ` is the pointer size in bits. For most of the supported
* targets, it is equal to `CPP_WORDSZ`.
*
* `MACH_TYPE` is a string representation of the machine type.
* `OS_TYPE` is analogous for the OS.
*
* `ALIGNMENT` is the largest `n`, such that all pointer are guaranteed
* to be aligned on `n`-byte boundary. Defining it to be 1 will always
* work, but will perform poorly. Should not be larger than size of
* a pointer.
*
* `DATASTART` is the beginning of the data segment. On some platforms
* `SEARCH_FOR_DATA_START` is defined. The latter will cause
* `GC_data_start` to be set to an address determined by accessing data
* backwards from `_end` until an unmapped page is found. `DATASTART` will
* be defined to be `GC_data_start`. On UNIX-like systems, the collector
* will scan the area between `DATASTART` and `DATAEND` for root pointers.
*
* `DATAEND`, if not `end`, where `end` is defined as `extern int end[]`.
* RTH suggests gaining access to linker script synth'd values with
* this idiom instead of `&end`, where `end` is defined as `extern int end`.
* Otherwise, "gcc will assume these are in .sdata/.sbss" and it will, e.g.,
* cause failures on `alpha*-*-*` with `-msmall-data` or `-fpic` or
* `mips-*-*` without any special options.
*
* `STACKBOTTOM` is the cold end of the stack, which is usually the
* highest address in the stack.
* Under OS/2, we have other ways of finding thread stacks.
* For each machine, the following should:
* 1. Define `STACK_GROWS_UP` if the stack grows toward higher addresses;
* 2. Define exactly one of
* - `STACKBOTTOM` (should be defined to be an expression),
* - `HEURISTIC1`,
* - `SPECIFIC_MAIN_STACKBOTTOM`,
* - `HEURISTIC2`.
*
* If `STACKBOTTOM` is defined, then its value will be used directly (as
* the stack bottom). If `SPECIFIC_MAIN_STACKBOTTOM` is defined, then it
* will be determined with a specific method appropriate for the operating
* system. Currently we look first for `__libc_stack_end` (currently only
* if `USE_LIBC_PRIVATES` is defined), and if that fails, read it from
* `/proc` pseudo-file. (If `USE_LIBC_PRIVATES` is not defined and
* `NO_PROC_STAT` is defined, we revert to `HEURISTIC2`.)
* If either of the last two macros are defined, then `STACKBOTTOM` is
* computed during collector startup using one of the following two
* heuristics:
* - `HEURISTIC1`: Take an address inside `GC_init`'s frame, and round it
* up to the next multiple of `STACK_GRAN`;
* - `HEURISTIC2`: Take an address inside `GC_init`'s frame, increment it
* repeatedly in small steps (decrement if `STACK_GROWS_UP`), and read
* the value at each location, remember the value when the first
* Segmentation violation or Bus error is signaled, round that to the
* nearest plausible page boundary, and use that instead of
* `STACKBOTTOM`.
*
* Gustavo Rodriguez-Rivera points out that on most (all?) UNIX machines,
* the value of `environ` is a pointer that can serve as `STACKBOTTOM`.
* I expect that `HEURISTIC2` can be replaced by this approach, which
* interferes far less with debugging. However it has the disadvantage that
* it is confused by a `putenv()` call before the collector is initialized.
* This could be dealt with by intercepting `putenv()`...
*
* If no expression for `STACKBOTTOM` can be found, and neither of the above
* heuristics are usable, the collector can still be used with all of the
* above undefined, provided one of the following is done:
* 1. `GC_mark_roots` can be changed to somehow mark from the correct
* stack(s) without reference to `STACKBOTTOM`. This is appropriate for
* use in conjunction with thread packages, since there will be multiple
* stacks. (Allocating thread stacks in the heap, and treating them as
* ordinary heap data objects is also possible as a last resort.
* However, this is likely to introduce significant amounts of excess
* storage retention unless the dead parts of the thread stacks are
* periodically cleared.)
* 2. Client code may set `GC_stackbottom` before calling any `GC_`
* routines. If the author of the client code owns the main program,
* this could be accomplished by introducing a new `main` function,
* calling `GC_call_with_gc_active()` which sets `GC_stackbottom` and
* then calls the original (real) `main` function.
*
* Each architecture may also define the style of virtual dirty bit
* implementation to be used:
* - `GWW_VDB`: use Win32 `GetWriteWatch` primitive;
* - `MPROTECT_VDB`: write-protect the heap and catch faults;
* - `PROC_VDB`: use the SVR4 `/proc` primitives to read dirty bits;
* - `SOFT_VDB`: use the Linux `/proc` primitives to track dirty bits.
*
* The first and second one may be combined, in which case a runtime
* selection will be made, based on `GetWriteWatch` availability.
*
* An architecture may define `DYNAMIC_LOADING` if `dyn_load.c` file
* implements `GC_register_dynamic_libraries()` for the architecture.
*
* An architecture may define `PREFETCH(x)` to preload the cache with `*x`.
* This defaults to gcc built-in operation (or a no-op for other compilers).
*
* `GC_PREFETCH_FOR_WRITE(x)` is used if `*x` is about to be written.
*
* An architecture may also define `CLEAR_DOUBLE(x)` to be a fast way to
* clear 2 pointers at `GC_malloc`-aligned address `x`. The default
* implementation is just to store two `NULL` pointers.
*
* `HEAP_START` may be defined as the initial address hint for `mmap`-based
* allocation.
*/
#ifdef LINUX
/* TODO: FreeBSD too? */
EXTERN_C_END
# include <features.h> /*< for `__GLIBC__` and `__GLIBC_MINOR__`, at least */
EXTERN_C_BEGIN
#endif
/* Convenient internal macro to test `glibc` version (if compiled against). */
#if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
# define GC_GLIBC_PREREQ(major, minor) \
((__GLIBC__ << 8) + __GLIBC_MINOR__ >= ((major) << 8) + (minor))
#else
# define GC_GLIBC_PREREQ(major, minor) 0 /*< `FALSE` */
#endif
/*
* Align a `ptr_t` pointer down/up to a given boundary. The latter should
* be a power of two.
*/
#if GC_CLANG_PREREQ(11, 0)
# define PTR_ALIGN_DOWN(p, b) __builtin_align_down(p, b)
# if defined(DARWIN) && defined(GC_SINGLE_OBJ_BUILD) && GC_CLANG_PREREQ(17, 0)
/* Workaround a crash in Apple clang-17. */
/* TODO: Disable for later clang versions when the bug is fixed. */
# define PTR_ALIGN_UP(p, b) \
((ptr_t)__builtin_align_up((GC_uintptr_t)(p), b))
# else
# define PTR_ALIGN_UP(p, b) __builtin_align_up(p, b)
# endif
#else
# define PTR_ALIGN_DOWN(p, b) \
((ptr_t)((GC_uintptr_t)(p) & ~((GC_uintptr_t)(b) - (GC_uintptr_t)1)))
# define PTR_ALIGN_UP(p, b) \
((ptr_t)(((GC_uintptr_t)(p) + (GC_uintptr_t)(b) - (GC_uintptr_t)1) \
& ~((GC_uintptr_t)(b) - (GC_uintptr_t)1)))
#endif
/*
* If available, we can use `__builtin_unwind_init()` to push the relevant
* registers onto the stack.
*/
#if GC_GNUC_PREREQ(2, 8) \
&& !GC_GNUC_PREREQ(11, 0) /*< broken at least in 11.2.0 on cygwin64 */ \
&& !defined(__INTEL_COMPILER) && !defined(__PATHCC__) \
&& !defined(__FUJITSU) /*< for FX10 system */ \
&& !(defined(POWERPC) && defined(DARWIN)) /*< for MacOS X 10.3.9 */ \
&& !defined(E2K) && !defined(RTEMS) \
&& !defined(__ARMCC_VERSION) /*< does not exist in armcc gnu emu */ \
&& !(defined(__clang__) \
&& defined(__ARM_ARCH_5TE__) /* clang-19 emits `vpush`/`vpop` */) \
&& (!defined(__clang__) \
|| GC_CLANG_PREREQ(8, 0) /* was no-op in clang-3 at least */)
# define HAVE_BUILTIN_UNWIND_INIT
#endif
#if (defined(__CC_ARM) || defined(CX_UX) || defined(DJGPP) || defined(EMBOX) \
|| defined(EWS4800) || defined(LINUX) || defined(OS2) || defined(RTEMS) \
|| defined(UTS4) || defined(MSWIN32) || defined(MSWINCE) \
|| (defined(NOSYS) && defined(RISCV))) \
&& !defined(NO_UNDERSCORE_SETJMP)
# define NO_UNDERSCORE_SETJMP
#endif
/*
* The common OS-specific definitions. Should be applicable to all
* (or most, at least) supported architectures.
*/
#ifdef CYGWIN32
# define OS_TYPE "CYGWIN32"
# define RETRY_GET_THREAD_CONTEXT
# ifdef USE_WINALLOC
# define GWW_VDB
# elif defined(USE_MMAP)
# define USE_MMAP_ANON
# endif
#endif /* CYGWIN32 */
#ifdef COSMO
# define OS_TYPE "COSMO"
# ifndef USE_GET_STACKBASE_FOR_MAIN
# define USE_GET_STACKBASE_FOR_MAIN
# endif
extern int __data_start[] __attribute__((__weak__));
# define DATASTART ((ptr_t)__data_start)
extern int _end[];
# define DATAEND ((ptr_t)_end)
# define USE_MMAP_ANON
# ifndef HAVE_CLOCK_GETTIME
# define HAVE_CLOCK_GETTIME 1
# endif
# ifndef HAVE_PTHREAD_SETNAME_NP_WITH_TID
/* Normally should be defined by `configure`, etc. */
# define HAVE_PTHREAD_SETNAME_NP_WITH_TID 1
# endif
# if !defined(GC_THREADS) || defined(NO_HANDLE_FORK) \
|| defined(GC_NO_CAN_CALL_ATFORK)
# define MPROTECT_VDB
/* FIXME: Otherwise `gctest` crashes in child process. */
# endif
/* FIXME: A deadlock occurs in markers, thus disabled for now. */
# undef PARALLEL_MARK
#endif /* COSMO */
#ifdef DARWIN
# define OS_TYPE "DARWIN"
# define DYNAMIC_LOADING
/*
* TODO: See `get_end(3)`, `get_etext` and `get_end` should not be used.
* These are not used when `dyld` support is enabled (it is the default).
*/
# define DATASTART ((ptr_t)get_etext())
# define DATAEND ((ptr_t)get_end())
# define USE_MMAP_ANON
/* There seems to be some issues with try-lock hanging on Darwin. */
/* TODO: This should be looked into some more. */
# define NO_PTHREAD_TRYLOCK
# ifndef TARGET_OS_XR
# define TARGET_OS_XR 0
# endif
# ifndef TARGET_OS_VISION
# define TARGET_OS_VISION 0
# endif
#endif /* DARWIN */
#ifdef EMBOX
# define OS_TYPE "EMBOX"
extern int _modules_data_start[], _apps_bss_end[];
# define DATASTART ((ptr_t)_modules_data_start)
# define DATAEND ((ptr_t)_apps_bss_end)
/*
* Note: the designated area might be quite large (several dozens of MBs)
* as it includes `.data` and `.bss` of all apps and modules of the built
* binary image.
*/
#endif /* EMBOX */
#ifdef FREEBSD
# define OS_TYPE "FREEBSD"
# define SPECIFIC_MAIN_STACKBOTTOM
# ifdef __ELF__
# define DYNAMIC_LOADING
# endif
# ifndef USE_MMAP
/* `sbrk()` is not available. */
# define USE_MMAP 1
# endif
# if !defined(ALPHA) && !defined(SPARC)
extern char etext[];
# define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)etext)
# define DATASTART_USES_XGETDATASTART
# ifndef REDIRECT_MALLOC
# define MPROTECT_VDB
# else
/* Similar as on Linux, `fread()` might use `malloc()`. */
# endif
# endif
#endif /* FREEBSD */
#ifdef HAIKU
# define OS_TYPE "HAIKU"
# define DYNAMIC_LOADING
/* Note: `DATASTART` is not used really, see `GC_register_main_static_data`. */
extern int etext[];
# define DATASTART PTR_ALIGN_UP((ptr_t)etext, 0x1000)
# ifndef USE_GET_STACKBASE_FOR_MAIN
# define USE_GET_STACKBASE_FOR_MAIN
# endif
# define USE_MMAP_ANON
/*
* On Haiku R1, at least, `pthreads` locks never spin but always call
* into the kernel if the lock cannot be acquired with a simple atomic
* operation. (Up to 5x overall performance improvement of the
* collector is observed by forcing use of spin locks.)
*/
# ifndef USE_SPIN_LOCK
# define USE_SPIN_LOCK
# endif
/*
* TODO: `MPROTECT_VDB` is not working correctly on anything other than
* recent nightly Haiku OS builds (as of Nov 2024), and also it is
* considerably slower than regular collecting, so do not enable it for now.
*/
EXTERN_C_END
# include <OS.h>
EXTERN_C_BEGIN
# define GETPAGESIZE() (unsigned)B_PAGE_SIZE
# ifndef HAVE_CLOCK_GETTIME
# define HAVE_CLOCK_GETTIME 1
# endif
#endif /* HAIKU */
#ifdef HPUX
# define OS_TYPE "HPUX"
extern int __data_start[];
# define DATASTART ((ptr_t)__data_start)
# ifdef USE_MMAP
# define USE_MMAP_ANON
# endif
# define DYNAMIC_LOADING
# define GETPAGESIZE() (unsigned)sysconf(_SC_PAGE_SIZE)
#endif /* HPUX */
#ifdef HURD
# define OS_TYPE "HURD"
# define HEURISTIC2
# define SEARCH_FOR_DATA_START
extern int _end[];
# define DATAEND ((ptr_t)_end)
/* TODO: `MPROTECT_VDB` is not quite working yet? */
# define DYNAMIC_LOADING
# define USE_MMAP_ANON
#endif /* HURD */
#ifdef LINUX
# define OS_TYPE "LINUX"
# if defined(FORCE_MPROTECT_BEFORE_MADVISE) || defined(PREFER_MMAP_PROT_NONE)
# define COUNT_UNMAPPED_REGIONS
# endif
# define RETRY_TKILL_ON_EAGAIN
# if !defined(MIPS) && !defined(POWERPC)
# define SPECIFIC_MAIN_STACKBOTTOM
# endif
# if defined(__ELF__) && !defined(IA64)
# define DYNAMIC_LOADING
# endif
# if defined(__ELF__) && !defined(ARC) && !defined(RISCV) && !defined(S390) \
&& !defined(TILEGX) && !defined(TILEPRO)
extern int _end[];
# define DATAEND ((ptr_t)_end)
# endif
# if !defined(REDIRECT_MALLOC) && !defined(E2K)
/* Requires Linux 2.3.47 or later. */
# define MPROTECT_VDB
# else
/*
* We seem to get random errors in the incremental mode, possibly because
* the Linux threads implementation itself is a `malloc` client and cannot
* deal with the signals. `fread()` uses `malloc()` too.
* In case of e2k, unless `-fsemi-spec-ld` (or `-O0`) option is passed
* to gcc (both when compiling the collector library and the client),
* a semi-speculative optimization may lead to `SIGILL` (with `ILL_ILLOPN`
* `si_code`) instead of `SIGSEGV`.
*/
# endif
#endif /* LINUX */
#ifdef KOS
# define OS_TYPE "KOS"
# ifndef USE_GET_STACKBASE_FOR_MAIN
/* Note: this requires `-lpthread` option. */
# define USE_GET_STACKBASE_FOR_MAIN
# endif
extern int __data_start[];
# define DATASTART ((ptr_t)__data_start)
#endif /* KOS */
#ifdef MSWIN32
# define OS_TYPE "MSWIN32"
/* `STACKBOTTOM` and `DATASTART` are handled specially in `os_dep.c` file. */
# if !defined(CPPCHECK)
# define DATAEND /*< not needed */
# endif
# if defined(USE_GLOBAL_ALLOC) && !defined(MSWINRT_FLAVOR)
/* Cannot pass `MEM_WRITE_WATCH` to `GlobalAlloc()`. */
# else
# define GWW_VDB
# endif
#endif
#ifdef MSWINCE
# define OS_TYPE "MSWINCE"
# if !defined(CPPCHECK)
# define DATAEND /*< not needed */
# endif
#endif
#ifdef NACL
# define OS_TYPE "NACL"
# if defined(__GLIBC__)
# define DYNAMIC_LOADING
# endif
# define DATASTART MAKE_CPTR(0x10020000)
extern int _end[];
# define DATAEND ((ptr_t)_end)
# define STACK_GRAN 0x10000
# define HEURISTIC1
# define NO_PTHREAD_GETATTR_NP
# define USE_MMAP_ANON
/* FIXME: Not real page size. */
# define GETPAGESIZE() 65536
# define MAX_NACL_GC_THREADS 1024
#endif /* NACL */
#ifdef NETBSD
# define OS_TYPE "NETBSD"
# define HEURISTIC2
# ifdef __ELF__
# define SEARCH_FOR_DATA_START
# define DYNAMIC_LOADING
# elif !defined(MIPS)
/* TODO: Probably do not exclude it. */
extern char etext[];
# define DATASTART ((ptr_t)etext)
# endif
# define MPROTECT_VDB
#endif /* NETBSD */
#ifdef NEXT
# define OS_TYPE "NEXT"
# define DATASTART ((ptr_t)get_etext())
# define DATAEND /*< not needed */
# undef USE_MUNMAP
#endif
#ifdef OPENBSD
# define OS_TYPE "OPENBSD"
# ifndef GC_THREADS
# define HEURISTIC2
# endif
# ifdef __ELF__
extern int __data_start[], _end[];
# define DATASTART ((ptr_t)__data_start)
# define DATAEND ((ptr_t)_end)
# define DYNAMIC_LOADING
# else
extern char etext[];
# define DATASTART ((ptr_t)etext)
# endif
# define MPROTECT_VDB
#endif /* OPENBSD */
#ifdef QNX
# define OS_TYPE "QNX"
# define SA_RESTART 0
# ifndef SPECIFIC_MAIN_STACKBOTTOM
/* TODO: This is not used by default. */
# define STACK_GRAN 0x1000000
# define HEURISTIC1
# endif
extern char etext[];
# define DATASTART ((ptr_t)etext)
extern int _end[];
# define DATAEND ((ptr_t)_end)
#endif /* QNX */
#ifdef SERENITY
# define OS_TYPE "SERENITY"
extern int etext[], _end[];
# define DATASTART PTR_ALIGN_UP((ptr_t)etext, 0x1000)
# define DATAEND ((ptr_t)_end)
# define DYNAMIC_LOADING
/* TODO: Enable `mprotect`-based VDB. */
# define USE_MMAP_ANON
#endif /* SERENITY */
#ifdef SOLARIS
# define OS_TYPE "SOLARIS"
extern int _end[];
# define DATAEND ((ptr_t)_end)
# if !defined(USE_MMAP) && defined(REDIRECT_MALLOC)
# define USE_MMAP 1
/*
* Otherwise we now use `calloc()`. `mmap()` may result in the heap
* interleaved with thread stacks, which can result in excessive
* black-listing. `sbrk()` is unusable since it does not interact
* correctly with the system `malloc()`.
*/
# endif
# ifdef USE_MMAP
# define HEAP_START ((word)0x40000000)
# else
# define HEAP_START ADDR(DATAEND)
# endif
# ifndef GC_THREADS
# define MPROTECT_VDB
# endif
# define DYNAMIC_LOADING
/*
* Define `STACKBOTTOM` as `(ptr_t)_start` worked through 2.7, but
* reportedly breaks under 2.8. It appears that the stack base is
* a property of the executable, so this should not break old executables.
* `HEURISTIC1` reportedly no longer works under Solaris 2.7.
* `HEURISTIC2` probably works, but this appears to be preferable.
* Apparently `USRSTACK` is defined to be `USERLIMIT`, but in some
* installations that is undefined. We work around this with a gross hack.
*/
EXTERN_C_END
# include <sys/vmparam.h>
EXTERN_C_BEGIN
# ifdef USERLIMIT
/* This should work everywhere, but does not. */
# define STACKBOTTOM ((ptr_t)USRSTACK)
# else
# define HEURISTIC2
# endif
#endif /* SOLARIS */
#ifdef SYMBIAN
# define OS_TYPE "SYMBIAN"
# define DATASTART ((ptr_t)ALIGNMENT) /*< cannot be `NULL` */
# define DATAEND ((ptr_t)ALIGNMENT)
# ifndef USE_MMAP
/* `sbrk()` is not available. */
# define USE_MMAP 1
# endif
#endif /* SYMBIAN */
/*
* Below are the definitions specific to each supported architecture
* and OS, grouped by the former.
*/
#ifdef M68K
# define MACH_TYPE "M68K"
# define CPP_WORDSZ 32
# define ALIGNMENT 2 /*< not 4 */
# ifdef OPENBSD
/* Nothing specific. */
# endif
# ifdef NETBSD
/* Nothing specific. */
# endif
# ifdef LINUX
# ifdef __ELF__
# if GC_GLIBC_PREREQ(2, 0)
# define SEARCH_FOR_DATA_START
# else
/*
* Hideous kludge: `__environ` is the first word in platform `crt0.o` file,
* and delimits the start of the data segment, no matter which `ld` options
* were passed through. We could use `_etext` instead, but that would
* include `.rodata`, which may contain large read-only data tables that
* we would rather not scan.
*/
extern char **__environ;
# define DATASTART ((ptr_t)(&__environ))
# endif
# else
extern int etext[];
# define DATASTART PTR_ALIGN_UP((ptr_t)etext, 0x1000)
# endif
# endif
# ifdef NEXT
# define STACKBOTTOM MAKE_CPTR(0x4000000)
# endif
#endif
#ifdef POWERPC
# define MACH_TYPE "POWERPC"
# ifdef LINUX
# if defined(__powerpc64__)
# define CPP_WORDSZ 64
# ifndef HBLKSIZE
# define HBLKSIZE 4096
# endif
# else
# define CPP_WORDSZ 32
# endif
/*
* `HEURISTIC1` has been reliably reported to fail for a 32-bit executable
* on a 64-bit kernel.
*/
# if defined(__bg__)
/*
* The Linux Compute Node Kernel (used on BlueGene systems) does not
* support the `SPECIFIC_MAIN_STACKBOTTOM` way.
*/
# define HEURISTIC2
# define NO_PTHREAD_GETATTR_NP
# else
# define SPECIFIC_MAIN_STACKBOTTOM
# endif
# define SEARCH_FOR_DATA_START
# ifndef SOFT_VDB
# define SOFT_VDB
# endif
# endif
# ifdef DARWIN
# if defined(__ppc64__)
# define CPP_WORDSZ 64
# define STACKBOTTOM MAKE_CPTR(0x7fff5fc00000)
# define CACHE_LINE_SIZE 64
# ifndef HBLKSIZE
# define HBLKSIZE 4096
# endif
# else
# define CPP_WORDSZ 32
# define STACKBOTTOM MAKE_CPTR(0xc0000000)
# endif
# define MPROTECT_VDB
# if defined(USE_PPC_PREFETCH) && defined(__GNUC__)
/* The performance impact of prefetches is untested. */
# define PREFETCH(x) \
__asm__ __volatile__("dcbt 0,%0" : : "r"((const void *)(x)))
# define GC_PREFETCH_FOR_WRITE(x) \
__asm__ __volatile__("dcbtst 0,%0" : : "r"((const void *)(x)))
# endif
# endif
# ifdef OPENBSD
# if defined(__powerpc64__)
# define CPP_WORDSZ 64
# else
# define CPP_WORDSZ 32
# endif
# endif
# ifdef FREEBSD
# if defined(__powerpc64__)
# define CPP_WORDSZ 64
# ifndef HBLKSIZE
# define HBLKSIZE 4096
# endif
# else
# define CPP_WORDSZ 32
# endif
# endif
# ifdef NETBSD
# define CPP_WORDSZ 32
# endif
# ifdef SN_TARGET_PS3
# define OS_TYPE "SN_TARGET_PS3"
# define CPP_WORDSZ 32
# define NO_GETENV
extern int _end[], __bss_start;
# define DATASTART ((ptr_t)__bss_start)
# define DATAEND ((ptr_t)_end)
# define STACKBOTTOM ((ptr_t)ps3_get_stack_bottom())
void *ps3_get_mem(size_t lb);
# define GET_MEM(lb) ps3_get_mem(lb)
/*
* The current `LOCK()` implementation for PS3 explicitly uses
* `pthread_mutex_lock()` for some reason.
*/
# define NO_PTHREAD_TRYLOCK
# endif
# ifdef AIX
# define OS_TYPE "AIX"
# undef ALIGNMENT /*< in case it is defined */
# undef IA64
/*
* DOB: some AIX installs stupidly define `IA64` in platform
* `sys/systemcfg.h` file.
*/
# ifdef __64BIT__
# define CPP_WORDSZ 64
# define STACKBOTTOM MAKE_CPTR(0x1000000000000000)
# else
# define CPP_WORDSZ 32
extern int errno;
# define STACKBOTTOM ((ptr_t)(&errno))
# endif
# define USE_MMAP_ANON
/*
* From AIX linker man page:
* - `_text` specifies the first location of the program;
* - `_etext` specifies the first location after the program;
* - `_data` specifies the first location of the data;
* - `_edata` specifies the first location after the initialized data;
* - `_end` (or `end`) specifies the first location after all data.
*/
extern int _data[], _end[];
# define DATASTART ((ptr_t)_data)
# define DATAEND ((ptr_t)_end)
# define MPROTECT_VDB
# define DYNAMIC_LOADING
/*
* Note: for really old versions of AIX, `DYNAMIC_LOADING` may have to
* be removed.
*/
# endif
# ifdef NOSYS
# define OS_TYPE "NOSYS"
# define CPP_WORDSZ 32
extern void __end[], __dso_handle[];
# define DATASTART ((ptr_t)__dso_handle) /*< OK, that is ugly */
# define DATAEND ((ptr_t)__end)
/* Note: stack starts at 0xE0000000 for the simulator. */
# define STACKBOTTOM PTR_ALIGN_UP(GC_approx_sp(), 0x10000000)
# endif
#endif /* POWERPC */
#ifdef VAX
# define MACH_TYPE "VAX"
# define CPP_WORDSZ 32
/* Pointers are `longword`-aligned by C compiler v4.2. */
extern char etext[];
# define DATASTART ((ptr_t)etext)
# ifdef BSD
# define OS_TYPE "BSD"
# define STACK_GRAN 0x1000000
# define HEURISTIC1
/* Note: `HEURISTIC2` may be OK, but it is hard to test. */
# endif
# ifdef ULTRIX
# define OS_TYPE "ULTRIX"
# define STACKBOTTOM MAKE_CPTR(0x7fffc800)
# endif
#endif /* VAX */
#ifdef SPARC
# define MACH_TYPE "SPARC"
# if defined(__arch64__) || defined(__sparcv9)
# define CPP_WORDSZ 64
# define ELF_CLASS ELFCLASS64
# else
# define CPP_WORDSZ 32
# define ALIGNMENT 4 /*< required by hardware */
# endif
# ifdef SOLARIS
extern int _etext[];
# define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext)
# define PROC_VDB
/*
* `getpagesize()` appeared to be missing from at least one Solaris 5.4
* installation. Weird.
*/
# define GETPAGESIZE() (unsigned)sysconf(_SC_PAGESIZE)
# endif
# ifdef DRSNX
# define OS_TYPE "DRSNX"
extern int etext[];
# define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)etext)
# define MPROTECT_VDB
# define STACKBOTTOM MAKE_CPTR(0xdfff0000)
# define DYNAMIC_LOADING
# endif
# ifdef LINUX
extern int _etext[];
# ifdef __arch64__
# define DATASTART GC_SysVGetDataStart(0x100000, (ptr_t)_etext)
# else
# define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext)
# endif
# endif
# ifdef OPENBSD
/* Nothing specific. */
# endif
# ifdef NETBSD
/* Nothing specific. */
# endif
# ifdef FREEBSD
extern char etext[];
# define DATASTART ((ptr_t)(&etext))
# define DATAEND ((ptr_t)GC_find_limit(DATASTART, TRUE))
# define DATAEND_IS_FUNC
# define GC_HAVE_DATAREGION2
extern char edata[], end[];
# define DATASTART2 ((ptr_t)(&edata))
# define DATAEND2 ((ptr_t)(&end))
# endif
#endif /* SPARC */
#ifdef I386
# define MACH_TYPE "I386"
# if (defined(__LP64__) || defined(_WIN64)) && !defined(CPPCHECK)
# error This should be handled as X86_64
# endif
# define CPP_WORDSZ 32
/*
* The 4-byte alignment appears to hold for all 32-bit compilers
* except Borland and Watcom. If using the Borland (bcc32) or
* Watcom (wcc386) compiler, `-a4` or `-zp4` option, respectively,
* should be passed to the compiler, both for building the library
* and client code. (The alternate solution is to define
* `FORCE_ALIGNMENT_ONE` macro but this would have significant
* negative performance implications.)
*/
# if defined(FORCE_ALIGNMENT_ONE) \
&& (defined(__BORLANDC__) || defined(__WATCOMC__))
# define ALIGNMENT 1
# endif
# ifdef SEQUENT
# define OS_TYPE "SEQUENT"
extern int etext[];
# define DATASTART PTR_ALIGN_UP((ptr_t)etext, 0x1000)
# define STACKBOTTOM MAKE_CPTR(0x3ffff000)
# endif
# ifdef HAIKU
/* Nothing specific. */
# endif
# ifdef HURD
/* Nothing specific. */
# endif
# ifdef EMBOX
/* Nothing specific. */
# endif
# ifdef NACL
/* Nothing specific. */
# endif
# ifdef QNX
/* Nothing specific. */
# endif
# ifdef SERENITY
/* Nothing specific. */
# endif
# ifdef SOLARIS
extern int _etext[];
# define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)_etext)
# define PROC_VDB
# endif
# ifdef SCO
# define OS_TYPE "SCO"
extern int etext[];
# define DATASTART \
(PTR_ALIGN_UP((ptr_t)etext, 0x400000) + (ADDR(etext) & 0xfff))
# define STACKBOTTOM MAKE_CPTR(0x7ffffffc)
# endif
# ifdef SCO_ELF
# define OS_TYPE "SCO_ELF"
extern int etext[];
# define DATASTART ((ptr_t)etext)
# define STACKBOTTOM MAKE_CPTR(0x8048000)
# define DYNAMIC_LOADING
# define ELF_CLASS ELFCLASS32
# endif
# ifdef DGUX
# define OS_TYPE "DGUX"
extern int _etext, _end;
# define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)(&_etext))
# define DATASTART_USES_XGETDATASTART
# define DATAEND ((ptr_t)(&_end))
# define HEURISTIC2
# define DYNAMIC_LOADING
# ifndef USE_MMAP
# define USE_MMAP 1
# endif
# define MAP_FAILED ((void *)(~(GC_uintptr_t)0))
# define HEAP_START ((word)0x40000000)
# endif /* DGUX */
# ifdef LINUX
/*
* This encourages `mmap()` to give us low addresses, thus allowing the
* heap to grow to ~3 GB.
*/
# define HEAP_START ((word)0x1000)
# ifdef __ELF__
# if GC_GLIBC_PREREQ(2, 0) || defined(HOST_ANDROID)
# define SEARCH_FOR_DATA_START
# else
/* See the comment of the Linux/m68k case. */
extern char **__environ;
# define DATASTART ((ptr_t)(&__environ))
# endif
# if !defined(GC_NO_SIGSETJMP) \
&& (defined(HOST_TIZEN) \
|| (defined(HOST_ANDROID) \
&& !(GC_GNUC_PREREQ(4, 8) || GC_CLANG_PREREQ(3, 2) \
|| __ANDROID_API__ >= 18)))
/*
* Older Android NDK releases lack `sigsetjmp` in x86 `libc` (`setjmp` is
* used instead to find `data_start`). The bug is fixed in Android NDK r8e
* (so, it is OK to use `sigsetjmp` if gcc-4.8+, clang-3.2+ or Android
* API level 18+).
*/
# define GC_NO_SIGSETJMP 1
# endif
# else
extern int etext[];
# define DATASTART PTR_ALIGN_UP((ptr_t)etext, 0x1000)
# endif
# ifdef USE_I686_PREFETCH
/*
* Empirically `prefetcht0` is much more effective at reducing cache miss
* stalls for the targeted load instructions. But it seems to interfere
* enough with other cache traffic that the net result is worse than
* `prefetchnta`.
*/
# define PREFETCH(x) \
__asm__ __volatile__("prefetchnta %0" : : "m"(*(char *)(x)))
# ifdef FORCE_WRITE_PREFETCH
/*
* Using prefetches for write seems to have a slight negative impact on
* performance, at least for a PIII/500.
*/
# define GC_PREFETCH_FOR_WRITE(x) \
__asm__ __volatile__("prefetcht0 %0" : : "m"(*(char *)(x)))
# else
# define GC_NO_PREFETCH_FOR_WRITE
# endif
# elif defined(USE_3DNOW_PREFETCH)
# define PREFETCH(x) \
__asm__ __volatile__("prefetch %0" : : "m"(*(char *)(x)))
# define GC_PREFETCH_FOR_WRITE(x) \
__asm__ __volatile__("prefetchw %0" : : "m"(*(char *)(x)))
# endif
# if defined(__GLIBC__) && !defined(__UCLIBC__) \
&& !defined(GLIBC_TSX_BUG_FIXED)
/* Workaround lock elision implementation for some `glibc`. */
# define GLIBC_2_19_TSX_BUG
EXTERN_C_END
# include <gnu/libc-version.h> /*< for `gnu_get_libc_version()` */
EXTERN_C_BEGIN
# endif
# ifndef SOFT_VDB
# define SOFT_VDB
# endif
# endif
# ifdef CYGWIN32
# define WOW64_THREAD_CONTEXT_WORKAROUND
# define DATASTART ((ptr_t)GC_DATASTART) /*< defined in `gc.h` file */
# define DATAEND ((ptr_t)GC_DATAEND)
# ifndef USE_WINALLOC
/* `MPROTECT_VDB` does not work, it leads to a spurious exit. */
# endif
# endif
# ifdef INTERIX
# define OS_TYPE "INTERIX"
extern int _data_start__[], _bss_end__[];
# define DATASTART ((ptr_t)_data_start__)
# define DATAEND ((ptr_t)_bss_end__)
# define STACKBOTTOM \
({ \
ptr_t rv; \
__asm__ __volatile__("movl %%fs:4, %%eax" : "=a"(rv)); \
rv; \
})
# define USE_MMAP_ANON
# endif
# ifdef OS2
# define OS_TYPE "OS2"
/*
* `STACKBOTTOM` and `DATASTART` are handled specially in `os_dep.c`
* file. OS/2 actually has the right system call!
*/
# define DATAEND /*< not needed */
# undef USE_MUNMAP
# define GETPAGESIZE() os2_getpagesize()
# endif
# ifdef MSWIN32
# define WOW64_THREAD_CONTEXT_WORKAROUND
# define RETRY_GET_THREAD_CONTEXT
# if defined(__BORLANDC__)
/*
* TODO: VDB based on `VirtualProtect` and `SetUnhandledExceptionFilter`
* does not work correctly.
*/
# else
# define MPROTECT_VDB
# endif
# endif
# ifdef MSWINCE
/* Nothing specific. */
# endif
# ifdef DJGPP
# define OS_TYPE "DJGPP"
EXTERN_C_END
# include "stubinfo.h"
EXTERN_C_BEGIN
extern int etext[];
# define DATASTART PTR_ALIGN_UP((ptr_t)etext, 0x200)
extern int __djgpp_stack_limit, _stklen;
# define STACKBOTTOM (MAKE_CPTR(__djgpp_stack_limit) + _stklen)
# endif
# ifdef OPENBSD
/* Nothing specific. */
# endif
# ifdef FREEBSD
# if defined(__GLIBC__)
extern int _end[];
# define DATAEND ((ptr_t)_end)
# endif
# endif
# ifdef NETBSD
/* Nothing specific. */
# endif
# ifdef THREE86BSD
# define OS_TYPE "THREE86BSD"
# define HEURISTIC2
extern char etext[];
# define DATASTART ((ptr_t)etext)
# endif
# ifdef BSDI
# define OS_TYPE "BSDI"
# define HEURISTIC2
extern char etext[];
# define DATASTART ((ptr_t)etext)
# endif
# ifdef NEXT
# define STACKBOTTOM MAKE_CPTR(0xc0000000)
# endif
# ifdef RTEMS
# define OS_TYPE "RTEMS"
EXTERN_C_END
# include <sys/unistd.h>
EXTERN_C_BEGIN
extern int etext[];
# define DATASTART ((ptr_t)etext)
void *rtems_get_stack_bottom(void);
# define InitStackBottom rtems_get_stack_bottom()
# define STACKBOTTOM ((ptr_t)InitStackBottom)
# undef USE_MUNMAP
# endif
# ifdef DOS4GW
# define OS_TYPE "DOS4GW"
extern long __nullarea;
extern char _end;
extern char *_STACKTOP;
/*
* Depending on calling conventions Watcom C either precedes or does not
* precede the names of the C variables with an underscore.
* Make sure the startup code variables always have the same name.
*/
# pragma aux __nullarea "*";
# pragma aux _end "*";
# define STACKBOTTOM ((ptr_t)_STACKTOP) /*< confused? me too */
# define DATASTART ((ptr_t)(&__nullarea))
# define DATAEND ((ptr_t)(&_end))
# undef USE_MUNMAP
# define GETPAGESIZE() 4096
# endif
# ifdef DARWIN
# define DARWIN_DONT_PARSE_STACK 1
# define STACKBOTTOM MAKE_CPTR(0xc0000000)
# define MPROTECT_VDB
# endif
#endif /* I386 */
#ifdef LOONGARCH
# define MACH_TYPE "LOONGARCH"
# define CPP_WORDSZ (__SIZEOF_SIZE_T__ * 8)
# ifdef LINUX
# pragma weak __data_start
extern int __data_start[];
# define DATASTART ((ptr_t)__data_start)
# endif
#endif /* LOONGARCH */
#ifdef SW_64
# define MACH_TYPE "SW_64"
# define CPP_WORDSZ 64
# ifdef LINUX
/* Nothing specific. */
# endif
#endif /* SW_64 */
#ifdef MIPS
# define MACH_TYPE "MIPS"
# ifdef LINUX
# ifdef _MIPS_SZPTR
# define CPP_WORDSZ _MIPS_SZPTR
# else
# define CPP_WORDSZ 32
# endif
# pragma weak __data_start
extern int __data_start[];
# define DATASTART ((ptr_t)__data_start)
# ifndef HBLKSIZE
# define HBLKSIZE 4096
# endif
# if GC_GLIBC_PREREQ(2, 2)
# define SPECIFIC_MAIN_STACKBOTTOM
# else
# define STACKBOTTOM MAKE_CPTR(0x7fff8000)
# endif
# endif
# ifdef EWS4800
# define OS_TYPE "EWS4800"
# define HEURISTIC2
# if defined(_MIPS_SZPTR) && (_MIPS_SZPTR == 64)
# define CPP_WORDSZ _MIPS_SZPTR
extern int _fdata[], _end[];
# define DATASTART ((ptr_t)_fdata)
# define DATAEND ((ptr_t)_end)
# else
# define CPP_WORDSZ 32
extern int etext[], edata[];
# define DATASTART \
(PTR_ALIGN_UP((ptr_t)etext, 0x40000) + (ADDR(etext) & 0xffff))
# define DATAEND ((ptr_t)edata)
# define GC_HAVE_DATAREGION2
extern int _DYNAMIC_LINKING[], _gp[];
# define DATASTART2 \
(_DYNAMIC_LINKING ? PTR_ALIGN_UP((ptr_t)_gp + 0x8000, 0x40000) \
: (ptr_t)edata)
extern int end[];
# define DATAEND2 ((ptr_t)end)
# endif
# endif
# ifdef ULTRIX
# define OS_TYPE "ULTRIX"
# define CPP_WORDSZ 32
# define HEURISTIC2
/*
* Note: the actual beginning of the data segment could probably be
* slightly higher since startup code allocates lots of stuff.
*/
# define DATASTART MAKE_CPTR(0x10000000)
# endif
# ifdef IRIX5
# define OS_TYPE "IRIX5"
# ifdef _MIPS_SZPTR
# define CPP_WORDSZ _MIPS_SZPTR
# else
# define CPP_WORDSZ 32
# endif
# define HEURISTIC2
extern int _fdata[];
# define DATASTART ((ptr_t)_fdata)
/*
* Lowest plausible heap address. In the `USE_MMAP` case, we map there.
* In either case it is used to identify heap sections so they are not
* considered as roots.
*/
# ifdef USE_MMAP
# define HEAP_START ((word)0x30000000)
# else
# define HEAP_START ADDR(DATASTART)
# endif
/* `MPROTECT_VDB` should work, but there is evidence of a breakage. */
# define DYNAMIC_LOADING
# endif
# ifdef MSWINCE
# define CPP_WORDSZ 32
# endif
# ifdef NETBSD
# define CPP_WORDSZ 32
# ifndef __ELF__
# define DATASTART MAKE_CPTR(0x10000000)
# define STACKBOTTOM MAKE_CPTR(0x7ffff000)
# endif
# endif
# ifdef OPENBSD
# define CPP_WORDSZ 64 /*< all OpenBSD/mips platforms are 64-bit */
# endif
# ifdef FREEBSD
# define CPP_WORDSZ 32
# endif
# ifdef NONSTOP
# define OS_TYPE "NONSTOP"
# define CPP_WORDSZ 32
# define DATASTART MAKE_CPTR(0x8000000)
extern char **environ;
# define DATAEND ((ptr_t)(environ - 0x10))
# define STACKBOTTOM MAKE_CPTR(0x4fffffff)
# undef USE_MUNMAP
# endif
#endif /* MIPS */
#ifdef NIOS2
# define MACH_TYPE "NIOS2"
# define CPP_WORDSZ 32
# ifndef HBLKSIZE
# define HBLKSIZE 4096
# endif
# ifdef LINUX
extern int __data_start[];
# define DATASTART ((ptr_t)__data_start)
# endif
#endif /* NIOS2 */
#ifdef OR1K
# define MACH_TYPE "OR1K"
# define CPP_WORDSZ 32
# ifndef HBLKSIZE
# define HBLKSIZE 4096
# endif
# ifdef LINUX
extern int __data_start[];
# define DATASTART ((ptr_t)__data_start)
# endif
#endif /* OR1K */
#ifdef HP_PA
# define MACH_TYPE "HP_PA"
# ifdef __LP64__
# define CPP_WORDSZ 64
# else
# define CPP_WORDSZ 32
# endif
# define STACK_GROWS_UP
# ifdef HPUX
# ifndef GC_THREADS
# define MPROTECT_VDB
# endif
# ifdef USE_HPUX_FIXED_STACKBOTTOM
/*
* The following appears to work for 7xx systems running HP/UX 9.xx.
* Furthermore, it might result in much faster collections than `HEURISTIC2`,
* which may involve scanning segments that directly precede the stack.
* It is not the default, since it may not work on older machine/OS
* combinations. (Thanks to Raymond X.T. Nijssen for uncovering this.)
* This technique also does not work with HP/UX 11.xx. The stack size is
* settable using the kernel `maxssiz` variable, and the size can be set
* dynamically in HP/UX 11.23 and later. It also does not handle
* `SHMEM_MAGIC` binaries that have stack and data in the first quadrant.
* This is from platform `/etc/conf/h/param.h` file.
*/
# define STACKBOTTOM MAKE_CPTR(0x7b033000)
# elif defined(USE_ENVIRON_POINTER)
/*
* Gustavo Rodriguez-Rivera suggested changing `HEURISTIC2` to this.
* Note that the collector must be initialized before the first `putenv()`
* call. Unfortunately, some clients do not obey.
*/
extern char **environ;
# define STACKBOTTOM ((ptr_t)environ)
# elif !defined(HEURISTIC2)
/* This uses `pst_vm_status` support. */
# define SPECIFIC_MAIN_STACKBOTTOM
# endif
# ifndef __GNUC__
# define PREFETCH(x) \
do { \
register long addr = (long)(x); \
(void)_asm("LDW", 0, 0, addr, 0); \
} while (0)
# endif
# endif /* HPUX */
# ifdef LINUX
# define SEARCH_FOR_DATA_START
# endif
# ifdef OPENBSD
/* Nothing specific. */
# endif
#endif /* HP_PA */
#ifdef ALPHA
# define MACH_TYPE "ALPHA"
# define CPP_WORDSZ 64
# ifdef NETBSD
# define ELFCLASS32 32
# define ELFCLASS64 64
# define ELF_CLASS ELFCLASS64
# endif
# ifdef OPENBSD
/* Nothing specific. */
# endif
# ifdef FREEBSD
extern char etext[];
# define DATASTART ((ptr_t)(&etext))
# define DATAEND ((ptr_t)GC_find_limit(DATASTART, TRUE))
# define DATAEND_IS_FUNC
/*
* Handle unmapped hole which `alpha*-*-freebsd[45]*` puts between
* `etext` and `edata`.
*/
# define GC_HAVE_DATAREGION2
extern char edata[], end[];
# define DATASTART2 ((ptr_t)(&edata))
# define DATAEND2 ((ptr_t)(&end))
/* `MPROTECT_VDB` is not yet supported at all on FreeBSD/alpha. */
# endif
# ifdef OSF1
# define OS_TYPE "OSF1"
# define DATASTART MAKE_CPTR(0x140000000)
extern int _end[];
# define DATAEND ((ptr_t)(&_end))
extern char **environ;
/*
* Round up from the value of `environ` to the nearest page boundary.
* Probably this is broken if `putenv()` is called before the collector
* initialization.
*/
# define STACKBOTTOM PTR_ALIGN_UP((ptr_t)environ, getpagesize())
/*
* Normally `HEURISTIC2` is too conservative, since the text segment
* immediately follows the stack. Hence we give an upper bound.
* This is currently unused, since `HEURISTIC2` is not defined.
*/
extern int __start[];
# define HEURISTIC2_LIMIT PTR_ALIGN_DOWN((ptr_t)__start, getpagesize())
# ifndef GC_THREADS
/* FIXME: Unresolved signal issues with threads. */
# define MPROTECT_VDB
# endif
# define DYNAMIC_LOADING
# endif
# ifdef LINUX
# ifdef __ELF__
# define SEARCH_FOR_DATA_START
# else
# define DATASTART MAKE_CPTR(0x140000000)
extern int _end[];
# define DATAEND ((ptr_t)_end)
# endif
# endif
#endif /* ALPHA */
#ifdef IA64
# define MACH_TYPE "IA64"
# ifdef HPUX
# ifdef _ILP32
# define CPP_WORDSZ 32
/* Note: requires 8-byte alignment (granularity) for `malloc()`. */
# define ALIGNMENT 4
# else
# if !defined(_LP64) && !defined(CPPCHECK)
# error Unknown ABI
# endif
# define CPP_WORDSZ 64
/* Note: requires 16-byte alignment (granularity) for `malloc()`. */
# define ALIGNMENT 8
# endif
/*
* Note that the collector must be initialized before the 1st `putenv`
* call.
*/
extern char **environ;
# define STACKBOTTOM ((ptr_t)environ)
/*
* The following was empirically determined, and is probably not very
* robust. Note that the backing store base seems to be at a nice address
* minus one page.
*/
# define BACKING_STORE_DISPLACEMENT 0x1000000
# define BACKING_STORE_ALIGNMENT 0x1000
/* Known to be wrong for recent HP/UX versions!!! */
# endif
# ifdef LINUX
# define CPP_WORDSZ 64
/*
* The following works on NUE and older kernels:
* `define STACKBOTTOM MAKE_CPTR(0xa000000000000000l)`.
*/
/* TODO: `SPECIFIC_MAIN_STACKBOTTOM` does not work on NUE. */
/* We also need the base address of the register stack backing store. */
# define SEARCH_FOR_DATA_START
# ifdef __GNUC__
# define DYNAMIC_LOADING
# else
/*
* In the Intel compiler environment, we seem to end up with statically
* linked executables and an undefined reference to `_DYNAMIC`.
*/
# endif
# ifdef __GNUC__
# ifndef __INTEL_COMPILER
# define PREFETCH(x) __asm__(" lfetch [%0]" : : "r"(x))
# define GC_PREFETCH_FOR_WRITE(x) \
__asm__(" lfetch.excl [%0]" : : "r"(x))
# define CLEAR_DOUBLE(x) \
__asm__(" stf.spill [%0]=f0" : : "r"((void *)(x)))
# else
EXTERN_C_END
# include <ia64intrin.h>
EXTERN_C_BEGIN
# define PREFETCH(x) __lfetch(__lfhint_none, (x))
# define GC_PREFETCH_FOR_WRITE(x) __lfetch(__lfhint_nta, (x))
# define CLEAR_DOUBLE(x) __stf_spill((void *)(x), 0)
# endif /* __INTEL_COMPILER */
# endif
# endif
# ifdef MSWIN32
/* FIXME: This is a very partial guess. There is no port, yet. */
# if defined(_WIN64)
# define CPP_WORDSZ 64
# else
/* TODO: Is this possible? */
# define CPP_WORDSZ 32
# endif
# endif
#endif /* IA64 */
#ifdef E2K
# define MACH_TYPE "E2K"
# ifdef __LP64__
# define CPP_WORDSZ 64
# else
# define CPP_WORDSZ 32
# endif
# ifndef HBLKSIZE
# define HBLKSIZE 4096
# endif
# ifdef LINUX
extern int __dso_handle[];
# define DATASTART ((ptr_t)__dso_handle)
# ifdef REDIRECT_MALLOC
# define NO_PROC_FOR_LIBRARIES
# endif
# endif
#endif /* E2K */
#ifdef M88K
# define MACH_TYPE "M88K"
# define CPP_WORDSZ 32
# define STACKBOTTOM MAKE_CPTR(0xf0000000) /*< determined empirically */
extern int etext[];
# ifdef CX_UX
# define OS_TYPE "CX_UX"
# define DATASTART (PTR_ALIGN_UP((ptr_t)etext, 0x400000) + 0x10000)
# endif
# ifdef DGUX
# define OS_TYPE "DGUX"
# define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)etext)
# define DATASTART_USES_XGETDATASTART
# endif
#endif /* M88K */
#ifdef S370
/*
* If this still works, and if anyone cares, this should probably be moved
* to the `S390` category.
*/
# define MACH_TYPE "S370"
# define CPP_WORDSZ 32
# define ALIGNMENT 4 /*< required by hardware */
# ifdef UTS4
# define OS_TYPE "UTS4"
extern int _etext[], _end[];
# define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext)
# define DATAEND ((ptr_t)_end)
# define HEURISTIC2
# endif
#endif /* S370 */
#ifdef S390
# define MACH_TYPE "S390"
# ifndef __s390x__
# define CPP_WORDSZ 32
# else
# define CPP_WORDSZ 64
# ifndef HBLKSIZE
# define HBLKSIZE 4096
# endif
# endif
# ifdef LINUX
extern int __data_start[] __attribute__((__weak__));
extern int _end[] __attribute__((__weak__));
# define DATASTART ((ptr_t)__data_start)
# define DATAEND ((ptr_t)_end)
# define CACHE_LINE_SIZE 256
# define GETPAGESIZE() 4096
# ifndef SOFT_VDB
# define SOFT_VDB
# endif
# endif
#endif /* S390 */
#ifdef AARCH64
# define MACH_TYPE "AARCH64"
# ifdef __ILP32__
# define CPP_WORDSZ 32
# else
# define CPP_WORDSZ 64
# endif
# ifndef HBLKSIZE
# define HBLKSIZE 4096
# endif
# ifdef LINUX
# if defined(HOST_ANDROID)
# define SEARCH_FOR_DATA_START
# else
extern int __data_start[] __attribute__((__weak__));
# define DATASTART ((ptr_t)__data_start)
# endif
# endif
# ifdef COSMO
/* Empty. */
# endif
# ifdef DARWIN
/* OS X, iOS, visionOS */
# define DARWIN_DONT_PARSE_STACK 1
# define STACKBOTTOM MAKE_CPTR(0x16fdfffff)
# if (TARGET_OS_IPHONE || TARGET_OS_XR || TARGET_OS_VISION)
/*
* `MPROTECT_VDB` causes use of non-public API like `exc_server`, this
* could be a reason for blocking the client application in the store.
*/
# elif TARGET_OS_OSX
# define MPROTECT_VDB
# endif
# endif
# ifdef FREEBSD
/* Nothing specific. */
# endif
# ifdef NETBSD
# define ELF_CLASS ELFCLASS64
# endif
# ifdef OPENBSD
/* Nothing specific. */
# endif
# ifdef NINTENDO_SWITCH
# define OS_TYPE "NINTENDO_SWITCH"
# define NO_HANDLE_FORK 1
extern int __bss_end[];
# define DATASTART ((ptr_t)ALIGNMENT) /*< cannot be `NULL` */
# define DATAEND ((ptr_t)(&__bss_end))
void *switch_get_stack_bottom(void);
# define STACKBOTTOM ((ptr_t)switch_get_stack_bottom())
void *switch_get_mem(size_t lb);
# define GET_MEM(lb) switch_get_mem(lb)
# define GETPAGESIZE() 4096
# undef USE_MMAP
# undef USE_MUNMAP
# ifndef HAVE_CLOCK_GETTIME
# define HAVE_CLOCK_GETTIME 1
# endif
# endif
# ifdef KOS
/* Nothing specific. */
# endif
# ifdef QNX
/* Nothing specific. */
# endif
# ifdef SERENITY
/* Nothing specific. */
# endif
# ifdef MSWIN32
/* UWP */
/* TODO: Enable `MPROTECT_VDB`. */
# endif
# ifdef NOSYS
# define OS_TYPE "NOSYS"
/* `__data_start` is usually defined in the target linker script. */
extern int __data_start[];
# define DATASTART ((ptr_t)__data_start)
extern void *__stack_base__;
# define STACKBOTTOM ((ptr_t)__stack_base__)
# endif
#endif /* AARCH64 */
#ifdef ARM32
# define MACH_TYPE "ARM32"
# define CPP_WORDSZ 32
# ifdef LINUX
# if GC_GLIBC_PREREQ(2, 0) || defined(HOST_ANDROID)
# define SEARCH_FOR_DATA_START
# else
/* See the comment of the Linux/m68k case. */
extern char **__environ;
# define DATASTART ((ptr_t)(&__environ))
# endif
# endif
# ifdef MSWINCE
/* Nothing specific. */
# endif
# ifdef FREEBSD
/* Nothing specific. */
# endif
# ifdef DARWIN
/* iOS */
# define DARWIN_DONT_PARSE_STACK 1
# define STACKBOTTOM MAKE_CPTR(0x30000000)
/* `MPROTECT_VDB` causes use of non-public API. */
# endif
# ifdef NETBSD
/* Nothing specific. */
# endif
# ifdef OPENBSD
/* Nothing specific. */
# endif
# ifdef QNX
/* Nothing specific. */
# endif
# ifdef SN_TARGET_PSP2
# define OS_TYPE "SN_TARGET_PSP2"
# define NO_HANDLE_FORK 1
# ifndef HBLKSIZE
# define HBLKSIZE 65536 /*< page size is 64 KB */
# endif
# define DATASTART ((ptr_t)ALIGNMENT)
# define DATAEND ((ptr_t)ALIGNMENT)
void *psp2_get_stack_bottom(void);
# define STACKBOTTOM ((ptr_t)psp2_get_stack_bottom())
void *psp2_get_mem(size_t lb);
# define GET_MEM(lb) psp2_get_mem(lb)
# endif
# ifdef NN_PLATFORM_CTR
# define OS_TYPE "NN_PLATFORM_CTR"
extern unsigned char Image$$ZI$$ZI$$Base[];
# define DATASTART ((ptr_t)Image$$ZI$$ZI$$Base)
extern unsigned char Image$$ZI$$ZI$$Limit[];
# define DATAEND ((ptr_t)Image$$ZI$$ZI$$Limit)
void *n3ds_get_stack_bottom(void);
# define STACKBOTTOM ((ptr_t)n3ds_get_stack_bottom())
# endif
# ifdef MSWIN32
/* UWP */
/* TODO: Enable `MPROTECT_VDB`. */
# endif
# ifdef NOSYS
# define OS_TYPE "NOSYS"
/* `__data_start` is usually defined in the target linker script. */
extern int __data_start[];
# define DATASTART ((ptr_t)__data_start)
/* `__stack_base__` is set in platform `newlib/libc/sys/arm/crt0.S` file. */
extern void *__stack_base__;
# define STACKBOTTOM ((ptr_t)__stack_base__)
# endif
# ifdef SYMBIAN
/* Nothing specific. */
# endif
#endif /* ARM32 */
#ifdef CRIS
# define MACH_TYPE "CRIS"
# define CPP_WORDSZ 32
# define ALIGNMENT 1
# ifdef LINUX
# define SEARCH_FOR_DATA_START
# endif
#endif /* CRIS */
#if defined(SH) && !defined(SH4)
# define MACH_TYPE "SH"
# define CPP_WORDSZ 32
# ifdef LINUX
# define SEARCH_FOR_DATA_START
# endif
# ifdef NETBSD
/* Nothing specific. */
# endif
# ifdef OPENBSD
/* Nothing specific. */
# endif
# ifdef MSWINCE
/* Nothing specific. */
# endif
#endif
#ifdef SH4
# define MACH_TYPE "SH4"
# define CPP_WORDSZ 32
# ifdef MSWINCE
/* Nothing specific. */
# endif
#endif /* SH4 */
#ifdef AVR32
# define MACH_TYPE "AVR32"
# define CPP_WORDSZ 32
# ifdef LINUX
# define SEARCH_FOR_DATA_START
# endif
#endif /* AVR32 */
#ifdef M32R
# define MACH_TYPE "M32R"
# define CPP_WORDSZ 32
# ifdef LINUX
# define SEARCH_FOR_DATA_START
# endif
#endif /* M32R */
#ifdef X86_64
# define MACH_TYPE "X86_64"
# ifdef __ILP32__
# define CPP_WORDSZ 32
# else
# define CPP_WORDSZ 64
# endif
# ifndef HBLKSIZE
# define HBLKSIZE 4096
# endif
# ifndef CACHE_LINE_SIZE
# define CACHE_LINE_SIZE 64
# endif
# ifdef PLATFORM_GETMEM
# define OS_TYPE "PLATFORM_GETMEM"
# define DATASTART ((ptr_t)ALIGNMENT)
# define DATAEND ((ptr_t)ALIGNMENT)
EXTERN_C_END
# include <pthread.h>
EXTERN_C_BEGIN
void *platform_get_stack_bottom(void);
# define STACKBOTTOM ((ptr_t)platform_get_stack_bottom())
void *platform_get_mem(size_t lb);
# define GET_MEM(lb) platform_get_mem(lb)
# endif
# ifdef LINUX
# define SEARCH_FOR_DATA_START
# if defined(__GLIBC__) && !defined(__UCLIBC__)
/*
* A workaround for GCF (Google Cloud Function) which does not support
* `mmap()` for `/dev/zero` pseudo-file. Should not cause any harm to
* other targets.
*/
# define USE_MMAP_ANON
# endif
# if defined(__GLIBC__) && !defined(__UCLIBC__) \
&& !defined(GETCONTEXT_FPU_BUG_FIXED)
/*
* At present, there is a bug in `glibc` `getcontext()` on Linux/x86_64
* (it clears FPU exception mask). We define this macro to workaround it.
*/
/* TODO: This seems to be fixed in `glibc` 2.14. */
# define GETCONTEXT_FPU_EXCMASK_BUG
# endif
# if defined(__GLIBC__) && !defined(__UCLIBC__) \
&& !defined(GLIBC_TSX_BUG_FIXED)
/* Workaround lock elision implementation for some `glibc`. */
# define GLIBC_2_19_TSX_BUG
EXTERN_C_END
# include <gnu/libc-version.h> /*< for `gnu_get_libc_version()` */
EXTERN_C_BEGIN
# endif
# ifndef SOFT_VDB
# define SOFT_VDB
# endif
# endif
# ifdef COSMO
/* Empty. */
# endif
# ifdef DARWIN
# define DARWIN_DONT_PARSE_STACK 1
# define STACKBOTTOM MAKE_CPTR(0x7fff5fc00000)
# define MPROTECT_VDB
# endif
# ifdef FREEBSD
# if defined(__GLIBC__)
extern int _end[];
# define DATAEND ((ptr_t)_end)
# endif
# if defined(__DragonFly__)
/*
* DragonFly BSD still has `vm.max_proc_mmap`, according to its
* `mmap(2)` man page.
*/
# define COUNT_UNMAPPED_REGIONS
# endif
# endif
# ifdef NETBSD
/* Nothing specific. */
# endif
# ifdef OPENBSD
/* Nothing specific. */
# endif
# ifdef HAIKU
/* Nothing specific. */
# endif
# ifdef HURD
/* Nothing specific. */
# endif
# ifdef QNX
/* Nothing specific. */
# endif
# ifdef SERENITY
/* Nothing specific. */
# endif
# ifdef SOLARIS
# define ELF_CLASS ELFCLASS64
extern int _etext[];
# define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)_etext)
# define PROC_VDB
# endif
# ifdef CYGWIN32
# ifndef USE_WINALLOC
# if defined(THREAD_LOCAL_ALLOC)
/*
* TODO: For an unknown reason, thread-local allocations lead to spurious
* process exit after the fault handler is once invoked.
*/
# else
# define MPROTECT_VDB
# endif
# endif
# endif
# ifdef MSWIN_XBOX1
# define OS_TYPE "MSWIN_XBOX1"
# define NO_GETENV
# define DATASTART ((ptr_t)ALIGNMENT)
# define DATAEND ((ptr_t)ALIGNMENT)
LONG64 durango_get_stack_bottom(void);
# define STACKBOTTOM ((ptr_t)durango_get_stack_bottom())
# define GETPAGESIZE() 4096
# ifndef USE_MMAP
# define USE_MMAP 1
# endif
/* The following is from platform `sys/mman.h` file. */
# define PROT_NONE 0
# define PROT_READ 1
# define PROT_WRITE 2
# define PROT_EXEC 4
# define MAP_PRIVATE 2
# define MAP_FIXED 0x10
# define MAP_FAILED ((void *)(~(GC_uintptr_t)0))
# endif
# ifdef MSWIN32
# define RETRY_GET_THREAD_CONTEXT
# if !defined(__GNUC__) || defined(__INTEL_COMPILER) \
|| (GC_GNUC_PREREQ(4, 7) && !defined(__MINGW64__))
/*
* Older gcc and Mingw-w64 (both gcc and clang) do not support
* `SetUnhandledExceptionFilter()` properly on x86_64.
*/
# define MPROTECT_VDB
# endif
# endif
#endif /* X86_64 */
#ifdef ARC
# define MACH_TYPE "ARC"
# define CPP_WORDSZ 32
# define CACHE_LINE_SIZE 64
# ifdef LINUX
extern int __data_start[] __attribute__((__weak__));
# define DATASTART ((ptr_t)__data_start)
# endif
#endif /* ARC */
#ifdef HEXAGON
# define MACH_TYPE "HEXAGON"
# define CPP_WORDSZ 32
# ifdef LINUX
# if defined(__GLIBC__)
# define SEARCH_FOR_DATA_START
# elif !defined(CPPCHECK)
# error Unknown Hexagon libc configuration
# endif
# endif
#endif /* HEXAGON */
#ifdef TILEPRO
# define MACH_TYPE "TILEPRO"
# define CPP_WORDSZ 32
# define PREFETCH(x) __insn_prefetch(x)
# define CACHE_LINE_SIZE 64
# ifdef LINUX
extern int __data_start[];
# define DATASTART ((ptr_t)__data_start)
# endif
#endif /* TILEPRO */
#ifdef TILEGX
# define MACH_TYPE "TILEGX"
# define CPP_WORDSZ (__SIZEOF_PTRDIFF_T__ * 8)
# if CPP_WORDSZ == 32
# define CLEAR_DOUBLE(x) (void)(*(long long *)(x) = 0)
# endif
# define PREFETCH(x) __insn_prefetch_l1(x)
# define CACHE_LINE_SIZE 64
# ifdef LINUX
extern int __data_start[];
# define DATASTART ((ptr_t)__data_start)
# endif
#endif /* TILEGX */
#ifdef RISCV
# define MACH_TYPE "RISCV"
# define CPP_WORDSZ (__SIZEOF_SIZE_T__ * 8) /*< 32 or 64 */
# ifdef FREEBSD
/* Nothing specific. */
# endif
# ifdef LINUX
extern int __data_start[] __attribute__((__weak__));
# define DATASTART ((ptr_t)__data_start)
# endif
# ifdef NETBSD
/* Nothing specific. */
# endif
# ifdef OPENBSD
/* Nothing specific. */
# endif
# ifdef NOSYS
# define OS_TYPE "NOSYS"
extern char etext[];
# define DATASTART ((ptr_t)etext)
/* FIXME: `STACKBOTTOM` is wrong! */
extern char **environ;
# define STACKBOTTOM ((ptr_t)environ)
/* TODO: Support 64K page size. */
# define GETPAGESIZE() 4096
# endif
#endif /* RISCV */
#ifdef WEBASSEMBLY
# define MACH_TYPE "WEBASSEMBLY"
# if defined(__wasm64__) && !defined(CPPCHECK)
# error 64-bit WebAssembly is not yet supported
# endif
# define CPP_WORDSZ 32
/*
* Emscripten does emulate `mmap` and `munmap`, but those should not be
* used in the collector, since WebAssembly lacks the native support of
* memory mapping. Use `sbrk()` instead (by default).
*/
# undef USE_MMAP
# undef USE_MUNMAP
# ifdef EMSCRIPTEN_TINY
void *emmalloc_memalign(size_t align, size_t lb);
# define GET_MEM(lb) emmalloc_memalign(GC_page_size, lb)
# endif
# ifdef EMSCRIPTEN
# define OS_TYPE "EMSCRIPTEN"
# define DATASTART ((ptr_t)ALIGNMENT)
# define DATAEND ((ptr_t)ALIGNMENT)
# if defined(GC_THREADS) && !defined(CPPCHECK)
# error No thread support yet
# endif
# endif
# ifdef WASI
# define OS_TYPE "WASI"
extern char __global_base, __heap_base;
# define DATASTART ((ptr_t)(&__global_base))
# define DATAEND ((ptr_t)(&__heap_base))
# define STACKBOTTOM DATASTART
# ifndef GC_NO_SIGSETJMP
# define GC_NO_SIGSETJMP 1 /*< no support of signals */
# endif
# ifndef NO_CLOCK
# define NO_CLOCK 1 /*< no support of `clock()` */
# endif
# if defined(GC_THREADS) && !defined(CPPCHECK)
# error No thread support yet
# endif
# endif
#endif /* WEBASSEMBLY */
#if defined(CYGWIN32) || defined(MSWIN32) || defined(MSWINCE)
/* Note: it does not include Xbox One. */
# define ANY_MSWIN
#endif
#if defined(GC_PTHREADS) || defined(GC_WIN32_THREADS) \
|| ((defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH) \
|| defined(SN_TARGET_PS3) || defined(SN_TARGET_PSP2)) \
&& defined(GC_THREADS))
# define THREADS
#endif
/*
* If the client app is known not to create threads (even indirectly in
* the used libraries) and the collector is not multi-threaded, then the
* collector could be built with `SINGLE_THREADED_PROCESS` macro defined.
* But in practice the macro should never be defined.
*/
#if defined(THREADS) && defined(SINGLE_THREADED_PROCESS)
# undef SINGLE_THREADED_PROCESS
#endif
#if defined(__CHERI_PURE_CAPABILITY__)
# define CHERI_PURECAP
#endif
#if defined(__GLIBC__) && !defined(DONT_USE_LIBC_PRIVATES)
/* Use the stack-end marker of `glibc`. */
# define USE_LIBC_PRIVATES
#endif
#ifdef NO_RETRY_GET_THREAD_CONTEXT
# undef RETRY_GET_THREAD_CONTEXT
#endif
#if defined(LINUX) && defined(SPECIFIC_MAIN_STACKBOTTOM) \
&& defined(NO_PROC_STAT) && !defined(USE_LIBC_PRIVATES)
/*
* This combination will fail, since we have no way to get the stack bottom.
* Use `HEURISTIC2` instead.
*/
# undef SPECIFIC_MAIN_STACKBOTTOM
# define HEURISTIC2
/* This may still fail on some architectures like `IA64`. We tried... */
#endif
#if defined(USE_MMAP_ANON) && !defined(USE_MMAP)
# define USE_MMAP 1
#elif (defined(LINUX) || defined(OPENBSD)) && defined(USE_MMAP)
/*
* The OS kernel may do a somewhat better job merging mappings with
* anonymous mappings.
*/
# define USE_MMAP_ANON
#endif
#if defined(CHERI_PURECAP) && defined(USE_MMAP)
/* TODO: Currently turned off to avoid downgrading permissions on CHERI. */
# undef USE_MUNMAP
#endif
#if (defined(E2K) && defined(USE_PTR_HWTAG) || defined(CHERI_PURECAP)) \
&& !defined(NO_BLACK_LISTING)
/*
* Misinterpreting of an integer is not possible on the platforms with
* H/W-tagged pointers, thus the black-listing mechanism is redundant.
*/
# define NO_BLACK_LISTING
#endif
#if defined(REDIRECT_MALLOC) && defined(THREADS) \
&& (defined(LINUX) || defined(NACL))
/* TODO: Unclear if NaCl really needs this. */
# define REDIR_MALLOC_AND_LINUXTHREADS
#endif
#if defined(REDIR_MALLOC_AND_LINUXTHREADS) && !defined(NO_PROC_FOR_LIBRARIES) \
&& !defined(USE_PROC_FOR_LIBRARIES)
/*
* NPTL allocates thread stacks with `mmap`, which is fine. But it
* keeps a cache of thread stacks. Each thread stack contains a thread
* control block (TCB). The latter, in turn, contains a pointer to
* (`sizeof(void*)` from the beginning of) the `dtv` for thread-local
* storage, which is `calloc`-allocated. If we do not scan the cached
* thread stacks, we appear to lose the `dtv`. This tends to result in
* something that looks like a bogus `dtv` count, which tends to result
* in a `memset()` call on a block that is way too large. Sometimes
* we are lucky and the process just dies... There seems to be
* a similar issue with some other memory allocated by the dynamic
* loader. This should be avoidable by either:
* - Defining `USE_PROC_FOR_LIBRARIES` here (that performs very poorly,
* precisely because we end up scanning cached stacks);
* - Have `calloc()` look at its callers (in spite of the fact that it
* is gross and disgusting).
*
* In fact, neither seems to suffice, probably in part because even
* with `USE_PROC_FOR_LIBRARIES`, we do not scan parts of stack
* segments that appear to be out of bounds. Thus we actually do both,
* which seems to yield the best results.
*/
# define USE_PROC_FOR_LIBRARIES
#endif
#ifndef OS_TYPE
# define OS_TYPE ""
#endif
#ifndef MACH_TYPE
# define MACH_TYPE ""
#endif
#ifndef DATAEND
extern int end[];
# define DATAEND ((ptr_t)end)
#endif
/*
* Workaround for Android NDK clang-3.5+ (as of NDK r10e) which does
* not provide correct `_end` symbol. Unfortunately, alternate `__end__`
* symbol is provided only by NDK `bfd` linker.
*/
#if defined(HOST_ANDROID) && defined(__clang__) && !defined(BROKEN_UUENDUU_SYM)
# undef DATAEND
# pragma weak __end__
extern int __end__[];
# define DATAEND (__end__ != 0 ? (ptr_t)__end__ : (ptr_t)_end)
#endif
#if defined(SOLARIS) || defined(DRSNX) || defined(UTS4) \
|| (defined(LINUX) && defined(SPARC))
/* OS has SVR4 generic features. Probably others also qualify. */
# define SVR4
# define DATASTART_USES_XGETDATASTART
#endif
#if defined(HAVE_SYS_TYPES_H) \
|| !(defined(__CC_ARM) || defined(GC_NO_TYPES) || defined(OS2) \
|| defined(MSWINCE) || defined(SN_TARGET_PSP2))
EXTERN_C_END
# if defined(COSMO) && defined(MPROTECT_VDB) && !defined(_GNU_SOURCE)
# define _GNU_SOURCE 1
# endif
# include <sys/types.h>
EXTERN_C_BEGIN
#endif /* HAVE_SYS_TYPES_H */
#if defined(HAVE_UNISTD_H) \
|| !(defined(GC_NO_TYPES) || defined(MSWIN32) || defined(MSWINCE) \
|| defined(MSWIN_XBOX1) || defined(NINTENDO_SWITCH) \
|| defined(NN_PLATFORM_CTR) || defined(OS2) || defined(SERENITY) \
|| defined(SN_TARGET_PSP2) || defined(__CC_ARM))
EXTERN_C_END
# include <unistd.h>
EXTERN_C_BEGIN
#endif /* HAVE_UNISTD_H */
#if !defined(ANY_MSWIN) && !defined(GETPAGESIZE)
# if defined(DGUX) || defined(HOST_ANDROID) || defined(HOST_TIZEN) \
|| defined(KOS) || defined(SERENITY) \
|| (defined(LINUX) && defined(SPARC))
# define GETPAGESIZE() (unsigned)sysconf(_SC_PAGESIZE)
# else
# define GETPAGESIZE() (unsigned)getpagesize()
# endif
#endif /* !ANY_MSWIN && !GETPAGESIZE */
#if defined(HOST_ANDROID) && !(__ANDROID_API__ >= 23) \
&& ((defined(MIPS) && (CPP_WORDSZ == 32)) || defined(ARM32) \
|| defined(I386) /* but not x32 */)
/*
* `tkill()` exists only on arm32/mips(32)/x86.
* NDK r11+ deprecates `tkill()` but keeps it for Mono clients.
*/
# define USE_TKILL_ON_ANDROID
#endif
#if defined(MPROTECT_VDB) && defined(__GLIBC__) && !GC_GLIBC_PREREQ(2, 2)
# error glibc too old?
#endif
#if defined(SOLARIS) || defined(DRSNX)
/* OS has Solaris-style semi-undocumented interface to dynamic loader. */
# define SOLARISDL
/* OS has Solaris-style signal handlers. */
# define SUNOS5SIGS
#endif
#if (defined(FREEBSD) \
&& (defined(__DragonFly__) || defined(__GLIBC__) \
|| __FreeBSD_kernel__ >= 4 || __FreeBSD__ >= 4)) \
|| defined(HPUX)
# define SUNOS5SIGS
#endif
#if defined(COSMO) || defined(HPUX) || defined(HURD) || defined(NETBSD) \
|| defined(SERENITY) || (defined(FREEBSD) && defined(SUNOS5SIGS)) \
|| (defined(IRIX5) && defined(_sigargs)) /*< Irix 5.x, not 6.x */
# define USE_SEGV_SIGACT
/* We may also get `SIGBUS`. */
# define USE_BUS_SIGACT
#elif defined(ANY_BSD) || defined(HAIKU) || defined(IRIX5) || defined(OSF1) \
|| defined(SUNOS5SIGS)
# define USE_SEGV_SIGACT
#endif
#if !defined(GC_EXPLICIT_SIGNALS_UNBLOCK) && defined(SUNOS5SIGS) \
&& !defined(GC_NO_PTHREAD_SIGMASK)
# define GC_EXPLICIT_SIGNALS_UNBLOCK
#endif
#if !defined(NO_SIGNALS_UNBLOCK_IN_MAIN) && defined(GC_NO_PTHREAD_SIGMASK)
# define NO_SIGNALS_UNBLOCK_IN_MAIN
#endif
#ifndef PARALLEL_MARK
# undef GC_PTHREADS_PARAMARK /*< just in case it is defined by client */
#elif defined(GC_PTHREADS) && !defined(GC_PTHREADS_PARAMARK) \
&& !defined(__MINGW32__)
/*
* Use `pthreads`-based parallel mark implementation. Except for
* MinGW 32/64 to workaround a deadlock in winpthreads-3.0b internals.
*/
# define GC_PTHREADS_PARAMARK
#endif
#if !defined(NO_MARKER_SPECIAL_SIGMASK) \
&& (defined(NACL) || defined(GC_WIN32_PTHREADS) \
|| (defined(GC_PTHREADS_PARAMARK) && defined(GC_WIN32_THREADS)) \
|| defined(GC_NO_PTHREAD_SIGMASK))
/*
* Either there is no `pthread_sigmask()`, or the GC marker thread cannot
* steal and drop user signal calls.
*/
# define NO_MARKER_SPECIAL_SIGMASK
#endif
#if defined(NETBSD) && defined(THREADS)
# define SIGRTMIN 33
# define SIGRTMAX 63
/*
* It seems to be necessary to wait until threads have restarted.
* But it is unclear why that is the case.
*/
# define GC_NETBSD_THREADS_WORKAROUND
#endif
#if defined(OPENBSD) && defined(THREADS)
EXTERN_C_END
# include <sys/param.h>
EXTERN_C_BEGIN
#endif
#if defined(AIX) || defined(ANY_BSD) || defined(BSD) || defined(COSMO) \
|| defined(DARWIN) || defined(DGUX) || defined(HAIKU) || defined(HPUX) \
|| defined(HURD) || defined(IRIX5) || defined(LINUX) || defined(OSF1) \
|| defined(QNX) || defined(SERENITY) || defined(SVR4)
/* Basic UNIX-like system calls work. */
# define UNIX_LIKE
#endif
#if defined(CPPCHECK)
# undef CPP_WORDSZ
# define CPP_WORDSZ (__SIZEOF_PTRDIFF_T__ * 8)
#elif CPP_WORDSZ != 32 && CPP_WORDSZ != 64
# error Bad word size
#endif
#ifndef CPP_PTRSZ
# ifdef CHERI_PURECAP
# define CPP_PTRSZ (__SIZEOF_POINTER__ * 8)
# else
# define CPP_PTRSZ CPP_WORDSZ
# endif
#endif
#ifndef CPPCHECK
# if GC_SIZEOF_PTR * 8 != CPP_PTRSZ
# error Bad pointer size
# endif
#endif /* !CPPCHECK */
#ifndef ALIGNMENT
# define ALIGNMENT (CPP_PTRSZ >> 3)
#endif
#if !defined(STACKBOTTOM) && (defined(ECOS) || defined(NOSYS)) \
&& !defined(CPPCHECK)
# error Undefined STACKBOTTOM
#endif
#ifdef IGNORE_DYNAMIC_LOADING
# undef DYNAMIC_LOADING
#endif
#if defined(SMALL_CONFIG) && !defined(GC_DISABLE_INCREMENTAL)
/* Presumably not worth the space it takes. */
# define GC_DISABLE_INCREMENTAL
#endif
/* `USE_WINALLOC` is only an option for Cygwin. */
#ifndef CYGWIN32
# undef USE_WINALLOC
#endif
#if defined(MSWIN32) || defined(MSWINCE)
# define USE_WINALLOC 1
#endif
#ifdef USE_WINALLOC
# undef USE_MMAP
#endif
#if defined(ANY_BSD) || defined(DARWIN) || defined(IRIX5) || defined(LINUX) \
|| defined(SERENITY) || defined(SOLARIS) \
|| ((defined(CYGWIN32) || defined(USE_MMAP) || defined(USE_MUNMAP)) \
&& !defined(USE_WINALLOC))
/* Try both `sbrk` and `mmap`, in that order. */
# define MMAP_SUPPORTED
#endif
/*
* Xbox One (DURANGO) may not need to be this aggressive, but the
* default is likely too lax under heavy allocation pressure.
* The platform does not have a virtual paging system, so it does not
* have a large virtual address space that a standard x86_64 platform has.
*/
#if defined(USE_MUNMAP) && !defined(MUNMAP_THRESHOLD) \
&& (defined(SN_TARGET_PS3) || defined(SN_TARGET_PSP2) \
|| defined(MSWIN_XBOX1))
# define MUNMAP_THRESHOLD 3
#endif
#if defined(GC_DISABLE_INCREMENTAL) || defined(DEFAULT_VDB)
# undef GWW_VDB
# undef MPROTECT_VDB
# undef PROC_VDB
# undef SOFT_VDB
#endif
#ifdef NO_GWW_VDB
# undef GWW_VDB
#endif
#ifdef NO_MPROTECT_VDB
# undef MPROTECT_VDB
#endif
#ifdef NO_SOFT_VDB
# undef SOFT_VDB
#endif
#if defined(SOFT_VDB) && defined(SOFT_VDB_LINUX_VER_STATIC_CHECK)
EXTERN_C_END
# include <linux/version.h> /*< for `LINUX_VERSION`, `LINUX_VERSION_CODE` */
EXTERN_C_BEGIN
# if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0)
/* Not reliable in Linux kernels prior to v3.18. */
# undef SOFT_VDB
# endif
#endif /* SOFT_VDB */
#ifdef GC_DISABLE_INCREMENTAL
# undef CHECKSUMS
#endif
#if defined(BASE_ATOMIC_OPS_EMULATED)
/*
* `GC_write_fault_handler()` cannot use lock-based atomic primitives
* as this could lead to a deadlock.
*/
# undef MPROTECT_VDB
#endif
#if defined(USE_PROC_FOR_LIBRARIES) && defined(LINUX) && defined(THREADS)
/* Incremental GC based on `mprotect` is incompatible with `/proc` roots. */
# undef MPROTECT_VDB
#endif
#if defined(MPROTECT_VDB) && defined(GC_PREFER_MPROTECT_VDB)
/* Choose `MPROTECT_VDB` manually (if multiple strategies available). */
# undef PROC_VDB
/* `GWW_VDB`, `SOFT_VDB` are handled in `os_dep.c` file. */
#endif
#ifdef PROC_VDB
/* Mutually exclusive VDB implementations (for now). */
# undef MPROTECT_VDB
/* For a test purpose only. */
# undef SOFT_VDB
#endif
#if defined(MPROTECT_VDB) && !defined(MSWIN32) && !defined(MSWINCE)
EXTERN_C_END
# include <signal.h> /*< for `SA_SIGINFO`, `SIGBUS` */
EXTERN_C_BEGIN
#endif
#if defined(SIGBUS) && !defined(HAVE_SIGBUS) && !defined(CPPCHECK)
# define HAVE_SIGBUS
#endif
#ifndef SA_SIGINFO
# define NO_SA_SIGACTION
#endif
#if (defined(NO_SA_SIGACTION) || defined(GC_NO_SIGSETJMP)) \
&& defined(MPROTECT_VDB) && !defined(DARWIN) && !defined(MSWIN32) \
&& !defined(MSWINCE)
# undef MPROTECT_VDB
#endif
#if !defined(DEFAULT_VDB) && !defined(GWW_VDB) && !defined(MPROTECT_VDB) \
&& !defined(PROC_VDB) && !defined(SOFT_VDB) \
&& !defined(GC_DISABLE_INCREMENTAL)
# define DEFAULT_VDB
#endif
#if defined(CHECK_SOFT_VDB) && !defined(CPPCHECK) \
&& (defined(GC_PREFER_MPROTECT_VDB) || !defined(SOFT_VDB) \
|| !defined(MPROTECT_VDB))
# error Invalid config for CHECK_SOFT_VDB
#endif
#if (defined(GC_DISABLE_INCREMENTAL) || defined(BASE_ATOMIC_OPS_EMULATED) \
|| defined(REDIRECT_MALLOC) || defined(SMALL_CONFIG) \
|| defined(REDIRECT_MALLOC_IN_HEADER) || defined(CHECKSUMS)) \
&& !defined(NO_MANUAL_VDB)
/* TODO: Implement `CHECKSUMS` for manual VDB. */
# define NO_MANUAL_VDB
#endif
#if !defined(PROC_VDB) && !defined(SOFT_VDB) \
&& !defined(NO_VDB_FOR_STATIC_ROOTS)
/* Cannot determine whether a static root page is dirty. */
# define NO_VDB_FOR_STATIC_ROOTS
#endif
#if defined(MPROTECT_VDB) && !defined(DONT_COUNT_PROTECTED_REGIONS) \
&& !defined(COUNT_PROTECTED_REGIONS) \
&& (defined(LINUX) || defined(__DragonFly__))
# define COUNT_PROTECTED_REGIONS
#endif
#if (defined(COUNT_PROTECTED_REGIONS) || defined(COUNT_UNMAPPED_REGIONS)) \
&& !defined(GC_UNMAPPED_REGIONS_SOFT_LIMIT)
/*
* The default limit of `vm.max_map_count` on Linux is ~65530.
* There is approximately one mapped region to every protected or
* unmapped region. Therefore if we aim to use up to half of
* `vm.max_map_count` for the collector (leaving half for the rest
* of the process), then the number of such regions should be one
* quarter of `vm.max_map_count`.
*/
# if defined(__DragonFly__)
# define GC_UNMAPPED_REGIONS_SOFT_LIMIT (1000000 / 4)
# else
# define GC_UNMAPPED_REGIONS_SOFT_LIMIT 16384
# endif
#endif
#if (((defined(ARM32) || defined(AVR32) || defined(MIPS) || defined(NIOS2) \
|| defined(OR1K)) \
&& defined(UNIX_LIKE)) \
|| defined(DARWIN) || defined(HAIKU) || defined(HURD) \
|| defined(OPENBSD) || defined(QNX) || defined(RTEMS) \
|| defined(SERENITY) || defined(HOST_ANDROID) \
|| (defined(LINUX) && !defined(__gnu_linux__))) \
&& !defined(NO_GETCONTEXT)
# define NO_GETCONTEXT 1
#endif
#if defined(MSWIN32) && !defined(CONSOLE_LOG) && defined(_MSC_VER) \
&& defined(_DEBUG) && !defined(NO_CRT)
/*
* This should be included before the platform `intrin.h` file to
* workaround some bug in Windows Kit (as of 10.0.17763) headers
* causing redefinition of `_malloca` macro.
*/
EXTERN_C_END
# include <crtdbg.h> /*< for `_CrtDbgReport` */
EXTERN_C_BEGIN
#endif
#ifndef PREFETCH
# if (GC_GNUC_PREREQ(3, 0) || defined(__clang__)) && !defined(NO_PREFETCH)
# define PREFETCH(x) __builtin_prefetch((x), 0, 0)
# elif defined(_MSC_VER) && !defined(NO_PREFETCH) \
&& (defined(_M_IX86) || defined(_M_X64)) && !defined(_CHPE_ONLY_) \
&& (_MSC_VER >= 1900 /* VS 2015+ */)
EXTERN_C_END
# include <intrin.h>
EXTERN_C_BEGIN
# define PREFETCH(x) _mm_prefetch((const char *)(x), _MM_HINT_T0)
/* TODO: Support also `_M_ARM` and `_M_ARM64` (`__prefetch`). */
# else
# define PREFETCH(x) (void)0
# endif
#endif /* !PREFETCH */
#ifndef GC_PREFETCH_FOR_WRITE
/*
* The default `GC_PREFETCH_FOR_WRITE(x)` is defined in `gc_inline.h` file,
* the latter one is included from `gc_priv.h` file.
*/
#endif
#ifndef CACHE_LINE_SIZE
# define CACHE_LINE_SIZE 32 /*< wild guess */
#endif
#ifndef STATIC
# ifdef GC_ASSERTIONS
# define STATIC /*< ignore to aid debugging (or profiling) */
# else
# define STATIC static
# endif
#endif
/*
* Do we need the `GC_find_limit` machinery to find the end of
* a data segment (or the backing store base)?
*/
#if defined(HEURISTIC2) || defined(SEARCH_FOR_DATA_START) || defined(IA64) \
|| defined(DGUX) || defined(FREEBSD) || defined(OPENBSD) || defined(SVR4) \
|| (defined(HPUX) && defined(SPECIFIC_MAIN_STACKBOTTOM)) \
|| (defined(CYGWIN32) && defined(I386) && defined(USE_MMAP) \
&& !defined(USE_WINALLOC)) \
|| (defined(NETBSD) && defined(__ELF__))
# define NEED_FIND_LIMIT
#endif
#if defined(LINUX) \
&& (defined(USE_PROC_FOR_LIBRARIES) || defined(IA64) \
|| !defined(SMALL_CONFIG))
# define NEED_PROC_MAPS
#endif
#if defined(LINUX) || defined(HURD) || defined(__GLIBC__)
# define REGISTER_LIBRARIES_EARLY
/*
* We sometimes use `dl_iterate_phdr`, which may acquire an internal lock.
* This is not safe after the world has stopped. So we must call
* `GC_register_dynamic_libraries` before stopping the world.
* For performance reasons, this may be beneficial on other platforms
* as well, though it should be avoided on Windows.
*/
#endif /* LINUX */
#if defined(SEARCH_FOR_DATA_START)
extern ptr_t GC_data_start;
# define DATASTART GC_data_start
#endif
#ifndef HEAP_START
# define HEAP_START 0
#endif
#ifndef CLEAR_DOUBLE
# define CLEAR_DOUBLE(x) \
(void)(((ptr_t *)(x))[0] = NULL, ((ptr_t *)(x))[1] = NULL)
#endif
/*
* Some `libc` implementations like `bionic`, `musl` and `glibc` 2.34+
* do not have `libpthread.so` file because the `pthreads`-related code
* is located in `libc.so` file, thus potential `calloc()` calls from
* such code are forwarded to real (`libc`) `calloc()` without any
* special handling on the collector side. Checking `glibc` version at
* compile time for the purpose seems to be fine.
*/
#if defined(REDIR_MALLOC_AND_LINUXTHREADS) && !defined(HAVE_LIBPTHREAD_SO) \
&& defined(__GLIBC__) && !GC_GLIBC_PREREQ(2, 34)
# define HAVE_LIBPTHREAD_SO
#endif
#if defined(REDIR_MALLOC_AND_LINUXTHREADS) \
&& !defined(INCLUDE_LINUX_THREAD_DESCR)
/*
* Will not work, since `libc` and the dynamic loader use thread locals,
* sometimes as the only reference.
*/
# define INCLUDE_LINUX_THREAD_DESCR
#endif
#ifndef CPPCHECK
# if defined(GC_AIX_THREADS) && !defined(AIX) \
|| (defined(GC_DARWIN_THREADS) && !defined(DARWIN)) \
|| (defined(GC_DGUX386_THREADS) && !defined(DGUX)) \
|| (defined(GC_FREEBSD_THREADS) && !defined(FREEBSD)) \
|| (defined(GC_HAIKU_THREADS) && !defined(HAIKU)) \
|| (defined(GC_HPUX_THREADS) && !defined(HPUX)) \
|| (defined(GC_IRIX_THREADS) && !defined(IRIX5)) \
|| (defined(GC_LINUX_THREADS) && !defined(LINUX) && !defined(NACL)) \
|| (defined(GC_NETBSD_THREADS) && !defined(NETBSD)) \
|| (defined(GC_OPENBSD_THREADS) && !defined(OPENBSD)) \
|| (defined(GC_OSF1_THREADS) && !defined(OSF1)) \
|| (defined(GC_RTEMS_PTHREADS) && !defined(RTEMS)) \
|| (defined(GC_SOLARIS_THREADS) && !defined(SOLARIS)) \
|| (defined(GC_WIN32_THREADS) && !defined(ANY_MSWIN) \
&& !defined(MSWIN_XBOX1))
# error Inconsistent configuration
# elif defined(GC_WIN32_PTHREADS) && defined(CYGWIN32)
# error Inconsistent configuration (GC_PTHREADS)
# endif
# if defined(PARALLEL_MARK) && !defined(THREADS)
# error Invalid config: PARALLEL_MARK requires GC_THREADS
# endif
# if defined(GWW_VDB) && !defined(USE_WINALLOC)
# error Invalid config: GWW_VDB requires USE_WINALLOC
# endif
# if (defined(GC_FINDLEAK_DELAY_FREE) && defined(SHORT_DBG_HDRS)) \
|| ((defined(FIND_LEAK) || defined(GC_FINDLEAK_DELAY_FREE)) \
&& defined(NO_FIND_LEAK))
# error Invalid config: FIND_LEAK and NO_FIND_LEAK are mutually exclusive
# endif
#endif /* !CPPCHECK */
#if defined(NO_FIND_LEAK) && !defined(DONT_USE_ATEXIT)
# define DONT_USE_ATEXIT
#endif
/* Whether `GC_page_size` is to be set to a value other than page size. */
#if defined(CYGWIN32) && (defined(MPROTECT_VDB) || defined(USE_MUNMAP)) \
|| (!defined(ANY_MSWIN) && !defined(WASI) && !defined(USE_MMAP) \
&& (defined(GC_DISABLE_INCREMENTAL) || defined(DEFAULT_VDB)))
/*
* Cygwin: use the allocation granularity instead.
* Other than WASI and Windows: use `HBLKSIZE` instead (unless `mmap()`
* is used).
*/
# define ALT_PAGESIZE_USED
# ifndef GC_NO_VALLOC
/* Nonetheless, we need the real page size is some extra functions. */
# define REAL_PAGESIZE_NEEDED
# endif
#endif
#if defined(GC_PTHREADS) && !defined(DARWIN) && !defined(GC_WIN32_THREADS) \
&& !defined(PLATFORM_STOP_WORLD) && !defined(SN_TARGET_PSP2)
# define PTHREAD_STOP_WORLD_IMPL
#endif
#if defined(PTHREAD_STOP_WORLD_IMPL) && !defined(NACL)
# define SIGNAL_BASED_STOP_WORLD
#endif
#if (defined(E2K) || defined(HP_PA) || defined(IA64) || defined(M68K) \
|| defined(NO_SA_SIGACTION)) \
&& defined(SIGNAL_BASED_STOP_WORLD)
# define SUSPEND_HANDLER_NO_CONTEXT
#endif
#if (defined(MSWIN32) || defined(MSWINCE) \
|| (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS))) \
&& !defined(NO_CRT) && !defined(NO_WRAP_MARK_SOME)
/*
* Under rare conditions, we may end up marking from nonexistent memory.
* Hence we need to be prepared to recover by running `GC_mark_some` with
* a suitable handler in place.
*/
/* TODO: Should we also define it for Cygwin? */
# define WRAP_MARK_SOME
#endif
#if !defined(MSWIN32) && !defined(MSWINCE) || defined(__GNUC__) \
|| defined(NO_CRT)
# define NO_SEH_AVAILABLE
#endif
#ifdef GC_WIN32_THREADS
/* The number of copied registers in `copy_ptr_regs()`. */
# if defined(I386)
# ifdef WOW64_THREAD_CONTEXT_WORKAROUND
# define PUSHED_REGS_COUNT 9
# else
# define PUSHED_REGS_COUNT 7
# endif
# elif defined(X86_64)
# ifdef XMM_CANT_STORE_PTRS
/* If pointers cannot be located in Xmm registers. */
# define PUSHED_REGS_COUNT 15
# else
/*
* gcc-13 may store pointers into SIMD registers when certain compiler
* optimizations are enabled.
*/
# define PUSHED_REGS_COUNT (15 + 32)
# endif
# elif defined(SHx)
# define PUSHED_REGS_COUNT 15
# elif defined(ARM32)
# define PUSHED_REGS_COUNT 13
# elif defined(AARCH64)
# define PUSHED_REGS_COUNT 30
# elif defined(MIPS) || defined(ALPHA)
# define PUSHED_REGS_COUNT 28
# elif defined(PPC)
# define PUSHED_REGS_COUNT 29
# endif
#endif /* GC_WIN32_THREADS */
#if !defined(GC_PTHREADS) && !defined(GC_PTHREADS_PARAMARK)
# undef HAVE_PTHREAD_SETNAME_NP_WITH_TID
# undef HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG
# undef HAVE_PTHREAD_SET_NAME_NP
#endif
#if !(defined(GC_PTHREADS) || defined(GC_PTHREADS_PARAMARK) \
|| (defined(MPROTECT_VDB) && defined(DARWIN)))
# undef HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID
#endif
#if defined(USE_RWLOCK) || defined(GC_DISABLE_SUSPEND_THREAD)
/*
* At least in the Linux threads implementation, `rwlock` primitives are
* not atomic in respect to signals, and suspending externally a thread
* which is running inside `pthread_rwlock_rdlock()` may lead to a deadlock.
*/
/* TODO: As a workaround `GC_suspend_thread()` API is disabled. */
# undef GC_ENABLE_SUSPEND_THREAD
#endif
#ifndef GC_NO_THREADS_DISCOVERY
# if defined(DARWIN) && defined(THREADS)
/* Task-based thread registration requires stack-frame-walking code. */
# if defined(DARWIN_DONT_PARSE_STACK)
# define GC_NO_THREADS_DISCOVERY
# endif
# elif defined(GC_WIN32_THREADS)
/*
* `DllMain`-based thread registration is currently incompatible with
* thread-local allocation, `pthreads` and WinCE.
*/
# if (!defined(GC_DLL) && !defined(GC_INSIDE_DLL)) || defined(GC_PTHREADS) \
|| defined(MSWINCE) || defined(NO_CRT) || defined(THREAD_LOCAL_ALLOC)
# define GC_NO_THREADS_DISCOVERY
# endif
# else
# define GC_NO_THREADS_DISCOVERY
# endif
#endif /* !GC_NO_THREADS_DISCOVERY */
#if defined(GC_DISCOVER_TASK_THREADS) && defined(GC_NO_THREADS_DISCOVERY) \
&& !defined(CPPCHECK)
# error Defined both GC_DISCOVER_TASK_THREADS and GC_NO_THREADS_DISCOVERY
#endif
#if defined(PARALLEL_MARK) && !defined(DEFAULT_STACK_MAYBE_SMALL) \
&& (defined(DGUX) || defined(HPUX) \
|| defined(NO_GETCONTEXT) /* e.g. musl */)
/* TODO: Test default stack size in configure. */
# define DEFAULT_STACK_MAYBE_SMALL
#endif
#ifdef PARALLEL_MARK
/* The minimum stack size for a marker thread. */
# define MIN_STACK_SIZE (8 * HBLKSIZE * sizeof(ptr_t))
#endif
#if defined(HOST_ANDROID) && !defined(THREADS) \
&& !defined(USE_GET_STACKBASE_FOR_MAIN)
/*
* Always use `pthread_attr_getstack` on Android (`-lpthread` option
* is not needed to be specified manually) since Linux-specific
* `os_main_stackbottom()` causes application crash if invoked inside
* Dalvik VM.
*/
# define USE_GET_STACKBASE_FOR_MAIN
#endif
/*
* Outline `pthreads` primitives to use in `GC_get_stack_base()` and
* `GC_get_main_stack_base()`.
*/
#if ((defined(FREEBSD) && defined(__GLIBC__) /* kFreeBSD */) \
|| defined(COSMO) || defined(HAIKU) || defined(LINUX) || defined(KOS) \
|| defined(NETBSD)) \
&& !defined(NO_PTHREAD_GETATTR_NP)
# define HAVE_PTHREAD_GETATTR_NP 1
#elif defined(FREEBSD) && !defined(__GLIBC__) \
&& !defined(NO_PTHREAD_ATTR_GET_NP)
# define HAVE_PTHREAD_NP_H 1 /*< requires include `pthread_np.h` file */
# define HAVE_PTHREAD_ATTR_GET_NP 1
#endif
#if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
&& defined(USE_GET_STACKBASE_FOR_MAIN) && !defined(STACKBOTTOM) \
&& !defined(HEURISTIC1) && !defined(HEURISTIC2) && !defined(STACK_GRAN) \
&& !defined(SPECIFIC_MAIN_STACKBOTTOM)
/* Dummy definitions; rely on `pthread_attr_getstack` actually. */
# define HEURISTIC1
# define STACK_GRAN 0x1000000
#endif
#if !defined(HAVE_CLOCK_GETTIME) && defined(_POSIX_TIMERS) \
&& (defined(CYGWIN32) || (defined(LINUX) && defined(__USE_POSIX199309)))
# define HAVE_CLOCK_GETTIME 1
#endif
#if defined(GC_PTHREADS) && !defined(E2K) && !defined(IA64) \
&& (!defined(DARWIN) || defined(DARWIN_DONT_PARSE_STACK)) \
&& !defined(SN_TARGET_PSP2) && !defined(REDIRECT_MALLOC)
/*
* Note: unimplemented in case of redirection of `malloc()` because
* the client-provided function might call some `pthreads` primitive
* which, in turn, may use `malloc()` internally.
*/
# define STACKPTR_CORRECTOR_AVAILABLE
#endif
#if defined(UNIX_LIKE) && defined(THREADS) && !defined(NO_CANCEL_SAFE) \
&& !defined(HOST_ANDROID)
/*
* Make the code cancellation-safe. This basically means that we ensure
* that cancellation requests are ignored while we are in the collector.
* This applies only to POSIX deferred cancellation; we do not handle POSIX
* asynchronous cancellation. Note that this only works if
* `pthread_setcancelstate` is async-signal-safe, at least in the absence of
* asynchronous cancellation. This appears to be true for `glibc`, though it
* is not documented. Without that assumption, there seems to be no way to
* safely wait in a signal handler, which we need to do for thread suspension.
* Also note that little other code appears to be cancellation-safe.
* Hence it may make sense to turn this off for performance.
*/
# define CANCEL_SAFE
#endif
#ifdef CANCEL_SAFE
# define IF_CANCEL(x) x
#else
# define IF_CANCEL(x) /*< empty */
#endif
#if defined(DARWIN) && defined(MPROTECT_VDB) \
&& !defined(NO_DESC_CATCH_EXCEPTION_RAISE) \
&& !defined(FORCE_DESC_CATCH_EXCEPTION_RAISE) && GC_CLANG_PREREQ(17, 0)
/*
* Workaround "REFERENCED_DYNAMICALLY flag on _catch_exception_raise"
* linker deprecation warnings on macOS 15.4.
*/
# define NO_DESC_CATCH_EXCEPTION_RAISE
#endif
#if !defined(CAN_HANDLE_FORK) && !defined(NO_HANDLE_FORK) \
&& !defined(HAVE_NO_FORK) \
&& ((defined(GC_PTHREADS) && !defined(NACL) \
&& !defined(GC_WIN32_PTHREADS) && !defined(USE_WINALLOC)) \
|| (defined(DARWIN) && defined(MPROTECT_VDB) /* `&& !THREADS` */) \
|| (defined(HANDLE_FORK) && defined(GC_PTHREADS)))
/*
* Attempts (where supported and requested) to make `GC_malloc` work in
* a child process forked from a multi-threaded parent process.
*/
# define CAN_HANDLE_FORK
#endif
/*
* Workaround "failed to create new win32 semaphore" Cygwin fatal error
* during semaphores fixup-after-fork.
*/
#if defined(CYGWIN32) && defined(THREADS) && defined(CAN_HANDLE_FORK) \
&& !defined(CYGWIN_SEM_FIXUP_AFTER_FORK_BUG_FIXED) \
&& !defined(EMULATE_PTHREAD_SEMAPHORE)
# define EMULATE_PTHREAD_SEMAPHORE
#endif
#if defined(CAN_HANDLE_FORK) && !defined(CAN_CALL_ATFORK) \
&& !defined(GC_NO_CAN_CALL_ATFORK) && !defined(HOST_TIZEN) \
&& !defined(HURD) && (!defined(HOST_ANDROID) || __ANDROID_API__ >= 21)
/* Have working `pthread_atfork()`. */
# define CAN_CALL_ATFORK
#endif
#if !defined(CAN_HANDLE_FORK) && !defined(HAVE_NO_FORK) \
&& !(defined(CYGWIN32) || defined(SOLARIS) || defined(UNIX_LIKE))
# define HAVE_NO_FORK
#endif
#if !defined(USE_MARK_BITS) && !defined(USE_MARK_BYTES) \
&& defined(PARALLEL_MARK)
/* Minimize compare-and-swap usage. */
# define USE_MARK_BYTES
#endif
#if (defined(MSWINCE) && !defined(__CEGCC__) || defined(MSWINRT_FLAVOR)) \
&& !defined(NO_GETENV)
# define NO_GETENV
#endif
#if (defined(NO_GETENV) || defined(MSWINCE)) && !defined(NO_GETENV_WIN32)
# define NO_GETENV_WIN32
#endif
#if !defined(MSGBOX_ON_ERROR) && !defined(NO_MSGBOX_ON_ERROR) \
&& defined(MSWIN32) && !defined(MSWINRT_FLAVOR) && !defined(MSWIN_XBOX1) \
&& !defined(SMALL_CONFIG)
/*
* Show a Windows message box with "OK" button on a GC fatal error.
* Client application is terminated once the user clicks the button.
*/
# define MSGBOX_ON_ERROR
#endif
#ifndef STRTOULL
# if defined(_WIN64) && !defined(__GNUC__)
# define STRTOULL _strtoui64
# elif defined(_LLP64) || defined(__LLP64__) || defined(_WIN64)
# define STRTOULL strtoull
# else
/* `strtoul()` fits since `sizeof(long)` is not less than `sizeof(word)`. */
# define STRTOULL strtoul
# endif
#endif /* !STRTOULL */
#ifndef GC_WORD_C
# if defined(_WIN64) && !defined(__GNUC__)
# define GC_WORD_C(val) val##ui64
# elif defined(_LLP64) || defined(__LLP64__) || defined(_WIN64)
# define GC_WORD_C(val) val##ULL
# else
# define GC_WORD_C(val) ((word)val##UL)
# endif
#endif /* !GC_WORD_C */
#if defined(__has_feature)
/* `__has_feature()` is supported. */
# if __has_feature(address_sanitizer)
# define ADDRESS_SANITIZER
# endif
# if __has_feature(memory_sanitizer)
# define MEMORY_SANITIZER
# endif
# if __has_feature(thread_sanitizer) && defined(THREADS)
# define THREAD_SANITIZER
# endif
#else
# ifdef __SANITIZE_ADDRESS__
/* gcc v4.8+ */
# define ADDRESS_SANITIZER
# endif
# if defined(__SANITIZE_THREAD__) && defined(THREADS)
/* gcc v7.1+ */
# define THREAD_SANITIZER
# endif
#endif /* !__has_feature */
#if defined(SPARC)
/* Stack clearing is crucial, and we include assembly code to do it well. */
# define ASM_CLEAR_CODE
#endif
/*
* Can we save call chain in objects for debugging? Set `NFRAMES`
* (number of saved frames) and `NARGS` (number of arguments for each
* frame) to reasonable values for the platform.
* Define `SAVE_CALL_CHAIN` if we can. `SAVE_CALL_COUNT` can be
* specified at build time, though we feel free to adjust it slightly.
* Define `NEED_CALLINFO` if we either save the call stack or
* `GC_ADD_CALLER` is defined. Note: `GC_CAN_SAVE_CALL_STACKS` is
* defined (for certain platforms) in `gc_config_macros.h` file.
*/
#if defined(SPARC) \
|| ((defined(I386) || defined(X86_64)) \
&& (defined(LINUX) || defined(__GLIBC__)))
/*
* Linux/x86: `SAVE_CALL_CHAIN` is supported if the code is compiled to save
* frame pointers by default, i.e. no `-fomit-frame-pointer` flag is given.
*/
# define CAN_SAVE_CALL_ARGS
#endif
#if defined(SAVE_CALL_COUNT) && !defined(GC_ADD_CALLER) \
&& defined(GC_CAN_SAVE_CALL_STACKS)
# define SAVE_CALL_CHAIN
#endif
#ifdef SAVE_CALL_CHAIN
/* Number of arguments to save for each call. */
# if defined(SAVE_CALL_NARGS) && defined(CAN_SAVE_CALL_ARGS)
# define NARGS SAVE_CALL_NARGS
# else
# define NARGS 0
# endif
/* Number of frames to save. Even for alignment reasons. */
# if !defined(SAVE_CALL_COUNT) || defined(CPPCHECK)
# define NFRAMES 6
# else
# define NFRAMES ((SAVE_CALL_COUNT + 1) & ~1)
# endif
# define NEED_CALLINFO
#elif defined(GC_ADD_CALLER)
# define NFRAMES 1
# define NARGS 0
# define NEED_CALLINFO
#endif
#if (defined(FREEBSD) || (defined(DARWIN) && !defined(_POSIX_C_SOURCE)) \
|| (defined(SOLARIS) \
&& (!defined(_XOPEN_SOURCE) || defined(__EXTENSIONS__))) \
|| defined(LINUX)) \
&& !defined(HAVE_DLADDR)
# define HAVE_DLADDR 1
#endif
#if defined(MAKE_BACK_GRAPH) && !defined(DBG_HDRS_ALL)
# define DBG_HDRS_ALL 1
#endif
#if defined(POINTER_MASK) && !defined(POINTER_SHIFT)
# define POINTER_SHIFT 0
#elif !defined(POINTER_MASK) && defined(POINTER_SHIFT)
# define POINTER_MASK GC_WORD_MAX
#endif
#if defined(FIXUP_POINTER)
/* Custom `FIXUP_POINTER(p)`. */
# define NEED_FIXUP_POINTER
#elif defined(DYNAMIC_POINTER_MASK)
# define FIXUP_POINTER(p) \
(p = (ptr_t)((((word)(p)) & GC_pointer_mask) << GC_pointer_shift))
# undef POINTER_MASK
# undef POINTER_SHIFT
# define NEED_FIXUP_POINTER
#elif defined(POINTER_MASK)
/*
* Note: extra parentheses around custom-defined `POINTER_MASK` and
*`POINTER_SHIFT` are intentional.
*/
# define FIXUP_POINTER(p) \
(p = (ptr_t)(((word)(p) & (POINTER_MASK)) << (POINTER_SHIFT)))
# define NEED_FIXUP_POINTER
#else
# define FIXUP_POINTER(p) (void)(p)
#endif
#ifdef LINT2
/*
* A macro (based on a tricky expression) to prevent false warnings
* like "Array compared to 0", "Comparison of identical expressions",
* "Untrusted loop bound" output by some static code analysis tools.
* The argument should not be a literal value. The result is
* converted to `word` type. (Actually, `GC_word` is used instead of
* `word` type as the latter might be undefined at the place of use.)
*/
# define COVERT_DATAFLOW(w) (~(GC_word)(w) ^ (~(GC_word)0))
#else
# define COVERT_DATAFLOW(w) ((GC_word)(w))
#endif
#if CPP_PTRSZ > CPP_WORDSZ
/* TODO: Cannot use tricky operations on a pointer. */
# define COVERT_DATAFLOW_P(p) ((ptr_t)(p))
#else
# define COVERT_DATAFLOW_P(p) ((ptr_t)COVERT_DATAFLOW(p))
#endif
#if defined(REDIRECT_MALLOC) && defined(THREADS) && !defined(LINUX) \
&& !defined(REDIRECT_MALLOC_IN_HEADER)
/*
* May work on other platforms (e.g. Darwin) provided the client
* ensures all the client threads are registered with the collector,
* e.g. by using the preprocessor-based interception of the thread
* primitives (i.e., define `GC_THREADS` and include `gc.h` file from
* all the client files those are using `pthread_create` and friends).
*/
#endif
EXTERN_C_END
#endif /* GCCONFIG_H */
#ifdef __cplusplus
typedef bool GC_bool;
#elif defined(__BORLANDC__) || defined(__WATCOMC__)
typedef int GC_bool;
#else
typedef char GC_bool;
#endif
#if defined(__cplusplus) && !defined(ANY_MSWIN)
/* Avoid macro redefinition on a Windows platform. */
# define TRUE true
# define FALSE false
#else
# define TRUE 1
# define FALSE 0
#endif
#if !defined(GC_ATOMIC_UNCOLLECTABLE) && defined(ATOMIC_UNCOLLECTABLE)
/* For compatibility with old-style naming. */
# define GC_ATOMIC_UNCOLLECTABLE
#endif
#ifndef GC_INNER
/*
* This tagging macro must be used at the start of every variable definition
* which is declared with `GC_EXTERN`. Should be also used for the GC-scope
* function definitions and prototypes. Must not be used in `gcconfig.h`
* file. Should not be used for the debugging-only functions.
*/
# if defined(GC_DLL) && defined(__GNUC__) && !defined(ANY_MSWIN)
# if GC_GNUC_PREREQ(4, 0) && !defined(GC_NO_VISIBILITY)
/* See the corresponding `GC_API` definition. */
# define GC_INNER __attribute__((__visibility__("hidden")))
# else
/* The attribute is unsupported. */
# define GC_INNER /*< empty */
# endif
# else
# define GC_INNER /*< empty */
# endif
# define GC_EXTERN extern GC_INNER
/*
* Used only for the GC-scope variables (prefixed with `GC_`) declared
* in the private header files. Must not be used for thread-local
* variables. Must not be used in `gcconfig.h` file.
* The corresponding variable definition must start with `GC_INNER`.
* Should not be used for the debugging- or profiling-only variables.
* As of now, there are some other exceptions, e.g. for the variables
* that are known to be used by some popular clients.
*/
#endif /* !GC_INNER */
#ifdef __cplusplus
/* `register` storage specifier is deprecated in C++11. */
# define REGISTER /*< empty */
#else
/*
* Used only for several local variables in the performance-critical
* functions. Should not be used for new code.
*/
# define REGISTER register
#endif
#if defined(CPPCHECK)
# define MACRO_BLKSTMT_BEGIN {
# define MACRO_BLKSTMT_END }
# define LOCAL_VAR_INIT_OK = 0 /*< to avoid "uninit var" false positive */
#else
# define MACRO_BLKSTMT_BEGIN do {
# define MACRO_BLKSTMT_END \
} \
while (0)
# define LOCAL_VAR_INIT_OK /*< empty */
#endif
#if defined(M68K) && defined(__GNUC__)
/*
* By default, `__alignof__(void *)` is 2 on m68k architecture.
* Use this attribute to have the machine-word alignment (i.e. 4-byte one
* on the given 32-bit architecture).
*/
# define GC_ATTR_PTRT_ALIGNED __attribute__((__aligned__(sizeof(ptr_t))))
#else
# define GC_ATTR_PTRT_ALIGNED /*< empty */
#endif
#ifdef CHERI_PURECAP
# include <cheriintrin.h>
#endif
EXTERN_C_BEGIN
typedef GC_uintptr_t GC_funcptr_uint;
#define FUNCPTR_IS_DATAPTR
typedef unsigned int unsigned32;
#define hblk GC_hblk_s
struct hblk;
typedef struct hblkhdr hdr;
EXTERN_C_END
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#ifndef GC_HEADERS_H
#define GC_HEADERS_H
#if !defined(GC_PRIVATE_H) && !defined(CPPCHECK)
# error gc_hdrs.h should be included from gc_priv.h
#endif
#if CPP_WORDSZ != 32 && CPP_WORDSZ < 36 && !defined(CPPCHECK)
# error Get a real machine
#endif
EXTERN_C_BEGIN
/*
* The 2-level tree data structure that is used to find block headers.
* If there are more than 32 bits in a pointer, the top level is a hash
* table.
*
* This defines `HDR()`, `GET_HDR()`, and `SET_HDR()`, the main macros
* used to retrieve and set object headers.
*
* We take advantage of a header lookup cache. This is a locally declared
* direct mapped cache, used inside the marker. The `HC_GET_HDR()` macro
* uses and maintains this cache. Assuming we get reasonable hit rates,
* this saves a few memory references from each pointer validation.
*/
#if CPP_WORDSZ > 32
# define HASH_TL
#endif
/* Define appropriate out-degrees for each of the two tree levels. */
#if defined(LARGE_CONFIG) || !defined(SMALL_CONFIG)
# define LOG_BOTTOM_SZ 10
#else
/* Keep top index size reasonable with smaller blocks. */
# define LOG_BOTTOM_SZ 11
#endif
#define BOTTOM_SZ (1 << LOG_BOTTOM_SZ)
#ifndef HASH_TL
# define LOG_TOP_SZ (CPP_WORDSZ - LOG_BOTTOM_SZ - LOG_HBLKSIZE)
#else
# define LOG_TOP_SZ 11
#endif
#define TOP_SZ (1 << LOG_TOP_SZ)
#ifdef COUNT_HDR_CACHE_HITS
extern word GC_hdr_cache_hits; /*< used for debugging/profiling */
extern word GC_hdr_cache_misses;
# define HC_HIT() (void)(++GC_hdr_cache_hits)
# define HC_MISS() (void)(++GC_hdr_cache_misses)
#else
# define HC_HIT() (void)0
# define HC_MISS() (void)0
#endif
typedef struct hce {
word block_addr; /*< right-shifted by `LOG_HBLKSIZE` */
hdr *hce_hdr;
} hdr_cache_entry;
#define HDR_CACHE_SIZE 8 /*< a power of two */
#define DECLARE_HDR_CACHE hdr_cache_entry hdr_cache[HDR_CACHE_SIZE]
#define INIT_HDR_CACHE BZERO(hdr_cache, sizeof(hdr_cache))
#define HCE(h) (hdr_cache + ((ADDR(h) >> LOG_HBLKSIZE) & (HDR_CACHE_SIZE - 1)))
#define HCE_VALID_FOR(hce, h) ((hce)->block_addr == (ADDR(h) >> LOG_HBLKSIZE))
#define HCE_HDR(h) ((hce)->hce_hdr)
#ifdef PRINT_BLACK_LIST
/*
* Handle a header cache miss. Returns a pointer to the header
* corresponding to `p`, if the latter can possibly be a valid object
* pointer, and `NULL` otherwise. Guaranteed to return `NULL` for
* a pointer past the first page of an object unless both
* `GC_all_interior_pointers` is set and `p` is in fact a valid object
* pointer. Never returns a pointer to a free `hblk`.
*/
GC_INNER hdr *GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce,
ptr_t source);
# define HEADER_CACHE_MISS(p, hce, source) \
GC_header_cache_miss(p, hce, source)
#else
GC_INNER hdr *GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce);
# define HEADER_CACHE_MISS(p, hce, source) GC_header_cache_miss(p, hce)
#endif
/*
* Set `hhdr` to the header for `p`. Analogous to `GET_HDR()` below,
* except that in the case of large objects, it gets the header for the
* object beginning if `GC_all_interior_pointers` is true. Sets `hhdr`
* to `NULL` if `p` points to somewhere other than the first page of an
* object, and it is not a valid pointer to the object.
*/
#define HC_GET_HDR(p, hhdr, source) \
{ /*< cannot use `do ... while (0)` here */ \
hdr_cache_entry *hce = HCE(p); \
if (LIKELY(HCE_VALID_FOR(hce, p))) { \
HC_HIT(); \
hhdr = hce->hce_hdr; \
} else { \
hhdr = HEADER_CACHE_MISS(p, hce, source); \
if (NULL == hhdr) \
break; /*< go to the enclosing loop end */ \
} \
}
typedef struct bi {
/*
* The bottom-level index contains one of three kinds of values:
* - 0 means we are not responsible for this block, or this is
* a block other than the first one in a free block;
* - 1 < (long)`x` <= `MAX_JUMP` means the block starts at least
* `x * HBLKSIZE` bytes before the current address;
* - a valid pointer points to a `hdr` structure (the above cannot
* be valid pointers due to the `GET_MEM()` return convention).
*/
hdr *index[BOTTOM_SZ];
/*
* All indices are linked in the ascending and descending orders,
* respectively.
*/
struct bi *asc_link;
struct bi *desc_link;
word key; /*< high-order address bits */
#ifdef HASH_TL
struct bi *hash_link; /*< hash chain link */
#endif
} bottom_index;
#define MAX_JUMP (HBLKSIZE - 1)
#define HDR_FROM_BI(bi, p) \
(bi)->index[(ADDR(p) >> LOG_HBLKSIZE) & (BOTTOM_SZ - 1)]
#ifndef HASH_TL
# define BI(p) GC_top_index[ADDR(p) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE)]
# define HDR_INNER(p) HDR_FROM_BI(BI(p), p)
# ifdef SMALL_CONFIG
# define HDR(p) GC_find_header(p)
# else
# define HDR(p) HDR_INNER(p)
# endif
# define GET_BI(p, bottom_indx) (void)((bottom_indx) = BI(p))
# define GET_HDR(p, hhdr) (void)((hhdr) = HDR(p))
# define SET_HDR(p, hhdr) (void)(HDR_INNER(p) = (hhdr))
# define GET_HDR_ADDR(p, ha) (void)((ha) = &HDR_INNER(p))
#else
/* A hash function for the tree top level. */
# define TL_HASH(hi) ((hi) & (TOP_SZ - 1))
/* Set `bottom_indx` to point to the bottom index for address `p`. */
# define GET_BI(p, bottom_indx) \
do { \
REGISTER word hi = ADDR(p) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE); \
REGISTER bottom_index *_bi = GC_top_index[TL_HASH(hi)]; \
while (_bi->key != hi && _bi != GC_all_nils) \
_bi = _bi->hash_link; \
(bottom_indx) = _bi; \
} while (0)
# define GET_HDR_ADDR(p, ha) \
do { \
REGISTER bottom_index *bi; \
GET_BI(p, bi); \
(ha) = &HDR_FROM_BI(bi, p); \
} while (0)
# define GET_HDR(p, hhdr) \
do { \
REGISTER hdr **_ha; \
GET_HDR_ADDR(p, _ha); \
(hhdr) = *_ha; \
} while (0)
# define SET_HDR(p, hhdr) \
do { \
REGISTER bottom_index *bi; \
GET_BI(p, bi); \
GC_ASSERT(bi != GC_all_nils); \
HDR_FROM_BI(bi, p) = (hhdr); \
} while (0)
# define HDR(p) GC_find_header(p)
#endif
/*
* Is the result a forwarding address to someplace closer to the
* beginning of the block or `NULL`?
*/
#define IS_FORWARDING_ADDR_OR_NIL(hhdr) ((size_t)ADDR(hhdr) <= MAX_JUMP)
/*
* Get an `HBLKSIZE`-aligned address closer to the beginning of the
* block `h`. Assumes that `hhdr` is equal to `HDR(h)`,
* `IS_FORWARDING_ADDR(hhdr)` is true and `hhdr` is not `NULL`.
* `HDR(result)` is expected to be non-`NULL`.
*/
#define FORWARDED_ADDR(h, hhdr) \
((struct hblk *)(h) - (size_t)(GC_uintptr_t)(hhdr))
EXTERN_C_END
#endif /* GC_HEADERS_H */
#ifndef GC_ATTR_NO_SANITIZE_ADDR
# ifndef ADDRESS_SANITIZER
# define GC_ATTR_NO_SANITIZE_ADDR /*< empty */
# elif GC_CLANG_PREREQ(3, 8)
# define GC_ATTR_NO_SANITIZE_ADDR \
__attribute__((__no_sanitize__("address")))
# else
# define GC_ATTR_NO_SANITIZE_ADDR __attribute__((__no_sanitize_address__))
# endif
#endif /* !GC_ATTR_NO_SANITIZE_ADDR */
#ifndef GC_ATTR_NO_SANITIZE_MEMORY
# ifndef MEMORY_SANITIZER
# define GC_ATTR_NO_SANITIZE_MEMORY /*< empty */
# elif GC_CLANG_PREREQ(3, 8)
# define GC_ATTR_NO_SANITIZE_MEMORY \
__attribute__((__no_sanitize__("memory")))
# else
# define GC_ATTR_NO_SANITIZE_MEMORY __attribute__((__no_sanitize_memory__))
# endif
#endif /* !GC_ATTR_NO_SANITIZE_MEMORY */
#ifndef GC_ATTR_NO_SANITIZE_THREAD
# ifndef THREAD_SANITIZER
# define GC_ATTR_NO_SANITIZE_THREAD /*< empty */
# elif GC_CLANG_PREREQ(3, 8)
# define GC_ATTR_NO_SANITIZE_THREAD \
__attribute__((__no_sanitize__("thread")))
# else
/*
* It seems that `no_sanitize_thread` attribute has no effect if the
* function is inlined (as of gcc-11.1.0, at least).
*/
# define GC_ATTR_NO_SANITIZE_THREAD \
GC_ATTR_NOINLINE __attribute__((__no_sanitize_thread__))
# endif
#endif /* !GC_ATTR_NO_SANITIZE_THREAD */
#define GC_ATTR_NO_SANITIZE_ADDR_MEM_THREAD \
GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY \
GC_ATTR_NO_SANITIZE_THREAD
#ifndef UNUSED_ARG
# define UNUSED_ARG(arg) ((void)(arg))
#endif
#ifdef HAVE_CONFIG_H
/* The `inline` keyword is determined by `AC_C_INLINE` of `autoconf`. */
# define GC_INLINE static inline
#elif defined(_MSC_VER) || defined(__INTEL_COMPILER) || defined(__DMC__) \
|| (GC_GNUC_PREREQ(3, 0) && defined(__STRICT_ANSI__)) \
|| defined(__BORLANDC__) || defined(__WATCOMC__)
# define GC_INLINE static __inline
#elif GC_GNUC_PREREQ(3, 0) || defined(__sun)
# define GC_INLINE static inline
#else
# define GC_INLINE static
#endif
#ifndef GC_ATTR_NOINLINE
# if GC_GNUC_PREREQ(4, 0)
# define GC_ATTR_NOINLINE __attribute__((__noinline__))
# elif _MSC_VER >= 1400
# define GC_ATTR_NOINLINE __declspec(noinline)
# else
# define GC_ATTR_NOINLINE /*< empty */
# endif
#endif
#ifndef GC_API_OSCALL
/* This is used to identify GC routines called by name from OS. */
# if defined(__GNUC__)
# if GC_GNUC_PREREQ(4, 0) && !defined(GC_NO_VISIBILITY)
/* Same as `GC_API` macro if `GC_DLL` one is defined. */
# define GC_API_OSCALL extern __attribute__((__visibility__("default")))
# else
/* The attribute is unsupported. */
# define GC_API_OSCALL extern
# endif
# else
# define GC_API_OSCALL GC_API
# endif
#endif
#ifndef GC_API_PRIV
# define GC_API_PRIV GC_API
#endif
#ifndef GC_API_PATCHABLE
# define GC_API_PATCHABLE GC_ATTR_NOINLINE GC_API
#endif
#if defined(THREADS) && !defined(NN_PLATFORM_CTR)
/*
* Copyright (c) 2017 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* This is a private collector header which provides an implementation of
* `libatomic_ops` subset primitives sufficient for the collector assuming
* that gcc atomic intrinsics are available (and have the correct
* implementation). This is enabled by defining `GC_BUILTIN_ATOMIC` macro.
* Otherwise, `libatomic_ops` library is used to define the primitives.
*/
#ifndef GC_ATOMIC_OPS_H
#define GC_ATOMIC_OPS_H
#ifdef GC_BUILTIN_ATOMIC
# ifdef __cplusplus
extern "C" {
# endif
typedef size_t AO_t;
# ifdef GC_PRIVATE_H /*< i.e. have `GC_INLINE` */
# define AO_INLINE GC_INLINE
# else
# define AO_INLINE static __inline
# endif
# if !defined(THREAD_SANITIZER) && !defined(GC_PRIVATE_H)
/* Similar to that in `gcconfig.h` file. */
# if defined(__has_feature)
# if __has_feature(thread_sanitizer)
# define THREAD_SANITIZER
# endif
# elif defined(__SANITIZE_THREAD__)
# define THREAD_SANITIZER
# endif
# endif /* !THREAD_SANITIZER && !GC_PRIVATE_H */
typedef unsigned char AO_TS_t;
# define AO_TS_CLEAR 0
# define AO_TS_INITIALIZER ((AO_TS_t)AO_TS_CLEAR)
# if defined(__GCC_ATOMIC_TEST_AND_SET_TRUEVAL) && !defined(CPPCHECK)
# define AO_TS_SET __GCC_ATOMIC_TEST_AND_SET_TRUEVAL
# else
# define AO_TS_SET (AO_TS_t)1 /*< true */
# endif
# define AO_CLEAR(p) __atomic_clear(p, __ATOMIC_RELEASE)
# define AO_test_and_set_acquire(p) \
(__atomic_test_and_set(p, __ATOMIC_ACQUIRE) ? AO_TS_SET : AO_TS_CLEAR)
# define AO_HAVE_test_and_set_acquire
# define AO_compiler_barrier() __atomic_signal_fence(__ATOMIC_SEQ_CST)
# if defined(THREAD_SANITIZER) && !defined(AO_USE_ATOMIC_THREAD_FENCE)
/*
* Workaround a compiler warning (reported by gcc-11, at least) that
* `__atomic_thread_fence` is unsupported with thread sanitizer.
*/
AO_INLINE void
AO_nop_full(void)
{
volatile AO_TS_t dummy = AO_TS_INITIALIZER;
(void)__atomic_test_and_set(&dummy, __ATOMIC_SEQ_CST);
}
# else
# define AO_nop_full() __atomic_thread_fence(__ATOMIC_SEQ_CST)
# endif
# define AO_HAVE_nop_full
# define AO_fetch_and_add(p, v) __atomic_fetch_add(p, v, __ATOMIC_RELAXED)
# define AO_HAVE_fetch_and_add
# define AO_fetch_and_add1(p) AO_fetch_and_add(p, 1)
# define AO_HAVE_fetch_and_add1
# define AO_fetch_and_sub1(p) AO_fetch_and_add(p, ~(AO_t)0 /* -1 */)
# define AO_HAVE_fetch_and_sub1
# define AO_or(p, v) (void)__atomic_or_fetch(p, v, __ATOMIC_RELAXED)
# define AO_HAVE_or
# define AO_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
# define AO_HAVE_load
# define AO_load_acquire(p) __atomic_load_n(p, __ATOMIC_ACQUIRE)
# define AO_HAVE_load_acquire
/*
* `AO_load_acquire_read(p)` is not defined as it is unused, but we need
* its `AO_HAVE_` macro defined.
*/
# define AO_HAVE_load_acquire_read
# define AO_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
# define AO_HAVE_store
# define AO_store_release(p, v) __atomic_store_n(p, v, __ATOMIC_RELEASE)
# define AO_HAVE_store_release
# define AO_store_release_write(p, v) AO_store_release(p, v)
# define AO_HAVE_store_release_write
# define AO_char_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
# define AO_HAVE_char_load
# define AO_char_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
# define AO_HAVE_char_store
# define AO_char_fetch_and_add1(p) __atomic_fetch_add(p, 1, __ATOMIC_RELAXED)
# define AO_HAVE_char_fetch_and_add1
# ifdef AO_REQUIRE_CAS
AO_INLINE int
AO_compare_and_swap_release(volatile AO_t *p, AO_t ov, AO_t nv)
{
return (int)__atomic_compare_exchange_n(p, &ov, nv, 0, __ATOMIC_RELEASE,
__ATOMIC_RELAXED /* on fail */);
}
# define AO_HAVE_compare_and_swap_release
# endif
# ifdef __cplusplus
} /* extern "C" */
# endif
# ifndef NO_LOCKFREE_AO_OR
/* `__atomic_or_fetch()` is assumed to be lock-free. */
# define HAVE_LOCKFREE_AO_OR 1
# endif
#else
/* Fall back to `libatomic_ops`. */
# include "atomic_ops.h"
/*
* `AO_compiler_barrier`, `AO_load` and `AO_store` should be defined
* for all targets; the rest of the primitives are guaranteed to exist
* only if `AO_REQUIRE_CAS` is defined (or if the corresponding
* `AO_HAVE_` macro is defined). i686 and x86_64 targets have
* `AO_nop_full`, `AO_load_acquire`, `AO_store_release`, at least.
*/
# if (!defined(AO_HAVE_load) || !defined(AO_HAVE_store)) && !defined(CPPCHECK)
# error AO_load or AO_store is missing; probably old version of atomic_ops
# endif
#endif /* !GC_BUILTIN_ATOMIC */
#if defined(GC_BUILTIN_ATOMIC) || defined(__CHERI_PURE_CAPABILITY__)
/*
* Assume that gcc atomic intrinsics are available (and have correct
* implementation). `p` should be of a pointer to `ptr_t` (`char *`) value.
*/
# define GC_cptr_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
# define GC_cptr_load_acquire(p) __atomic_load_n(p, __ATOMIC_ACQUIRE)
# define GC_cptr_load_acquire_read(p) GC_cptr_load_acquire(p)
# define GC_cptr_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
# define GC_cptr_store_release(p, v) __atomic_store_n(p, v, __ATOMIC_RELEASE)
# define GC_cptr_store_release_write(p, v) GC_cptr_store_release(p, v)
# ifdef AO_REQUIRE_CAS
AO_INLINE int
GC_cptr_compare_and_swap(char *volatile *p, char *ov, char *nv)
{
return (int)__atomic_compare_exchange_n(p, &ov, nv, 0, __ATOMIC_RELAXED,
__ATOMIC_RELAXED);
}
# endif
#else
/*
* Redirect to the `AO_` primitives. Assume the size of `AO_t` matches
* that of a pointer.
*/
# define GC_cptr_load(p) (char *)AO_load((volatile AO_t *)(p))
# define GC_cptr_load_acquire(p) (char *)AO_load_acquire((volatile AO_t *)(p))
# define GC_cptr_load_acquire_read(p) \
(char *)AO_load_acquire_read((volatile AO_t *)(p))
# define GC_cptr_store(p, v) AO_store((volatile AO_t *)(p), (AO_t)(v))
# define GC_cptr_store_release(p, v) \
AO_store_release((volatile AO_t *)(p), (AO_t)(v))
# define GC_cptr_store_release_write(p, v) \
AO_store_release_write((volatile AO_t *)(p), (AO_t)(v))
# ifdef AO_REQUIRE_CAS
# define GC_cptr_compare_and_swap(p, ov, nv) \
AO_compare_and_swap((volatile AO_t *)(p), (AO_t)(ov), (AO_t)(nv))
# endif
#endif /* !GC_BUILTIN_ATOMIC */
#endif /* GC_ATOMIC_OPS_H */
# ifndef AO_HAVE_compiler_barrier
# define AO_HAVE_compiler_barrier 1
# endif
#endif
#ifdef ANY_MSWIN
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN 1
# endif
# define NOSERVICE
# include <windows.h>
/* This is included strictly after the platform `windows.h` file. */
# include <winbase.h>
#endif /* ANY_MSWIN */
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
* Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#ifndef GC_LOCKS_H
#define GC_LOCKS_H
#if !defined(GC_PRIVATE_H) && !defined(CPPCHECK)
# error gc_locks.h should be included from gc_priv.h
#endif
/*
* Mutual exclusion between allocator/collector routines. Needed if
* there is more than one allocator thread. Note that `I_HOLD_LOCK`,
* `I_DONT_HOLD_LOCK` and `I_HOLD_READER_LOCK` macros are used only
* positively in assertions, and may return `TRUE` in the "do not know"
* case.
*/
#ifdef THREADS
EXTERN_C_BEGIN
# if defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH)
extern void GC_lock(void);
extern void GC_unlock(void);
# define UNCOND_LOCK() GC_lock()
# define UNCOND_UNLOCK() GC_unlock()
# ifdef GC_ASSERTIONS
# define SET_LOCK_HOLDER() (void)0
# endif
# endif
# if (!defined(AO_HAVE_test_and_set_acquire) || defined(GC_WIN32_THREADS) \
|| defined(LINT2) || defined(RTEMS) || defined(SN_TARGET_PS3) \
|| defined(BASE_ATOMIC_OPS_EMULATED) || defined(USE_RWLOCK)) \
&& defined(GC_PTHREADS)
# define USE_PTHREAD_LOCKS
# undef USE_SPIN_LOCK
# if (defined(GC_WIN32_THREADS) || defined(LINT2) || defined(USE_RWLOCK)) \
&& !defined(NO_PTHREAD_TRYLOCK)
/*
* `pthread_mutex_trylock` may not win in `GC_lock` on Win32, due to
* built-in support for spinning first?
*/
# define NO_PTHREAD_TRYLOCK
# endif
# endif
# if defined(GC_WIN32_THREADS) && !defined(USE_PTHREAD_LOCKS) \
|| defined(GC_PTHREADS)
/* A value which is not equal to `NUMERIC_THREAD_ID(id)` for any thread. */
# define NO_THREAD ((unsigned long)(-1L))
# ifdef GC_ASSERTIONS
GC_EXTERN unsigned long GC_lock_holder;
# define UNSET_LOCK_HOLDER() (void)(GC_lock_holder = NO_THREAD)
# endif
# endif /* GC_WIN32_THREADS || GC_PTHREADS */
# if defined(GC_WIN32_THREADS) && !defined(USE_PTHREAD_LOCKS)
# ifdef USE_RWLOCK
GC_EXTERN SRWLOCK GC_allocate_ml;
# else
GC_EXTERN CRITICAL_SECTION GC_allocate_ml;
# endif
# ifdef GC_ASSERTIONS
# define SET_LOCK_HOLDER() (void)(GC_lock_holder = GetCurrentThreadId())
# define I_HOLD_LOCK() \
(!GC_need_to_lock || GC_lock_holder == GetCurrentThreadId())
# ifdef THREAD_SANITIZER
# define I_DONT_HOLD_LOCK() TRUE /*< conservatively say yes */
# else
# define I_DONT_HOLD_LOCK() \
(!GC_need_to_lock || GC_lock_holder != GetCurrentThreadId())
# endif
# ifdef USE_RWLOCK
# define UNCOND_READER_LOCK() \
{ \
GC_ASSERT(I_DONT_HOLD_LOCK()); \
AcquireSRWLockShared(&GC_allocate_ml); \
}
# define UNCOND_READER_UNLOCK() \
{ \
GC_ASSERT(I_DONT_HOLD_LOCK()); \
ReleaseSRWLockShared(&GC_allocate_ml); \
}
# define UNCOND_LOCK() \
{ \
GC_ASSERT(I_DONT_HOLD_LOCK()); \
AcquireSRWLockExclusive(&GC_allocate_ml); \
SET_LOCK_HOLDER(); \
}
# define UNCOND_UNLOCK() \
{ \
GC_ASSERT(I_HOLD_LOCK()); \
UNSET_LOCK_HOLDER(); \
ReleaseSRWLockExclusive(&GC_allocate_ml); \
}
# else
# define UNCOND_LOCK() \
{ \
GC_ASSERT(I_DONT_HOLD_LOCK()); \
EnterCriticalSection(&GC_allocate_ml); \
SET_LOCK_HOLDER(); \
}
# define UNCOND_UNLOCK() \
{ \
GC_ASSERT(I_HOLD_LOCK()); \
UNSET_LOCK_HOLDER(); \
LeaveCriticalSection(&GC_allocate_ml); \
}
# endif
# else
# ifdef USE_RWLOCK
# define UNCOND_READER_LOCK() AcquireSRWLockShared(&GC_allocate_ml)
# define UNCOND_READER_UNLOCK() ReleaseSRWLockShared(&GC_allocate_ml)
# define UNCOND_LOCK() AcquireSRWLockExclusive(&GC_allocate_ml)
# define UNCOND_UNLOCK() ReleaseSRWLockExclusive(&GC_allocate_ml)
# else
# define UNCOND_LOCK() EnterCriticalSection(&GC_allocate_ml)
# define UNCOND_UNLOCK() LeaveCriticalSection(&GC_allocate_ml)
# endif
# endif /* !GC_ASSERTIONS */
# elif defined(GC_PTHREADS)
EXTERN_C_END
# include <pthread.h>
EXTERN_C_BEGIN
/*
* POSIX allows `pthread_t` to be a structure type, though it rarely is.
* Unfortunately, we need to use a `pthread_t` to index a data structure.
* It also helps if comparisons do not involve a function call.
* Hence we introduce platform-dependent macros to compare `pthread_t` ids
* and to map them to integers (of `unsigned long` type). This mapping
* does not need to result in different values for each thread, though
* that should be true as much as possible.
*/
# if !defined(GC_WIN32_PTHREADS)
# define NUMERIC_THREAD_ID(id) ((unsigned long)(GC_uintptr_t)(id))
# define THREAD_EQUAL(id1, id2) ((id1) == (id2))
# define NUMERIC_THREAD_ID_UNIQUE
# elif defined(__WINPTHREADS_VERSION_MAJOR) /*< winpthreads */
# define NUMERIC_THREAD_ID(id) ((unsigned long)(id))
# define THREAD_EQUAL(id1, id2) ((id1) == (id2))
/* `NUMERIC_THREAD_ID()` is 32-bit and, thus, not unique on Win64. */
# ifndef _WIN64
# define NUMERIC_THREAD_ID_UNIQUE
# endif
# else /* pthreads-win32 */
# define NUMERIC_THREAD_ID(id) ((unsigned long)(word)(id.p))
/*
* The platform on which `pthread_t` is a structure.
* Using documented internal details of pthreads-win32 library.
* Faster than `pthread_equal()`. Should not change with the
* future versions of pthreads-win32 library.
*/
# define THREAD_EQUAL(id1, id2) (id1.p == id2.p && id1.x == id2.x)
/*
* Generic definitions based on `pthread_equal()` always work but will
* result in poor performance (as `NUMERIC_THREAD_ID()` might give
* a constant value) and weak assertion checking.
*/
# undef NUMERIC_THREAD_ID_UNIQUE
# endif
# ifdef SN_TARGET_PSP2
EXTERN_C_END
# include "psp2-support.h"
EXTERN_C_BEGIN
GC_EXTERN WapiMutex GC_allocate_ml_PSP2;
# define UNCOND_LOCK() \
{ \
int res; \
GC_ASSERT(I_DONT_HOLD_LOCK()); \
res = PSP2_MutexLock(&GC_allocate_ml_PSP2); \
GC_ASSERT(0 == res); \
(void)res; \
SET_LOCK_HOLDER(); \
}
# define UNCOND_UNLOCK() \
{ \
int res; \
GC_ASSERT(I_HOLD_LOCK()); \
UNSET_LOCK_HOLDER(); \
res = PSP2_MutexUnlock(&GC_allocate_ml_PSP2); \
GC_ASSERT(0 == res); \
(void)res; \
}
# elif (!defined(THREAD_LOCAL_ALLOC) || defined(USE_SPIN_LOCK)) \
&& !defined(USE_PTHREAD_LOCKS) && !defined(THREAD_SANITIZER) \
&& !defined(USE_RWLOCK)
/*
* In the `THREAD_LOCAL_ALLOC` case, the allocator lock tends to
* be held for long periods, if it is held at all.
* Thus spinning and sleeping for fixed periods are likely to result
* in significant wasted time. We thus rely mostly on queued locks.
*/
# undef USE_SPIN_LOCK
# define USE_SPIN_LOCK
GC_INNER void GC_lock(void);
# ifdef GC_ASSERTIONS
# define UNCOND_LOCK() \
{ \
GC_ASSERT(I_DONT_HOLD_LOCK()); \
if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_SET) \
GC_lock(); \
SET_LOCK_HOLDER(); \
}
# define UNCOND_UNLOCK() \
{ \
GC_ASSERT(I_HOLD_LOCK()); \
UNSET_LOCK_HOLDER(); \
AO_CLEAR(&GC_allocate_lock); \
}
# else
# define UNCOND_LOCK() \
{ \
if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_SET) \
GC_lock(); \
}
# define UNCOND_UNLOCK() AO_CLEAR(&GC_allocate_lock)
# endif /* !GC_ASSERTIONS */
# else
# ifndef USE_PTHREAD_LOCKS
# define USE_PTHREAD_LOCKS
# endif
# endif /* THREAD_LOCAL_ALLOC || USE_PTHREAD_LOCKS */
# ifdef USE_PTHREAD_LOCKS
EXTERN_C_END
# include <pthread.h>
EXTERN_C_BEGIN
# ifdef GC_ASSERTIONS
GC_INNER void GC_lock(void);
# define UNCOND_LOCK() \
{ \
GC_ASSERT(I_DONT_HOLD_LOCK()); \
GC_lock(); \
SET_LOCK_HOLDER(); \
}
# endif
# ifdef USE_RWLOCK
GC_EXTERN pthread_rwlock_t GC_allocate_ml;
# ifdef GC_ASSERTIONS
# define UNCOND_READER_LOCK() \
{ \
GC_ASSERT(I_DONT_HOLD_LOCK()); \
(void)pthread_rwlock_rdlock(&GC_allocate_ml); \
}
# define UNCOND_READER_UNLOCK() \
{ \
GC_ASSERT(I_DONT_HOLD_LOCK()); \
(void)pthread_rwlock_unlock(&GC_allocate_ml); \
}
# define UNCOND_UNLOCK() \
{ \
GC_ASSERT(I_HOLD_LOCK()); \
UNSET_LOCK_HOLDER(); \
(void)pthread_rwlock_unlock(&GC_allocate_ml); \
}
# else
# define UNCOND_READER_LOCK() \
(void)pthread_rwlock_rdlock(&GC_allocate_ml)
# define UNCOND_READER_UNLOCK() UNCOND_UNLOCK()
# define UNCOND_LOCK() (void)pthread_rwlock_wrlock(&GC_allocate_ml)
# define UNCOND_UNLOCK() (void)pthread_rwlock_unlock(&GC_allocate_ml)
# endif /* !GC_ASSERTIONS */
# else
GC_EXTERN pthread_mutex_t GC_allocate_ml;
# ifdef GC_ASSERTIONS
# define UNCOND_UNLOCK() \
{ \
GC_ASSERT(I_HOLD_LOCK()); \
UNSET_LOCK_HOLDER(); \
pthread_mutex_unlock(&GC_allocate_ml); \
}
# else
# if defined(NO_PTHREAD_TRYLOCK)
# define UNCOND_LOCK() pthread_mutex_lock(&GC_allocate_ml)
# else
GC_INNER void GC_lock(void);
# define UNCOND_LOCK() \
{ \
if (pthread_mutex_trylock(&GC_allocate_ml) != 0) \
GC_lock(); \
}
# endif
# define UNCOND_UNLOCK() pthread_mutex_unlock(&GC_allocate_ml)
# endif /* !GC_ASSERTIONS */
# endif
# endif /* USE_PTHREAD_LOCKS */
# ifdef GC_ASSERTIONS
/* The allocator lock holder. */
# define SET_LOCK_HOLDER() \
(void)(GC_lock_holder = NUMERIC_THREAD_ID(pthread_self()))
# define I_HOLD_LOCK() \
(!GC_need_to_lock \
|| GC_lock_holder == NUMERIC_THREAD_ID(pthread_self()))
# if !defined(NUMERIC_THREAD_ID_UNIQUE) || defined(THREAD_SANITIZER)
# define I_DONT_HOLD_LOCK() TRUE /*< conservatively say yes */
# else
# define I_DONT_HOLD_LOCK() \
(!GC_need_to_lock \
|| GC_lock_holder != NUMERIC_THREAD_ID(pthread_self()))
# endif
# endif /* GC_ASSERTIONS */
# if !defined(GC_WIN32_THREADS)
/*
* A hint that we are in the collector and holding the allocator lock
* for an extended period.
*/
GC_EXTERN volatile unsigned char GC_collecting;
# ifdef AO_HAVE_char_store
# if defined(GC_ASSERTIONS) && defined(AO_HAVE_char_fetch_and_add1)
/* Ensure `ENTER_GC()` is not used recursively. */
# define ENTER_GC() GC_ASSERT(!AO_char_fetch_and_add1(&GC_collecting))
# else
# define ENTER_GC() AO_char_store(&GC_collecting, TRUE)
# endif
# define EXIT_GC() AO_char_store(&GC_collecting, FALSE)
# else
# define ENTER_GC() (void)(GC_collecting = TRUE)
# define EXIT_GC() (void)(GC_collecting = FALSE)
# endif
# endif
# endif /* GC_PTHREADS */
# if defined(GC_ALWAYS_MULTITHREADED) \
&& (defined(USE_PTHREAD_LOCKS) || defined(USE_SPIN_LOCK))
# define GC_need_to_lock TRUE
# define set_need_to_lock() (void)0
# else
# if defined(GC_ALWAYS_MULTITHREADED) && !defined(CPPCHECK)
# error Runtime initialization of the allocator lock is needed!
# endif
# undef GC_ALWAYS_MULTITHREADED
# ifdef THREAD_SANITIZER
/*
* To workaround TSan false positive (e.g., when `GC_pthread_create()` is
* called from multiple threads in parallel), do not set `GC_need_to_lock`
* if it is already set.
*/
# define set_need_to_lock() \
(void)(*(GC_bool volatile *)&GC_need_to_lock \
? FALSE \
: (GC_need_to_lock = TRUE))
# else
# define set_need_to_lock() (void)(GC_need_to_lock = TRUE)
/* We are multi-threaded now. */
# endif
# endif
EXTERN_C_END
#else /* !THREADS */
# define LOCK() (void)0
# define UNLOCK() (void)0
# ifdef GC_ASSERTIONS
/*
* `I_HOLD_LOCK()` and `I_DONT_HOLD_LOCK()` are used only in positive
* assertions or to test whether we still need to acquire the allocator
* lock; `TRUE` works in either case.
*/
# define I_HOLD_LOCK() TRUE
# define I_DONT_HOLD_LOCK() TRUE
# endif
#endif /* !THREADS */
#if defined(UNCOND_LOCK) && !defined(LOCK)
# if (defined(LINT2) && defined(USE_PTHREAD_LOCKS)) \
|| defined(GC_ALWAYS_MULTITHREADED)
/*
* Instruct code analysis tools not to care about `GC_need_to_lock`
* influence to `LOCK`/`UNLOCK` semantic.
*/
# define LOCK() UNCOND_LOCK()
# define UNLOCK() UNCOND_UNLOCK()
# ifdef UNCOND_READER_LOCK
# define READER_LOCK() UNCOND_READER_LOCK()
# define READER_UNLOCK() UNCOND_READER_UNLOCK()
# endif
# else
/* At least two thread running; need to lock. */
# define LOCK() \
do { \
if (GC_need_to_lock) \
UNCOND_LOCK(); \
} while (0)
# define UNLOCK() \
do { \
if (GC_need_to_lock) \
UNCOND_UNLOCK(); \
} while (0)
# ifdef UNCOND_READER_LOCK
# define READER_LOCK() \
do { \
if (GC_need_to_lock) \
UNCOND_READER_LOCK(); \
} while (0)
# define READER_UNLOCK() \
do { \
if (GC_need_to_lock) \
UNCOND_READER_UNLOCK(); \
} while (0)
# endif
# endif
#endif /* UNCOND_LOCK && !LOCK */
#ifdef READER_LOCK
# define HAS_REAL_READER_LOCK
/* TODO: Implement I_HOLD_READER_LOCK, conservatively say yes for now. */
# define I_HOLD_READER_LOCK() TRUE
#else
# define READER_LOCK() LOCK()
# define READER_UNLOCK() UNLOCK()
# ifdef GC_ASSERTIONS
/*
* A macro to check that the allocator lock is held at least in the
* reader mode.
*/
# define I_HOLD_READER_LOCK() I_HOLD_LOCK()
# endif
#endif /* !READER_LOCK */
/*
* A variant of `READER_UNLOCK()` which ensures that data written
* before the unlock will be visible to the thread which acquires the
* allocator lock in the exclusive mode. But according to some `rwlock`
* documentation: writers synchronize with prior writers and readers.
*/
#define READER_UNLOCK_RELEASE() READER_UNLOCK()
#ifndef ENTER_GC
# define ENTER_GC()
# define EXIT_GC()
#endif
#endif /* GC_LOCKS_H */
#ifdef GC_ASSERTIONS
# define GC_ASSERT(e) \
do { \
if (UNLIKELY(!(e))) { \
GC_err_printf("Assertion failure: %s:%d\n", __FILE__, __LINE__); \
ABORT("assertion failure"); \
} \
} while (0)
#else
# define GC_ASSERT(e)
#endif
#include "gc/gc_inline.h"
/*
* Prevent certain compiler warnings by making a pointer-related cast
* through a "pointer-sized" numeric type.
*/
#define CAST_THRU_UINTPTR(t, x) ((t)(GC_uintptr_t)(x))
#define CAST_AWAY_VOLATILE_PVOID(p) \
CAST_THRU_UINTPTR(/* no volatile */ void *, p)
/*
* Convert an `unsigned` value to a `void` pointer. Typically used to
* print a numeric value using "%p" format specifier. The pointer is not
* supposed to be dereferenced.
*/
#define NUMERIC_TO_VPTR(v) ((void *)(GC_uintptr_t)(v))
/* Create a `ptr_t` pointer from a number (of `word` type). */
#define MAKE_CPTR(w) ((ptr_t)(GC_uintptr_t)(word)(w))
#define GC_WORD_MAX (~(word)0)
/* Convert given pointer to its address. Result is of `word` type. */
#ifdef CHERI_PURECAP
# define ADDR(p) cheri_address_get(p)
#else
# define ADDR(p) ((word)(GC_uintptr_t)(p))
#endif
#define ADDR_LT(p, q) GC_ADDR_LT(p, q)
#define ADDR_GE(p, q) (!ADDR_LT(p, q))
/*
* Check whether pointer `p` is in range [`s`, `e_p1`).
* `p` should not have side effects.
*/
#define ADDR_INSIDE(p, s, e_p1) (ADDR_GE(p, s) && ADDR_LT(p, e_p1))
/* Handy definitions to compare and adjust pointers in a stack. */
#ifdef STACK_GROWS_UP
# define HOTTER_THAN(p, q) ADDR_LT(q, p) /*< inverse */
# define MAKE_COOLER(p, d) \
(void)((p) -= ADDR(p) > (word)((d) * sizeof(*(p))) ? (d) : 0)
# define MAKE_HOTTER(p, d) (void)((p) += (d))
#else
# define HOTTER_THAN(p, q) ADDR_LT(p, q)
# define MAKE_COOLER(p, d) \
(void)((p) \
+= ADDR(p) <= (word)(GC_WORD_MAX - (d) * sizeof(*(p))) ? (d) : 0)
# define MAKE_HOTTER(p, d) (void)((p) -= (d))
#endif /* !STACK_GROWS_UP */
/* Clear/set flags (given by a mask) in a pointer. */
#define CPTR_CLEAR_FLAGS(p, mask) \
(ptr_t)((GC_uintptr_t)(p) & ~(GC_uintptr_t)(word)(mask))
#define CPTR_SET_FLAGS(p, mask) (ptr_t)((GC_uintptr_t)(p) | (word)(mask))
/* Easily changeable parameters are below. */
#ifdef ALL_INTERIOR_POINTERS
/*
* Forces all pointers into the interior of an object to be considered valid.
* Also causes the sizes of all objects to be inflated by at least one byte.
* This should suffice to guarantee that in the presence of a compiler that
* does not perform garbage-collector-unsafe optimizations, all portable,
* strictly ANSI conforming C programs should be safely usable with `malloc`
* replaced by `GC_malloc` and `free` calls removed. There are several
* disadvantages:
* 1. There are probably no interesting, portable, strictly ANSI-conforming
* C programs;
* 2. This option makes it hard for the collector to allocate space that is
* not "pointed to" by integers, etc. (Under SunOS 4.x with a statically
* linked `libc`, we empirically observed that it would be difficult to
* allocate individual objects larger than 100 KB; even if only smaller
* objects are allocated, more swap space is likely to be needed;
* fortunately, much of this will never be touched.)
*
* If you can easily avoid using this option, do. If not, try to keep
* individual objects small. This is really controlled at startup, through
* `GC_all_interior_pointers` variable.
*/
#endif
EXTERN_C_BEGIN
#ifndef GC_NO_FINALIZATION
/*
* If `GC_finalize_on_demand` is not set, invoke eligible finalizers.
* Otherwise: call `(*GC_finalizer_notifier)()` if there are finalizers to
* be run, and we have not called this procedure yet this collection cycle.
*/
GC_INNER void GC_notify_or_invoke_finalizers(void);
/*
* Perform all indicated finalization actions on unmarked objects.
* Unreachable finalizable objects are enqueued for processing by
* `GC_invoke_finalizers()`. Cause disappearing links to disappear
* and unreachable objects to be enqueued for finalization.
* Invoked with the allocator lock held but the world is running.
*/
GC_INNER void GC_finalize(void);
# ifndef GC_TOGGLE_REFS_NOT_NEEDED
/* Process the "toggle-refs" before GC starts. */
GC_INNER void GC_process_togglerefs(void);
# endif
# ifndef SMALL_CONFIG
GC_INNER void GC_print_finalization_stats(void);
# endif
#else
# define GC_notify_or_invoke_finalizers() (void)0
#endif /* GC_NO_FINALIZATION */
#if !defined(DONT_ADD_BYTE_AT_END)
# ifdef LINT2
/*
* Explicitly instruct the code analysis tool that `GC_all_interior_pointers`
* is assumed to have only value of 0 or 1.
*/
# define EXTRA_BYTES ((size_t)(GC_all_interior_pointers ? 1 : 0))
# else
# define EXTRA_BYTES ((size_t)GC_all_interior_pointers)
# endif
# define MAX_EXTRA_BYTES 1
#else
# define EXTRA_BYTES 0
# define MAX_EXTRA_BYTES 0
#endif
#ifdef LARGE_CONFIG
# define MINHINCR 64
# define MAXHINCR 4096
#else
/*
* Minimum heap increment, in blocks of `HBLKSIZE`. Note: must be multiple
* of largest page size.
*/
# define MINHINCR 16
/* Maximum heap increment, in blocks. */
# define MAXHINCR 2048
#endif /* !LARGE_CONFIG */
/* Stack saving for debugging. */
#ifdef NEED_CALLINFO
struct callinfo {
GC_return_addr_t ci_pc; /*< `pc` of caller, not callee */
# if NARGS > 0
GC_hidden_pointer ci_arg[NARGS]; /*< hide to avoid retention */
# endif
# if (NFRAMES * (NARGS + 1)) % 2 == 1
/* Likely alignment problem. */
ptr_t ci_dummy;
# endif
};
# ifdef SAVE_CALL_CHAIN
/*
* Fill in the `pc` and argument information for up to `NFRAMES` of
* my callers. Ignore my frame and my callers frame.
*/
GC_INNER void GC_save_callers(struct callinfo info[NFRAMES]);
# endif
/* Print `info` to `stderr`. We do not hold the allocator lock. */
GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]);
#endif /* NEED_CALLINFO */
EXTERN_C_END
/*
* Macros to ensure same formatting of C array/struct/union initializer
* across multiple versions of clang-format.
*/
#define C_INITIALIZER_BEGIN {
#define C_INITIALIZER_END }
/* OS interface routines. */
#ifndef NO_CLOCK
# ifdef BSD_TIME
# undef CLOCK_TYPE
# undef GET_TIME
# undef MS_TIME_DIFF
# define CLOCK_TYPE struct timeval
# define CLOCK_TYPE_INITIALIZER C_INITIALIZER_BEGIN 0, 0 C_INITIALIZER_END
# define GET_TIME(x) \
do { \
struct rusage rusage; \
getrusage(RUSAGE_SELF, &rusage); \
x = rusage.ru_utime; \
} while (0)
/*
* Compute time difference. `a` time is expected to be not earlier
* than `b` one; the result has `unsigned long` type.
*/
# define MS_TIME_DIFF(a, b) \
((unsigned long)((long)(a.tv_sec - b.tv_sec) * 1000 \
+ (long)(a.tv_usec - b.tv_usec) / 1000 \
- (a.tv_usec < b.tv_usec \
&& (long)(a.tv_usec - b.tv_usec) % 1000 \
!= 0 \
? 1 \
: 0)))
/*
* The nanosecond part of the time difference. The total time difference
* could be computed as:
* `MS_TIME_DIFF(a, b) * 1000000 + NS_FRAC_TIME_DIFF(a, b)`.
*/
# define NS_FRAC_TIME_DIFF(a, b) \
((unsigned long)((a.tv_usec < b.tv_usec \
&& (long)(a.tv_usec - b.tv_usec) % 1000 != 0 \
? 1000L \
: 0) \
+ (long)(a.tv_usec - b.tv_usec) % 1000) \
* 1000)
# elif defined(MSWIN32) || defined(MSWINCE) || defined(WINXP_USE_PERF_COUNTER)
# if defined(MSWINRT_FLAVOR) || defined(WINXP_USE_PERF_COUNTER)
# define CLOCK_TYPE ULONGLONG
/*
* Note: two standalone `if` statements below are used to avoid MS VC
* false warning (FP) about potentially uninitialized `tc` variable.
*/
# define GET_TIME(x) \
do { \
LARGE_INTEGER freq, tc; \
if (!QueryPerformanceFrequency(&freq)) \
ABORT("QueryPerformanceFrequency requires WinXP+"); \
if (!QueryPerformanceCounter(&tc)) \
ABORT("QueryPerformanceCounter failed"); \
x = (CLOCK_TYPE)((double)tc.QuadPart / freq.QuadPart * 1e9); \
/* TODO: Call QueryPerformanceFrequency once at GC init. */ \
} while (0)
# define MS_TIME_DIFF(a, b) ((unsigned long)(((a) - (b)) / 1000000UL))
# define NS_FRAC_TIME_DIFF(a, b) \
((unsigned long)(((a) - (b)) % 1000000UL))
# else
# define CLOCK_TYPE DWORD
# define GET_TIME(x) (void)(x = GetTickCount())
# define MS_TIME_DIFF(a, b) ((unsigned long)((a) - (b)))
# define NS_FRAC_TIME_DIFF(a, b) 0UL
# endif /* !WINXP_USE_PERF_COUNTER */
# elif defined(NN_PLATFORM_CTR)
# define CLOCK_TYPE long long
EXTERN_C_BEGIN
CLOCK_TYPE n3ds_get_system_tick(void);
CLOCK_TYPE n3ds_convert_tick_to_ms(CLOCK_TYPE tick);
EXTERN_C_END
# define GET_TIME(x) (void)(x = n3ds_get_system_tick())
# define MS_TIME_DIFF(a, b) \
((unsigned long)n3ds_convert_tick_to_ms((a) - (b)))
/* TODO: Implement NS_FRAC_TIME_DIFF(). */
# define NS_FRAC_TIME_DIFF(a, b) 0UL
# elif defined(HAVE_CLOCK_GETTIME)
# include <time.h>
# define CLOCK_TYPE struct timespec
# define CLOCK_TYPE_INITIALIZER C_INITIALIZER_BEGIN 0, 0 C_INITIALIZER_END
# if defined(_POSIX_MONOTONIC_CLOCK) && !defined(NINTENDO_SWITCH)
# define GET_TIME(x) \
do { \
if (clock_gettime(CLOCK_MONOTONIC, &x) == -1) \
ABORT("clock_gettime failed"); \
} while (0)
# else
# define GET_TIME(x) \
do { \
if (clock_gettime(CLOCK_REALTIME, &x) == -1) \
ABORT("clock_gettime failed"); \
} while (0)
# endif
# define MS_TIME_DIFF(a, b) \
/* `a.tv_nsec - b.tv_nsec` is in range -1e9 to 1e9, exclusively. */ \
((unsigned long)((a).tv_nsec + (1000000L * 1000 - (b).tv_nsec)) \
/ 1000000UL \
+ ((unsigned long)((a).tv_sec - (b).tv_sec) * 1000UL) - 1000UL)
# define NS_FRAC_TIME_DIFF(a, b) \
((unsigned long)((a).tv_nsec + (1000000L * 1000 - (b).tv_nsec)) \
% 1000000UL)
# else /* !BSD_TIME && !LINUX && !NN_PLATFORM_CTR && !MSWIN32 */
# include <time.h>
# if defined(FREEBSD) && !defined(CLOCKS_PER_SEC)
# include <machine/limits.h>
# define CLOCKS_PER_SEC CLK_TCK
# endif
# if !defined(CLOCKS_PER_SEC)
/*
* This is technically a bug in the implementation.
* ANSI requires that `CLOCKS_PER_SEC` be defined. But at least under
* SunOS 4.1.1, it is not. Also note that the combination of ANSI C
* and POSIX is incredibly gross here. The type `clock_t` is used by
* both `clock()` and `times()`. But on some machines these use
* different notions of a clock tick, `CLOCKS_PER_SEC` seems to apply
* only to `clock()`. Hence we use it here. On many machines,
* including SunOS, `clock()` actually uses units of microseconds (that
* are not really clock ticks).
*/
# define CLOCKS_PER_SEC 1000000
# endif
# define CLOCK_TYPE clock_t
# define GET_TIME(x) (void)(x = clock())
# define MS_TIME_DIFF(a, b) \
(CLOCKS_PER_SEC % 1000 == 0 \
? (unsigned long)((a) - (b)) \
/ (unsigned long)(CLOCKS_PER_SEC / 1000) \
: ((unsigned long)((a) - (b)) * 1000) \
/ (unsigned long)CLOCKS_PER_SEC)
/*
* Avoid using `double` type since some targets (like ARM) might
* require `-lm` option for `double`-to-`long` conversion.
*/
# define NS_FRAC_TIME_DIFF(a, b) \
(CLOCKS_PER_SEC <= 1000 \
? 0UL \
: (unsigned long)(CLOCKS_PER_SEC <= (clock_t)1000000UL \
? (((a) - (b)) \
* ((clock_t)1000000UL / CLOCKS_PER_SEC) \
% 1000) \
* 1000 \
: (CLOCKS_PER_SEC \
<= (clock_t)1000000UL * 1000 \
? ((a) - (b)) \
* ((clock_t)1000000UL * 1000 \
/ CLOCKS_PER_SEC) \
: (((a) - (b)) * (clock_t)1000000UL \
* 1000) \
/ CLOCKS_PER_SEC) \
% (clock_t)1000000UL))
# endif /* !BSD_TIME && !MSWIN32 */
# ifndef CLOCK_TYPE_INITIALIZER
/*
* This is used to initialize `CLOCK_TYPE` variables (to some value)
* to avoid "variable might be uninitialized" compiler warnings.
*/
# define CLOCK_TYPE_INITIALIZER 0
# endif
#endif /* !NO_CLOCK */
/* We use `bzero()` and `bcopy()` internally. They may not be available. */
#if defined(M68K) && defined(NEXT) || defined(VAX)
# define BCOPY_EXISTS
#elif defined(DARWIN)
# include <string.h>
# define BCOPY_EXISTS
#endif
#if !defined(BCOPY_EXISTS) || defined(CPPCHECK)
# include <string.h>
# define BCOPY(x, y, n) memcpy(y, x, (size_t)(n))
# define BZERO(x, n) memset(x, 0, (size_t)(n))
#else
# define BCOPY(x, y, n) bcopy((void *)(x), (void *)(y), (size_t)(n))
# define BZERO(x, n) bzero((void *)(x), (size_t)(n))
#endif
EXTERN_C_BEGIN
#if defined(CPPCHECK) && defined(ANY_MSWIN)
# undef TEXT
# ifdef UNICODE
# define TEXT(s) L##s
# else
# define TEXT(s) s
# endif
#endif /* CPPCHECK && ANY_MSWIN */
/* Stop and restart mutator threads. */
#if defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH) \
|| defined(GC_WIN32_THREADS) || defined(GC_PTHREADS)
GC_INNER void GC_stop_world(void);
GC_INNER void GC_start_world(void);
# define STOP_WORLD() GC_stop_world()
# define START_WORLD() GC_start_world()
#else
/* Just do a sanity check: we are not inside `GC_do_blocking()`. */
# define STOP_WORLD() GC_ASSERT(GC_blocked_sp == NULL)
# define START_WORLD()
#endif
/* Abandon ship. */
#ifdef SMALL_CONFIG
# define GC_on_abort(msg) (void)0 /*< be silent on abort */
#else
GC_API_PRIV GC_abort_func GC_on_abort;
#endif
#if defined(CPPCHECK)
# define ABORT(msg) \
{ \
GC_on_abort(msg); \
abort(); \
}
#else
# if defined(MSWIN_XBOX1) && !defined(DebugBreak)
# define DebugBreak() __debugbreak()
# elif defined(MSWINCE) && !defined(DebugBreak) \
&& (!defined(UNDER_CE) || (defined(__MINGW32CE__) && !defined(ARM32)))
/*
* This simplifies linking for WinCE (and, probably, does not
* hurt debugging much); use `-D DebugBreak=DebugBreak` to override
* this behavior if really needed. This is also a workaround for
* x86mingw32ce toolchain (if it is still declaring `DebugBreak()`
* instead of defining it as a macro).
*/
# define DebugBreak() _exit(-1) /*< there is no `abort()` in WinCE */
# endif
# if defined(MSWIN32) && (defined(NO_DEBUGGING) || defined(LINT2))
/*
* A more user-friendly abort after showing fatal message.
* Exit on error without running "at-exit" callbacks.
*/
# define ABORT(msg) (GC_on_abort(msg), _exit(-1))
# elif defined(MSWINCE) && defined(NO_DEBUGGING)
# define ABORT(msg) (GC_on_abort(msg), ExitProcess(-1))
# elif defined(MSWIN32) || defined(MSWINCE)
# if defined(_CrtDbgBreak) && defined(_DEBUG) && defined(_MSC_VER)
# define ABORT(msg) \
{ \
GC_on_abort(msg); \
_CrtDbgBreak() /*< `__debugbreak()` */; \
}
# else
# define ABORT(msg) \
{ \
GC_on_abort(msg); \
DebugBreak(); \
}
/*
* Note: on a WinCE box, this could be silently ignored (i.e., the program
* is not aborted); `DebugBreak()` is a statement in some toolchains.
*/
# endif
# else /* !MSWIN32 */
# define ABORT(msg) (GC_on_abort(msg), abort())
# endif
#endif /* !CPPCHECK */
/*
* For the abort message with 1 .. 3 arguments. `C_msg` and `C_fmt`
* should be literals. `C_msg` should not contain format specifiers.
* Arguments should match their format specifiers.
*/
#define ABORT_ARG1(C_msg, C_fmt, arg1) \
MACRO_BLKSTMT_BEGIN \
GC_ERRINFO_PRINTF(C_msg /* + */ C_fmt "\n", arg1); \
ABORT(C_msg); \
MACRO_BLKSTMT_END
#define ABORT_ARG2(C_msg, C_fmt, arg1, arg2) \
MACRO_BLKSTMT_BEGIN \
GC_ERRINFO_PRINTF(C_msg /* + */ C_fmt "\n", arg1, arg2); \
ABORT(C_msg); \
MACRO_BLKSTMT_END
#define ABORT_ARG3(C_msg, C_fmt, arg1, arg2, arg3) \
MACRO_BLKSTMT_BEGIN \
GC_ERRINFO_PRINTF(C_msg /* + */ C_fmt "\n", arg1, arg2, arg3); \
ABORT(C_msg); \
MACRO_BLKSTMT_END
/*
* Same as `ABORT` but does not have a `noreturn` attribute.
* I.e. `ABORT` on a dummy condition (which is always true).
*/
#define ABORT_RET(msg) \
if ((GC_funcptr_uint)GC_current_warn_proc == ~(GC_funcptr_uint)0) { \
} else \
ABORT(msg)
/* Exit process abnormally, but without making a mess (e.g. out of memory). */
#define EXIT() (GC_on_abort(NULL), exit(1 /* `EXIT_FAILURE` */))
/*
* Print warning message, e.g. almost out of memory. The argument (if any)
* format specifier should be: "%s", "%p", "%"`WARN_PRIdPTR` or
* "%"`WARN_PRIuPTR`.
*/
#define WARN(msg, arg) \
GC_current_warn_proc("GC Warning: " msg, (GC_uintptr_t)(arg))
GC_EXTERN GC_warn_proc GC_current_warn_proc;
/*
* Print format type macro for decimal `GC_signed_word` value passed to
* `WARN()`. This could be redefined for Win64 or LLP64, but typically
* should not be done as the `WARN` format string is, possibly,
* processed on the client side, so non-standard print type modifiers
* (like MS "I64d") should be avoided here if possible.
* TODO: Assuming `sizeof(void *)` is equal to `sizeof(long)` or this
* is a little-endian machine.
*/
#ifndef WARN_PRIdPTR
# define WARN_PRIdPTR "ld"
# define WARN_PRIuPTR "lu"
#endif
/*
* A tagging macro (for a code static analyzer) to indicate that the
* string obtained from an untrusted source (e.g., `argv[]`, `getenv`)
* is safe to use in a vulnerable operation (e.g., `open`, `exec`).
*/
#define TRUSTED_STRING(s) COVERT_DATAFLOW_P(s)
#ifdef GC_READ_ENV_FILE
/*
* This routine scans `GC_envfile_content` for the specified environment
* variable (and returns its value if found).
*/
GC_INNER char *GC_envfile_getenv(const char *name);
/* Get the process environment entry. */
# define GETENV(name) GC_envfile_getenv(name)
#elif defined(NO_GETENV) && !defined(CPPCHECK)
# define GETENV(name) NULL
#elif defined(EMPTY_GETENV_RESULTS) && !defined(CPPCHECK)
/* Workaround for a reputed Wine bug. */
GC_INLINE char *
fixed_getenv(const char *name)
{
char *value = getenv(name);
return value != NULL && *value != '\0' ? value : NULL;
}
# define GETENV(name) fixed_getenv(name)
#else
# define GETENV(name) getenv(name)
#endif
EXTERN_C_END
#if defined(DARWIN)
# include <mach/thread_status.h>
# ifndef MAC_OS_X_VERSION_MAX_ALLOWED
/* Include this header just to import the above macro. */
# include <AvailabilityMacros.h>
# endif
# if defined(POWERPC)
# if CPP_WORDSZ == 32
# define GC_THREAD_STATE_T ppc_thread_state_t
# else
# define GC_THREAD_STATE_T ppc_thread_state64_t
# define GC_MACH_THREAD_STATE PPC_THREAD_STATE64
# define GC_MACH_THREAD_STATE_COUNT PPC_THREAD_STATE64_COUNT
# endif
# elif defined(I386) || defined(X86_64)
# if CPP_WORDSZ == 32
# if defined(i386_THREAD_STATE_COUNT) \
&& !defined(x86_THREAD_STATE32_COUNT)
/* Use old naming convention for i686. */
# define GC_THREAD_STATE_T i386_thread_state_t
# define GC_MACH_THREAD_STATE i386_THREAD_STATE
# define GC_MACH_THREAD_STATE_COUNT i386_THREAD_STATE_COUNT
# else
# define GC_THREAD_STATE_T x86_thread_state32_t
# define GC_MACH_THREAD_STATE x86_THREAD_STATE32
# define GC_MACH_THREAD_STATE_COUNT x86_THREAD_STATE32_COUNT
# endif
# else
# define GC_THREAD_STATE_T x86_thread_state64_t
# define GC_MACH_THREAD_STATE x86_THREAD_STATE64
# define GC_MACH_THREAD_STATE_COUNT x86_THREAD_STATE64_COUNT
# endif
# elif defined(ARM32) && defined(ARM_UNIFIED_THREAD_STATE) \
&& !defined(CPPCHECK)
# define GC_THREAD_STATE_T arm_unified_thread_state_t
# define GC_MACH_THREAD_STATE ARM_UNIFIED_THREAD_STATE
# define GC_MACH_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT
# elif defined(ARM32)
# define GC_THREAD_STATE_T arm_thread_state_t
# ifdef ARM_MACHINE_THREAD_STATE_COUNT
# define GC_MACH_THREAD_STATE ARM_MACHINE_THREAD_STATE
# define GC_MACH_THREAD_STATE_COUNT ARM_MACHINE_THREAD_STATE_COUNT
# endif
# elif defined(AARCH64)
# define GC_THREAD_STATE_T arm_thread_state64_t
# define GC_MACH_THREAD_STATE ARM_THREAD_STATE64
# define GC_MACH_THREAD_STATE_COUNT ARM_THREAD_STATE64_COUNT
# elif !defined(CPPCHECK)
# error define GC_THREAD_STATE_T
# endif
# ifndef GC_MACH_THREAD_STATE
# define GC_MACH_THREAD_STATE MACHINE_THREAD_STATE
# define GC_MACH_THREAD_STATE_COUNT MACHINE_THREAD_STATE_COUNT
# endif
/*
* Try to work out the right way to access thread state structure members.
* The structure has different definition in different Darwin versions.
* This now defaults to the (older) names without `__`, thus hopefully
* not breaking any existing `Makefile.direct` builds.
*/
# if __DARWIN_UNIX03
# define THREAD_FLD_NAME(x) __##x
# else
# define THREAD_FLD_NAME(x) x
# endif
# if defined(ARM32) && defined(ARM_UNIFIED_THREAD_STATE)
# define THREAD_FLD(x) ts_32.THREAD_FLD_NAME(x)
# else
# define THREAD_FLD(x) THREAD_FLD_NAME(x)
# endif
#endif /* DARWIN */
#ifndef WASI
# include <setjmp.h>
#endif
#include <stdio.h>
#if defined(CAN_HANDLE_FORK) && defined(GC_PTHREADS)
# include <pthread.h> /*< for `pthread_t` */
#endif
#if __STDC_VERSION__ >= 201112L
# include <assert.h> /*< for `static_assert` */
#endif
EXTERN_C_BEGIN
/* Definitions depending on `word` size. */
#define modWORDSZ(n) ((n) & (CPP_WORDSZ - 1)) /*< `n mod size_of_word` */
#define divWORDSZ(n) ((n) / CPP_WORDSZ)
#define SIGNB ((word)1 << (CPP_WORDSZ - 1))
#define SIZET_SIGNB (GC_SIZE_MAX ^ (GC_SIZE_MAX >> 1))
#if CPP_PTRSZ / 8 != ALIGNMENT
# define UNALIGNED_PTRS
#endif
#define BYTES_TO_GRANULES(lb) ((lb) / GC_GRANULE_BYTES)
#define GRANULES_TO_BYTES(lg) (GC_GRANULE_BYTES * (lg))
#define BYTES_TO_PTRS(lb) ((lb) / sizeof(ptr_t))
#define PTRS_TO_BYTES(lpw) ((lpw) * sizeof(ptr_t))
#define GRANULES_TO_PTRS(lg) (GC_GRANULE_PTRS * (lg))
/*
* Convert size in bytes to that in pointers rounding up (but not adding
* extra byte at end).
*/
#define BYTES_TO_PTRS_ROUNDUP(lb) BYTES_TO_PTRS((lb) + sizeof(ptr_t) - 1)
/* Size parameters. */
/*
* Heap block size, in bytes. Should be a power of two.
* Incremental collection with `MPROTECT_VDB` currently requires the
* page size to be a multiple of `HBLKSIZE`. Since most modern
* architectures support variable page sizes down to 4 KB, and i686 and
* x86_64 are generally 4 KB, we now default to 4 KB, except for:
* - Alpha: seems to be used with 8 KB pages;
* - `SMALL_CONFIG`: want less block-level fragmentation.
*/
#ifndef HBLKSIZE
# if defined(SMALL_CONFIG) && !defined(LARGE_CONFIG)
# define CPP_LOG_HBLKSIZE 10
# elif defined(ALPHA)
# define CPP_LOG_HBLKSIZE 13
# else
# define CPP_LOG_HBLKSIZE 12
# endif
#else
# if HBLKSIZE == 512
# define CPP_LOG_HBLKSIZE 9
# elif HBLKSIZE == 1024
# define CPP_LOG_HBLKSIZE 10
# elif HBLKSIZE == 2048
# define CPP_LOG_HBLKSIZE 11
# elif HBLKSIZE == 4096
# define CPP_LOG_HBLKSIZE 12
# elif HBLKSIZE == 8192
# define CPP_LOG_HBLKSIZE 13
# elif HBLKSIZE == 16384
# define CPP_LOG_HBLKSIZE 14
# elif HBLKSIZE == 32768
# define CPP_LOG_HBLKSIZE 15
# elif HBLKSIZE == 65536
# define CPP_LOG_HBLKSIZE 16
# elif !defined(CPPCHECK)
# error Bad HBLKSIZE value
# endif
# undef HBLKSIZE
#endif
#define LOG_HBLKSIZE ((size_t)CPP_LOG_HBLKSIZE)
#define HBLKSIZE ((size_t)1 << CPP_LOG_HBLKSIZE)
#define GC_SQRT_SIZE_MAX ((((size_t)1) << (sizeof(size_t) * 8 / 2)) - 1)
/*
* Maximum size of objects supported by free list (larger objects are
* allocated directly with `GC_alloc_large()`, by rounding to the next
* multiple of `HBLKSIZE`).
*/
#define MAXOBJBYTES (HBLKSIZE >> 1)
#define MAXOBJGRANULES BYTES_TO_GRANULES(MAXOBJBYTES)
#define divHBLKSZ(n) ((n) >> LOG_HBLKSIZE)
/*
* Equivalent to subtracting one `hblk` pointer from another. We do it
* this way because a compiler should find it hard to use an integer
* division instead of a shift. The bundled SunOS 4.1 otherwise sometimes
* pessimizes the subtraction to involve a call to `.div`.
*/
#define HBLK_PTR_DIFF(p, q) divHBLKSZ((ptr_t)p - (ptr_t)q)
#define modHBLKSZ(n) ((n) & (HBLKSIZE - 1))
#define HBLKPTR(objptr) \
((struct hblk *)PTR_ALIGN_DOWN((ptr_t)(objptr), HBLKSIZE))
#define HBLKDISPL(objptr) modHBLKSZ((size_t)ADDR(objptr))
/* Same as `HBLKPTR` but points to the first block in the page. */
#define HBLK_PAGE_ALIGNED(objptr) \
((struct hblk *)PTR_ALIGN_DOWN((ptr_t)(objptr), GC_page_size))
/* Round up allocation size (in bytes) to a multiple of a granule. */
#define ROUNDUP_GRANULE_SIZE(lb) /*< `lb` should have no side-effect */ \
(SIZET_SAT_ADD(lb, GC_GRANULE_BYTES - 1) & ~(size_t)(GC_GRANULE_BYTES - 1))
/*
* Round up byte allocation request (after adding `EXTRA_BYTES`) to
* a multiple of a granule, then convert it to granules.
*/
#define ALLOC_REQUEST_GRANS(lb) /*< `lb` should have no side-effect */ \
BYTES_TO_GRANULES(SIZET_SAT_ADD(lb, GC_GRANULE_BYTES - 1 + EXTRA_BYTES))
#if MAX_EXTRA_BYTES == 0
# define ADD_EXTRA_BYTES(lb) (lb)
# define SMALL_OBJ(lb) LIKELY((lb) <= MAXOBJBYTES)
#else
# define ADD_EXTRA_BYTES(lb) /*< `lb` should have no side-effect */ \
SIZET_SAT_ADD(lb, EXTRA_BYTES)
/*
* This really just tests that `lb` is not greater than
* `MAXOBJBYTES - EXTRA_BYTES`, but we try to avoid looking up `EXTRA_BYTES`.
*/
# define SMALL_OBJ(lb) /*< `lb` should have no side-effect */ \
(LIKELY((lb) <= MAXOBJBYTES - MAX_EXTRA_BYTES) \
|| (lb) <= MAXOBJBYTES - EXTRA_BYTES)
#endif
/*
* Hash table representation of sets of pages. Implements a map from
* `HBLKSIZE`-aligned chunks of the address space to one bit each.
* This assumes it is OK to spuriously set bits, e.g. because multiple
* addresses are represented by a single location. Used by
* black-listing code, and perhaps by dirty bit maintenance code.
*/
#ifndef LOG_PHT_ENTRIES
# ifdef LARGE_CONFIG
# if CPP_WORDSZ == 32
/*
* Collisions are impossible (because of a 4 GB space limit).
* Each table takes 128 KB, some of which may never be touched.
*/
# define LOG_PHT_ENTRIES 20
# else
/*
* Collisions likely at 2M blocks, which is greater than 8 GB.
* Each table takes 256 KB, some of which may never be touched.
*/
# define LOG_PHT_ENTRIES 21
# endif
# elif !defined(SMALL_CONFIG)
/*
* Collisions are likely if heap grows to more than 256K blocks,
* which is greater than 1 GB. Each hash table occupies 32 KB.
* Even for somewhat smaller heaps, say half of that, collisions may
* be an issue because we blacklist addresses outside the heap.
*/
# define LOG_PHT_ENTRIES 18
# else
/*
* Collisions are likely if heap grows to more than 32K blocks,
* which is 128 MB. Each hash table occupies 4 KB.
*/
# define LOG_PHT_ENTRIES 15
# endif
#endif /* !LOG_PHT_ENTRIES */
#define PHT_ENTRIES (1 << LOG_PHT_ENTRIES)
#define PHT_SIZE (PHT_ENTRIES > CPP_WORDSZ ? PHT_ENTRIES / CPP_WORDSZ : 1)
typedef word page_hash_table[PHT_SIZE];
#define PHT_HASH(p) ((size_t)((ADDR(p) >> LOG_HBLKSIZE) & (PHT_ENTRIES - 1)))
#define get_pht_entry_from_index(bl, index) \
(((bl)[divWORDSZ(index)] >> modWORDSZ(index)) & 1)
#define set_pht_entry_from_index(bl, index) \
(void)((bl)[divWORDSZ(index)] |= (word)1 << modWORDSZ(index))
#if defined(THREADS) && defined(AO_HAVE_or)
/*
* And, one more variant for `GC_add_to_black_list_normal` and
*`GC_add_to_black_list_stack` (invoked indirectly by `GC_do_local_mark()`)
* and `async_set_pht_entry_from_index()` (invoked by `GC_dirty()` or the
* write fault handler).
*/
# define set_pht_entry_from_index_concurrent(bl, index) \
AO_or((volatile AO_t *)&(bl)[divWORDSZ(index)], \
(AO_t)1 << modWORDSZ(index))
# ifdef MPROTECT_VDB
# define set_pht_entry_from_index_concurrent_volatile(bl, index) \
set_pht_entry_from_index_concurrent(bl, index)
# endif
#else
# define set_pht_entry_from_index_concurrent(bl, index) \
set_pht_entry_from_index(bl, index)
# ifdef MPROTECT_VDB
/*
* Same as `set_pht_entry_from_index` but avoiding the compound assignment
* for a `volatile` array.
*/
# define set_pht_entry_from_index_concurrent_volatile(bl, index) \
(void)((bl)[divWORDSZ(index)] \
= (bl)[divWORDSZ(index)] | ((word)1 << modWORDSZ(index)))
# endif
#endif
/* Heap blocks. */
/*
* The upper bound. We allocate 1 bit per allocation granule.
* If `MARK_BIT_PER_OBJ` is not defined, we use every `n`-th bit, where
* `n` is the number of allocation granules per object. Otherwise, we only
* use the initial group of mark bits, and it is safe to allocate smaller
* header for large objects.
*/
#define MARK_BITS_PER_HBLK (HBLKSIZE / GC_GRANULE_BYTES)
#ifndef MARK_BIT_PER_OBJ
/*
* We maintain layout maps for heap blocks containing objects of
* a given size. Each entry in this map describes a byte offset
* (displacement) and has the following type.
*/
# if (1 << (CPP_LOG_HBLKSIZE - 1)) / GC_GRANULE_BYTES <= 0x100
typedef unsigned char hb_map_entry_t;
# else
typedef unsigned short hb_map_entry_t;
# endif
#endif /* !MARK_BIT_PER_OBJ */
struct hblkhdr {
/*
* Link field for `hblk` free list and for lists of chunks waiting to
* be reclaimed.
*/
struct hblk *hb_next;
struct hblk *hb_prev; /*< backwards link for free list */
struct hblk *hb_block; /*< the corresponding block */
/*
* Kind of objects in the block. Each kind identifies a mark
* procedure and a set of list headers. Sometimes called regions.
*/
unsigned char hb_obj_kind;
unsigned char hb_flags;
/* Ignore pointers that do not point to the first `hblk` of this object. */
#define IGNORE_OFF_PAGE 1
/*
* This is a free block, which has been unmapped from the address space.
* `GC_remap()` must be invoked on it before it can be reallocated.
* Set only if `USE_MUNMAP` macro is defined.
*/
#define WAS_UNMAPPED 2
/* Block is free, i.e. not in use. */
#define FREE_BLK 4
#ifdef ENABLE_DISCLAIM
/* This kind has a callback on reclaim. */
# define HAS_DISCLAIM 8
/*
* Mark from all objects, marked or not. Used to mark objects needed
* by reclaim notifier.
*/
# define MARK_UNCONDITIONALLY 0x10
#endif
#ifndef MARK_BIT_PER_OBJ
# define LARGE_BLOCK 0x20
#endif
/*
* Value of `GC_gc_no` when block was last allocated or swept.
* May wrap. For a free block, this is maintained only for `USE_MUNMAP`,
* and indicates when the header was allocated, or when the size of the
* block last changed.
*/
unsigned short hb_last_reclaimed;
#ifdef MARK_BIT_PER_OBJ
# define LARGE_INV_SZ ((unsigned32)1 << 16)
/*
* A good upper bound for `2**32 / hb_sz`.
* For large objects, we use `LARGE_INV_SZ`.
*/
unsigned32 hb_inv_sz;
#endif
/*
* If in use, size in bytes, of objects in the block.
* Otherwise, the size of the whole free block. We assume that this is
* convertible to `GC_signed_word` without generating a negative result.
* We avoid generating free blocks larger than that.
*/
size_t hb_sz;
/* Object descriptor for marking. See `gc_mark.h` file. */
word hb_descr;
#ifndef MARK_BIT_PER_OBJ
/*
* A table of remainders `mod BYTES_TO_GRANULES(hb_sz)` essentially,
* except for large blocks. See `GC_obj_map`.
*/
hb_map_entry_t *hb_map;
#endif
#ifdef PARALLEL_MARK
/*
* Number of set mark bits, excluding the one always set at the end.
* Currently it is updated concurrently and hence only approximate.
* But a zero value does guarantee that the block contains
* no marked objects. Ensuring this property means that we never
* decrement it to zero during a collection, and hence the count may
* be one too high. Due to concurrent updates, an arbitrary number
* of increments, but not all of them (!) may be lost, hence it may,
* in theory, be much too low. The count may also be too high
* if multiple mark threads mark the same object due to a race.
*/
volatile AO_t hb_n_marks;
#else
/* Without parallel marking, the count is accurate. */
size_t hb_n_marks;
#endif
#ifdef USE_MARK_BYTES
/*
* Unlike the other case, this is in units of bytes. Since we force
* certain alignment, we need at most one mark bit per a granule.
* But we do allocate and set one extra mark bit to avoid
* an explicit check for the partial object at the end of each block.
*/
# define HB_MARKS_SZ (MARK_BITS_PER_HBLK + 1)
union {
/*
* The `i`-th byte is 1 if the object starting at granule `i`
* or object `i` is marked, 0 otherwise. The mark bit for the
* "one past the end" object is always set to avoid a special case
* test in the marker.
*/
char _hb_marks[HB_MARKS_SZ];
word dummy; /*< force word alignment of mark bytes */
} _mark_byte_union;
# define hb_marks _mark_byte_union._hb_marks
#else
# define HB_MARKS_SZ (MARK_BITS_PER_HBLK / CPP_WORDSZ + 1)
# if defined(PARALLEL_MARK) || (defined(THREAD_SANITIZER) && defined(THREADS))
volatile AO_t hb_marks[HB_MARKS_SZ];
# else
word hb_marks[HB_MARKS_SZ];
# endif
#endif /* !USE_MARK_BYTES */
};
/* A "random" mark bit index for assertions. */
#define ANY_INDEX 23
/* Heap block body. */
#define HBLK_WORDS (HBLKSIZE / sizeof(word))
#define HBLK_GRANULES (HBLKSIZE / GC_GRANULE_BYTES)
/*
* The number of objects in a block dedicated to a certain size.
* May erroneously yield zero (instead of one) for large objects.
*/
#define HBLK_OBJS(sz_in_bytes) (HBLKSIZE / (sz_in_bytes))
struct hblk {
char hb_body[HBLKSIZE];
};
#define HBLK_IS_FREE(hhdr) (((hhdr)->hb_flags & FREE_BLK) != 0)
#define OBJ_SZ_TO_BLOCKS(lb) divHBLKSZ((lb) + HBLKSIZE - 1)
/*
* Size of block (in units of `HBLKSIZE`) needed to hold objects of
* given `lb` (in bytes). The checked variant prevents wrap around.
*/
#define OBJ_SZ_TO_BLOCKS_CHECKED(lb) /*< `lb` should have no side-effect */ \
divHBLKSZ(SIZET_SAT_ADD(lb, HBLKSIZE - 1))
/* The object free-list link. */
#define obj_link(p) (*(void **)(p))
/*
* Root sets. Logically private to `mark_rts.c` file. But we do not
* want the tables scanned, so we put them here.
*/
/* The maximum number of ranges that can be registered as static roots. */
#ifdef LARGE_CONFIG
# define MAX_ROOT_SETS 8192
#elif !defined(SMALL_CONFIG)
# define MAX_ROOT_SETS 2048
#else
# define MAX_ROOT_SETS 512
#endif
/* Maximum number of segments that can be excluded from root sets. */
#define MAX_EXCLUSIONS (MAX_ROOT_SETS / 4)
/* A data structure for excluded static roots. */
struct exclusion {
ptr_t e_start;
ptr_t e_end;
};
/*
* A data structure for list of root sets. We keep a hash table, so that
* we can filter out duplicate additions. Under Win32, we need to do
* a better job of filtering overlaps, so we resort to sequential search,
* and pay the price.
*/
struct roots {
ptr_t r_start; /*< multiple of pointer size */
ptr_t r_end; /*< multiple of pointer size and greater than `r_start` */
#ifndef ANY_MSWIN
struct roots *r_next;
#endif
/* Delete before registering new dynamic libraries if set. */
GC_bool r_tmp;
};
#ifndef ANY_MSWIN
/* Size of hash table index to roots. */
# define LOG_RT_SIZE 6
/* `RT_SIZE` should be a power of 2, may be not equal to `MAX_ROOT_SETS`. */
# define RT_SIZE (1 << LOG_RT_SIZE)
#endif
#if (!defined(MAX_HEAP_SECTS) || defined(CPPCHECK)) \
&& (defined(ANY_MSWIN) || defined(USE_PROC_FOR_LIBRARIES))
# ifdef LARGE_CONFIG
# if CPP_WORDSZ > 32
# define MAX_HEAP_SECTS 81920
# else
# define MAX_HEAP_SECTS 7680
# endif
# elif defined(SMALL_CONFIG) && !defined(USE_PROC_FOR_LIBRARIES)
# if defined(PARALLEL_MARK) && (defined(MSWIN32) || defined(CYGWIN32))
# define MAX_HEAP_SECTS 384
# else
# define MAX_HEAP_SECTS 128 /*< roughly 256 MB (`128 * 2048 * 1024`) */
# endif
# elif CPP_WORDSZ > 32
# define MAX_HEAP_SECTS 1024 /*< roughly 8 GB */
# else
# define MAX_HEAP_SECTS 512 /*< roughly 4 GB */
# endif
#endif /* !MAX_HEAP_SECTS */
typedef struct GC_ms_entry {
ptr_t mse_start; /*< beginning of object, pointer-aligned one */
#ifdef PARALLEL_MARK
volatile AO_t mse_descr;
#else
/*
* The descriptor; the low-order two bits are tags, as described
* in `gc_mark.h` file.
*/
word mse_descr;
#endif
} mse;
/*
* Current state of marking. Used to remember where we are during the
* concurrent marking.
*/
typedef int mark_state_t;
struct disappearing_link;
struct finalizable_object;
struct dl_hashtbl_s {
struct disappearing_link **head;
size_t entries;
unsigned log_size;
};
struct fnlz_roots_s {
struct finalizable_object **fo_head;
/* List of objects that should be finalized now. */
struct finalizable_object *finalize_now;
};
union toggle_ref_u {
/* The least significant bit is used to distinguish between choices. */
void *strong_ref;
GC_hidden_pointer weak_ref;
};
/*
* Extended descriptors. `GC_typed_mark_proc` understands these.
* These are used for simple objects that are larger than what can
* be described by a `BITMAP_BITS`-sized bitmap.
*/
typedef struct {
/*
* The least significant bit corresponds to the first "pointer-sized"
* word.
*/
word ed_bitmap;
GC_bool ed_continued; /*< next entry is continuation */
} typed_ext_descr_t;
struct HeapSect {
ptr_t hs_start;
size_t hs_bytes;
};
#ifdef MAKE_BACK_GRAPH
/* The maximum in-degree we handle directly. */
# define BACKGRAPH_MAX_IN 10
struct back_edges_s {
/* Number of edges, including those in continuation structures. */
word n_edges;
unsigned short flags;
/*
* If `height` is greater than zero, then keeps the `GC_gc_no` value
* when it was computed. If it was computed this cycle, then it is
* current. If it was computed during the last cycle, then it belongs
* to the old height, which is only saved for live objects referenced by
* dead ones. This may grow due to references from newly dead objects.
*/
unsigned short height_gc_no;
/*
* Longest path through unreachable nodes to this node that we found
* using depth first search.
*/
GC_signed_word height;
ptr_t edges[BACKGRAPH_MAX_IN];
/*
* Pointer to continuation structure; we use only the edges field in
* the continuation. Also used as a free-list link.
*/
struct back_edges_s *cont;
};
#endif
/*
* Lists of all heap blocks and free lists as well as other random data
* structures that should not be scanned by the collector. These are
* grouped together in a structure so that they can be easily skipped by
* `GC_push_conditional_with_exclusions()`. The ordering is weird to
* make `GC_malloc` faster by keeping the important fields sufficiently
* close together that a single load of a base register will do.
* Scalars that could easily appear to be pointers are also put here.
* The main fields should precede any conditionally included fields, where
* possible.
*/
struct _GC_arrays {
#define GC_heapsize GC_arrays._heapsize
word _heapsize; /*< heap size in bytes (value never goes down) */
#define GC_requested_heapsize GC_arrays._requested_heapsize
word _requested_heapsize; /*< heap size due to explicit expansion */
#define GC_heapsize_on_gc_disable GC_arrays._heapsize_on_gc_disable
word _heapsize_on_gc_disable;
#define GC_heapsize_at_forced_unmap GC_arrays._heapsize_at_forced_unmap
word _heapsize_at_forced_unmap; /*< accessed with the allocator lock held */
#define GC_last_heap_addr GC_arrays._last_heap_addr
word _last_heap_addr;
/*
* Total bytes contained in blocks on the free list of large objects.
* (A large object is the one that occupies a block of at least
* two `HBLKSIZE`.)
*/
#define GC_large_free_bytes GC_arrays._large_free_bytes
word _large_free_bytes;
/* Total number of bytes in allocated large objects blocks. */
#define GC_large_allocd_bytes GC_arrays._large_allocd_bytes
word _large_allocd_bytes;
/*
* Maximum number of bytes that were ever allocated in large object blocks.
* This is used to help decide when it is safe to split up a large block.
*/
#define GC_max_large_allocd_bytes GC_arrays._max_large_allocd_bytes
word _max_large_allocd_bytes;
/* Number of bytes allocated before this collection cycle. */
#define GC_bytes_allocd_before_gc GC_arrays._bytes_allocd_before_gc
word _bytes_allocd_before_gc;
#define GC_our_mem_bytes GC_arrays._our_mem_bytes
word _our_mem_bytes;
/* Number of bytes allocated during this collection cycle. */
#define GC_bytes_allocd GC_arrays._bytes_allocd
word _bytes_allocd;
/*
* Number of black-listed bytes dropped during GC cycle as a result
* of repeated scanning during allocation attempts. These are treated
* largely as allocated, even though they are not useful to the client.
*/
#define GC_bytes_dropped GC_arrays._bytes_dropped
word _bytes_dropped;
/*
* Approximate number of bytes in objects (and headers) that became
* ready for finalization in the last collection.
*/
#define GC_bytes_finalized GC_arrays._bytes_finalized
word _bytes_finalized;
/*
* Number of explicitly deallocated bytes of memory since last
* collection.
*/
#define GC_bytes_freed GC_arrays._bytes_freed
word _bytes_freed;
/*
* Bytes of memory explicitly deallocated while finalizers were running.
* Used to approximate size of memory explicitly deallocated by finalizers.
*/
#define GC_finalizer_bytes_freed GC_arrays._finalizer_bytes_freed
word _finalizer_bytes_freed;
/*
* Pointer to the first (lowest address) `bottom_index` entity;
* assumes the allocator lock is held.
*/
#define GC_all_bottom_indices GC_arrays._all_bottom_indices
bottom_index *_all_bottom_indices;
/*
* Pointer to the last (highest address) `bottom_index` entity;
* assumes the allocator lock is held.
*/
#define GC_all_bottom_indices_end GC_arrays._all_bottom_indices_end
bottom_index *_all_bottom_indices_end;
#define GC_scratch_free_ptr GC_arrays._scratch_free_ptr
ptr_t _scratch_free_ptr;
#define GC_hdr_free_list GC_arrays._hdr_free_list
hdr *_hdr_free_list;
#define GC_scratch_end_addr GC_arrays._scratch_end_addr
word _scratch_end_addr; /*< the end point of the current scratch area */
#if defined(IRIX5) || (defined(USE_PROC_FOR_LIBRARIES) && !defined(LINUX))
# define USE_SCRATCH_LAST_END_PTR
/*
* The address of the end point of the last obtained scratch area.
* Used by `GC_register_dynamic_libraries()`.
*/
# define GC_scratch_last_end_addr GC_arrays._scratch_last_end_addr
word _scratch_last_end_addr;
#endif
#if defined(GC_ASSERTIONS) || defined(MAKE_BACK_GRAPH) \
|| defined(INCLUDE_LINUX_THREAD_DESCR) \
|| (defined(KEEP_BACK_PTRS) && ALIGNMENT == 1)
# define SET_REAL_HEAP_BOUNDS
/*
* Similar to `GC_least_plausible_heap_addr` and
* `GC_greatest_plausible_heap_addr` but do not include future
* (potential) heap expansion. Both variables are zero initially.
*/
# define GC_least_real_heap_addr GC_arrays._least_real_heap_addr
# define GC_greatest_real_heap_addr GC_arrays._greatest_real_heap_addr
word _least_real_heap_addr;
word _greatest_real_heap_addr;
#endif
#define GC_used_heap_size_after_full GC_arrays._used_heap_size_after_full
word _used_heap_size_after_full;
/* Number of explicitly managed bytes of storage at last collection. */
#define GC_non_gc_bytes_at_gc GC_arrays._non_gc_bytes_at_gc
word _non_gc_bytes_at_gc;
/* The number of extra calls to `GC_mark_some` that we have made. */
#define GC_mark_deficit GC_arrays._mark_deficit
size_t _mark_deficit;
#ifndef NO_BLACK_LISTING
/*
* Counter of the cases when found block by `GC_allochblk_nth` is
* black-listed completely.
*/
# define GC_drop_blacklisted_count GC_arrays._drop_blacklisted_count
unsigned _drop_blacklisted_count;
/* Number of warnings suppressed so far. */
# define GC_large_alloc_warn_suppressed GC_arrays._large_alloc_warn_suppressed
long _large_alloc_warn_suppressed;
/* Non-stack false references seen at last full collection. */
# define GC_old_normal_bl GC_arrays._old_normal_bl
word *_old_normal_bl;
/* Non-stack false references seen since last full collection. */
# define GC_incomplete_normal_bl GC_arrays._incomplete_normal_bl
word *_incomplete_normal_bl;
# define GC_old_stack_bl GC_arrays._old_stack_bl
# define GC_incomplete_stack_bl GC_arrays._incomplete_stack_bl
word *_old_stack_bl;
word *_incomplete_stack_bl;
#endif
/* The limits of stack for `GC_mark_some()` and friends. */
#define GC_mark_stack GC_arrays._mark_stack
#define GC_mark_stack_limit GC_arrays._mark_stack_limit
mse *_mark_stack;
mse *_mark_stack_limit;
/*
* All ranges between `GC_mark_stack` (incl.) and `GC_mark_stack_top`
* (incl.) still need to be marked from.
*/
#define GC_mark_stack_top GC_arrays._mark_stack_top
#ifdef PARALLEL_MARK
/* Updated only with the mark lock held, but read asynchronously. */
mse *volatile _mark_stack_top;
# define GC_mark_no GC_arrays._mark_no
word _mark_no; /*< protected by the mark lock */
/*
* Number of bytes of memory allocated since we released the allocator lock.
* Instead of reacquiring the allocator lock just to add this in, we add it
* in the next time we reacquire the allocator lock. (Atomically adding it
* does not work, since we would have to atomically update it in
* `GC_malloc`, which is too expensive.)
*/
# define GC_bytes_allocd_tmp GC_arrays._bytes_allocd_tmp
volatile AO_t _bytes_allocd_tmp;
#else
mse *_mark_stack_top;
#endif
#ifdef DYNAMIC_POINTER_MASK
/*
* Both mask and shift are zeros by default; if mask is zero, then
* correct it to ~0 at the collector initialization.
*/
# define GC_pointer_mask GC_arrays._pointer_mask
# define GC_pointer_shift GC_arrays._pointer_shift
word _pointer_mask;
unsigned char _pointer_shift;
#endif
#ifdef THREADS
# ifdef USE_SPIN_LOCK
# define GC_allocate_lock GC_arrays._allocate_lock
volatile AO_TS_t _allocate_lock;
# endif
# if !defined(HAVE_LOCKFREE_AO_OR) && defined(AO_HAVE_test_and_set_acquire) \
&& (!defined(NO_MANUAL_VDB) || defined(MPROTECT_VDB))
# define NEED_FAULT_HANDLER_LOCK
# define GC_fault_handler_lock GC_arrays._fault_handler_lock
volatile AO_TS_t _fault_handler_lock;
# endif
# if !(defined(GC_ALWAYS_MULTITHREADED) \
&& (defined(USE_PTHREAD_LOCKS) || defined(USE_SPIN_LOCK)))
# define GC_need_to_lock GC_arrays._need_to_lock
GC_bool _need_to_lock;
# endif
# ifdef GC_ASSERTIONS
# define GC_thr_initialized GC_arrays._thr_initialized
GC_bool _thr_initialized;
# endif
# if defined(GC_USE_DLOPEN_WRAP) && !defined(GC_WIN32_THREADS) \
&& !defined(PLATFORM_THREADS) && !defined(SN_TARGET_PSP2)
# define GC_syms_wrap_initialized GC_arrays._syms_wrap_initialized
GC_bool _syms_wrap_initialized;
# endif
# define GC_roots_were_cleared GC_arrays._roots_were_cleared
GC_bool _roots_were_cleared;
#else
# ifndef GC_NO_FINALIZATION
/*
* The variables to minimize the level of recursion when a client
* finalizer allocates memory.
*/
# define GC_finalizer_nested GC_arrays._finalizer_nested
# define GC_finalizer_skipped GC_arrays._finalizer_skipped
unsigned char _finalizer_nested;
unsigned short _finalizer_skipped;
# endif
#endif
/* Has `GC_init()` been run? */
#define GC_is_initialized GC_arrays._is_initialized
GC_bool _is_initialized;
/*
* Do we need a larger mark stack? May be set by client-supplied
* mark routines.
*/
#define GC_mark_stack_too_small GC_arrays._mark_stack_too_small
GC_bool _mark_stack_too_small;
/* Are there collectible marked objects in the heap? */
#define GC_objects_are_marked GC_arrays._objects_are_marked
GC_bool _objects_are_marked;
#ifndef GC_DISABLE_INCREMENTAL
# define GC_should_start_incremental_collection \
GC_arrays._should_start_incremental_collection
GC_bool _should_start_incremental_collection;
#endif
#ifndef GC_NO_FINALIZATION
/* Avoid the work if unreachable finalizable objects are not used. */
/* TODO: Turn this variable into a counter. */
# define GC_need_unreachable_finalization \
GC_arrays._need_unreachable_finalization
GC_bool _need_unreachable_finalization;
#endif
#ifndef DONT_USE_ATEXIT
# ifdef SMALL_CONFIG
# define GC_skip_collect_atexit FALSE
# else
/*
* A dedicated variable to avoid a garbage collection on abort.
* `GC_find_leak` cannot be used for this purpose as otherwise
* TSan finds a data race (between `GC_default_on_abort` and, e.g.,
* `GC_finish_collection`).
*/
# define GC_skip_collect_atexit GC_arrays._skip_collect_atexit
GC_bool _skip_collect_atexit;
# endif
#endif
#if defined(NO_FIND_LEAK) && defined(SHORT_DBG_HDRS)
# define GC_debugging_initialized GC_arrays._debugging_initialized
GC_bool _debugging_initialized;
# define GC_debugging_started FALSE
#else
# define GC_debugging_initialized GC_debugging_started
/* `GC_debug_malloc()` has been called, once at least. */
# define GC_debugging_started GC_arrays._debugging_started
GC_bool _debugging_started;
# define GC_have_errors GC_arrays._have_errors
# ifdef AO_HAVE_store
volatile AO_t _have_errors;
# else
GC_bool _have_errors;
# endif
#endif
#define GC_explicit_typing_initialized GC_arrays._explicit_typing_initialized
#ifdef AO_HAVE_load_acquire
volatile AO_t _explicit_typing_initialized;
#else
GC_bool _explicit_typing_initialized;
#endif
/* Indicate whether a full collection due to heap growth is needed. */
#define GC_need_full_gc GC_arrays._need_full_gc
GC_bool _need_full_gc;
#ifndef NO_CLOCK
/*
* Do performance measurements if set to true (e.g., accumulation of the
* total time of full collections).
*/
# define GC_measure_performance GC_arrays._measure_performance
GC_bool _measure_performance;
/*
* Variables for world-stop average delay time statistic computation.
* `GC_world_stopped_total_divisor` is incremented every world stop and
* halved when reached its maximum (or upon `GC_world_stopped_total_time`
* overflow). In milliseconds.
*/
/* TODO: Store the nanosecond part. */
# define GC_world_stopped_total_time GC_arrays._world_stopped_total_time
# define GC_world_stopped_total_divisor GC_arrays._world_stopped_total_divisor
unsigned _world_stopped_total_time;
unsigned _world_stopped_total_divisor;
#endif
#ifndef NO_FIND_LEAK
# define GC_n_leaked GC_arrays._n_leaked
unsigned _n_leaked;
#endif
#ifndef SHORT_DBG_HDRS
# define GC_n_smashed GC_arrays._n_smashed
unsigned _n_smashed;
#endif
/* Number of bytes in the accessible composite objects. */
#define GC_composite_in_use GC_arrays._composite_in_use
word _composite_in_use;
/* Number of bytes in the accessible atomic objects. */
#define GC_atomic_in_use GC_arrays._atomic_in_use
word _atomic_in_use;
/* GC number of latest successful `GC_expand_hp_inner()` call. */
#define GC_last_heap_growth_gc_no GC_arrays._last_heap_growth_gc_no
word _last_heap_growth_gc_no;
/*
* Number of bytes of memory reclaimed minus the number of bytes originally
* on free lists that we had to drop. Protected by the allocator lock.
*/
#define GC_bytes_found GC_arrays._bytes_found
GC_signed_word _bytes_found;
#ifndef GC_GET_HEAP_USAGE_NOT_NEEDED
/*
* Number of bytes reclaimed before this collection cycle; used for
* statistics only.
*/
# define GC_reclaimed_bytes_before_gc GC_arrays._reclaimed_bytes_before_gc
word _reclaimed_bytes_before_gc;
#endif
#ifdef USE_MUNMAP
# define GC_unmapped_bytes GC_arrays._unmapped_bytes
word _unmapped_bytes;
#else
# define GC_unmapped_bytes 0
#endif
#if defined(COUNT_UNMAPPED_REGIONS) && defined(USE_MUNMAP)
# define GC_num_unmapped_regions GC_arrays._num_unmapped_regions
GC_signed_word _num_unmapped_regions;
#else
# define GC_num_unmapped_regions 0
#endif
#define GC_all_nils GC_arrays._all_nils
bottom_index *_all_nils;
#define GC_scan_ptr GC_arrays._scan_ptr
struct hblk *_scan_ptr;
#ifdef PARALLEL_MARK
# define GC_main_local_mark_stack GC_arrays._main_local_mark_stack
mse *_main_local_mark_stack;
/*
* The lowest entry on mark stack that may not be empty.
* Updated only by the initiating thread.
*/
# define GC_first_nonempty GC_arrays._first_nonempty
volatile ptr_t _first_nonempty;
#endif
#ifdef ENABLE_TRACE
# define GC_trace_ptr GC_arrays._trace_ptr
ptr_t _trace_ptr;
#endif
#if !defined(ALWAYS_SMALL_CLEAR_STACK) && !defined(STACK_NOT_SCANNED) \
&& !defined(THREADS)
/*
* Coolest stack pointer value from which we have already cleared
* the stack.
*/
# define GC_min_sp GC_arrays._min_sp
ptr_t _min_sp;
/*
* The "hottest" stack pointer value we have seen recently.
* Degrades over time.
*/
# define GC_high_water GC_arrays._high_water
ptr_t _high_water;
#endif
#if CPP_PTRSZ > CPP_WORDSZ
# define GC_noop_sink_ptr GC_arrays._noop_sink_ptr
volatile ptr_t _noop_sink_ptr;
#endif
#define GC_noop_sink GC_arrays._noop_sink
#if defined(AO_HAVE_store) && defined(THREAD_SANITIZER)
volatile AO_t _noop_sink;
#else
volatile word _noop_sink;
#endif
#if !defined(SMALL_CONFIG) && !defined(GC_NO_FINALIZATION)
/* Saved number of disappearing links for stats printing. */
# define GC_old_dl_entries GC_arrays._old_dl_entries
size_t _old_dl_entries;
# ifndef GC_LONG_REFS_NOT_NEEDED
# define GC_old_ll_entries GC_arrays._old_ll_entries
size_t _old_ll_entries;
# endif
#endif
#define GC_mark_stack_size GC_arrays._mark_stack_size
size_t _mark_stack_size;
#define GC_mark_state GC_arrays._mark_state
mark_state_t _mark_state; /*< initialized to `MS_NONE` (0) */
#define GC_capacity_heap_sects GC_arrays._capacity_heap_sects
size_t _capacity_heap_sects;
#define GC_n_heap_sects GC_arrays._n_heap_sects
size_t _n_heap_sects; /*< number of separately added heap sections */
#ifdef ANY_MSWIN
# define GC_n_heap_bases GC_arrays._n_heap_bases
size_t _n_heap_bases; /*< see `GC_heap_bases[]` */
#endif
#ifdef USE_PROC_FOR_LIBRARIES
/* Number of `GET_MEM`-allocated memory sections. */
# define GC_n_memory GC_arrays._n_memory
word _n_memory;
#endif
#ifdef GC_GCJ_SUPPORT
# define GC_last_finalized_no GC_arrays._last_finalized_no
# define GC_gcjobjfreelist GC_arrays._gcjobjfreelist
word _last_finalized_no;
ptr_t *_gcjobjfreelist;
#endif
#define GC_fo_entries GC_arrays._fo_entries
size_t _fo_entries;
#ifndef GC_NO_FINALIZATION
# define GC_dl_hashtbl GC_arrays._dl_hashtbl
# define GC_fnlz_roots GC_arrays._fnlz_roots
# define GC_log_fo_table_size GC_arrays._log_fo_table_size
# ifndef GC_LONG_REFS_NOT_NEEDED
# define GC_ll_hashtbl GC_arrays._ll_hashtbl
struct dl_hashtbl_s _ll_hashtbl;
# endif
struct dl_hashtbl_s _dl_hashtbl;
struct fnlz_roots_s _fnlz_roots;
unsigned _log_fo_table_size;
# ifndef GC_TOGGLE_REFS_NOT_NEEDED
# define GC_toggleref_arr GC_arrays._toggleref_arr
# define GC_toggleref_array_size GC_arrays._toggleref_array_size
# define GC_toggleref_array_capacity GC_arrays._toggleref_array_capacity
union toggle_ref_u *_toggleref_arr;
size_t _toggleref_array_size;
size_t _toggleref_array_capacity;
# endif
#endif
#ifdef TRACE_BUF
# define GC_trace_buf_pos GC_arrays._trace_buf_pos
size_t _trace_buf_pos; /*< an index in the circular buffer */
#endif
/*
* How many consecutive collection/expansion failures?
* Reset by `GC_allochblk()`.
*/
#define GC_alloc_fail_count GC_arrays._alloc_fail_count
unsigned _alloc_fail_count;
#ifdef ENABLE_DISCLAIM
# define GC_finalized_kind GC_arrays._finalized_kind
unsigned _finalized_kind;
#endif
#ifndef NO_CLOCK
# define GC_full_gc_total_time GC_arrays._full_gc_total_time
# define GC_stopped_mark_total_time GC_arrays._stopped_mark_total_time
# define GC_full_gc_total_ns_frac GC_arrays._full_gc_total_ns_frac
# define GC_stopped_mark_total_ns_frac GC_arrays._stopped_mark_total_ns_frac
unsigned long _full_gc_total_time; /*< in ms, may wrap */
unsigned long _stopped_mark_total_time;
unsigned32 _full_gc_total_ns_frac; /*< fraction of 1 ms */
unsigned32 _stopped_mark_total_ns_frac;
#endif
#ifdef GC_WIN32_THREADS
/* Largest index in `dll_thread_table` that was ever used. */
# define GC_max_thread_index GC_arrays._max_thread_index
volatile LONG _max_thread_index;
#endif
/* Total size of registered root sections. */
#define GC_root_size GC_arrays._root_size
word _root_size;
/* `GC_static_roots[0..n_root_sets-1]` contains the valid root sets. */
#define n_root_sets GC_arrays._n_root_sets
size_t _n_root_sets;
#define GC_excl_table_entries GC_arrays._excl_table_entries
size_t _excl_table_entries; /*< number of entries in use */
#define GC_ed_size GC_arrays._ed_size
size_t _ed_size; /*< current size of above arrays */
#define GC_avail_descr GC_arrays._avail_descr
size_t _avail_descr; /*< next available slot */
#if defined(CAN_HANDLE_FORK) && defined(GC_PTHREADS)
/* Value of `pthread_self()` of the thread which called `fork()`. */
# define GC_parent_pthread_self GC_arrays._parent_pthread_self
pthread_t _parent_pthread_self;
#endif
/* Points to array of extended descriptors. */
#define GC_ext_descriptors GC_arrays._ext_descriptors
typed_ext_descr_t *_ext_descriptors;
/*
* Table of user-defined mark procedures. There is a small number
* of these, which can be referenced by `DS_PROC` mark descriptors.
* See `gc_mark.h` file.
*/
#define GC_mark_procs GC_arrays._mark_procs
GC_mark_proc _mark_procs[GC_MAX_MARK_PROCS];
/*
* `GC_valid_offsets[i]` implies
* `GC_modws_valid_offsets[i % sizeof(ptr_t)]`.
*/
#define GC_modws_valid_offsets GC_arrays._modws_valid_offsets
char _modws_valid_offsets[sizeof(ptr_t)];
#ifdef MAKE_BACK_GRAPH
/* Points to never-used `back_edges` space. */
# define GC_n_back_edge_structs GC_arrays._n_back_edge_structs
int _n_back_edge_structs;
# define GC_back_edge_space GC_arrays._back_edge_space
struct back_edges_s *_back_edge_space;
/* Pointer to free list of deallocated `back_edges` structures. */
# define GC_avail_back_edges GC_arrays._avail_back_edges
struct back_edges_s *_avail_back_edges;
/*
* Table of objects that are currently on the depth-first search stack.
* Only objects with in-degree one are in this table. Other objects are
* identified using `HEIGHT_IN_PROGRESS`.
*/
/* FIXME: This data structure needs improvement. */
# define GC_backgraph_in_progress_space GC_arrays._backgraph_in_progress_space
# define GC_backgraph_in_progress_size GC_arrays._backgraph_in_progress_size
# define GC_backgraph_n_in_progress GC_arrays._backgraph_n_in_progress
ptr_t *_backgraph_in_progress_space;
size_t _backgraph_in_progress_size;
size_t _backgraph_n_in_progress;
# define GC_backgraph_max_deepest_h GC_arrays._backgraph_max_deepest_h
# define GC_backgraph_deepest_height GC_arrays._backgraph_deepest_height
# define GC_backgraph_deepest_obj GC_arrays._backgraph_deepest_obj
word _backgraph_max_deepest_h;
word _backgraph_deepest_height;
ptr_t _backgraph_deepest_obj;
#endif
#ifdef GC_READ_ENV_FILE
/*
* The content of the `.gc.env` file with CR and LF replaced to '\0'.
* `NULL` if the file is missing or empty. Otherwise, always ends
* with '\0' (designating the end of the file).
*/
# define GC_envfile_content GC_arrays._envfile_content
char *_envfile_content;
/* Length of `GC_envfile_content` (if non-`NULL`). */
# define GC_envfile_length GC_arrays._envfile_length
size_t _envfile_length;
#endif
#ifndef ANY_MSWIN
/*
* The hash table header. Used only to check whether a range
* is already present.
*/
# define GC_root_index GC_arrays._root_index
struct roots *_root_index[RT_SIZE];
#endif
#if defined(SAVE_CALL_CHAIN) && !defined(DONT_SAVE_TO_LAST_STACK) \
&& (!defined(REDIRECT_MALLOC) || !defined(GC_HAVE_BUILTIN_BACKTRACE))
/*
* Stack at last garbage collection. Useful for debugging mysterious
* object disappearances. In the multi-threaded case, we currently only
* save the calling stack. Not supported in case of `malloc` redirection
* because `backtrace()` may call `malloc()`.
*/
struct callinfo _last_stack[NFRAMES];
# define SAVE_CALLERS_TO_LAST_STACK() GC_save_callers(GC_arrays._last_stack)
#else
# define SAVE_CALLERS_TO_LAST_STACK() (void)0
#endif
/* Free list for `NORMAL` objects. */
#define GC_objfreelist GC_arrays._objfreelist
void *_objfreelist[MAXOBJGRANULES + 1];
/* Free list for atomic objects. */
#define GC_aobjfreelist GC_arrays._aobjfreelist
void *_aobjfreelist[MAXOBJGRANULES + 1];
/*
* Uncollectible but traced objects. Objects on this and `_auobjfreelist`
* are always marked, except during garbage collections.
*/
#define GC_uobjfreelist GC_arrays._uobjfreelist
void *_uobjfreelist[MAXOBJGRANULES + 1];
#ifdef GC_ATOMIC_UNCOLLECTABLE
/* Atomic uncollectible but traced objects. */
# define GC_auobjfreelist GC_arrays._auobjfreelist
void *_auobjfreelist[MAXOBJGRANULES + 1];
#endif
/*
* Number of granules to allocate when asked for a certain number of bytes
* (plus `EXTRA_BYTES`). Should be accessed with the allocator lock held.
*/
#define GC_size_map GC_arrays._size_map
size_t _size_map[MAXOBJBYTES + 1];
#ifndef MARK_BIT_PER_OBJ
/*
* If the element is not `NULL`, then it points to a map of valid object
* addresses. `GC_obj_map[lg][i]` is `i % lg`. This is now used purely
* to replace a division in the marker by a table lookup.
* `GC_obj_map[0]` is used for large objects and contains all nonzero
* entries. This gets us out of the marker fast path without an extra test.
*/
# define GC_obj_map GC_arrays._obj_map
hb_map_entry_t *_obj_map[MAXOBJGRANULES + 1];
# define OBJ_MAP_LEN BYTES_TO_GRANULES(HBLKSIZE)
#endif
#define VALID_OFFSET_SZ HBLKSIZE
/*
* A nonzero `GC_valid_offsets[i]` means `i` is registered as
* a displacement.
*/
#define GC_valid_offsets GC_arrays._valid_offsets
char _valid_offsets[VALID_OFFSET_SZ];
#ifndef GC_DISABLE_INCREMENTAL
/* Pages that were dirty at last `GC_read_dirty()` call. */
# define GC_grungy_pages GC_arrays._grungy_pages
page_hash_table _grungy_pages;
/* Pages dirtied since last `GC_read_dirty()` call. */
# define GC_dirty_pages GC_arrays._dirty_pages
# ifdef MPROTECT_VDB
volatile
# endif
page_hash_table _dirty_pages;
#endif
#if (defined(CHECKSUMS) && (defined(GWW_VDB) || defined(SOFT_VDB))) \
|| defined(PROC_VDB)
/* A table to indicate the pages ever dirtied. */
# define GC_written_pages GC_arrays._written_pages
page_hash_table _written_pages;
#endif
/* Heap segments potentially containing client objects. */
#define GC_heap_sects GC_arrays._heap_sects
struct HeapSect *_heap_sects;
#if defined(USE_PROC_FOR_LIBRARIES)
/* All `GET_MEM`-allocated memory. Includes block headers and the like. */
# define GC_our_memory GC_arrays._our_memory
struct HeapSect _our_memory[MAX_HEAP_SECTS];
#endif
#ifdef ANY_MSWIN
/* Start address of memory regions obtained from OS. */
# define GC_heap_bases GC_arrays._heap_bases
ptr_t _heap_bases[MAX_HEAP_SECTS];
#endif
#ifdef MSWINCE
/* Committed lengths of memory regions obtained from OS. */
# define GC_heap_lengths GC_arrays._heap_lengths
word _heap_lengths[MAX_HEAP_SECTS];
#endif
#define GC_static_roots GC_arrays._static_roots
struct roots _static_roots[MAX_ROOT_SETS];
/* Array of exclusions, ascending address order. */
#define GC_excl_table GC_arrays._excl_table
struct exclusion _excl_table[MAX_EXCLUSIONS];
/*
* The block header index. Each entry points to a `bottom_index` entity.
* On a 32-bit machine, it points to the index for a set of the high-order
* bits equal to the index. For longer addresses, we hash the high-order
* bits to compute the index in `GC_top_index`, and each entry points to
* a hash chain. The last entry in each chain is `GC_all_nils`.
*/
#define GC_top_index GC_arrays._top_index
bottom_index *_top_index[TOP_SZ];
#ifdef ECOS
# ifndef ECOS_GC_MEMORY_SIZE
# define ECOS_GC_MEMORY_SIZE (448 * 1024)
# endif
# define GC_ecos_brk_idx GC_arrays._ecos_brk_idx
# define GC_ecos_memory GC_arrays._ecos_memory
size_t _ecos_brk_idx;
char _ecos_memory[ECOS_GC_MEMORY_SIZE];
#endif
};
GC_API_PRIV struct _GC_arrays GC_arrays;
#define beginGC_arrays ((ptr_t)(&GC_arrays))
#define endGC_arrays (beginGC_arrays + sizeof(GC_arrays))
/* Object kinds. */
#ifndef MAXOBJKINDS
# ifdef SMALL_CONFIG
# define MAXOBJKINDS 16
# else
# define MAXOBJKINDS 24
# endif
#endif
GC_EXTERN struct obj_kind {
/*
* Array of free-list headers for this kind of object. Point either
* to `GC_arrays` or to storage allocated with `GC_scratch_alloc()`.
*/
void **ok_freelist;
/*
* List headers for lists of blocks waiting to be swept.
* Indexed by object size in granules.
*/
struct hblk **ok_reclaim_list;
/* Descriptor template for objects in this block. */
word ok_descriptor;
/*
* Add object size in bytes to descriptor template to obtain descriptor.
* Otherwise the template is used as is.
*/
GC_bool ok_relocate_descr;
/* Clear objects before putting them on the free list. */
GC_bool ok_init;
#ifdef ENABLE_DISCLAIM
/*
* Mark from all, including unmarked, objects in block.
* Used to protect objects reachable from reclaim notifiers.
*/
GC_bool ok_mark_unconditionally;
/*
* The disclaim procedure is called before `obj` is reclaimed, but
* must also tolerate being called with object from free list.
* A nonzero exit prevents object from being reclaimed.
*/
int(GC_CALLBACK *ok_disclaim_proc)(void * /* `obj` */);
# define OK_DISCLAIM_INITZ /* comma */ , FALSE, 0
#else
# define OK_DISCLAIM_INITZ /*< empty */
#endif
} GC_obj_kinds[MAXOBJKINDS];
#define beginGC_obj_kinds ((ptr_t)(&GC_obj_kinds[0]))
#define endGC_obj_kinds (beginGC_obj_kinds + sizeof(GC_obj_kinds))
/* The predefined kinds. */
#define PTRFREE GC_I_PTRFREE
#define NORMAL GC_I_NORMAL
#define UNCOLLECTABLE 2
#ifdef GC_ATOMIC_UNCOLLECTABLE
# define AUNCOLLECTABLE 3
# define IS_UNCOLLECTABLE(k) (((k) & ~1) == UNCOLLECTABLE)
# define GC_N_KINDS_INITIAL_VALUE 4
#else
# define IS_UNCOLLECTABLE(k) ((k) == UNCOLLECTABLE)
# define GC_N_KINDS_INITIAL_VALUE 3
#endif
GC_EXTERN unsigned GC_n_kinds;
/* May mean the allocation granularity size, not page size. */
GC_EXTERN size_t GC_page_size;
#ifdef REAL_PAGESIZE_NEEDED
GC_EXTERN size_t GC_real_page_size;
#else
# define GC_real_page_size GC_page_size
#endif
/*
* Get heap memory from the OS.
* Note that `sbrk`-like allocation is preferred, since it usually
* makes it possible to merge consecutively allocated chunks.
* It also avoids unintended recursion with `REDIRECT_MALLOC` macro
* defined. `GET_MEM()` argument should be of `size_t` type and
* have no side-effect. `GET_MEM()` returns `HBLKSIZE`-aligned chunk
* (`NULL` means a failure). In case of `MMAP_SUPPORTED`, the argument
* must also be a multiple of a physical page size.
* `GET_MEM` is currently not assumed to retrieve zero-filled space.
*/
/* TODO: Take advantage of `GET_MEM()` returning a zero-filled space. */
#if defined(ANY_MSWIN) || defined(MSWIN_XBOX1) || defined(OS2)
GC_INNER void *GC_get_mem(size_t lb);
# define GET_MEM(lb) GC_get_mem(lb)
# if defined(CYGWIN32) && !defined(USE_WINALLOC)
# define NEED_UNIX_GET_MEM
# endif
#elif defined(DOS4GW) || defined(EMBOX) || defined(KOS) || defined(NEXT) \
|| defined(NONSTOP) || defined(RTEMS) || defined(__CC_ARM) \
|| (defined(SOLARIS) && !defined(USE_MMAP))
/* TODO: Use `page_alloc()` directly on Embox. */
# if defined(REDIRECT_MALLOC) && !defined(CPPCHECK)
# error Malloc redirection is unsupported
# endif
# define GET_MEM(lb) \
((void *)HBLKPTR((ptr_t)calloc(1, SIZET_SAT_ADD(lb, GC_page_size)) \
+ GC_page_size - 1))
#elif !defined(GET_MEM)
GC_INNER void *GC_unix_get_mem(size_t lb);
# define GET_MEM(lb) GC_unix_get_mem(lb)
# define NEED_UNIX_GET_MEM
#endif
/*
* Round up allocation size to a multiple of a page size.
* `GC_setpagesize()` is assumed to be already invoked.
*/
#define ROUNDUP_PAGESIZE(lb) /*< `lb` should have no side-effect */ \
(SIZET_SAT_ADD(lb, GC_page_size - 1) & ~(GC_page_size - 1))
/*
* Same as `ROUNDUP_PAGESIZE` but is used to make `GET_MEM()` argument
* safe.
*/
#ifdef MMAP_SUPPORTED
# define ROUNDUP_PAGESIZE_IF_MMAP(lb) ROUNDUP_PAGESIZE(lb)
#else
# define ROUNDUP_PAGESIZE_IF_MMAP(lb) (lb)
#endif
#ifdef ANY_MSWIN
GC_EXTERN SYSTEM_INFO GC_sysinfo;
/*
* Is `p` the start of either the `malloc` heap, or of one of the collector
* heap sections?
*/
GC_INNER GC_bool GC_is_heap_base(const void *p);
#endif
#ifdef GC_GCJ_SUPPORT
/* Note: `GC_hblkfreelist` and `GC_free_bytes` remain visible to GNU `gcj`. */
extern struct hblk *GC_hblkfreelist[];
extern word GC_free_bytes[];
#endif
/* This is used by `GC_do_blocking()`. */
struct blocking_data {
GC_fn_type fn;
void *client_data; /*< and result */
};
/* This is used by `GC_call_with_gc_active`, `GC_push_all_stack_sections`. */
struct GC_traced_stack_sect_s {
ptr_t saved_stack_ptr;
#ifdef IA64
ptr_t saved_backing_store_ptr;
ptr_t backing_store_end;
#endif
struct GC_traced_stack_sect_s *prev;
};
#ifdef THREADS
/*
* Process all "traced stack sections" - scan entire stack except for
* frames belonging to the user functions invoked by `GC_do_blocking`.
*/
GC_INNER void
GC_push_all_stack_sections(ptr_t lo, ptr_t hi,
struct GC_traced_stack_sect_s *traced_stack_sect);
/*
* The total size, in bytes, of all stacks.
* Updated on every `GC_push_all_stacks()` call.
*/
GC_EXTERN word GC_total_stacksize;
#else
/* Note: `NULL` value means we are not inside `GC_do_blocking()` call. */
GC_EXTERN ptr_t GC_blocked_sp;
/*
* Points to the "frame" data held in stack by the innermost
* `GC_call_with_gc_active()`. `NULL` if no such "frame" active.
*/
GC_EXTERN struct GC_traced_stack_sect_s *GC_traced_stack_sect;
#endif /* !THREADS */
#if defined(E2K) && defined(THREADS) || defined(IA64)
/*
* The bottom of the register stack of the primordial thread.
* E2K: holds the offset (`ps_ofs`) instead of a pointer.
*/
GC_EXTERN ptr_t GC_register_stackbottom;
#endif
#ifdef IA64
/* Similar to `GC_push_all_stack_sections` but for IA-64 registers store. */
GC_INNER void GC_push_all_register_sections(
ptr_t bs_lo, ptr_t bs_hi, GC_bool eager,
struct GC_traced_stack_sect_s *traced_stack_sect);
#endif
/*
* Mark bit operations.
*
* The marks are in a reserved area of each heap block.
* Each object or granule has one mark bit associated with it.
* Only those corresponding to the beginning of an object are used.
*/
/*
* Retrieve, set, clear the `n`-th mark bit in a given heap block.
* (Recall that bit `n` corresponds to `n`-th object or allocation granule
* relative to the beginning of the block, including unused space.)
*/
#ifdef USE_MARK_BYTES
# define mark_bit_from_hdr(hhdr, n) ((hhdr)->hb_marks[n])
# define set_mark_bit_from_hdr(hhdr, n) (void)((hhdr)->hb_marks[n] = 1)
# define clear_mark_bit_from_hdr(hhdr, n) (void)((hhdr)->hb_marks[n] = 0)
#else
/* Set mark bit correctly, even if mark bits may be concurrently accessed. */
# if defined(PARALLEL_MARK) || (defined(THREAD_SANITIZER) && defined(THREADS))
/*
* Workaround TSan false positive: there is no race between
* `mark_bit_from_hdr` and `set_mark_bit_from_hdr` when `n` is different
* (alternatively, `USE_MARK_BYTES` could be used). If TSan is off, then
* `AO_or()` is used only if we define `USE_MARK_BITS` macro explicitly.
*/
# define OR_WORD(addr, bits) AO_or(addr, bits)
# else
# define OR_WORD(addr, bits) (void)(*(addr) |= (bits))
# endif
# define mark_bit_from_hdr(hhdr, n) \
(((hhdr)->hb_marks[divWORDSZ(n)] >> modWORDSZ(n)) & (word)1)
# define set_mark_bit_from_hdr(hhdr, n) \
OR_WORD((hhdr)->hb_marks + divWORDSZ(n), (word)1 << modWORDSZ(n))
# define clear_mark_bit_from_hdr(hhdr, n) \
(void)(((word *)CAST_AWAY_VOLATILE_PVOID((hhdr)->hb_marks))[divWORDSZ(n)] \
&= ~((word)1 << modWORDSZ(n)))
#endif /* !USE_MARK_BYTES */
#ifdef MARK_BIT_PER_OBJ
/*
* Get the mark bit index corresponding to the given byte offset and
* size (in bytes).
*/
# define MARK_BIT_NO(offset, sz) ((offset) / (sz))
/* Spacing between useful mark bits. */
# define MARK_BIT_OFFSET(sz) 1
/* Position of final, always set, mark bit. */
# define FINAL_MARK_BIT(sz) ((sz) > MAXOBJBYTES ? 1 : HBLK_OBJS(sz))
#else
# define MARK_BIT_NO(offset, sz) BYTES_TO_GRANULES(offset)
# define MARK_BIT_OFFSET(sz) BYTES_TO_GRANULES(sz)
# define FINAL_MARK_BIT(sz) \
((sz) > MAXOBJBYTES ? MARK_BITS_PER_HBLK \
: BYTES_TO_GRANULES(HBLK_OBJS(sz) * (sz)))
#endif /* !MARK_BIT_PER_OBJ */
/* Important internal collector routines. */
/* Return the current stack pointer, approximately. */
GC_INNER ptr_t GC_approx_sp(void);
/*
* Same as `GC_approx_sp` but a macro. `sp` should be a local variable
* of `volatile` `ptr_t` type.
*/
#if (defined(E2K) && defined(__clang__) \
|| (defined(S390) && __clang_major__ < 8)) \
&& !defined(CPPCHECK)
/*
* Workaround some bugs in clang:
* - "undefined reference to llvm.frameaddress" error (clang-9/e2k);
* - a crash in SystemZTargetLowering of libLLVM-3.8 (s390).
*/
# define STORE_APPROX_SP_TO(sp) (void)(sp = (ptr_t)(&sp))
#elif defined(CPPCHECK) \
|| ((__GNUC__ >= 4 /* `GC_GNUC_PREREQ(4, 0)` */) \
&& !defined(STACK_NOT_SCANNED))
/* TODO: Use `GC_GNUC_PREREQ` after fixing a bug in cppcheck. */
/* Note: l-value is passed instead of pointer to `sp` (because of cppcheck). */
# define STORE_APPROX_SP_TO(sp) (void)(sp = (ptr_t)__builtin_frame_address(0))
#else
# define STORE_APPROX_SP_TO(sp) (void)(sp = (ptr_t)(&sp))
#endif
/* Have we allocated enough to amortize a collection? */
GC_INNER GC_bool GC_should_collect(void);
/*
* Get the next block whose address is at least `h`. Returned block
* is managed by the collector. The block must be in use unless
* `allow_free` is TRUE. Return `NULL` if there is no such block.
*/
GC_INNER struct hblk *GC_next_block(struct hblk *h, GC_bool allow_free);
/*
* Get the last (highest address) block whose address is at most `h`.
* Returned block is managed by the collector, but may or may not be in use.
* Return `NULL` if there is no such block.
*/
GC_INNER struct hblk *GC_prev_block(struct hblk *h);
GC_INNER void GC_mark_init(void);
/*
* Clear mark bits in all allocated heap blocks (i.e. for all heap objects).
* This invalidates the marker invariant, and sets `GC_mark_state` to
* reflect this. (This implicitly starts marking to reestablish the
* invariant.)
*/
GC_INNER void GC_clear_marks(void);
/*
* Tell the marker that marked objects may point to unmarked ones, and
* roots may point to unmarked objects. Reset mark stack.
*/
GC_INNER void GC_invalidate_mark_state(void);
/*
* Perform a small amount of marking. We try to touch roughly a page
* of memory. Returns quickly if no collection is in progress.
* Returns `TRUE` if we just finished a mark phase.
* `cold_gc_frame` argument is an address inside a frame of the
* collector that remains valid until all marking is complete;
* `NULL` value indicates that it is OK to miss some register values.
* In the case of an incremental collection, the world may be running.
*/
GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame);
/*
* Initiate a garbage collection. Initiates a full collection if the
* mark state is invalid; otherwise it is a partial one.
*/
GC_INNER void GC_initiate_gc(void);
/*
* Is a collection in progress? Note that this can return `TRUE` in
* the non-incremental case, if a collection has been abandoned and
* the mark state is now `MS_INVALID`.
*/
GC_INNER GC_bool GC_collection_in_progress(void);
/*
* Push contents of the symbol residing in the static roots area excluded
* from scanning by the collector for a reason. Note: it should be used only
* for symbols of relatively small size (containing one or several pointers).
*/
#define GC_PUSH_ALL_SYM(sym) GC_push_all_eager(&(sym), &(sym) + 1)
/* Same as `GC_push_all` but consider interior pointers as valid. */
GC_INNER void GC_push_all_stack(ptr_t b, ptr_t t);
#ifdef NO_VDB_FOR_STATIC_ROOTS
# define GC_push_conditional_static(b, t, all) \
((void)(all), GC_push_all(b, t))
#else
/*
* Same as `GC_push_conditional` (does either of `GC_push_all` or
* `GC_push_selected` depending on the third argument) but the caller
* guarantees the region belongs to the registered static roots.
*/
GC_INNER void GC_push_conditional_static(void *b, void *t, GC_bool all);
#endif
#if defined(WRAP_MARK_SOME) && defined(PARALLEL_MARK)
/*
* Similar to `GC_push_conditional` but scans the whole region immediately.
* `GC_mark_local` does not handle memory protection faults yet.
* So, the static data regions are scanned immediately by `GC_push_roots`.
*/
GC_INNER void GC_push_conditional_eager(void *bottom, void *top, GC_bool all);
#endif
/*
* In the multi-threaded case, we push part of the current thread stack
* with `GC_push_all_eager` when we push the registers. This gets the
* callee-save registers that may disappear. The remainder of the stacks
* are scheduled for scanning in `(*GC_push_other_roots)()`, which is
* thread-package-specific.
*/
/*
* Push all or dirty roots. Call the mark routine (`GC_push_one` for
* a single pointer, `GC_push_conditional` on groups of pointers) on every
* top level accessible pointer. If not `all`, then arrange to push only
* possibly altered values. `cold_gc_frame` is an address inside
* a collector frame that remains valid until all marking is complete;
* a `NULL` pointer indicates that it is OK to miss some register values.
*/
GC_INNER void GC_push_roots(GC_bool all, ptr_t cold_gc_frame);
/*
* Push system or application specific roots onto the mark stack.
* In some environments (e.g. a multi-threaded one) this is predefined
* to be nonzero. A client-supplied replacement should also call the
* original function. Remains externally visible as used by some
* well-known 3rd-party software (e.g., ECL) currently.
*/
GC_API_PRIV GC_push_other_roots_proc GC_push_other_roots;
#ifdef THREADS
GC_INNER void GC_push_thread_structures(void);
#endif
/*
* A pointer set to `GC_push_typed_structures_proc` lazily so that we can
* avoid linking in the typed allocation support if the latter is unused.
*/
GC_EXTERN void (*GC_push_typed_structures)(void);
typedef void (*GC_with_callee_saves_func)(ptr_t arg, void *context);
/*
* Ensure that either registers are pushed, or callee-save registers are
* somewhere on the stack, and then call `fn(arg, ctxt)`. `ctxt` is either
* a pointer to a `ucontext_t` entity we generated, or `NULL`. Could be
* called with or w/o the allocator lock held; could be called from a signal
* handler as well.
*/
GC_INNER void GC_with_callee_saves_pushed(GC_with_callee_saves_func fn,
ptr_t arg);
#if defined(IA64) || defined(SPARC)
/*
* Cause all stacked registers to be saved in memory. Return a pointer to
* the top of the corresponding memory stack.
*/
ptr_t GC_save_regs_in_stack(void);
#endif
#ifdef E2K
# include <asm/e2k_syswork.h>
# include <errno.h>
# include <sys/syscall.h>
# if defined(CPPCHECK)
/*
* Workaround "Uninitialized bs_lo" and "obsolete alloca() called"
* false positive (FP) warnings.
*/
# define PS_ALLOCA_BUF(pbuf, sz) \
(void)(GC_noop1_ptr(pbuf), *(pbuf) = (ptr_t)__builtin_alloca(sz))
# else
# define PS_ALLOCA_BUF(pbuf, sz) (void)(*(pbuf) = (ptr_t)alloca(sz))
# endif
/*
* Approximate size (in bytes) of the obtained procedure stack part
* belonging to `syscall()` itself.
*/
# define PS_SYSCALL_TAIL_BYTES 0x100
/*
* Determine the current size of the whole procedure stack. The size
* is valid only within the current function.
*/
# define GET_PROCEDURE_STACK_SIZE_INNER(psz_ull) \
do { \
*(psz_ull) = 0; /*< might be redundant */ \
if (syscall(__NR_access_hw_stacks, E2K_GET_PROCEDURE_STACK_SIZE, NULL, \
NULL, 0, psz_ull) \
== -1) \
ABORT_ARG1("Cannot get size of procedure stack", ": errno= %d", \
errno); \
GC_ASSERT(*(psz_ull) > 0 && *(psz_ull) % sizeof(ptr_t) == 0); \
} while (0)
# ifdef THREADS
# define PS_COMPUTE_ADJUSTED_OFS(padj_ps_ofs, ps_ofs, ofs_sz_ull) \
do { \
if ((ofs_sz_ull) <= (ps_ofs) /* `&& ofs_sz_ull > 0` */) \
ABORT_ARG2("Incorrect size of procedure stack", \
": ofs= %lu, size= %lu", (unsigned long)(ps_ofs), \
(unsigned long)(ofs_sz_ull)); \
*(padj_ps_ofs) = (ps_ofs) > (unsigned)PS_SYSCALL_TAIL_BYTES \
? (ps_ofs) - (unsigned)PS_SYSCALL_TAIL_BYTES \
: 0; \
} while (0)
# else
/* A simplified variant of the above assuming `ps_ofs` is a zero const. */
# define PS_COMPUTE_ADJUSTED_OFS(padj_ps_ofs, ps_ofs, ofs_sz_ull) \
do { \
GC_STATIC_ASSERT((ps_ofs) == 0); \
(void)(ofs_sz_ull); \
*(padj_ps_ofs) = 0; \
} while (0)
# endif /* !THREADS */
/*
* Copy procedure (register) stack to a stack-allocated buffer.
* Usable from a signal handler. The buffer (`*pbuf`) is valid only
* within the current function. `ps_ofs` designates the offset in the
* procedure stack to copy the contents from. Note: this macro cannot
* be changed to a function because `alloca()` and both `syscall()`
* should be called in the context of the caller.
*/
# define GET_PROCEDURE_STACK_LOCAL(ps_ofs, pbuf, psz) \
do { \
unsigned long long ofs_sz_ull; \
size_t adj_ps_ofs; \
\
GET_PROCEDURE_STACK_SIZE_INNER(&ofs_sz_ull); \
PS_COMPUTE_ADJUSTED_OFS(&adj_ps_ofs, ps_ofs, ofs_sz_ull); \
*(psz) = (size_t)ofs_sz_ull - adj_ps_ofs; \
/* Allocate buffer on the stack; cannot return `NULL`. */ \
PS_ALLOCA_BUF(pbuf, *(psz)); \
/* Copy the procedure stack at the given offset to the buffer. */ \
for (;;) { \
ofs_sz_ull = adj_ps_ofs; \
if (syscall(__NR_access_hw_stacks, E2K_READ_PROCEDURE_STACK_EX, \
&ofs_sz_ull, *(pbuf), *(psz), NULL) \
!= -1) \
break; \
if (errno != EAGAIN) \
ABORT_ARG2("Cannot read procedure stack", ": sz= %lu, errno= %d", \
(unsigned long)(*(psz)), errno); \
} \
} while (0)
#endif /* E2K */
#if defined(E2K) && defined(USE_PTR_HWTAG)
/* Load value and get tag of the target memory. */
# if defined(__ptr64__)
# define LOAD_TAGGED_VALUE(v, tag, p) \
do { \
ptr_t val; \
__asm__ __volatile__("ldd, sm %[adr], 0x0, %[val]\n\t" \
"gettagd %[val], %[tag]\n" \
: [val] "=r"(val), [tag] "=r"(tag) \
: [adr] "r"(p)); \
v = val; \
} while (0)
# elif !defined(CPPCHECK)
# error Unsupported -march for e2k target
# endif
# define LOAD_PTR_OR_CONTINUE(v, p) \
{ \
int tag LOCAL_VAR_INIT_OK; \
LOAD_TAGGED_VALUE(v, tag, p); \
if (tag != 0) \
continue; \
}
#elif defined(CHERI_PURECAP)
# define HAS_TAG_AND_PERM_LOAD(cap) \
(cheri_tag_get(cap) != 0 && (cheri_perms_get(cap) & CHERI_PERM_LOAD) != 0)
# define LOAD_PTR_OR_CONTINUE(v, p) \
{ \
word base_addr; \
v = *(ptr_t *)(p); \
if (!HAS_TAG_AND_PERM_LOAD(v)) \
continue; \
base_addr = cheri_base_get(v); \
if (ADDR(v) < base_addr || ADDR(v) >= base_addr + cheri_length_get(v)) \
continue; \
}
# define CAPABILITY_COVERS_RANGE(cap, b_addr, e_addr) \
(cheri_base_get(cap) <= (b_addr) \
&& cheri_base_get(cap) + cheri_length_get(cap) >= (e_addr))
# define SPANNING_CAPABILITY(cap, b_addr, e_addr) \
(cheri_tag_get(cap) && CAPABILITY_COVERS_RANGE(cap, b_addr, e_addr) \
&& (cheri_perms_get(cap) & (CHERI_PERM_LOAD | CHERI_PERM_LOAD_CAP)) \
!= 0)
#else
# define LOAD_PTR_OR_CONTINUE(v, p) (void)(v = *(ptr_t *)(p))
#endif /* !CHERI_PURECAP */
#if defined(DARWIN) && defined(THREADS)
/*
* If `p` points to an object, mark it and push contents on the mark stack.
* Pointer recognition test always accepts interior pointers, i.e. this is
* appropriate for pointers found on the thread stack.
*/
void GC_push_one(word p);
#endif
/*
* Mark and push (i.e. gray) a single object `p` onto the main mark stack.
* Consider `p` to be valid if it is an interior pointer. The object `p`
* has passed a preliminary pointer validity test, but we do not definitely
* know whether it is valid. Mark bits are not atomically updated; thus
* this must be the only thread setting them.
*/
#if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS)
GC_INNER void GC_mark_and_push_stack(ptr_t p, ptr_t source);
#else
GC_INNER void GC_mark_and_push_stack(ptr_t p);
#endif
/* Is the block with the given header containing no pointers? */
#define IS_PTRFREE(hhdr) (0 == (hhdr)->hb_descr)
/* Clear all mark bits in the header. */
GC_INNER void GC_clear_hdr_marks(hdr *hhdr);
/* Set all mark bits in the header. Used for uncollectible blocks. */
GC_INNER void GC_set_hdr_marks(hdr *hhdr);
/* Set all mark bits associated with a free list. */
GC_INNER void GC_set_fl_marks(ptr_t);
#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
/*
* Check that all mark bits associated with a free list are set.
* Abort if not.
*/
void GC_check_fl_marks(void **);
#endif
/*
* Add [`b`,`e`) to the root set. Adding the same interval a second
* time is a moderately fast no-op, and hence benign. We do not handle
* different but overlapping intervals efficiently. (But we do handle
* them correctly.) `tmp` specifies that the interval may be deleted
* before re-registering dynamic libraries.
*/
GC_INNER void GC_add_roots_inner(ptr_t b, ptr_t e, GC_bool tmp);
#ifdef USE_PROC_FOR_LIBRARIES
/*
* Remove given range from every static root which intersects with the range.
* `GC_remove_tmp_roots` is assumed to be called before this function is
* called (repeatedly) by `GC_register_map_entries`.
*/
GC_INNER void GC_remove_roots_subregion(ptr_t b, ptr_t e);
#endif
/*
* Inform the collector that a certain section of statically allocated
* memory contains no pointers to garbage-collected memory.
* The range boundaries should be properly aligned and valid.
*/
GC_INNER void GC_exclude_static_roots_inner(ptr_t start, ptr_t finish);
#if defined(ANY_MSWIN) || defined(DYNAMIC_LOADING)
/* Add dynamic library data sections to the root set. */
GC_INNER void GC_register_dynamic_libraries(void);
#endif
/*
* Remove and re-register dynamic libraries if we are configured to do
* that at each collection.
*/
GC_INNER void GC_cond_register_dynamic_libraries(void);
/* Machine-dependent startup routines. */
/*
* Get the cold end of the stack of the primordial thread. This is always
* called from the main (primordial) thread.
*/
GC_INNER ptr_t GC_get_main_stack_base(void);
#ifdef IA64
/* Get the cold end of register stack. */
GC_INNER ptr_t GC_get_register_stack_base(void);
#endif
GC_INNER void GC_register_data_segments(void);
#ifdef THREADS
/* Both are invoked from `GC_init()` only. */
GC_INNER void GC_thr_init(void);
/*
* Perform all initializations, including those that may require allocation,
* e.g. initialize thread-local free lists if used. Called by `GC_init()`.
*/
GC_INNER void GC_init_parallel(void);
# ifndef DONT_USE_ATEXIT
GC_INNER GC_bool GC_is_main_thread(void);
# endif
#else
# ifdef TRACE_BUF
void GC_add_trace_entry(const char *caller_fn_name, ptr_t arg1, ptr_t arg2);
# endif
#endif /* !THREADS */
#ifdef NO_BLACK_LISTING
# define GC_bl_init() (void)0
/* Do not define `GC_bl_init_no_interiors()`. */
# define GC_ADD_TO_BLACK_LIST_NORMAL(p, source) ((void)(p))
# define GC_ADD_TO_BLACK_LIST_STACK(p, source) ((void)(p))
# define GC_promote_black_lists() (void)0
# define GC_unpromote_black_lists() (void)0
#else
/*
* If we need a block of `n` bytes, and we have a block of `n + BL_LIMIT`
* bytes available, and `n` is greater than `BL_LIMIT`, but all possible
* positions in it are black-listed, we just use it anyway (and print
* a warning, if warnings are enabled). This risks subsequently leaking
* the block due to a false reference. But not using the block risks
* unreasonable immediate heap growth.
*/
# define BL_LIMIT GC_black_list_spacing
/*
* Average number of bytes between black-listed blocks. Approximate.
* Counts only blocks that are "stack black-listed", i.e. that are
* problematic in the interior of an object.
*/
GC_EXTERN word GC_black_list_spacing;
/*
* The interval between unsuppressed warnings about repeated allocation
* of a very large block.
*/
GC_EXTERN long GC_large_alloc_warn_interval;
/* Initialize the black listing mechanism. */
GC_INNER void GC_bl_init(void);
GC_INNER void GC_bl_init_no_interiors(void);
# ifdef PRINT_BLACK_LIST
/*
* Register bits as a possible future false reference from the heap
* or static data. The argument `p` is not a valid pointer reference,
* but it falls inside the plausible heap bounds.
*/
GC_INNER void GC_add_to_black_list_normal(ptr_t p, ptr_t source);
# define GC_ADD_TO_BLACK_LIST_NORMAL(p, source) \
if (GC_all_interior_pointers) { \
GC_add_to_black_list_stack(p, source); \
} else \
GC_add_to_black_list_normal(p, source)
GC_INNER void GC_add_to_black_list_stack(ptr_t p, ptr_t source);
# define GC_ADD_TO_BLACK_LIST_STACK(p, source) \
GC_add_to_black_list_stack(p, source)
# else
GC_INNER void GC_add_to_black_list_normal(ptr_t p);
# define GC_ADD_TO_BLACK_LIST_NORMAL(p, source) \
if (GC_all_interior_pointers) { \
GC_add_to_black_list_stack(p); \
} else \
GC_add_to_black_list_normal(p)
GC_INNER void GC_add_to_black_list_stack(ptr_t p);
# define GC_ADD_TO_BLACK_LIST_STACK(p, source) GC_add_to_black_list_stack(p)
# endif /* PRINT_BLACK_LIST */
/*
* Declare an end to a black listing phase. (I.e. signal the completion of
* a collection.) Turn the incomplete black lists into new black lists, etc.
*/
GC_INNER void GC_promote_black_lists(void);
/*
* Approximately undo the effect of `GC_promote_black_lists()`.
* This actually loses some information, but only in a reasonably safe way.
*/
GC_INNER void GC_unpromote_black_lists(void);
#endif
/*
* The collector internal memory allocation for small objects.
* Deallocation is not possible. May return `NULL`.
*/
GC_INNER ptr_t GC_scratch_alloc(size_t bytes);
#ifdef GWW_VDB
/* `GC_scratch_recycle_no_gww()` is not used. */
#else
# define GC_scratch_recycle_no_gww GC_scratch_recycle_inner
#endif
/* Reuse the memory region by the heap. */
GC_INNER void GC_scratch_recycle_inner(void *ptr, size_t sz);
#ifndef MARK_BIT_PER_OBJ
/*
* Add a heap block map for objects of a size in granules to `GC_obj_map`.
* A size of zero is used for large objects. Returns `FALSE` on failure.
*/
GC_INNER GC_bool GC_add_map_entry(size_t lg);
#endif
/*
* Same as `GC_register_displacement` but assuming the allocator lock
* is already held.
*/
GC_INNER void GC_register_displacement_inner(size_t offset);
/*
* Allocate a new heap block for small objects of size `lg` (in granules)
* and `kind`. Add all of the block's objects to the free list for objects
* of that size. Set all mark bits if objects are uncollectible.
* Will fail to do anything if out of memory.
*/
GC_INNER void GC_new_hblk(size_t lg, int kind);
/*
* Build a free list for objects of size `lg` (in granules) inside heap
* block `h`. Clear objects inside `h` if `clear` argument is set.
* Add `list` to the end of the free list we build. Return the new
* free list. Normally called by `GC_new_hblk()`, but this could also
* be called without the allocator lock, if we ensure that there is no
* concurrent collection which might reclaim objects that we have not
* yet allocated.
*/
GC_INNER ptr_t GC_build_fl(struct hblk *h, ptr_t list, size_t lg,
GC_bool clear);
/*
* Allocate (and return pointer to) a heap block for objects of the
* given size and alignment (in bytes), searching over the appropriate
* free block lists; inform the marker that the found block is valid
* for objects of the indicated size. Assumes (as implied by the argument
* name) that `EXTRA_BYTES` value is already added to the size, if needed.
* The client is responsible for clearing the block, if needed.
* Note: we set `GC_obj_map` field in the header correctly; the caller
* is responsible for building an object's free list in the block.
*/
GC_INNER struct hblk *GC_allochblk(size_t lb_adjusted, int kind,
unsigned flags, size_t align_m1);
/*
* Deallocate (free) a heap block and mark it as invalid. Coalesce it
* with its neighbors if possible. All mark words are assumed to be cleared.
*/
GC_INNER void GC_freehblk(struct hblk *p);
/* Miscellaneous GC routines. */
/*
* This explicitly increases the size of the heap. It is used internally,
* but may also be invoked from `GC_expand_hp` by client. The argument is
* in units of `HBLKSIZE`. (An argument of zero is treated as 1.)
* Returns `FALSE` on failure.
*/
GC_INNER GC_bool GC_expand_hp_inner(word n);
/*
* Restore unmarked objects to free lists, or (if `abort_if_found` is `TRUE`)
* report them. (I.e. perform `GC_reclaim_block()` on the entire heap,
* after first clearing small-object free lists if we are not just looking
* for leaks.) Sweeping of small object pages is largely deferred.
*/
GC_INNER void GC_start_reclaim(GC_bool abort_if_found);
/*
* Sweep blocks of the indicated object size (in granules) and kind
* until either the appropriate nonempty free list is found, or there
* are no more blocks to sweep.
*/
GC_INNER void GC_continue_reclaim(size_t lg, int kind);
/*
* Reclaim all small blocks waiting to be reclaimed. Abort and return
* `FALSE` when/if `(*stop_func)()` returns `TRUE`. If this returns `TRUE`,
* then it is safe to restart the world with incorrectly cleared mark bits.
* If `ignore_old`, then reclaim only blocks that have been reclaimed
* recently, and discard the rest. `stop_func` may be 0.
*/
GC_INNER GC_bool GC_reclaim_all(GC_stop_func stop_func, GC_bool ignore_old);
/*
* Generic procedure to rebuild a free list in `hbp` with header `hhdr`,
* with objects of size `sz` bytes. Add `list` to the end of the free list.
* Add the number of reclaimed bytes to `*pcount`. Note: it could be called
* directly from `GC_malloc_many`.
*/
GC_INNER ptr_t GC_reclaim_generic(struct hblk *hbp, hdr *hhdr, size_t sz,
GC_bool init, ptr_t list, word *pcount);
/*
* Is given heap block completely unmarked (i.e. contains no marked objects)?
* This does not require the block to be in physical memory.
*/
GC_INNER GC_bool GC_block_empty(const hdr *hhdr);
/* Always returns 0 (`FALSE`). */
GC_INNER int GC_CALLBACK GC_never_stop_func(void);
/*
* Stop-the-world garbage collection. The caller must have acquired
* the allocator lock. If `stop_func` is not `GC_never_stop_func`, then
* abort if `stop_func` returns `TRUE`. Return `TRUE` if we successfully
* completed the collection (otherwise the collection is aborted).
*/
GC_INNER GC_bool GC_try_to_collect_inner(GC_stop_func stop_func);
#define GC_gcollect_inner() (void)GC_try_to_collect_inner(GC_never_stop_func)
#ifdef THREADS
/*
* We may currently be in thread creation or destruction. Only set to `TRUE`
* while the allocator lock is held. When set, it is OK to run the garbage
* collection from an unknown thread. Protected by the allocator lock.
*/
GC_EXTERN GC_bool GC_in_thread_creation;
#endif
/*
* Do `n_blocks` units of a garbage collection work, if appropriate.
* A unit is an amount appropriate for `HBLKSIZE` bytes of allocation.
*/
GC_INNER void GC_collect_a_little_inner(size_t n_blocks);
GC_INNER void *GC_malloc_kind_aligned_global(size_t lb, int kind,
size_t align_m1);
GC_INNER void *GC_generic_malloc_aligned(size_t lb, int kind, unsigned flags,
size_t align_m1);
/*
* Allocate an object of the given `kind` but assuming the allocator
* lock is already held. Should not be used to directly allocate
* objects requiring special handling on allocation. `flags` argument
* should be 0 or `IGNORE_OFF_PAGE`; in the latter case the client
* guarantees there will always be a pointer to the beginning (i.e.
* within the first `hblk`) of the object while it is live.
*/
GC_INNER void *GC_generic_malloc_inner(size_t lb, int kind, unsigned flags);
/*
* Collect or expand heap in an attempt make the indicated number of
* free blocks available. Should be called until the blocks are
* available (setting `retry` value to `TRUE` unless this is the first
* call in a loop) or until it fails by returning `FALSE`. The `flags`
* argument should be `IGNORE_OFF_PAGE` or 0.
*/
GC_INNER GC_bool GC_collect_or_expand(word needed_blocks, unsigned flags,
GC_bool retry);
/*
* Make the indicated object free list nonempty, and return its head (the
* first object on the free list). The object must be removed from the free
* list by the caller. The size is in granules.
*/
GC_INNER ptr_t GC_allocobj(size_t lg, int kind);
#ifdef GC_ADD_CALLER
/*
* `GC_DBG_EXTRAS` is used by the collector debug API functions (unlike
* `GC_EXTRAS` used by the debug API macros) thus `GC_RETURN_ADDR_PARENT`
* (pointing to client caller) should be used if possible.
*/
# ifdef GC_HAVE_RETURN_ADDR_PARENT
# define GC_DBG_EXTRAS GC_RETURN_ADDR_PARENT, NULL, 0
# else
# define GC_DBG_EXTRAS GC_RETURN_ADDR, NULL, 0
# endif
#else
# define GC_DBG_EXTRAS "unknown", 0
#endif /* !GC_ADD_CALLER */
#ifdef GC_COLLECT_AT_MALLOC
/*
* Parameter to force collection at every `malloc` of size greater or
* equal to the given value. This might be handy during debugging.
* Note: this variable is visible outside for debugging purpose.
*/
extern size_t GC_dbg_collect_at_malloc_min_lb;
# define GC_DBG_COLLECT_AT_MALLOC(lb) \
(void)((lb) >= GC_dbg_collect_at_malloc_min_lb ? (GC_gcollect(), 0) : 0)
#else
# define GC_DBG_COLLECT_AT_MALLOC(lb) (void)0
#endif /* !GC_COLLECT_AT_MALLOC */
/* Allocation routines that bypass the thread-local cache. */
#if defined(THREAD_LOCAL_ALLOC) && defined(GC_GCJ_SUPPORT)
/*
* Allocate an object, clear it, and store the pointer to the type
* structure ("vtable" in `gcj`). This adds a byte at the end of the
* object if `GC_malloc` would.
*/
GC_INNER void *GC_core_gcj_malloc(size_t lb, const void *vtable_ptr,
unsigned flags);
#endif
GC_INNER void GC_init_headers(void);
/*
* Install a header for block `h`. Return `NULL` on failure, or the
* uninitialized header otherwise.
*/
GC_INNER hdr *GC_install_header(struct hblk *h);
/*
* Set up forwarding counts for block `h` of size `sz`. Return `FALSE`
* on failure.
*/
GC_INNER GC_bool GC_install_counts(struct hblk *h, size_t sz);
/* Remove the header for block `h`. */
GC_INNER void GC_remove_header(struct hblk *h);
/* Remove forwarding counts for `h`. */
GC_INNER void GC_remove_counts(struct hblk *h, size_t sz);
/* A non-macro variant of the header location routine. */
GC_INNER hdr *GC_find_header(const void *h);
/*
* Get `HBLKSIZE`-aligned heap memory chunk from the OS and add the
* chunk to `GC_our_memory`. Return `NULL` if out of memory.
*/
GC_INNER ptr_t GC_os_get_mem(size_t bytes);
#if defined(NO_FIND_LEAK) && defined(SHORT_DBG_HDRS)
# define GC_print_all_errors() (void)0
# define GC_check_heap() (void)0
# define GC_print_all_smashed() (void)0
#else
/*
* Print smashed and leaked objects, if any. Clear the lists of such
* objects. Called without the allocator lock held.
*/
GC_INNER void GC_print_all_errors(void);
/*
* Check that all objects in the heap with debugging info are intact.
* Add any that are not to `GC_smashed` list.
*/
GC_EXTERN void (*GC_check_heap)(void);
/* Print `GC_smashed` list if it is not empty. Then clear the list. */
GC_EXTERN void (*GC_print_all_smashed)(void);
#endif
/*
* If possible, print (using `GC_err_printf()`) a more detailed
* description (terminated with "\n") of the object referred to by `p`.
*/
GC_EXTERN void (*GC_print_heap_obj)(ptr_t p);
GC_INNER void GC_default_print_heap_obj_proc(ptr_t p);
#if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG)
/*
* Print an address map of the process. The caller should hold the
* allocator lock.
*/
void GC_print_address_map(void);
#endif
#ifdef NO_FIND_LEAK
# define GC_find_leak_inner FALSE
#else
# define GC_find_leak_inner GC_find_leak
# ifndef SHORT_DBG_HDRS
/*
* Do not immediately deallocate object on `free()` in the find-leak mode,
* just mark it as freed (and deallocate it after collection).
*/
GC_EXTERN GC_bool GC_findleak_delay_free;
# endif
#endif /* !NO_FIND_LEAK */
#if defined(NO_FIND_LEAK) && defined(SHORT_DBG_HDRS)
# define get_have_errors() FALSE
#elif defined(AO_HAVE_store)
# define GC_SET_HAVE_ERRORS() AO_store(&GC_have_errors, (AO_t)TRUE)
# define get_have_errors() \
((GC_bool)AO_load(&GC_have_errors)) /*< no barrier */
#else
# define GC_SET_HAVE_ERRORS() (void)(GC_have_errors = TRUE)
/*
* We saw a smashed or leaked object. Call error printing routine
* occasionally. It is OK to read it not acquiring the allocator lock.
* Once set to `TRUE`, it is never cleared.
*/
# define get_have_errors() GC_have_errors
#endif /* !AO_HAVE_store */
#define VERBOSE 2
#if !defined(NO_CLOCK) || !defined(SMALL_CONFIG)
/*
* Value of 1 generates basic collector log; `VERBOSE` generates additional
* messages.
*/
GC_EXTERN int GC_print_stats;
#else /* SMALL_CONFIG */
/*
* Defined as a macro to aid the compiler to remove the relevant message
* character strings from the executable (with a particular level of
* optimizations).
*/
# define GC_print_stats 0
#endif
#ifdef KEEP_BACK_PTRS
/* Number of random backtraces to generate for each collection. */
GC_EXTERN long GC_backtraces;
#endif
/*
* A trivial (linear congruential) pseudo-random numbers generator,
* safe for the concurrent usage.
*/
#define GC_RAND_MAX ((int)(~0U >> 1))
#if defined(AO_HAVE_store) && defined(THREAD_SANITIZER)
# define GC_RAND_STATE_T volatile AO_t
# define GC_RAND_NEXT(pseed) GC_rand_next(pseed)
GC_INLINE int
GC_rand_next(GC_RAND_STATE_T *pseed)
{
AO_t next = (AO_t)((AO_load(pseed) * (unsigned32)1103515245UL + 12345)
& (unsigned32)((unsigned)GC_RAND_MAX));
AO_store(pseed, next);
return (int)next;
}
#else
# define GC_RAND_STATE_T unsigned32
# define GC_RAND_NEXT(pseed) /*< overflow and race are OK */ \
(int)(*(pseed) = (*(pseed) * (unsigned32)1103515245UL + 12345) \
& (unsigned32)((unsigned)GC_RAND_MAX))
#endif
#ifdef MAKE_BACK_GRAPH
GC_EXTERN GC_bool GC_print_back_height;
void GC_print_back_graph_stats(void);
#endif
#ifdef THREADS
/*
* Explicitly deallocate the object when we already hold the allocator lock.
* Only used for internally allocated objects.
*/
GC_INNER void GC_free_inner(void *p);
#endif
#ifdef VALGRIND_TRACKING
# define FREE_PROFILER_HOOK(p) GC_free_profiler_hook(p)
#else
# define FREE_PROFILER_HOOK(p) (void)(p)
#endif
/*
* Macros used for collector internal allocation. These assume the
* allocator lock is held.
*/
#ifdef DBG_HDRS_ALL
/*
* An allocation function for internal use. Normally internally allocated
* objects do not have debug information. But in this case, we need to make
* sure that all objects have debug headers.
*/
GC_INNER void *GC_debug_generic_malloc_inner(size_t lb, int kind,
unsigned flags);
# define GC_INTERNAL_MALLOC(lb, k) GC_debug_generic_malloc_inner(lb, k, 0)
# define GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(lb, k) \
GC_debug_generic_malloc_inner(lb, k, IGNORE_OFF_PAGE)
# ifdef THREADS
/* Used internally; we assume it is called correctly. */
GC_INNER void GC_debug_free_inner(void *p);
# define GC_INTERNAL_FREE GC_debug_free_inner
# else
# define GC_INTERNAL_FREE GC_debug_free
# endif
#else
# define GC_INTERNAL_MALLOC(lb, k) GC_generic_malloc_inner(lb, k, 0)
# define GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(lb, k) \
GC_generic_malloc_inner(lb, k, IGNORE_OFF_PAGE)
# ifdef THREADS
# define GC_INTERNAL_FREE GC_free_inner
# else
# define GC_INTERNAL_FREE GC_free
# endif
#endif /* !DBG_HDRS_ALL */
/* Memory unmapping routines. */
#ifdef USE_MUNMAP
/*
* Unmap blocks that have not been recently touched. This is the only
* way blocks are ever unmapped.
*/
GC_INNER void GC_unmap_old(unsigned threshold);
/*
* Merge all unmapped blocks that are adjacent to other free blocks.
* This may involve remapping, since all blocks are either fully mapped
* or fully unmapped. Returns `TRUE` if at least one block was merged.
*/
GC_INNER GC_bool GC_merge_unmapped(void);
GC_INNER void GC_unmap(ptr_t start, size_t bytes);
GC_INNER void GC_remap(ptr_t start, size_t bytes);
/*
* Two adjacent blocks have already been unmapped and are about to be merged.
* Unmap the whole block. This typically requires that we unmap a small
* section in the middle that was not previously unmapped due to alignment
* constraints.
*/
GC_INNER void GC_unmap_gap(ptr_t start1, size_t bytes1, ptr_t start2,
size_t bytes2);
#endif
#ifdef CAN_HANDLE_FORK
/*
* Fork-handling mode:
* - 0 means no `fork` handling is requested (but client could anyway
* call `fork()` provided it is surrounded with `GC_atfork_prepare`,
* `GC_atfork_parent`, `GC_atfork_child` calls);
* - (-1) means the collector tries to use `pthread_at_fork()` if it is
* available (if it succeeds, then `GC_handle_fork` value is changed to
* one), a portable client should nonetheless surround `fork()` with
* `GC_atfork_prepare()` and the accompanying routines (for the case
* of `pthread_at_fork()` failure or absence);
* - 1 (or other values) means client fully relies on `pthread_at_fork`
* (so if it is missing or failed, then `abort` occurs in `GC_init()`),
* `GC_atfork_prepare` and the accompanying routines are no-op in such
* a case.
*
* Note: the value is examined by `GC_thr_init`.
*/
GC_EXTERN int GC_handle_fork;
# ifdef THREADS
# if defined(SOLARIS) && !defined(_STRICT_STDC)
/* Update `pthreads` id in the child process right after `fork`. */
GC_INNER void GC_stackbase_info_update_after_fork(void);
# else
# define GC_stackbase_info_update_after_fork() (void)0
# endif
# endif
#endif /* CAN_HANDLE_FORK */
#ifdef NO_MANUAL_VDB
# define GC_manual_vdb FALSE
# define GC_auto_incremental GC_incremental
# define GC_dirty(p) (void)(p)
# define REACHABLE_AFTER_DIRTY(p) (void)(p)
#else
/*
* The incremental collection is in the manual VDB mode.
* Assumes `GC_incremental` is `TRUE`. Should not be modified once
* `GC_incremental` is set to `TRUE`.
*/
GC_EXTERN GC_bool GC_manual_vdb;
# define GC_auto_incremental (GC_incremental && !GC_manual_vdb)
/*
* Manually mark the page containing `p` as dirty. Logically, this
* dirties the entire object. Does not require locking.
* Exported and marked as `noinline` for the purpose of some clients that
* need to patch the symbol when using write barrier validation.
*/
GC_API_PATCHABLE void GC_dirty_inner(const void *p);
# define GC_dirty(p) (GC_manual_vdb ? GC_dirty_inner(p) : (void)0)
# define REACHABLE_AFTER_DIRTY(p) GC_reachable_here(p)
#endif /* !NO_MANUAL_VDB */
#ifdef GC_DISABLE_INCREMENTAL
# define GC_incremental FALSE
#else
/*
* Using incremental/generational collection. Assumes dirty bits are
* being maintained.
*/
GC_EXTERN GC_bool GC_incremental;
/* Virtual dirty bit (VDB) implementations; each one exports the following. */
/*
* Initialize the virtual dirty bit implementation. Returns `TRUE` if
* virtual dirty bits are maintained (otherwise it is OK to be called again
* if the client calls `GC_enable_incremental()` once more).
*/
GC_INNER GC_bool GC_dirty_init(void);
/*
* Retrieve system dirty bits for the heap to a local buffer (unless
* `output_unneeded`). The caller should set `output_unneeded` to indicate
* that reading of the retrieved dirty bits is not planned till the next
* retrieval. Restore the systems notion of which pages are dirty.
* We assume that either the world is stopped or it is OK to lose dirty bits
* while it is happening (`GC_enable_incremental()` is the caller and
* `output_unneeded` is `TRUE` at least if the multi-threading support is on).
*/
GC_INNER void GC_read_dirty(GC_bool output_unneeded);
/*
* Is the `HBLKSIZE`-sized page at `h` marked dirty in the local buffer?
* If the actual page size is different, this returns `TRUE` if any of
* the pages overlapping `h` are dirty. This routine may err on the side
* of labeling pages as dirty (and this implementation does).
*/
GC_INNER GC_bool GC_page_was_dirty(struct hblk *h);
/*
* Block `h` is about to be written or allocated shortly. Ensure that
* all pages containing any part of the `nblocks` `hblk` entities starting
* at `h` are no longer write-protected (by the virtual dirty bit
* implementation). I.e., this is a call that:
* - hints that [`h`, `h + nblocks`) is about to be written;
* - guarantees that protection is removed;
* - may speed up some virtual dirty bit implementations;
* - may be essential if we need to ensure that pointer-free system
* call buffers in the heap are not protected.
*/
GC_INNER void GC_remove_protection(struct hblk *h, size_t nblocks,
GC_bool is_ptrfree);
# if !defined(NO_VDB_FOR_STATIC_ROOTS) && !defined(PROC_VDB)
/* Is VDB working for static roots? */
GC_INNER GC_bool GC_is_vdb_for_static_roots(void);
# endif
# ifdef CAN_HANDLE_FORK
# if defined(PROC_VDB) || defined(SOFT_VDB) \
|| (defined(MPROTECT_VDB) && defined(DARWIN) && defined(THREADS))
/*
* Update pid-specific resources (like `/proc` file descriptors) needed
* by the dirty bits implementation after `fork` in the child process.
*/
GC_INNER void GC_dirty_update_child(void);
# else
# define GC_dirty_update_child() (void)0
# endif
# endif /* CAN_HANDLE_FORK */
# if defined(MPROTECT_VDB) && defined(DARWIN)
EXTERN_C_END
# include <pthread.h>
EXTERN_C_BEGIN
# ifdef THREADS
GC_INNER int GC_inner_pthread_create(pthread_t *t,
GC_PTHREAD_CREATE_CONST pthread_attr_t *a,
void *(*fn)(void *), void *arg);
# else
# define GC_inner_pthread_create pthread_create
# endif
# endif
#endif /* !GC_DISABLE_INCREMENTAL */
#if defined(COUNT_PROTECTED_REGIONS) && defined(MPROTECT_VDB)
/*
* Do actions on heap growth, if needed, to prevent hitting the OS kernel
* limit on the VM map regions.
*/
GC_INNER void GC_handle_protected_regions_limit(void);
#else
# define GC_handle_protected_regions_limit() (void)0
#endif
/* Same as `GC_base` but accepts and returns a pointer to `const` object. */
#define GC_base_C(p) ((const void *)GC_base(GC_CAST_AWAY_CONST_PVOID(p)))
/* Some debugging print routines. */
void GC_print_block_list(void);
void GC_print_hblkfreelist(void);
void GC_print_heap_sects(void);
void GC_print_static_roots(void);
#ifdef KEEP_BACK_PTRS
/*
* Store back pointer to `source` in `dest`, if that appears to be
* possible. This is not completely safe, since we may mistakenly
* conclude that `dest` has a debugging wrapper. But the error
* probability is very small, and this should not be used in
* production code. We assume that `dest` is the real base pointer.
* `source` should usually be a pointer to the interior of an object.
*/
GC_INNER void GC_store_back_pointer(ptr_t source, ptr_t dest);
GC_INNER void GC_marked_for_finalization(ptr_t dest);
# define GC_STORE_BACK_PTR(source, dest) GC_store_back_pointer(source, dest)
# define GC_MARKED_FOR_FINALIZATION(dest) GC_marked_for_finalization(dest)
#else
# define GC_STORE_BACK_PTR(source, dest) (void)(source)
# define GC_MARKED_FOR_FINALIZATION(dest)
#endif /* !KEEP_BACK_PTRS */
/* Make arguments appear live to compiler. */
void GC_noop6(word, word, word, word, word, word);
#ifndef GC_ATTR_FORMAT_PRINTF
# if GC_GNUC_PREREQ(3, 0)
# define GC_ATTR_FORMAT_PRINTF(spec_argnum, first_checked) \
__attribute__((__format__(__printf__, spec_argnum, first_checked)))
# else
# define GC_ATTR_FORMAT_PRINTF(spec_argnum, first_checked)
# endif
#endif
/* Logging and diagnostic output. */
/*
* `GC_printf` is used typically on client explicit print requests.
* A variant of `printf` that does not allocate, 1 KB total output length.
* (It uses `sprintf()` internally; hopefully the latter does not allocate
* memory for `long` arguments.) For all `GC_*_printf` routines,
* it is recommended to put "\n" at the end of `format` string (for the
* output atomicity).
*/
GC_API_PRIV void GC_printf(const char *format, ...)
GC_ATTR_FORMAT_PRINTF(1, 2);
GC_API_PRIV void GC_err_printf(const char *format, ...)
GC_ATTR_FORMAT_PRINTF(1, 2);
/*
* The basic logging routine. Typically, it is called directly only inside
* various `DEBUG_*` blocks.
*/
GC_API_PRIV void GC_log_printf(const char *format, ...)
GC_ATTR_FORMAT_PRINTF(1, 2);
#ifndef GC_ANDROID_LOG
# define GC_PRINT_STATS_FLAG (GC_print_stats != 0)
# define GC_INFOLOG_PRINTF GC_COND_LOG_PRINTF
/*
* The "verbose" logging routine which is called only if `GC_print_stats`
* is `VERBOSE`.
*/
# define GC_verbose_log_printf GC_log_printf
#else
extern GC_bool GC_quiet;
# define GC_PRINT_STATS_FLAG (!GC_quiet)
/* `INFO`/`DBG` loggers are enabled even if `GC_print_stats` is off. */
# ifndef GC_INFOLOG_PRINTF
# define GC_INFOLOG_PRINTF \
if (GC_quiet) { \
} else \
GC_info_log_printf
# endif
GC_INNER void GC_info_log_printf(const char *format, ...)
GC_ATTR_FORMAT_PRINTF(1, 2);
GC_INNER void GC_verbose_log_printf(const char *format, ...)
GC_ATTR_FORMAT_PRINTF(1, 2);
#endif /* GC_ANDROID_LOG */
#if defined(SMALL_CONFIG) || defined(GC_ANDROID_LOG)
# define GC_ERRINFO_PRINTF GC_INFOLOG_PRINTF
#else
# define GC_ERRINFO_PRINTF GC_log_printf
#endif
/*
* Convenient wrapper macros over `GC_log_printf()` and
* `GC_verbose_log_printf()`.
*/
#define GC_COND_LOG_PRINTF \
if (LIKELY(!GC_print_stats)) { \
} else \
GC_log_printf
#define GC_VERBOSE_LOG_PRINTF \
if (LIKELY(GC_print_stats != VERBOSE)) { \
} else \
GC_verbose_log_printf
#ifndef GC_DBGLOG_PRINTF
# define GC_DBGLOG_PRINTF \
if (!GC_PRINT_STATS_FLAG) { \
} else \
GC_log_printf
#endif
/* Write `s` to `stderr`, but do not buffer, do not add newlines, do not... */
void GC_err_puts(const char *s);
/*
* A handy macro for logging size values (of `word` type) in KiB, rounding
* to nearest value.
*/
#define TO_KiB_UL(v) ((unsigned long)(((v) + ((1 << 9) - 1)) >> 10))
#ifdef USE_MUNMAP
GC_EXTERN unsigned GC_unmap_threshold; /*< defined in `alloc.c` file */
/*
* Force memory unmapping on every collection. Has no effect on
* implicitly-initiated collections.
*/
GC_EXTERN GC_bool GC_force_unmap_on_gcollect;
#endif
#ifdef MSWIN32
GC_EXTERN GC_bool GC_no_win32_dlls; /*< defined in `os_dep.c` file */
/* Is this a Windows NT derivative (i.e. NT, Win2K, XP or later)? */
GC_EXTERN GC_bool GC_wnt;
#endif
#ifdef THREADS
# if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
GC_EXTERN CRITICAL_SECTION GC_write_cs;
# ifdef GC_ASSERTIONS
/*
* Set to `TRUE` only if `GC_stop_world()` has acquired `GC_write_cs`.
* Protected by `GC_write_cs`.
*/
GC_EXTERN GC_bool GC_write_disabled;
# endif
# endif /* MSWIN32 || MSWINCE */
# ifdef NEED_FAULT_HANDLER_LOCK
/*
* Acquire the spin lock we use to update dirty bits. Threads should
* not get stopped holding it. But we may acquire and release it during
* `GC_remove_protection()` call.
*/
# define GC_acquire_dirty_lock() \
do { /* Empty. */ \
} while (AO_test_and_set_acquire(&GC_fault_handler_lock) == AO_TS_SET)
# define GC_release_dirty_lock() AO_CLEAR(&GC_fault_handler_lock)
# else
# define GC_acquire_dirty_lock() (void)0
# define GC_release_dirty_lock() (void)0
# endif
# ifdef MSWINCE
GC_EXTERN GC_bool GC_dont_query_stack_min;
# endif
#elif defined(IA64)
/* Value returned from register flushing routine (`ar.bsp`). */
GC_EXTERN ptr_t GC_save_regs_ret_val;
#endif /* !THREADS */
#ifdef THREAD_LOCAL_ALLOC
GC_EXTERN GC_bool GC_world_stopped; /*< defined in `alloc.c` file */
/*
* We must explicitly mark `ptrfree` and `gcj` free lists, since the
* free list links would not otherwise be found. We also set them in
* the normal free lists, since that involves touching less memory than
* if we scanned them normally.
*/
GC_INNER void GC_mark_thread_local_free_lists(void);
#endif
#if defined(GLIBC_2_19_TSX_BUG) && defined(GC_PTHREADS_PARAMARK)
/* Parse string like `<major>[.<minor>[<tail>]]` and return `major` value. */
GC_INNER int GC_parse_version(int *pminor, const char *pverstr);
#endif
#if defined(MPROTECT_VDB) && defined(GWW_VDB)
/*
* Returns `TRUE` if `GetWriteWatch()` is available. May be called
* repeatedly. May be called with or without the allocator lock held.
*/
GC_INNER GC_bool GC_gww_dirty_init(void);
#endif
#if defined(CHECKSUMS) || defined(PROC_VDB)
/* Could any valid GC heap pointer ever have been written to this page? */
GC_INNER GC_bool GC_page_was_ever_dirty(struct hblk *h);
#endif
#ifndef GC_NO_DEINIT
GC_INNER void GC_reset_freelist(void);
GC_INNER void GC_reset_obj_kinds(void);
# ifdef CHECKSUMS
void GC_reset_check_page(void);
# endif
# ifdef THREADS
GC_INNER void GC_reset_threads(void);
# endif
# ifdef THREAD_LOCAL_ALLOC
GC_INNER void GC_reset_thread_local_initialization(void);
# endif
#endif
#ifdef CHECKSUMS
# ifdef MPROTECT_VDB
void GC_record_fault(struct hblk *h);
# endif
void GC_check_dirty(void);
#endif
GC_INNER void GC_setpagesize(void);
GC_INNER void GC_initialize_offsets(void);
#if defined(REDIR_MALLOC_AND_LINUXTHREADS) \
&& !defined(REDIRECT_MALLOC_IN_HEADER)
GC_INNER void GC_init_lib_bounds(void);
#else
# define GC_init_lib_bounds() (void)0
#endif
#ifdef REDIR_MALLOC_AND_LINUXTHREADS
/*
* Find the text (code) mapping for the library whose name, after
* stripping the directory part, starts with `nm`.
*/
GC_INNER GC_bool GC_text_mapping(const char *nm, ptr_t *startp, ptr_t *endp);
#endif
#if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
GC_INNER void GC_add_current_malloc_heap(void);
#endif
#ifdef MAKE_BACK_GRAPH
/*
* Rebuild the representation of the backward reachability graph.
* Does not examine mark bits. Could be called before collection.
*/
GC_INNER void GC_build_back_graph(void);
GC_INNER void GC_traverse_back_graph(void);
#endif
#ifdef MSWIN32
GC_INNER void GC_init_win32(void);
#endif
#ifndef ANY_MSWIN
/*
* Is a particular static root (with the given start) registered?
* If so, then return a pointer to it, else `NULL`. The type is a lie,
* since the real type does not make sense here, and we only test for `NULL`.
*/
GC_INNER void *GC_roots_present(ptr_t);
#endif
#if defined(GC_WIN32_THREADS)
/* Same as `GC_push_one` but for a sequence of registers. */
GC_INNER void GC_push_many_regs(const word *regs, unsigned count);
/*
* Find stack with the lowest address which overlaps the interval
* [`start`, `limit`). Return stack bounds in `*plo` and `*phi`.
* If no such stack is found, both `*phi` and `*plo` will be set to an
* address higher than `limit`.
*/
GC_INNER void GC_get_next_stack(ptr_t start, ptr_t limit, ptr_t *plo,
ptr_t *phi);
# if defined(MPROTECT_VDB) && !defined(CYGWIN32)
GC_INNER void GC_set_write_fault_handler(void);
# endif
# if defined(WRAP_MARK_SOME) && !defined(GC_PTHREADS)
/*
* Did we invalidate mark phase with an unexpected thread start?
* Return `TRUE` if a thread was attached since we last asked or since
* `GC_attached_thread` was explicitly reset.
*/
GC_INNER GC_bool GC_started_thread_while_stopped(void);
# endif
#endif /* GC_WIN32_THREADS */
#if defined(MPROTECT_VDB) && defined(DARWIN) && defined(THREADS)
GC_INNER void GC_mprotect_stop(void);
GC_INNER void GC_mprotect_resume(void);
# ifndef GC_NO_THREADS_DISCOVERY
GC_INNER void GC_darwin_register_self_mach_handler(void);
# endif
#endif
#ifndef NOT_GCBUILD
/*
* Iterate over forwarding addresses, if any, to get the beginning of
* the block and its header. Assumes `*phhdr` is non-`NULL` on entry,
* and guarantees `*phhdr` is non-`NULL` on return.
*/
GC_INLINE struct hblk *
GC_find_starting_hblk(struct hblk *h, hdr **phhdr)
{
hdr *hhdr = *phhdr;
GC_ASSERT(HDR(h) == hhdr);
for (; IS_FORWARDING_ADDR_OR_NIL(hhdr); hhdr = HDR(h)) {
GC_ASSERT(hhdr != NULL);
h = FORWARDED_ADDR(h, hhdr);
}
*phhdr = hhdr;
return h;
}
#endif /* !NOT_GCBUILD */
#if (defined(PARALLEL_MARK) \
&& !defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG) \
&& (defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) \
|| defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID) \
|| defined(HAVE_PTHREAD_SET_NAME_NP))) \
|| (defined(DYNAMIC_LOADING) \
&& ((defined(USE_PROC_FOR_LIBRARIES) && !defined(LINUX)) \
|| defined(DARWIN) || defined(IRIX5))) \
|| defined(PROC_VDB) || defined(SOFT_VDB)
/*
* A function to convert a long integer value `lv` to a string adding
* the `prefix` and optional `suffix`. The resulting string is put to
* `buf` of the designated size (`buf_sz`). Guaranteed to append
* a trailing "\0" and not to exceed the buffer size. (Note that it is
* recommended to reserve at least 20 characters for the number part of
* the string in `buf` to avoid a compiler warning about potential
* number truncation.)
*/
# ifndef GC_DISABLE_SNPRINTF
# define GC_snprintf_s_ld_s(buf, buf_sz, prefix, lv, suffix) \
(void)(snprintf(buf, buf_sz, "%s%ld%s", prefix, lv, suffix), \
(buf)[(buf_sz) - (size_t)1] = '\0')
# else
# define NEED_SNPRINTF_SLDS
GC_INNER void GC_snprintf_s_ld_s(char *buf, size_t buf_sz, const char *prefix,
long lv, const char *suffix);
# endif
#endif
#ifdef THREADS
# ifndef GC_NO_FINALIZATION
/* Called by `GC_finalize()` (in case of an allocation failure observed). */
GC_INNER void GC_reset_finalizer_nested(void);
/*
* Check and update the thread-local level of finalizers recursion.
* Returns `NULL` if `GC_invoke_finalizers()` should not be called by
* the collector (to minimize the risk of a deep finalizers recursion),
* otherwise returns a pointer to the thread-local `finalizer_nested`.
* Called by `GC_notify_or_invoke_finalizers()` only.
*/
GC_INNER unsigned char *GC_check_finalizer_nested(void);
# endif
GC_INNER void GC_do_blocking_inner(ptr_t data, void *context);
/*
* Should do exactly the right thing if the world is stopped; should
* not fail if it is not stopped.
*/
GC_INNER void GC_push_all_stacks(void);
# ifdef USE_PROC_FOR_LIBRARIES
GC_INNER GC_bool GC_segment_is_thread_stack(ptr_t lo, ptr_t hi);
# endif
# if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
&& defined(IA64)
/*
* Find the largest stack base smaller than `bound`. May be used to find
* the boundary between a register stack and adjacent immediately preceding
* memory stack.
*/
GC_INNER ptr_t GC_greatest_stack_base_below(ptr_t bound);
# endif
#endif /* THREADS */
#ifdef DYNAMIC_LOADING
/* Do we need to separately register the main static data segment? */
GC_INNER GC_bool GC_register_main_static_data(void);
# ifdef DARWIN
GC_INNER void GC_init_dyld(void);
# endif
#endif /* DYNAMIC_LOADING */
#ifdef SEARCH_FOR_DATA_START
GC_INNER void GC_init_linux_data_start(void);
#endif
#ifdef NEED_PROC_MAPS
# if defined(DYNAMIC_LOADING) && defined(USE_PROC_FOR_LIBRARIES) \
|| defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) \
|| (defined(CHECK_SOFT_VDB) && defined(MPROTECT_VDB)) \
|| defined(REDIR_MALLOC_AND_LINUXTHREADS)
/*
* Assign various fields of the first line in `maps_ptr` to `*p_start`,
* `*p_end`, `*p_prot`, `*p_maj_dev` and `*p_mapping_name`.
* `p_mapping_name` may be `NULL`. `*p_prot` and `*p_mapping_name` are
* assigned pointers into the original buffer.
*/
GC_INNER const char *GC_parse_map_entry(const char *maps_ptr, ptr_t *p_start,
ptr_t *p_end, const char **p_prot,
unsigned *p_maj_dev,
const char **p_mapping_name);
# endif
# if defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) \
|| (defined(CHECK_SOFT_VDB) && defined(MPROTECT_VDB))
/*
* Try to read the backing store base from `/proc/self/maps` file.
* Return the bounds of the writable mapping with a zero major device,
* which includes the address passed as data. Return `FALSE` if there
* is no such mapping.
*/
GC_INNER GC_bool GC_enclosing_writable_mapping(ptr_t addr, ptr_t *startp,
ptr_t *endp);
# endif
/*
* Copy the content of `/proc/self/maps` file to a buffer in our
* address space. Return the address of the buffer.
*/
GC_INNER const char *GC_get_maps(void);
#endif /* NEED_PROC_MAPS */
#ifdef GC_ASSERTIONS
/* Should return the same value as `GC_large_free_bytes`. */
GC_INNER word GC_compute_large_free_bytes(void);
/* Should return the same value as `GC_root_size`. */
GC_INNER word GC_compute_root_size(void);
#endif
/* Check a compile time assertion at compile time. */
#if defined(_MSC_VER) && (_MSC_VER >= 1700)
# define GC_STATIC_ASSERT(e) static_assert(e, "static assertion failed: " #e)
#elif defined(static_assert) && !defined(CPPCHECK) \
&& (__STDC_VERSION__ >= 201112L)
# define GC_STATIC_ASSERT(e) \
do { /* placed in `do`-`while` for proper formatting by clang-format */ \
static_assert(e, #e); \
} while (0)
#elif defined(mips) && !defined(__GNUC__) && !defined(CPPCHECK)
/*
* DOB: MIPSPro C gets an internal error taking the `sizeof` an array type.
* This code works correctly (ugliness is to avoid "unused var" warnings).
*/
# define GC_STATIC_ASSERT(e) \
do { \
if (0) { \
char j[(e) ? 1 : -1]; \
j[0] = '\0'; \
j[0] = j[0]; \
} \
} while (0)
#else
/* The error message for failure is a bit baroque, but... */
# define GC_STATIC_ASSERT(e) (void)sizeof(char[(e) ? 1 : -1])
#endif
/*
* Runtime check for an argument declared as non-`NULL` is actually
* not `NULL`.
*/
#if GC_GNUC_PREREQ(4, 0)
/* Workaround tautological-pointer-compare Clang warning. */
# define NONNULL_ARG_NOT_NULL(arg) \
(*CAST_THRU_UINTPTR(volatile void **, &(arg)) != NULL)
#else
# define NONNULL_ARG_NOT_NULL(arg) ((arg) != NULL)
#endif
#define COND_DUMP_CHECKS \
do { \
GC_ASSERT(I_HOLD_LOCK()); \
GC_ASSERT(GC_compute_large_free_bytes() == GC_large_free_bytes); \
GC_ASSERT(GC_compute_root_size() == GC_root_size); \
} while (0)
#ifndef NO_DEBUGGING
/* A flag to generate regular debugging dumps. */
GC_EXTERN GC_bool GC_dump_regularly;
# define COND_DUMP \
if (UNLIKELY(GC_dump_regularly)) { \
GC_dump_named(NULL); \
} else \
COND_DUMP_CHECKS
#else
# define COND_DUMP COND_DUMP_CHECKS
#endif
/*
* We need additional synchronization facilities from the thread support.
* We believe these are less performance critical than the allocator lock;
* standard `pthreads`-based implementations should be sufficient.
*/
#ifdef PARALLEL_MARK
/*
* Number of mark threads we would like to have excluding the initiating
* thread.
*/
# define GC_markers_m1 GC_parallel
/* A flag to temporarily avoid parallel marking. */
GC_EXTERN GC_bool GC_parallel_mark_disabled;
/*
* The routines to deal with the mark lock and condition variables.
* If the allocator lock is also acquired, it must be done first.
* The mark lock is used to both protect some variables used by the
* parallel marker, and to protect `GC_fl_builder_count`.
* `GC_notify_all_marker()` is called when the state of the parallel marker
* changes in some significant way (see `gc_mark.h` file for details).
* The latter set of events includes incrementing `GC_mark_no`.
* `GC_notify_all_builder()` is called when `GC_fl_builder_count` reaches
* zero.
*/
/*
* Wait all markers to finish initialization (i.e. store `marker_sp`,
* `marker_bsp`, `marker_mach_threads`, `GC_marker_Id`).
*/
GC_INNER void GC_wait_for_markers_init(void);
GC_INNER void GC_acquire_mark_lock(void);
GC_INNER void GC_release_mark_lock(void);
GC_INNER void GC_notify_all_builder(void);
GC_INNER void GC_wait_for_reclaim(void);
/*
* Number of threads currently building free lists without holding
* the allocator lock. It is not safe to collect if this is nonzero.
* Also, together with the mark lock, it is used as a semaphore during
* marker threads startup. Protected by the mark lock.
*/
GC_EXTERN GC_signed_word GC_fl_builder_count;
GC_INNER void GC_notify_all_marker(void);
GC_INNER void GC_wait_marker(void);
/*
* Try to help out parallel marker, if it is running, for mark cycle
* `my_mark_no`. Returns if the mark cycle finishes or was already
* done, or there was nothing to do for some other reason. We hold the
* mark lock only, the initiating thread holds the allocator lock.
*/
GC_INNER void GC_help_marker(word my_mark_no);
GC_INNER void GC_start_mark_threads_inner(void);
# define INCR_MARKS(hhdr) \
AO_store(&(hhdr)->hb_n_marks, AO_load(&(hhdr)->hb_n_marks) + 1)
#else
# define INCR_MARKS(hhdr) (void)(++(hhdr)->hb_n_marks)
#endif /* !PARALLEL_MARK */
#if defined(SIGNAL_BASED_STOP_WORLD) && !defined(SIG_SUSPEND)
/*
* We define the thread suspension signal here, so that we can refer
* to it in the virtual dirty bit (VDB) implementation, if necessary.
* Ideally we would allocate a (real-time?) signal using the standard
* mechanism. Unfortunately, there is no such one. (There is one in
* Linux `glibc`, but it is not exported.) Thus we continue to use
* the same hard-coded signals we have always used.
*/
# ifdef THREAD_SANITIZER
/*
* Unfortunately, use of an asynchronous signal to suspend threads leads to
* the situation when the signal is not delivered (it is stored to
* `pending_signals` in TSan runtime actually) while the destination thread
* is blocked in `pthread_mutex_lock()`. Thus, we use some synchronous one
* instead (which is again unlikely to be used by clients directly).
*/
# define SIG_SUSPEND SIGSYS
# elif (defined(DGUX) || defined(LINUX)) && !defined(GC_USESIGRT_SIGNALS)
# if defined(SPARC) && !defined(SIGPWR)
/*
* Linux/SPARC does not properly define `SIGPWR` in platform `signal.h` file.
* It is aliased to `SIGLOST` in platform `asm/signal.h` file, though.
*/
# define SIG_SUSPEND SIGLOST
# else
/* LinuxThreads itself uses `SIGUSR1` and `SIGUSR2`. */
# define SIG_SUSPEND SIGPWR
# endif
# elif defined(FREEBSD) && defined(__GLIBC__) && !defined(GC_USESIGRT_SIGNALS)
# define SIG_SUSPEND (32 + 6)
# elif (defined(FREEBSD) || defined(HURD) || defined(RTEMS)) \
&& !defined(GC_USESIGRT_SIGNALS)
# define SIG_SUSPEND SIGUSR1
/* `SIGTSTP` and `SIGCONT` could be used alternatively on FreeBSD. */
# elif (defined(OPENBSD) && !defined(GC_USESIGRT_SIGNALS)) \
|| defined(SERENITY)
# define SIG_SUSPEND SIGXFSZ
# elif defined(_SIGRTMIN) && !defined(CPPCHECK)
# define SIG_SUSPEND _SIGRTMIN + 6
# else
# define SIG_SUSPEND SIGRTMIN + 6
# endif
#endif /* GC_PTHREADS && !SIG_SUSPEND */
#if defined(GC_PTHREADS) && !defined(GC_SEM_INIT_PSHARED)
# define GC_SEM_INIT_PSHARED 0
#endif
/*
* Some macros for `setjmp()` working across signal handlers, where
* possible.
*/
#if (defined(UNIX_LIKE) || (defined(NEED_FIND_LIMIT) && defined(CYGWIN32))) \
&& !defined(GC_NO_SIGSETJMP)
# if defined(SUNOS5SIGS) && !defined(FREEBSD) && !defined(LINUX)
EXTERN_C_END
# include <sys/siginfo.h>
EXTERN_C_BEGIN
# endif
/*
* Define `SETJMP()` and friends to be the variant restoring the signal
* mask.
*/
# define SETJMP(env) sigsetjmp(env, 1)
# define LONGJMP(env, val) siglongjmp(env, val)
# define JMP_BUF sigjmp_buf
#else
# ifdef ECOS
# define SETJMP(env) hal_setjmp(env)
# else
# define SETJMP(env) setjmp(env)
# endif
# define LONGJMP(env, val) longjmp(env, val)
# define JMP_BUF jmp_buf
#endif /* !UNIX_LIKE || GC_NO_SIGSETJMP */
#ifdef DATASTART_USES_XGETDATASTART
# ifdef FREEBSD
EXTERN_C_END
# include <machine/trap.h>
EXTERN_C_BEGIN
# endif
GC_INNER ptr_t GC_SysVGetDataStart(size_t, ptr_t);
#endif /* DATASTART_USES_XGETDATASTART */
#if defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS) \
|| defined(NEED_FIND_LIMIT) || defined(SEARCH_FOR_DATA_START)
# if (defined(HOST_ANDROID) || defined(__ANDROID__)) \
&& defined(IGNORE_DYNAMIC_LOADING)
/* Declared as public one in `gc.h` file. */
# else
void *GC_find_limit(void *p, int up);
# endif
#endif
#if defined(NEED_FIND_LIMIT) \
|| (defined(UNIX_LIKE) && !defined(NO_DEBUGGING)) \
|| (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) \
|| (defined(WRAP_MARK_SOME) && defined(NO_SEH_AVAILABLE))
typedef void (*GC_fault_handler_t)(int);
GC_INNER void GC_set_and_save_fault_handler(GC_fault_handler_t);
#endif
#if defined(NEED_FIND_LIMIT) \
|| (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) \
|| (defined(WRAP_MARK_SOME) && defined(NO_SEH_AVAILABLE))
GC_EXTERN JMP_BUF GC_jmp_buf;
/*
* Set up a handler for address faults which will `longjmp`
* to `GC_jmp_buf`.
*/
GC_INNER void GC_setup_temporary_fault_handler(void);
/* Undo the effect of `GC_setup_temporary_fault_handler`. */
GC_INNER void GC_reset_fault_handler(void);
#endif /* NEED_FIND_LIMIT || USE_PROC_FOR_LIBRARIES || WRAP_MARK_SOME */
/* Some convenience macros for cancellation support. */
#ifdef CANCEL_SAFE
# if defined(GC_ASSERTIONS) \
&& (defined(USE_COMPILER_TLS) \
|| (defined(LINUX) && !defined(ARM32) && GC_GNUC_PREREQ(3, 3) \
|| defined(HPUX) /* and probably others... */))
extern __thread unsigned char GC_cancel_disable_count;
# define NEED_CANCEL_DISABLE_COUNT
# define INCR_CANCEL_DISABLE() ++GC_cancel_disable_count
# define DECR_CANCEL_DISABLE() --GC_cancel_disable_count
# define ASSERT_CANCEL_DISABLED() GC_ASSERT(GC_cancel_disable_count > 0)
# else
# define INCR_CANCEL_DISABLE()
# define DECR_CANCEL_DISABLE()
# define ASSERT_CANCEL_DISABLED() (void)0
# endif /* !GC_ASSERTIONS */
# define DISABLE_CANCEL(state) \
do { \
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &state); \
INCR_CANCEL_DISABLE(); \
} while (0)
# define RESTORE_CANCEL(state) \
do { \
ASSERT_CANCEL_DISABLED(); \
pthread_setcancelstate(state, NULL); \
DECR_CANCEL_DISABLE(); \
} while (0)
#else
# define DISABLE_CANCEL(state) (void)0
# define RESTORE_CANCEL(state) (void)0
# define ASSERT_CANCEL_DISABLED() (void)0
#endif /* !CANCEL_SAFE */
/* Multiply 32-bit unsigned values (used by `GC_push_contents_hdr()`). */
#ifdef NO_LONGLONG64
# define LONG_MULT(hprod, lprod, x, y) \
do { \
unsigned32 lx = (x) & (0xffffU); \
unsigned32 ly = (y) & (0xffffU); \
unsigned32 hx = (x) >> 16; \
unsigned32 hy = (y) >> 16; \
unsigned32 lxhy = lx * hy; \
unsigned32 mid = hx * ly + lxhy; /*< may overflow */ \
unsigned32 lxly = lx * ly; \
\
lprod = (mid << 16) + lxly; /*< may overflow */ \
hprod = hx * hy + ((lprod) < lxly ? 1U : 0) \
+ (mid < lxhy ? (unsigned32)0x10000UL : 0) + (mid >> 16); \
} while (0)
#elif defined(I386) && defined(__GNUC__) && !defined(NACL)
# define LONG_MULT(hprod, lprod, x, y) \
__asm__ __volatile__("mull %2" : "=a"(lprod), "=d"(hprod) : "r"(y), "0"(x))
#else
# if (defined(__int64) && !defined(__GNUC__) || defined(__BORLANDC__)) \
&& !defined(CPPCHECK)
# define ULONG_MULT_T unsigned __int64
# else
# define ULONG_MULT_T unsigned long long
# endif
# define LONG_MULT(hprod, lprod, x, y) \
do { \
ULONG_MULT_T prod = (ULONG_MULT_T)(x) * (ULONG_MULT_T)(y); \
\
GC_STATIC_ASSERT(sizeof(x) + sizeof(y) <= sizeof(prod)); \
hprod = (unsigned32)(prod >> 32); \
lprod = (unsigned32)prod; \
} while (0)
#endif /* !I386 && !NO_LONGLONG64 */
EXTERN_C_END
#endif /* GC_PRIVATE_H */
#ifdef KEEP_BACK_PTRS
# include "gc/gc_backptr.h"
#endif
EXTERN_C_BEGIN
#ifndef GC_FREED_MEM_MARKER
# if CPP_WORDSZ == 32
# define GC_FREED_MEM_MARKER (GC_uintptr_t)0xdeadbeef
# else
# define GC_FREED_MEM_MARKER ((GC_uintptr_t)GC_WORD_C(0xEFBEADDEdeadbeef))
# endif
#endif /* !GC_FREED_MEM_MARKER */
/*
* Stored both one past the end of user object, and one before the end of
* the object as seen by the allocator.
*/
#if CPP_WORDSZ == 32
# define START_FLAG (GC_uintptr_t)0xfedcedcb
# define END_FLAG (GC_uintptr_t)0xbcdecdef
#else
# define START_FLAG ((GC_uintptr_t)GC_WORD_C(0xFEDCEDCBfedcedcb))
# define END_FLAG ((GC_uintptr_t)GC_WORD_C(0xBCDECDEFbcdecdef))
#endif
#if defined(KEEP_BACK_PTRS) || defined(PRINT_BLACK_LIST)
/*
* Pointer "source" variants that are not real locations. Used in
* `oh_back_ptr` fields and as `source` argument to some marking functions.
*/
/* Object was marked because it is finalizable. */
# define MARKED_FOR_FINALIZATION ((ptr_t)NUMERIC_TO_VPTR(2))
/*
* Object was marked from a register. Hence the "source" of the reference
* does not have an address.
*/
# define MARKED_FROM_REGISTER ((ptr_t)NUMERIC_TO_VPTR(4))
# define NOT_MARKED ((ptr_t)NUMERIC_TO_VPTR(8))
#endif /* KEEP_BACK_PTRS || PRINT_BLACK_LIST */
/*
* Object debug header. The size of the structure is assumed not to
* de-align things, and to be a multiple of a double-pointer length.
*/
typedef struct {
#if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
/*
* We potentially keep two different kinds of back pointers.
* `KEEP_BACK_PTRS` stores a single back pointer in each reachable
* object to allow reporting of why an object was retained.
* `MAKE_BACK_GRAPH` builds a graph containing the inverse of all
* "points-to" edges including those involving objects that have just
* become unreachable. This allows detection of growing chains of
* unreachable objects. It may be possible to eventually combine both,
* but for now we keep them separate. Both kinds of back pointers are
* hidden using the following macros. In both cases, the plain variant
* is constrained to have the least significant bit of 1, to allow it
* to be distinguished from a free-list link. This means the plain
* variant must have the least significant bit of zero.
* Note that blocks dropped by black-listing will also have the least
* significant bit clear once debugging has started; we are careful never
* to overwrite such a value.
*/
# if ALIGNMENT == 1
/* Fudge back pointer to be even. */
# define HIDE_BACK_PTR(p) \
GC_HIDE_POINTER((ptr_t)(~(GC_uintptr_t)1 & (GC_uintptr_t)(p)))
# else
# define HIDE_BACK_PTR(p) GC_HIDE_POINTER(p)
# endif
/*
* Always define either none or both of the fields to ensure
* double-pointer alignment.
*/
GC_hidden_pointer oh_back_ptr;
GC_hidden_pointer oh_bg_ptr;
#endif
const char *oh_string; /*< object descriptor string (file name) */
GC_signed_word oh_int; /*< object descriptor integer (line number) */
#ifdef NEED_CALLINFO
struct callinfo oh_ci[NFRAMES];
#endif
#ifndef SHORT_DBG_HDRS
GC_uintptr_t oh_sz; /*< the original `malloc` argument */
GC_uintptr_t oh_sf; /*< the "start" flag (marker) */
#endif
} oh;
#define GET_OH_LINENUM(ohdr) ((int)(ohdr)->oh_int)
#ifdef SHORT_DBG_HDRS
# define DEBUG_BYTES sizeof(oh)
# define UNCOLLECTABLE_DEBUG_BYTES DEBUG_BYTES
#else
/*
* Add space for `END_FLAG`, but use any extra space that was already
* added to catch off-the-end pointers. For uncollectible objects, the
* extra byte is not added.
*/
# define UNCOLLECTABLE_DEBUG_BYTES (sizeof(oh) + sizeof(GC_uintptr_t))
# define DEBUG_BYTES (UNCOLLECTABLE_DEBUG_BYTES - EXTRA_BYTES)
#endif
/*
* `ADD_CALL_CHAIN` stores a (partial) call chain into an object header;
* it should be called with the allocator lock held.
* `PRINT_CALL_CHAIN` prints the call chain stored in an object to `stderr`;
* it requires we do not hold the allocator lock.
*/
#if defined(SAVE_CALL_CHAIN)
# define ADD_CALL_CHAIN(base, ra) GC_save_callers(((oh *)(base))->oh_ci)
# if defined(REDIRECT_MALLOC) && defined(THREADS) && defined(DBG_HDRS_ALL) \
&& NARGS == 0 && NFRAMES % 2 == 0 && defined(GC_HAVE_BUILTIN_BACKTRACE)
/*
* A dummy variant of `GC_save_callers()` which does not call
* `backtrace()`.
*/
GC_INNER void GC_save_callers_no_unlock(struct callinfo info[NFRAMES]);
# define ADD_CALL_CHAIN_INNER(base) \
GC_save_callers_no_unlock(((oh *)(base))->oh_ci)
# endif
# define PRINT_CALL_CHAIN(base) GC_print_callers(((oh *)(base))->oh_ci)
#elif defined(GC_ADD_CALLER)
# define ADD_CALL_CHAIN(base, ra) ((oh *)(base))->oh_ci[0].ci_pc = (ra)
# define PRINT_CALL_CHAIN(base) GC_print_callers(((oh *)(base))->oh_ci)
#else
# define ADD_CALL_CHAIN(base, ra)
# define PRINT_CALL_CHAIN(base)
#endif
#if !defined(ADD_CALL_CHAIN_INNER) && defined(DBG_HDRS_ALL)
/* A variant of `ADD_CALL_CHAIN()` used for internal allocations. */
# define ADD_CALL_CHAIN_INNER(base) ADD_CALL_CHAIN(base, GC_RETURN_ADDR)
#endif
#ifdef GC_ADD_CALLER
# define OPT_RA ra,
#else
# define OPT_RA
#endif
/*
* Check whether object given by its base pointer has debugging info.
* The argument (`base`) is assumed to point to a legitimate object in the
* collector heap. This excludes the check as to whether the back pointer
* is odd, which is added by the `GC_HAS_DEBUG_INFO` macro. Note that
* if `DBG_HDRS_ALL` is defined, uncollectible objects on free lists
* may not have debug information set. Thus, it is not always safe to
* return 1 (true), even if the client does its part. Return -1 if the
* object with debug info has been marked as deallocated.
*/
#ifdef SHORT_DBG_HDRS
# define GC_has_other_debug_info(base) 1
#else
GC_INNER int GC_has_other_debug_info(ptr_t base);
GC_INNER void GC_add_smashed(ptr_t smashed);
/*
* Use `GC_err_printf()` and friends to print a description of the object
* whose client-visible address is `p`, and which was smashed at memory
* location pointed by `clobbered`.
*/
GC_INNER void GC_print_smashed_obj(const char *msg, void *p, ptr_t clobbered);
/* Print all objects on the list. Clear the list. */
GC_INNER void GC_print_all_smashed_proc(void);
#endif /* !SHORT_DBG_HDRS */
#if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
# if defined(SHORT_DBG_HDRS) && !defined(CPPCHECK)
# error Non-ptr stored in object results in GC_HAS_DEBUG_INFO malfunction
/* We may mistakenly conclude that base has a debugging wrapper. */
# endif
# if defined(PARALLEL_MARK) && defined(KEEP_BACK_PTRS)
/*
* Note: the atomic load is used as `GC_store_back_pointer` stores
* `oh_back_ptr` atomically (`base` might point to the field); this prevents
* a TSan warning.
*/
# define GC_HAS_DEBUG_INFO(base) \
(((GC_uintptr_t)GC_cptr_load((volatile ptr_t *)(base)) & 1) != 0 \
&& GC_has_other_debug_info(base) > 0)
# else
# define GC_HAS_DEBUG_INFO(base) \
(((*(GC_uintptr_t *)(base)) & 1) != 0 \
&& GC_has_other_debug_info(base) > 0)
# endif
#else
# define GC_HAS_DEBUG_INFO(base) (GC_has_other_debug_info(base) > 0)
#endif /* !KEEP_BACK_PTRS && !MAKE_BACK_GRAPH */
EXTERN_C_END
#endif /* GC_DBG_MLC_H */
/*
* This implements a full, though not well-tuned, representation of the
* backwards points-to graph. This is used to test for non-GC-robust
* data structures; the code is not used during normal garbage collection.
*
* One restriction is that we drop all back-edges from nodes with very
* high in-degree, and simply add them to a list of such nodes. They are
* then treated as permanent roots. If this by itself does not introduce
* a space leak, then such nodes cannot contribute to a growing space leak.
*/
#ifdef MAKE_BACK_GRAPH
# if (!defined(DBG_HDRS_ALL) \
|| (ALIGNMENT != CPP_PTRSZ / 8) /* `|| !defined(UNIX_LIKE)` */) \
&& !defined(CPPCHECK)
# error The configuration does not support MAKE_BACK_GRAPH
# endif
/*
* We store single back pointers directly in the object's `oh_bg_ptr` field.
* If there is more than one pointer to an object, we store `q` or'ed with
* `FLAG_MANY`, where `q` is a pointer to a `back_edges` object.
* Every once in a while we use a `back_edges` object even for a single
* pointer, since we need the other fields in the `back_edges` structure to
* be present in some fraction of the objects. Otherwise we get serious
* performance issues.
*/
# define FLAG_MANY 2
/* Directly points to a reachable object; retain for the next collection. */
# define RETAIN 1
/* Special values of `height` field of `back_edges`. */
# define HEIGHT_UNKNOWN (-2)
# define HEIGHT_IN_PROGRESS (-1)
typedef struct back_edges_s back_edges;
# define MAX_BACK_EDGE_STRUCTS 100000
/*
* Allocate a new back edge structure. Should be more sophisticated
* if this were production code.
*/
static back_edges *
new_back_edges(void)
{
GC_ASSERT(I_HOLD_LOCK());
if (NULL == GC_back_edge_space) {
size_t bytes_to_get;
GC_ASSERT(0 == GC_n_back_edge_structs);
GC_ASSERT(NULL == GC_avail_back_edges);
GC_ASSERT(GC_page_size != 0);
bytes_to_get
= ROUNDUP_PAGESIZE_IF_MMAP(MAX_BACK_EDGE_STRUCTS * sizeof(back_edges));
GC_back_edge_space = (back_edges *)GC_os_get_mem(bytes_to_get);
if (NULL == GC_back_edge_space)
ABORT("Insufficient memory for back edges");
}
if (GC_avail_back_edges != NULL) {
back_edges *result = GC_avail_back_edges;
GC_avail_back_edges = result->cont;
result->cont = NULL;
return result;
}
if (GC_n_back_edge_structs >= MAX_BACK_EDGE_STRUCTS - 1) {
ABORT("Needed too much space for back edges: adjust "
"MAX_BACK_EDGE_STRUCTS");
}
return &GC_back_edge_space[GC_n_back_edge_structs++];
}
/* Deallocate `p` and its associated continuation structures. */
static void
deallocate_back_edges(back_edges *p)
{
back_edges *last;
for (last = p; last->cont != NULL;)
last = last->cont;
last->cont = GC_avail_back_edges;
GC_avail_back_edges = p;
}
# define INITIAL_IN_PROGRESS 10000
static void
push_in_progress(ptr_t p)
{
GC_ASSERT(I_HOLD_LOCK());
if (GC_backgraph_n_in_progress >= GC_backgraph_in_progress_size) {
ptr_t *new_in_progress_space;
GC_ASSERT(GC_page_size != 0);
if (NULL == GC_backgraph_in_progress_space) {
GC_ASSERT(0 == GC_backgraph_n_in_progress);
GC_ASSERT(0 == GC_backgraph_in_progress_size);
GC_backgraph_in_progress_size
= ROUNDUP_PAGESIZE_IF_MMAP(INITIAL_IN_PROGRESS * sizeof(ptr_t))
/ sizeof(ptr_t);
new_in_progress_space = (ptr_t *)GC_os_get_mem(
GC_backgraph_in_progress_size * sizeof(ptr_t));
} else {
GC_backgraph_in_progress_size *= 2;
new_in_progress_space = (ptr_t *)GC_os_get_mem(
GC_backgraph_in_progress_size * sizeof(ptr_t));
if (new_in_progress_space != NULL)
BCOPY(GC_backgraph_in_progress_space, new_in_progress_space,
GC_backgraph_n_in_progress * sizeof(ptr_t));
}
# ifndef GWW_VDB
GC_scratch_recycle_no_gww(GC_backgraph_in_progress_space,
GC_backgraph_n_in_progress * sizeof(ptr_t));
# elif defined(LINT2)
/* TODO: Implement GWW-aware recycling as in `alloc_mark_stack`. */
GC_noop1_ptr(GC_backgraph_in_progress_space);
# endif
GC_backgraph_in_progress_space = new_in_progress_space;
}
if (NULL == GC_backgraph_in_progress_space)
ABORT("MAKE_BACK_GRAPH: Out of in-progress space: "
"Huge linear data structure?");
GC_backgraph_in_progress_space[GC_backgraph_n_in_progress++] = p;
}
static GC_bool
is_in_progress(const char *p)
{
size_t i;
for (i = 0; i < GC_backgraph_n_in_progress; ++i) {
if (GC_backgraph_in_progress_space[i] == p)
return TRUE;
}
return FALSE;
}
GC_INLINE void
pop_in_progress(ptr_t p)
{
# ifndef GC_ASSERTIONS
UNUSED_ARG(p);
# endif
--GC_backgraph_n_in_progress;
GC_ASSERT(GC_backgraph_in_progress_space[GC_backgraph_n_in_progress] == p);
}
# define GET_OH_BG_PTR(p) (ptr_t) GC_REVEAL_POINTER(((oh *)(p))->oh_bg_ptr)
# define SET_OH_BG_PTR(p, q) (((oh *)(p))->oh_bg_ptr = GC_HIDE_POINTER(q))
/* Ensure that `p` has a `back_edges` structure associated with it. */
static void
ensure_struct(ptr_t p)
{
ptr_t old_back_ptr = GET_OH_BG_PTR(p);
GC_ASSERT(I_HOLD_LOCK());
if ((ADDR(old_back_ptr) & FLAG_MANY) == 0) {
back_edges *be = new_back_edges();
be->flags = 0;
# if defined(CPPCHECK)
GC_noop1_ptr(&old_back_ptr);
/* Workaround a false positive that `old_back_ptr` cannot be `NULL`. */
# endif
if (NULL == old_back_ptr) {
be->n_edges = 0;
} else {
be->n_edges = 1;
be->edges[0] = old_back_ptr;
}
be->height = HEIGHT_UNKNOWN;
be->height_gc_no = (unsigned short)(GC_gc_no - 1);
GC_ASSERT(ADDR_GE((ptr_t)be, (ptr_t)GC_back_edge_space));
SET_OH_BG_PTR(p, CPTR_SET_FLAGS(be, FLAG_MANY));
}
}
/*
* Add the (forward) edge from `p` to `q` to the backward graph. Both `p`
* and `q` are pointers to the object base, i.e. pointers to an `oh`.
*/
static void
add_edge(ptr_t p, ptr_t q)
{
ptr_t pred = GET_OH_BG_PTR(q);
back_edges *be, *be_cont;
word i;
GC_ASSERT(p == GC_base(p) && q == GC_base(q));
GC_ASSERT(I_HOLD_LOCK());
if (!GC_HAS_DEBUG_INFO(q) || !GC_HAS_DEBUG_INFO(p)) {
/*
* This is really a misinterpreted free-list link, since we saw
* a pointer to a free list. Do not overwrite it!
*/
return;
}
# if defined(CPPCHECK)
GC_noop1_ptr(&pred);
# endif
if (NULL == pred) {
/*
* A not very random number we use to occasionally allocate
* a `back_edges` structure even for a single backward edge.
* This prevents us from repeatedly tracing back through very long
* chains, since we will have some place to store `height` and
* `HEIGHT_IN_PROGRESS` flag along the way.
*/
# define GOT_LUCKY_NUMBER (((++random_number) & 0x7f) == 0)
static unsigned random_number = 13;
SET_OH_BG_PTR(q, p);
if (GOT_LUCKY_NUMBER)
ensure_struct(q);
return;
}
/* Check whether it was already in the list of predecessors. */
{
back_edges *e = (back_edges *)CPTR_CLEAR_FLAGS(pred, FLAG_MANY);
word n_edges;
word total;
int local = 0;
if ((ADDR(pred) & FLAG_MANY) != 0) {
n_edges = e->n_edges;
} else if ((COVERT_DATAFLOW(ADDR(pred)) & 1) == 0) {
/* A misinterpreted free-list link. */
n_edges = 1;
local = -1;
} else {
n_edges = 0;
}
for (total = 0; total < n_edges; ++total) {
if (local == BACKGRAPH_MAX_IN) {
e = e->cont;
local = 0;
}
if (local >= 0)
pred = e->edges[local++];
if (pred == p)
return;
}
}
ensure_struct(q);
be = (back_edges *)CPTR_CLEAR_FLAGS(GET_OH_BG_PTR(q), FLAG_MANY);
for (i = be->n_edges, be_cont = be; i > BACKGRAPH_MAX_IN;
i -= BACKGRAPH_MAX_IN)
be_cont = be_cont->cont;
if (i == BACKGRAPH_MAX_IN) {
be_cont->cont = new_back_edges();
be_cont = be_cont->cont;
i = 0;
}
be_cont->edges[i] = p;
be->n_edges++;
# ifdef DEBUG_PRINT_BIG_N_EDGES
if (GC_print_stats == VERBOSE && be->n_edges == 100) {
GC_err_printf("The following object has big in-degree:\n");
# ifdef THREADS
/*
* Note: we cannot call the debug variant of `GC_print_heap_obj` here
* because the allocator lock is held.
*/
GC_default_print_heap_obj_proc(q);
# else
GC_print_heap_obj(q);
# endif
}
# endif
}
typedef void (*per_object_func)(ptr_t p, size_t sz, word descr);
static GC_CALLBACK void
per_object_helper(struct hblk *h, void *fn_ptr)
{
const hdr *hhdr = HDR(h);
word descr = hhdr->hb_descr;
per_object_func fn = *(per_object_func *)fn_ptr;
size_t sz = hhdr->hb_sz;
size_t i = 0;
do {
fn((ptr_t)(h->hb_body + i), sz, descr);
i += sz;
} while (i + sz <= HBLKSIZE);
}
GC_INLINE void
GC_apply_to_each_object(per_object_func fn)
{
GC_apply_to_all_blocks(per_object_helper, &fn);
}
static void
reset_back_edge(ptr_t p, size_t sz, word descr)
{
UNUSED_ARG(sz);
UNUSED_ARG(descr);
GC_ASSERT(I_HOLD_LOCK());
/* Skip any free-list links, or dropped blocks. */
if (GC_HAS_DEBUG_INFO(p)) {
ptr_t old_back_ptr = GET_OH_BG_PTR(p);
if ((ADDR(old_back_ptr) & FLAG_MANY) != 0) {
back_edges *be = (back_edges *)CPTR_CLEAR_FLAGS(old_back_ptr, FLAG_MANY);
if (!(be->flags & RETAIN)) {
deallocate_back_edges(be);
SET_OH_BG_PTR(p, NULL);
} else {
GC_ASSERT(GC_is_marked(p));
/*
* Back edges may point to objects that will not be retained.
* Delete them for now, but remember the height. Some will be
* added back at next collection.
*/
be->n_edges = 0;
if (be->cont != NULL) {
deallocate_back_edges(be->cont);
be->cont = NULL;
}
GC_ASSERT(GC_is_marked(p));
/* We only retain things for one collection cycle at a time. */
be->flags &= (unsigned short)~RETAIN;
}
} else /* simple back pointer */ {
/* Clear to avoid dangling pointer. */
SET_OH_BG_PTR(p, NULL);
}
}
}
static void
add_back_edges(ptr_t p, size_t sz, word descr)
{
ptr_t current_p = p + sizeof(oh);
/* For now, fix up non-length descriptors conservatively. */
if ((descr & GC_DS_TAGS) != GC_DS_LENGTH) {
descr = sz;
}
for (; ADDR_LT(current_p, p + descr); current_p += sizeof(ptr_t)) {
ptr_t q;
LOAD_PTR_OR_CONTINUE(q, current_p);
FIXUP_POINTER(q);
if (GC_least_real_heap_addr < ADDR(q)
&& ADDR(q) < GC_greatest_real_heap_addr) {
ptr_t target = (ptr_t)GC_base(q);
if (target != NULL)
add_edge(p, target);
}
}
}
GC_INNER void
GC_build_back_graph(void)
{
GC_ASSERT(I_HOLD_LOCK());
GC_apply_to_each_object(add_back_edges);
}
/*
* Return an approximation to the length of the longest simple path through
* unreachable objects to `p`. We refer to this as the height of `p`.
*/
static word
backwards_height(ptr_t p)
{
word result;
ptr_t pred = GET_OH_BG_PTR(p);
back_edges *be;
GC_ASSERT(I_HOLD_LOCK());
# if defined(CPPCHECK)
GC_noop1_ptr(&pred);
# endif
if (NULL == pred)
return 1;
if ((ADDR(pred) & FLAG_MANY) == 0) {
if (is_in_progress(p)) {
/*
* DFS (depth-first search) back edge, i.e. we followed an edge to
* an object already on our stack. Ignore.
*/
return 0;
}
push_in_progress(p);
result = backwards_height(pred) + 1;
pop_in_progress(p);
return result;
}
be = (back_edges *)CPTR_CLEAR_FLAGS(pred, FLAG_MANY);
if (be->height >= 0 && be->height_gc_no == (unsigned short)GC_gc_no)
return (word)be->height;
/* Ignore back edges in DFS. */
if (be->height == HEIGHT_IN_PROGRESS)
return 0;
result = be->height > 0 ? (word)be->height : 1U;
be->height = HEIGHT_IN_PROGRESS;
{
back_edges *e = be;
word n_edges;
word total;
int local = 0;
if ((ADDR(pred) & FLAG_MANY) != 0) {
n_edges = e->n_edges;
} else if ((ADDR(pred) & 1) == 0) {
/* A misinterpreted free-list link. */
n_edges = 1;
local = -1;
} else {
n_edges = 0;
}
for (total = 0; total < n_edges; ++total) {
word this_height;
if (local == BACKGRAPH_MAX_IN) {
e = e->cont;
local = 0;
}
if (local >= 0)
pred = e->edges[local++];
/*
* Execute the following once for each predecessor `pred` of `p`
* in the points-to graph.
*/
if (GC_is_marked(pred) && (ADDR(GET_OH_BG_PTR(p)) & FLAG_MANY) == 0) {
GC_COND_LOG_PRINTF("Found bogus pointer from %p to %p\n", (void *)pred,
(void *)p);
/*
* Reachable object "points to" unreachable one. Could be caused
* by our lax treatment of the collector descriptors.
*/
this_height = 1;
} else {
this_height = backwards_height(pred);
}
if (this_height >= result)
result = this_height + 1;
}
}
be->height = (GC_signed_word)result;
be->height_gc_no = (unsigned short)GC_gc_no;
return result;
}
/*
* Compute the maximum height of every unreachable predecessor `p` of
* a reachable object. Arrange to save the heights of all such objects `p`
* so that they can be used in calculating the height of objects in the next
* collection. Set `GC_backgraph_deepest_height` to be the maximum height we
* encounter, and `GC_backgraph_deepest_obj` to be the corresponding object.
*/
static void
update_deepest_height(ptr_t p, size_t sz, word descr)
{
UNUSED_ARG(sz);
UNUSED_ARG(descr);
GC_ASSERT(I_HOLD_LOCK());
if (GC_is_marked(p) && GC_HAS_DEBUG_INFO(p)) {
word p_height = 0;
ptr_t p_deepest_obj = NULL;
ptr_t back_ptr;
back_edges *be = NULL;
/*
* If we remembered a height last time, use it as a minimum.
* It may have increased due to newly unreachable chains pointing
* to `p`, but it cannot have decreased.
*/
back_ptr = GET_OH_BG_PTR(p);
# if defined(CPPCHECK)
GC_noop1_ptr(&back_ptr);
# endif
if (back_ptr != NULL && (ADDR(back_ptr) & FLAG_MANY) != 0) {
be = (back_edges *)CPTR_CLEAR_FLAGS(back_ptr, FLAG_MANY);
if (be->height != HEIGHT_UNKNOWN)
p_height = (word)be->height;
}
{
ptr_t pred = back_ptr;
back_edges *e = (back_edges *)CPTR_CLEAR_FLAGS(pred, FLAG_MANY);
word n_edges;
word total;
int local = 0;
if ((ADDR(pred) & FLAG_MANY) != 0) {
n_edges = e->n_edges;
} else if (pred != NULL && (ADDR(pred) & 1) == 0) {
/* A misinterpreted free-list link. */
n_edges = 1;
local = -1;
} else {
n_edges = 0;
}
for (total = 0; total < n_edges; ++total) {
if (local == BACKGRAPH_MAX_IN) {
e = e->cont;
local = 0;
}
if (local >= 0)
pred = e->edges[local++];
/*
* Execute the following once for each predecessor `pred` of `p`
* in the points-to graph.
*/
if (!GC_is_marked(pred) && GC_HAS_DEBUG_INFO(pred)) {
word this_height = backwards_height(pred);
if (this_height > p_height) {
p_height = this_height;
p_deepest_obj = pred;
}
}
}
}
if (p_height > 0) {
/* Remember the height for next time. */
if (NULL == be) {
ensure_struct(p);
back_ptr = GET_OH_BG_PTR(p);
be = (back_edges *)CPTR_CLEAR_FLAGS(back_ptr, FLAG_MANY);
}
be->flags |= RETAIN;
be->height = (GC_signed_word)p_height;
be->height_gc_no = (unsigned short)GC_gc_no;
}
if (p_height > GC_backgraph_deepest_height) {
GC_backgraph_deepest_height = p_height;
GC_backgraph_deepest_obj = p_deepest_obj;
}
}
}
GC_INNER void
GC_traverse_back_graph(void)
{
GC_ASSERT(I_HOLD_LOCK());
GC_backgraph_deepest_height = 0;
GC_apply_to_each_object(update_deepest_height);
if (GC_backgraph_deepest_obj != NULL) {
/* Keep the pointer until we can print it. */
GC_set_mark_bit(GC_backgraph_deepest_obj);
}
}
void
GC_print_back_graph_stats(void)
{
GC_ASSERT(I_HOLD_LOCK());
GC_printf("Maximum backwards height of reachable objects"
" at GC #%lu is %lu\n",
(unsigned long)GC_gc_no,
(unsigned long)GC_backgraph_deepest_height);
if (GC_backgraph_deepest_height > GC_backgraph_max_deepest_h) {
ptr_t obj = GC_backgraph_deepest_obj;
GC_backgraph_max_deepest_h = GC_backgraph_deepest_height;
UNLOCK();
GC_err_printf(
"The following unreachable object is last in a longest chain "
"of unreachable objects:\n");
GC_print_heap_obj(obj);
LOCK();
}
GC_COND_LOG_PRINTF("Needed max total of %d back-edge structs\n",
GC_n_back_edge_structs);
GC_apply_to_each_object(reset_back_edge);
GC_backgraph_deepest_obj = NULL;
}
#endif /* MAKE_BACK_GRAPH */
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#ifndef NO_BLACK_LISTING
/*
* We maintain several hash tables of `hblk` entities that have had false
* hits. Each contains one bit per hash bucket. If any page in the bucket
* has had a false hit, we assume that all of them have.
* See the definition of `page_hash_table` in `gc_priv.h` file.
* False hits from the stack(s) are much more dangerous than false hits
* from elsewhere, since the former can pin a large object that spans the
* block, even though it does not start on the dangerous block.
*/
/*
* Externally callable routines are:
* - `GC_add_to_black_list_normal`,
* - `GC_add_to_black_list_stack`,
* - `GC_promote_black_lists`.
*/
GC_INNER word GC_black_list_spacing = 0;
STATIC void
GC_clear_bl(word *bl)
{
BZERO(bl, sizeof(page_hash_table));
}
STATIC void
GC_copy_bl(const word *old, word *dest)
{
BCOPY(old, dest, sizeof(page_hash_table));
}
# ifdef PRINT_BLACK_LIST
STATIC void
GC_print_blacklisted_ptr(ptr_t p, ptr_t source, const char *kind_str)
{
ptr_t base = (ptr_t)GC_base(source);
if (0 == base) {
GC_err_printf("Black listing (%s) %p referenced from %p in %s\n", kind_str,
(void *)p, (void *)source,
NULL != source ? "root set" : "register");
} else {
/*
* FIXME: We cannot call the debug variant of `GC_print_heap_obj`
* (with `PRINT_CALL_CHAIN`) here because the allocator lock is held
* and the world is stopped.
*/
GC_err_printf("Black listing (%s) %p referenced from %p in"
" object at %p of appr. %lu bytes\n",
kind_str, (void *)p, (void *)source, (void *)base,
(unsigned long)GC_size(base));
}
}
# endif /* PRINT_BLACK_LIST */
GC_INNER void
GC_bl_init_no_interiors(void)
{
GC_ASSERT(I_HOLD_LOCK());
if (NULL == GC_incomplete_normal_bl) {
GC_old_normal_bl = (word *)GC_scratch_alloc(sizeof(page_hash_table));
GC_incomplete_normal_bl
= (word *)GC_scratch_alloc(sizeof(page_hash_table));
if (NULL == GC_old_normal_bl || NULL == GC_incomplete_normal_bl) {
GC_err_printf("Insufficient memory for black list\n");
EXIT();
}
GC_clear_bl(GC_old_normal_bl);
GC_clear_bl(GC_incomplete_normal_bl);
}
}
GC_INNER void
GC_bl_init(void)
{
GC_ASSERT(I_HOLD_LOCK());
GC_black_list_spacing = MINHINCR * HBLKSIZE; /*< initial guess */
if (!GC_all_interior_pointers) {
GC_bl_init_no_interiors();
}
GC_ASSERT(NULL == GC_old_stack_bl && NULL == GC_incomplete_stack_bl);
GC_old_stack_bl = (word *)GC_scratch_alloc(sizeof(page_hash_table));
GC_incomplete_stack_bl = (word *)GC_scratch_alloc(sizeof(page_hash_table));
if (NULL == GC_old_stack_bl || NULL == GC_incomplete_stack_bl) {
GC_err_printf("Insufficient memory for black list\n");
EXIT();
}
GC_clear_bl(GC_old_stack_bl);
GC_clear_bl(GC_incomplete_stack_bl);
}
static word compute_total_stack_black_listed(void);
GC_INNER void
GC_promote_black_lists(void)
{
word *very_old_normal_bl = GC_old_normal_bl;
word *very_old_stack_bl = GC_old_stack_bl;
word total_stack_black_listed; /*< number of bytes on stack blacklist */
GC_ASSERT(I_HOLD_LOCK());
GC_old_normal_bl = GC_incomplete_normal_bl;
GC_old_stack_bl = GC_incomplete_stack_bl;
if (!GC_all_interior_pointers) {
GC_clear_bl(very_old_normal_bl);
}
GC_clear_bl(very_old_stack_bl);
GC_incomplete_normal_bl = very_old_normal_bl;
GC_incomplete_stack_bl = very_old_stack_bl;
total_stack_black_listed = compute_total_stack_black_listed();
GC_VERBOSE_LOG_PRINTF(
"%lu bytes in heap blacklisted for interior pointers\n",
(unsigned long)total_stack_black_listed);
if (total_stack_black_listed != 0)
GC_black_list_spacing
= HBLKSIZE * (GC_heapsize / total_stack_black_listed);
if (GC_black_list_spacing < 3 * HBLKSIZE)
GC_black_list_spacing = 3 * HBLKSIZE;
if (GC_black_list_spacing > MAXHINCR * HBLKSIZE) {
/*
* Make it easier to allocate really huge blocks, which otherwise may
* have problems with nonuniform blacklist distributions.
* This way we should always succeed immediately after growing the heap.
*/
GC_black_list_spacing = MAXHINCR * HBLKSIZE;
}
}
GC_INNER void
GC_unpromote_black_lists(void)
{
if (!GC_all_interior_pointers) {
GC_copy_bl(GC_old_normal_bl, GC_incomplete_normal_bl);
}
GC_copy_bl(GC_old_stack_bl, GC_incomplete_stack_bl);
}
# if defined(PARALLEL_MARK) && defined(THREAD_SANITIZER)
# define backlist_set_pht_entry_from_index(db, index) \
set_pht_entry_from_index_concurrent(db, index)
# else
/*
* It is safe to set a bit in a blacklist even without synchronization,
* the only drawback is that we might have to redo black-listing sometimes.
*/
# define backlist_set_pht_entry_from_index(bl, index) \
set_pht_entry_from_index(bl, index)
# endif
# ifdef PRINT_BLACK_LIST
GC_INNER void
GC_add_to_black_list_normal(ptr_t p, ptr_t source)
# else
GC_INNER void
GC_add_to_black_list_normal(ptr_t p)
# endif
{
# ifndef PARALLEL_MARK
GC_ASSERT(I_HOLD_LOCK());
# endif
if (GC_modws_valid_offsets[ADDR(p) & (sizeof(ptr_t) - 1)]) {
size_t index = PHT_HASH(p);
if (NULL == HDR(p) || get_pht_entry_from_index(GC_old_normal_bl, index)) {
# ifdef PRINT_BLACK_LIST
if (!get_pht_entry_from_index(GC_incomplete_normal_bl, index)) {
GC_print_blacklisted_ptr(p, source, "normal");
}
# endif
backlist_set_pht_entry_from_index(GC_incomplete_normal_bl, index);
} else {
/*
* This is probably just an interior pointer to an allocated object,
* and is not worth black listing.
*/
}
}
}
# ifdef PRINT_BLACK_LIST
GC_INNER void
GC_add_to_black_list_stack(ptr_t p, ptr_t source)
# else
GC_INNER void
GC_add_to_black_list_stack(ptr_t p)
# endif
{
size_t index = PHT_HASH(p);
# ifndef PARALLEL_MARK
GC_ASSERT(I_HOLD_LOCK());
# endif
if (NULL == HDR(p) || get_pht_entry_from_index(GC_old_stack_bl, index)) {
# ifdef PRINT_BLACK_LIST
if (!get_pht_entry_from_index(GC_incomplete_stack_bl, index)) {
GC_print_blacklisted_ptr(p, source, "stack");
}
# endif
backlist_set_pht_entry_from_index(GC_incomplete_stack_bl, index);
}
}
#endif /* !NO_BLACK_LISTING */
GC_API struct GC_hblk_s *GC_CALL
GC_is_black_listed(struct GC_hblk_s *h, size_t len)
{
#ifdef NO_BLACK_LISTING
UNUSED_ARG(h);
UNUSED_ARG(len);
#else
size_t index = PHT_HASH(h);
size_t i, nblocks;
if (!GC_all_interior_pointers
&& (get_pht_entry_from_index(GC_old_normal_bl, index)
|| get_pht_entry_from_index(GC_incomplete_normal_bl, index))) {
return h + 1;
}
nblocks = divHBLKSZ(len);
for (i = 0;;) {
if (GC_old_stack_bl[divWORDSZ(index)] == 0
&& GC_incomplete_stack_bl[divWORDSZ(index)] == 0) {
/* An easy case. */
i += CPP_WORDSZ - modWORDSZ(index);
} else {
if (get_pht_entry_from_index(GC_old_stack_bl, index)
|| get_pht_entry_from_index(GC_incomplete_stack_bl, index)) {
return &h[i + 1];
}
i++;
}
if (i >= nblocks)
break;
index = PHT_HASH(h + i);
}
#endif
return NULL;
}
#ifndef NO_BLACK_LISTING
/*
* Return the number of black-listed blocks in a given range. Used only
* for statistical purposes. Looks only at the `GC_incomplete_stack_bl`.
*/
STATIC word
GC_number_stack_black_listed(struct hblk *start, struct hblk *endp1)
{
struct hblk *h;
word result = 0;
for (h = start; ADDR_LT((ptr_t)h, (ptr_t)endp1); h++) {
size_t index = PHT_HASH(h);
if (get_pht_entry_from_index(GC_old_stack_bl, index))
result++;
}
return result;
}
/* Return the total number of (stack) black-listed bytes. */
static word
compute_total_stack_black_listed(void)
{
size_t i;
word total = 0;
for (i = 0; i < GC_n_heap_sects; i++) {
struct hblk *start = (struct hblk *)GC_heap_sects[i].hs_start;
struct hblk *endp1 = start + divHBLKSZ(GC_heap_sects[i].hs_bytes);
total += GC_number_stack_black_listed(start, endp1);
}
return total * HBLKSIZE;
}
#endif /* !NO_BLACK_LISTING */
/*
* Copyright (c) 1992-1994 by Xerox Corporation. All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#ifdef CHECKSUMS
/*
* This is debugging code intended to verify the results of dirty bit
* computations. Currently works only in a single-threaded environment.
*/
# define NSUMS 10000
# define OFFSET 0x10000
typedef struct {
GC_bool new_valid;
word old_sum;
word new_sum;
/*
* Block to which this refers plus `OFFSET` to hide it from the
* garbage collector.
*/
struct hblk *block;
} page_entry;
page_entry GC_sums[NSUMS] = { { FALSE, 0, 0, NULL } };
/* Record of pages on which we saw a write fault. */
STATIC word GC_faulted[NSUMS] = { 0 };
STATIC size_t GC_n_faulted = 0;
# ifndef GC_NO_DEINIT
void
GC_reset_check_page(void)
{
BZERO(GC_sums, sizeof(GC_sums));
GC_n_faulted = 0;
}
# endif
# ifdef MPROTECT_VDB
void
GC_record_fault(struct hblk *h)
{
GC_ASSERT(GC_page_size != 0);
if (GC_n_faulted >= NSUMS)
ABORT("write fault log overflowed");
GC_faulted[GC_n_faulted++] = ADDR(HBLK_PAGE_ALIGNED(h));
}
# endif
STATIC GC_bool
GC_was_faulted(struct hblk *h)
{
size_t i;
word page = ADDR(HBLK_PAGE_ALIGNED(h));
for (i = 0; i < GC_n_faulted; ++i) {
if (GC_faulted[i] == page)
return TRUE;
}
return FALSE;
}
STATIC word
GC_checksum(struct hblk *h)
{
word *p;
word *lim = (word *)(h + 1);
word result = 0;
for (p = (word *)h; ADDR_LT((ptr_t)p, (ptr_t)lim); p++) {
result += *p;
}
return result | SIGNB; /*< does not look like pointer */
}
int GC_n_dirty_errors = 0;
int GC_n_faulted_dirty_errors = 0;
unsigned long GC_n_clean = 0;
unsigned long GC_n_dirty = 0;
STATIC void
GC_update_check_page(struct hblk *h, int index)
{
page_entry *pe = &GC_sums[index];
hdr *hhdr = HDR(h);
if (pe->block != 0 && pe->block != h + OFFSET)
ABORT("goofed");
pe->old_sum = pe->new_sum;
pe->new_sum = GC_checksum(h);
# if !defined(MSWIN32) && !defined(MSWINCE)
if (pe->new_sum != SIGNB && !GC_page_was_ever_dirty(h)) {
GC_err_printf("GC_page_was_ever_dirty(%p) is wrong\n", (void *)h);
}
# endif
if (GC_page_was_dirty(h)) {
GC_n_dirty++;
} else {
GC_n_clean++;
}
if (hhdr != NULL) {
(void)GC_find_starting_hblk(h, &hhdr);
if (pe->new_valid
# ifdef SOFT_VDB
&& !HBLK_IS_FREE(hhdr)
# endif
&& !IS_PTRFREE(hhdr) && pe->old_sum != pe->new_sum) {
if (!GC_page_was_dirty(h) || !GC_page_was_ever_dirty(h)) {
GC_bool was_faulted = GC_was_faulted(h);
GC_n_dirty_errors++; /*< set breakpoint here */
if (was_faulted)
GC_n_faulted_dirty_errors++;
}
}
}
pe->new_valid = TRUE;
pe->block = h + OFFSET;
}
/* Should be called immediately after `GC_read_dirty`. */
void
GC_check_dirty(void)
{
int index;
size_t i;
GC_n_dirty_errors = 0;
GC_n_faulted_dirty_errors = 0;
GC_n_clean = 0;
GC_n_dirty = 0;
index = 0;
for (i = 0; i < GC_n_heap_sects; i++) {
ptr_t start = GC_heap_sects[i].hs_start;
struct hblk *h;
for (h = (struct hblk *)start;
ADDR_LT((ptr_t)h, start + GC_heap_sects[i].hs_bytes); h++) {
GC_update_check_page(h, index);
index++;
if (index >= NSUMS) {
i = GC_n_heap_sects;
break;
}
}
}
GC_COND_LOG_PRINTF("Checked %lu clean and %lu dirty pages\n", GC_n_clean,
GC_n_dirty);
if (GC_n_dirty_errors > 0) {
GC_err_printf("Found %d dirty bit errors (%d were faulted)\n",
GC_n_dirty_errors, GC_n_faulted_dirty_errors);
}
if (GC_n_faulted > 0) {
/* Do not expose block addresses to the garbage collector. */
BZERO(GC_faulted, GC_n_faulted * sizeof(word));
GC_n_faulted = 0;
}
}
#endif /* CHECKSUMS */
/*
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 2001 by Hewlett-Packard Company. All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* Private declarations of the collector marker data structures (like the
* mark stack) and macros. Needed by the marker and the client-supplied
* mark routines. Transitively includes `gc_priv.h` file.
*/
#ifndef GC_PMARK_H
#define GC_PMARK_H
#if defined(HAVE_CONFIG_H) && !defined(GC_PRIVATE_H)
/*
* When `gc_pmark.h` file is included from `gc_priv.h` file, some of
* macros might be undefined in `gcconfig.h` file, so skip `config.h`
* file in this case.
*/
#endif
#ifndef GC_BUILD
# define GC_BUILD
#endif
#if (defined(__linux__) || defined(__GLIBC__) || defined(__GNU__)) \
&& !defined(_GNU_SOURCE) && defined(GC_PTHREADS) \
&& !defined(GC_NO_PTHREAD_SIGMASK)
# define _GNU_SOURCE 1
#endif
#if defined(KEEP_BACK_PTRS) || defined(PRINT_BLACK_LIST)
#endif
EXTERN_C_BEGIN
/*
* The real declarations of the following is in `gc_priv.h` file, so
* that we can avoid scanning `GC_mark_procs` table.
*/
/*
* Mark descriptor stuff that should remain private for now, mostly
* because it is hard to export `CPP_WORDSZ` macro without include
* `gcconfig.h` file.
*/
#define BITMAP_BITS (CPP_WORDSZ - GC_DS_TAG_BITS)
#define PROC(descr) \
(GC_mark_procs[((descr) >> GC_DS_TAG_BITS) & (GC_MAX_MARK_PROCS - 1)])
#define ENV(descr) ((descr) >> (GC_DS_TAG_BITS + GC_LOG_MAX_MARK_PROCS))
#define MAX_ENV (((word)1 << (BITMAP_BITS - GC_LOG_MAX_MARK_PROCS)) - 1)
GC_EXTERN unsigned GC_n_mark_procs;
/* Number of mark stack entries to discard on overflow. */
#define GC_MARK_STACK_DISCARDS (INITIAL_MARK_STACK_SIZE / 8)
#ifdef PARALLEL_MARK
/*
* Allow multiple threads to participate in the marking process.
* This works roughly as follows:
* - The main mark stack never shrinks, but it can grow.
* - The initiating threads holds the allocator lock, sets
* `GC_help_wanted`.
* - Other threads:
* 1. Update `GC_helper_count` (while holding the mark lock).
* 2. Allocate a local mark stack repeatedly:
* 2.1. Steal a global mark stack entry by atomically replacing
* its descriptor with 0;
* 2.2. Copy it to the local stack;
* 2.3. Mark on the local stack until it is empty, or it may be
* profitable to copy it back;
* 2.4. If necessary, copy local stack to global one, holding the
* mark lock;
* 2.5. Stop when the global mark stack is empty.
* 3. Decrement `GC_helper_count` (holding the mark lock).
*
* This is an experiment to see if we can do something along the lines
* of the University of Tokyo SGC in a less intrusive, though probably
* also less performant, way.
*/
/* `GC_mark_stack_top` is protected by the mark lock. */
/*
* `GC_notify_all_marker()` is used when `GC_help_wanted` is first set,
* when the last helper becomes inactive, when something is added to the
* global mark stack, and just after `GC_mark_no` is incremented.
* This could be split into multiple conditional variables (and probably
* should be) to scale to really large numbers of processors.
*/
#endif /* PARALLEL_MARK */
/*
* Push the object `obj` with corresponding heap block header `hhdr`
* onto the mark stack. Returns the updated `mark_stack_top` value.
*/
GC_INLINE mse *
GC_push_obj(ptr_t obj, const hdr *hhdr, mse *mark_stack_top,
mse *mark_stack_limit)
{
GC_ASSERT(!HBLK_IS_FREE(hhdr));
if (!IS_PTRFREE(hhdr)) {
mark_stack_top = GC_custom_push_proc(hhdr->hb_descr, obj, mark_stack_top,
mark_stack_limit);
}
return mark_stack_top;
}
/*
* Push the contents of `current` onto the mark stack if it is a valid
* pointer to a currently unmarked object. Mark it.
*/
#define PUSH_CONTENTS(current, mark_stack_top, mark_stack_limit, source) \
do { \
hdr *my_hhdr; \
HC_GET_HDR(current, my_hhdr, source); /*< contains `break` */ \
mark_stack_top = GC_push_contents_hdr( \
current, mark_stack_top, mark_stack_limit, source, my_hhdr, TRUE); \
} while (0)
/* Set mark bit, exit (using `break` statement) if it is already set. */
#ifdef USE_MARK_BYTES
# if defined(PARALLEL_MARK) && defined(AO_HAVE_char_store) \
&& !defined(BASE_ATOMIC_OPS_EMULATED)
/*
* There is a race here, and we may set the bit twice in the concurrent
* case. This can result in the object being pushed twice. But that is
* only a performance issue.
*/
# define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \
{ /*< cannot use `do ... while (0)` here */ \
volatile unsigned char *mark_byte_addr \
= (unsigned char *)(hhdr)->hb_marks + (bit_no); \
/* Unordered atomic load and store are sufficient here. */ \
if (AO_char_load(mark_byte_addr) != 0) \
break; /*< go to the enclosing loop end */ \
AO_char_store(mark_byte_addr, 1); \
}
# else
# define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \
{ /*< cannot use `do ... while (0)` here */ \
ptr_t mark_byte_addr = (ptr_t)(hhdr)->hb_marks + (bit_no); \
\
if (*mark_byte_addr != 0) \
break; /*< go to the enclosing loop end */ \
*mark_byte_addr = 1; \
}
# endif /* !PARALLEL_MARK */
#else
# if defined(PARALLEL_MARK) || (defined(THREAD_SANITIZER) && defined(THREADS))
# ifdef THREAD_SANITIZER
# define MARK_WORD_READ(addr) AO_load(addr)
# else
# define MARK_WORD_READ(addr) (*(addr))
# endif
/*
* This is used only if we explicitly define `USE_MARK_BITS` macro.
* The following may fail to exit even if the bit was already set.
* For our uses, that is benign.
*/
# define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \
{ /*< cannot use `do ... while (0)` here */ \
volatile AO_t *mark_word_addr = (hhdr)->hb_marks + divWORDSZ(bit_no); \
word my_bits = (word)1 << modWORDSZ(bit_no); \
\
if ((MARK_WORD_READ(mark_word_addr) & my_bits) != 0) \
break; /*< go to the enclosing loop end */ \
AO_or(mark_word_addr, my_bits); \
}
# else /* !PARALLEL_MARK */
# define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \
{ /*< cannot use `do ... while (0)` here */ \
word *mark_word_addr = (hhdr)->hb_marks + divWORDSZ(bit_no); \
word old = *mark_word_addr; \
word my_bits = (word)1 << modWORDSZ(bit_no); \
\
if ((old & my_bits) != 0) \
break; /*< go to the enclosing loop end */ \
*(mark_word_addr) = old | my_bits; \
}
# endif
#endif /* !USE_MARK_BYTES */
#ifdef ENABLE_TRACE
# define TRACE(source, cmd) \
if (GC_trace_ptr != NULL && (ptr_t)(source) == GC_trace_ptr) \
cmd
# define TRACE_TARGET(target, cmd) \
if (GC_trace_ptr != NULL && GC_is_heap_ptr(GC_trace_ptr) \
&& (target) == *(ptr_t *)GC_trace_ptr) \
cmd
#else
# define TRACE(source, cmd)
# define TRACE_TARGET(source, cmd)
#endif
/*
* If the mark bit corresponding to `current` is not set, set it, and
* push the contents of the object on the mark stack. `current` points
* to the beginning of the object. We rely on the fact that the
* preceding header calculation will succeed for a pointer past the
* first page of an object, only if it is in fact a valid pointer
* to the object. Thus we can omit the otherwise necessary tests here.
*/
GC_INLINE mse *
GC_push_contents_hdr(ptr_t current, mse *mark_stack_top, mse *mark_stack_limit,
ptr_t source, hdr *hhdr, GC_bool do_offset_check)
{
do {
/*
* Displacement in the block, in bytes; always within range.
* Note, in particular, that this value is the displacement from the
* beginning of the heap block, which may itself be in the interior
* of a large object. If `current` does not point to the first block,
* then we are in the all-interior-pointers mode, and it is safe to
* use any displacement value.
*/
size_t displ = HBLKDISPL(current);
ptr_t base = current;
#ifdef MARK_BIT_PER_OBJ
unsigned32 gran_displ; /*< `high_prod` */
unsigned32 inv_sz = hhdr->hb_inv_sz;
#else
size_t gran_displ = BYTES_TO_GRANULES(displ);
size_t gran_offset = hhdr->hb_map[gran_displ];
size_t byte_offset = displ & (GC_GRANULE_BYTES - 1);
/* The following always fails for large block references. */
if (UNLIKELY((gran_offset | byte_offset) != 0))
#endif
{
#ifdef MARK_BIT_PER_OBJ
if (UNLIKELY(inv_sz == LARGE_INV_SZ))
#else
if ((hhdr->hb_flags & LARGE_BLOCK) != 0)
#endif
{
/* `gran_offset` is bogus. */
size_t obj_displ;
base = (ptr_t)hhdr->hb_block;
obj_displ = (size_t)(current - base);
if (obj_displ != displ) {
GC_ASSERT(obj_displ < hhdr->hb_sz);
/*
* Must be in the all-interior-pointers mode, non-first block
* already did validity check on cache miss.
*/
} else if (do_offset_check && !GC_valid_offsets[obj_displ]) {
GC_ADD_TO_BLACK_LIST_NORMAL(current, source);
break;
}
GC_ASSERT(hhdr->hb_sz > HBLKSIZE
|| hhdr->hb_block == HBLKPTR(current));
GC_ASSERT(ADDR_GE(current, (ptr_t)hhdr->hb_block));
gran_displ = 0;
} else {
#ifdef MARK_BIT_PER_OBJ
unsigned32 low_prod;
LONG_MULT(gran_displ, low_prod, (unsigned32)displ, inv_sz);
if ((low_prod >> 16) != 0)
#endif
{
size_t obj_displ;
#ifdef MARK_BIT_PER_OBJ
/* Accurate enough if `HBLKSIZE` is not greater than 2**15. */
GC_STATIC_ASSERT(HBLKSIZE <= (1 << 15));
obj_displ = (((low_prod >> 16) + 1) * hhdr->hb_sz) >> 16;
#else
obj_displ = GRANULES_TO_BYTES(gran_offset) + byte_offset;
#endif
if (do_offset_check && !GC_valid_offsets[obj_displ]) {
GC_ADD_TO_BLACK_LIST_NORMAL(current, source);
break;
}
#ifndef MARK_BIT_PER_OBJ
gran_displ -= gran_offset;
#endif
base -= obj_displ;
}
}
}
#ifdef MARK_BIT_PER_OBJ
/*
* May get here for pointer to start of block not at the beginning
* of object. If so, it is valid, and we are fine.
*/
GC_ASSERT(gran_displ <= HBLK_OBJS(hhdr->hb_sz));
#else
GC_ASSERT(hhdr == GC_find_header(base));
GC_ASSERT(gran_displ % BYTES_TO_GRANULES(hhdr->hb_sz) == 0);
#endif
TRACE(source, GC_log_printf("GC #%lu: passed validity tests\n",
(unsigned long)GC_gc_no));
SET_MARK_BIT_EXIT_IF_SET(hhdr, gran_displ); /*< contains `break` */
TRACE(source, GC_log_printf("GC #%lu: previously unmarked\n",
(unsigned long)GC_gc_no));
TRACE_TARGET(base, GC_log_printf("GC #%lu: marking %p from %p instead\n",
(unsigned long)GC_gc_no, (void *)base,
(void *)source));
INCR_MARKS(hhdr);
GC_STORE_BACK_PTR(source, base);
mark_stack_top = GC_push_obj(base, hhdr, mark_stack_top, mark_stack_limit);
} while (0);
return mark_stack_top;
}
#if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS)
# define PUSH_ONE_CHECKED_STACK(p, source) \
GC_mark_and_push_stack(p, (ptr_t)(source))
#else
# define PUSH_ONE_CHECKED_STACK(p, source) GC_mark_and_push_stack(p)
#endif
/*
* Push a single value onto mark stack. Mark from the object
* pointed to by `p`. The argument should be of `ptr_t` type.
* Invoke `FIXUP_POINTER()` before any further processing.
* p` is considered valid even if it is an interior pointer.
* Previously marked objects are not pushed. Hence we make progress
* even if the mark stack overflows.
*/
#ifdef NEED_FIXUP_POINTER
/* Try both the raw variant and the fixed up one. */
# define GC_PUSH_ONE_STACK(p, source) \
do { \
ptr_t pp = (p); \
\
if (ADDR_LT((ptr_t)GC_least_plausible_heap_addr, p) \
&& ADDR_LT(p, (ptr_t)GC_greatest_plausible_heap_addr)) { \
PUSH_ONE_CHECKED_STACK(p, source); \
} \
FIXUP_POINTER(pp); \
if (ADDR_LT((ptr_t)GC_least_plausible_heap_addr, pp) \
&& ADDR_LT(pp, (ptr_t)GC_greatest_plausible_heap_addr)) { \
PUSH_ONE_CHECKED_STACK(pp, source); \
} \
} while (0)
#else /* !NEED_FIXUP_POINTER */
# define GC_PUSH_ONE_STACK(p, source) \
do { \
if (ADDR_LT((ptr_t)GC_least_plausible_heap_addr, p) \
&& ADDR_LT(p, (ptr_t)GC_greatest_plausible_heap_addr)) { \
PUSH_ONE_CHECKED_STACK(p, source); \
} \
} while (0)
#endif
/*
* Same as `GC_PUSH_ONE_STACK`, but the interior pointers recognition as
* for normal heap pointers.
*/
#define GC_PUSH_ONE_HEAP(p, source, mark_stack_top) \
do { \
FIXUP_POINTER(p); \
if (ADDR_LT((ptr_t)GC_least_plausible_heap_addr, p) \
&& ADDR_LT(p, (ptr_t)GC_greatest_plausible_heap_addr)) \
mark_stack_top = GC_mark_and_push( \
p, mark_stack_top, GC_mark_stack_limit, (void **)(source)); \
} while (0)
/*
* Mark objects pointed to by the regions described by mark stack entries
* between `mark_stack` and `mark_stack_top`, inclusive. Assumes the upper
* limit of a mark stack entry is never `NULL`. A mark stack entry never
* has zero size. Return the new value of `mark_stack_top`.
* We try to traverse on the order of a `hblk` of memory before we return.
* Caller is responsible for calling this until the mark stack is empty.
* Note that this is the most performance critical routine in the collector.
* Hence it contains all sorts of ugly hacks to speed things up.
* In particular, we avoid procedure calls on the common path, we take
* advantage of peculiarities of the mark descriptor encoding, we optionally
* maintain a cache for the block address to header mapping, we prefetch
* when an object is "grayed", etc.
*/
GC_INNER mse *GC_mark_from(mse *mark_stack_top, mse *mark_stack,
mse *mark_stack_limit);
#define MARK_FROM_MARK_STACK() \
GC_mark_stack_top = GC_mark_from(GC_mark_stack_top, GC_mark_stack, \
GC_mark_stack + GC_mark_stack_size);
#define GC_mark_stack_empty() \
ADDR_LT((ptr_t)GC_mark_stack_top, (ptr_t)GC_mark_stack)
/*
* The current state of marking, as follows. We say something is dirty
* if it was written since the last time we retrieved dirty bits.
* We say it is grungy if it was marked dirty in the last set of bits
* we retrieved. Invariant "I": all roots and marked objects `p` are
* either dirty, or point to objects `q` that are either marked or
* a pointer to `q` appears in a range on the mark stack.
*/
/* No marking in progress. "I" holds. Mark stack is empty. */
#define MS_NONE 0
/*
* Rescuing objects are currently being pushed. "I" holds, except that
* grungy roots may point to unmarked objects, as may marked grungy objects
* above `GC_scan_ptr`.
*/
#define MS_PUSH_RESCUERS 1
/*
* Uncollectible objects are currently being pushed. "I" holds, except
* that marked uncollectible objects above `GC_scan_ptr` may point to
* unmarked objects. Roots may point to unmarked objects too.
*/
#define MS_PUSH_UNCOLLECTABLE 2
/* "I" holds, mark stack may be nonempty. */
#define MS_ROOTS_PUSHED 3
/*
* "I" may not hold, e.g. because of the mark stack overflow. However,
* marked heap objects below `GC_scan_ptr` point to marked or stacked
* objects.
*/
#define MS_PARTIALLY_INVALID 4
/* "I" may not hold. */
#define MS_INVALID 5
EXTERN_C_END
#endif /* GC_PMARK_H */
#ifdef GC_GCJ_SUPPORT
/*
* This is an allocator interface tuned for `gcj` (the GNU static Java
* compiler).
*
* Each allocated object has a pointer in its beginning to a "vtable",
* which for our purposes is simply a structure describing the type of
* the object. This descriptor structure contains a GC marking
* descriptor at offset `GC_GCJ_MARK_DESCR_OFFSET`.
*
* It is hoped that this interface may also be useful for other systems,
* possibly with some tuning of the constants. But the immediate goal
* is to get better `gcj` performance.
*
* We assume: counting on explicit initialization of this interface is OK.
*/
# include "gc/gc_gcj.h"
/* Object kind for objects with descriptors in "vtable". */
int GC_gcj_kind = 0;
/* The kind of objects that are always marked with a mark procedure call. */
int GC_gcj_debug_kind = 0;
STATIC struct GC_ms_entry *GC_CALLBACK
GC_gcj_fake_mark_proc(word *addr, struct GC_ms_entry *mark_stack_top,
struct GC_ms_entry *mark_stack_limit, word env)
{
UNUSED_ARG(addr);
UNUSED_ARG(mark_stack_limit);
UNUSED_ARG(env);
# if defined(FUNCPTR_IS_DATAPTR) && defined(CPPCHECK)
GC_noop1((word)(GC_funcptr_uint)(&GC_init_gcj_malloc));
# endif
ABORT_RET("No client gcj mark proc is specified");
return mark_stack_top;
}
# ifdef FUNCPTR_IS_DATAPTR
GC_API void GC_CALL
GC_init_gcj_malloc(int mp_index, void *mp)
{
GC_init_gcj_malloc_mp((unsigned)mp_index,
CAST_THRU_UINTPTR(GC_mark_proc, mp),
GC_GCJ_MARK_DESCR_OFFSET);
}
# endif /* FUNCPTR_IS_DATAPTR */
GC_API void GC_CALL
GC_init_gcj_malloc_mp(unsigned mp_index, GC_mark_proc mp, size_t descr_offset)
{
# ifndef GC_IGNORE_GCJ_INFO
GC_bool ignore_gcj_info;
# endif
GC_STATIC_ASSERT(GC_GCJ_MARK_DESCR_OFFSET >= sizeof(ptr_t));
if (0 == mp) {
/* In case `GC_DS_PROC` is unused. */
mp = GC_gcj_fake_mark_proc;
}
/* Initialize the collector just in case it is not done yet. */
GC_init();
if (descr_offset != GC_GCJ_MARK_DESCR_OFFSET)
ABORT("GC_init_gcj_malloc_mp: bad offset");
LOCK();
if (GC_gcjobjfreelist != NULL) {
/* Already initialized. */
UNLOCK();
return;
}
# ifdef GC_IGNORE_GCJ_INFO
/* This is useful for debugging on platforms with missing `getenv()`. */
# define ignore_gcj_info TRUE
# else
ignore_gcj_info = GETENV("GC_IGNORE_GCJ_INFO") != NULL;
# endif
if (ignore_gcj_info) {
GC_COND_LOG_PRINTF("Gcj-style type information is disabled!\n");
}
GC_ASSERT(GC_mark_procs[mp_index] == (GC_mark_proc)0); /*< unused */
GC_mark_procs[mp_index] = mp;
if (mp_index >= GC_n_mark_procs)
ABORT("GC_init_gcj_malloc_mp: bad index");
/* Set up object kind `gcj`-style indirect descriptor. */
GC_gcjobjfreelist = (ptr_t *)GC_new_free_list_inner();
if (ignore_gcj_info) {
/*
* Use a simple length-based descriptor, thus forcing a fully
* conservative scan.
*/
GC_gcj_kind = (int)GC_new_kind_inner((void **)GC_gcjobjfreelist,
/* 0 | */ GC_DS_LENGTH, TRUE, TRUE);
GC_gcj_debug_kind = GC_gcj_kind;
} else {
GC_gcj_kind = (int)GC_new_kind_inner(
(void **)GC_gcjobjfreelist,
(((word)(-(GC_signed_word)GC_GCJ_MARK_DESCR_OFFSET
- GC_INDIR_PER_OBJ_BIAS))
| GC_DS_PER_OBJECT),
FALSE, TRUE);
/* Set up object kind for objects that require mark procedure call. */
GC_gcj_debug_kind = (int)GC_new_kind_inner(
GC_new_free_list_inner(),
GC_MAKE_PROC(mp_index, 1 /* allocated with debug info */), FALSE,
TRUE);
}
UNLOCK();
# undef ignore_gcj_info
}
# ifdef THREAD_LOCAL_ALLOC
GC_INNER
# else
STATIC
# endif
void *
GC_core_gcj_malloc(size_t lb, const void *vtable_ptr, unsigned flags)
{
ptr_t op;
size_t lg;
GC_DBG_COLLECT_AT_MALLOC(lb);
LOCK();
if (SMALL_OBJ(lb)
&& (op = GC_gcjobjfreelist[lg = GC_size_map[lb]], LIKELY(op != NULL))) {
GC_gcjobjfreelist[lg] = (ptr_t)obj_link(op);
GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
GC_ASSERT(NULL == ((void **)op)[1]);
} else {
/*
* A mechanism to release the allocator lock and invoke finalizers.
* We do not really have an opportunity to do this on a rarely
* executed path on which the allocator lock is not held. Thus we
* check at a rarely executed point at which it is safe to release
* the allocator lock; we do this even where we could just call
* `GC_notify_or_invoke_finalizers()`, since it is probably cheaper
* and certainly more uniform.
*/
/* TODO: Consider doing the same elsewhere? */
if (GC_gc_no != GC_last_finalized_no) {
UNLOCK();
GC_notify_or_invoke_finalizers();
LOCK();
GC_last_finalized_no = GC_gc_no;
}
op = (ptr_t)GC_generic_malloc_inner(lb, GC_gcj_kind, flags);
if (NULL == op) {
GC_oom_func oom_fn = GC_oom_fn;
UNLOCK();
return (*oom_fn)(lb);
}
}
*(const void **)op = vtable_ptr;
UNLOCK();
GC_dirty(op);
REACHABLE_AFTER_DIRTY(vtable_ptr);
return GC_clear_stack(op);
}
# ifndef THREAD_LOCAL_ALLOC
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_gcj_malloc(size_t lb, const void *vtable_ptr)
{
return GC_core_gcj_malloc(lb, vtable_ptr, 0 /* `flags` */);
}
# endif /* !THREAD_LOCAL_ALLOC */
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_gcj_malloc_ignore_off_page(size_t lb, const void *vtable_ptr)
{
return GC_core_gcj_malloc(lb, vtable_ptr, IGNORE_OFF_PAGE);
}
#endif /* GC_GCJ_SUPPORT */
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996 by Silicon Graphics. All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#if defined(KEEP_BACK_PTRS) && defined(GC_ASSERTIONS)
#endif
/*
* This implements:
* 1. Allocation of heap block headers;
* 2. A map from addresses to heap block addresses to heap block headers.
*
* Access speed is crucial. We implement an index structure based on
* a two-level tree.
*/
GC_INNER hdr *
GC_find_header(const void *h)
{
#ifdef HASH_TL
hdr *result;
GET_HDR(h, result);
return result;
#else
return HDR_INNER(h);
#endif
}
GC_INNER hdr *
#ifdef PRINT_BLACK_LIST
GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce, ptr_t source)
#else
GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce)
#endif
{
hdr *hhdr;
HC_MISS();
GET_HDR(p, hhdr);
if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
if (GC_all_interior_pointers) {
if (hhdr != NULL) {
/* Pointer to near the start of the large object. */
ptr_t current = (ptr_t)GC_find_starting_hblk(HBLKPTR(p), &hhdr);
if (hhdr->hb_flags & IGNORE_OFF_PAGE)
return 0;
if (HBLK_IS_FREE(hhdr) || p - current >= (GC_signed_word)hhdr->hb_sz) {
GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
/* The pointer is past the end of the block. */
return 0;
}
} else {
GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
/* And return `NULL`. */
}
GC_ASSERT(NULL == hhdr || !HBLK_IS_FREE(hhdr));
/*
* Pointers past the first page are probably too rare to add them to
* the cache. We do not. And correctness relies on the fact that
* we do not.
*/
return hhdr;
} else {
if (NULL == hhdr) {
GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
}
return 0;
}
} else {
if (HBLK_IS_FREE(hhdr)) {
GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
return 0;
} else {
hce->block_addr = ADDR(p) >> LOG_HBLKSIZE;
hce->hce_hdr = hhdr;
return hhdr;
}
}
}
/*
* Routines to dynamically allocate collector data structures that will
* never be freed.
*/
GC_INNER ptr_t
GC_scratch_alloc(size_t bytes)
{
ptr_t result = GC_scratch_free_ptr;
size_t bytes_to_get;
GC_ASSERT(I_HOLD_LOCK());
bytes = ROUNDUP_GRANULE_SIZE(bytes);
for (;;) {
GC_ASSERT(GC_scratch_end_addr >= ADDR(result));
if (bytes <= GC_scratch_end_addr - ADDR(result)) {
/* Unallocated space of scratch buffer has enough size. */
GC_scratch_free_ptr = result + bytes;
return result;
}
GC_ASSERT(GC_page_size != 0);
if (bytes >= MINHINCR * HBLKSIZE) {
bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(bytes);
result = GC_os_get_mem(bytes_to_get);
if (result != NULL) {
#if defined(KEEP_BACK_PTRS) && (GC_GRANULE_BYTES < 0x10)
GC_ASSERT(ADDR(result) > (word)NOT_MARKED);
#endif
/* No update of scratch free area pointer; get memory directly. */
#ifdef USE_SCRATCH_LAST_END_PTR
/*
* Update end point of last obtained area (needed only by
* `GC_register_dynamic_libraries` for some targets).
*/
GC_scratch_last_end_addr = ADDR(result) + bytes;
#endif
}
return result;
}
/* This is rounded up for a safety reason. */
bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(MINHINCR * HBLKSIZE);
result = GC_os_get_mem(bytes_to_get);
if (UNLIKELY(NULL == result)) {
WARN("Out of memory - trying to allocate requested amount"
" (%" WARN_PRIuPTR " bytes)...\n",
bytes);
bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(bytes);
result = GC_os_get_mem(bytes_to_get);
if (result != NULL) {
#ifdef USE_SCRATCH_LAST_END_PTR
GC_scratch_last_end_addr = ADDR(result) + bytes;
#endif
}
return result;
}
/* TODO: Some amount of unallocated space may remain unused forever. */
/* Update scratch area pointers and retry. */
GC_scratch_free_ptr = result;
GC_scratch_end_addr = ADDR(GC_scratch_free_ptr) + bytes_to_get;
#ifdef USE_SCRATCH_LAST_END_PTR
GC_scratch_last_end_addr = GC_scratch_end_addr;
#endif
}
}
/* Return an uninitialized header. */
static hdr *
alloc_hdr(void)
{
hdr *result;
GC_ASSERT(I_HOLD_LOCK());
if (NULL == GC_hdr_free_list) {
result = (hdr *)GC_scratch_alloc(sizeof(hdr));
} else {
result = GC_hdr_free_list;
GC_hdr_free_list = (hdr *)result->hb_next;
}
return result;
}
GC_INLINE void
free_hdr(hdr *hhdr)
{
hhdr->hb_next = (struct hblk *)GC_hdr_free_list;
GC_hdr_free_list = hhdr;
}
#ifdef COUNT_HDR_CACHE_HITS
/* Used for debugging/profiling (the symbols are externally visible). */
word GC_hdr_cache_hits = 0;
word GC_hdr_cache_misses = 0;
#endif
GC_INNER void
GC_init_headers(void)
{
unsigned i;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(NULL == GC_all_nils);
GC_all_nils = (bottom_index *)GC_scratch_alloc(sizeof(bottom_index));
if (GC_all_nils == NULL) {
GC_err_printf("Insufficient memory for GC_all_nils\n");
EXIT();
}
BZERO(GC_all_nils, sizeof(bottom_index));
for (i = 0; i < TOP_SZ; i++) {
GC_top_index[i] = GC_all_nils;
}
}
/*
* Make sure that there is a bottom-level index block for address `addr`.
* Returns `FALSE` on failure.
*/
static GC_bool
get_index(word addr)
{
word hi = addr >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE);
bottom_index *r;
bottom_index *p;
bottom_index **prev;
bottom_index *pi; /*< `old_p` */
word i;
GC_ASSERT(I_HOLD_LOCK());
#ifdef HASH_TL
i = TL_HASH(hi);
pi = GC_top_index[i];
for (p = pi; p != GC_all_nils; p = p->hash_link) {
if (p->key == hi)
return TRUE;
}
#else
if (GC_top_index[hi] != GC_all_nils)
return TRUE;
i = hi;
#endif
r = (bottom_index *)GC_scratch_alloc(sizeof(bottom_index));
if (UNLIKELY(NULL == r))
return FALSE;
BZERO(r, sizeof(bottom_index));
r->key = hi;
#ifdef HASH_TL
r->hash_link = pi;
#endif
/* Add it to the list of bottom indices. */
prev = &GC_all_bottom_indices; /*< pointer to `p` */
pi = NULL; /*< `bottom_index` preceding `p` */
while ((p = *prev) != 0 && p->key < hi) {
pi = p;
prev = &p->asc_link;
}
r->desc_link = pi;
if (NULL == p) {
GC_all_bottom_indices_end = r;
} else {
p->desc_link = r;
}
r->asc_link = p;
*prev = r;
GC_top_index[i] = r;
return TRUE;
}
GC_INNER hdr *
GC_install_header(struct hblk *h)
{
hdr *result;
GC_ASSERT(I_HOLD_LOCK());
if (UNLIKELY(!get_index(ADDR(h))))
return NULL;
result = alloc_hdr();
if (LIKELY(result != NULL)) {
GC_ASSERT(!IS_FORWARDING_ADDR_OR_NIL(result));
SET_HDR(h, result);
#ifdef USE_MUNMAP
result->hb_last_reclaimed = (unsigned short)GC_gc_no;
#endif
}
return result;
}
GC_INNER GC_bool
GC_install_counts(struct hblk *h, size_t sz /* bytes */)
{
struct hblk *hbp;
for (hbp = h; ADDR_LT((ptr_t)hbp, (ptr_t)h + sz); hbp += BOTTOM_SZ) {
if (!get_index(ADDR(hbp)))
return FALSE;
/* Is overflow of `hbp` expected? */
if (ADDR(hbp) > GC_WORD_MAX - (word)BOTTOM_SZ * HBLKSIZE)
break;
}
if (!get_index(ADDR(h) + sz - 1))
return FALSE;
GC_ASSERT(!IS_FORWARDING_ADDR_OR_NIL(HDR(h)));
for (hbp = h + 1; ADDR_LT((ptr_t)hbp, (ptr_t)h + sz); hbp++) {
word i = (word)HBLK_PTR_DIFF(hbp, h);
SET_HDR(hbp, (hdr *)NUMERIC_TO_VPTR(i > MAX_JUMP ? MAX_JUMP : i));
}
return TRUE;
}
GC_INNER void
GC_remove_header(struct hblk *h)
{
hdr **ha;
GET_HDR_ADDR(h, ha);
free_hdr(*ha);
*ha = 0;
}
GC_INNER void
GC_remove_counts(struct hblk *h, size_t sz /* bytes */)
{
struct hblk *hbp;
if (sz <= HBLKSIZE)
return;
if (NULL == HDR(h + 1)) {
#ifdef GC_ASSERTIONS
for (hbp = h + 2; ADDR_LT((ptr_t)hbp, (ptr_t)h + sz); hbp++) {
GC_ASSERT(NULL == HDR(hbp));
}
#endif
return;
}
for (hbp = h + 1; ADDR_LT((ptr_t)hbp, (ptr_t)h + sz); hbp++) {
SET_HDR(hbp, NULL);
}
}
#define HBLK_ADDR(bi, j) \
((((bi)->key << LOG_BOTTOM_SZ) + (word)(j)) << LOG_HBLKSIZE)
GC_API void GC_CALL
GC_apply_to_all_blocks(GC_walk_hblk_fn fn, void *client_data)
{
bottom_index *bi;
for (bi = GC_all_bottom_indices; bi != NULL; bi = bi->asc_link) {
GC_signed_word j;
for (j = BOTTOM_SZ - 1; j >= 0;) {
hdr *hhdr = bi->index[j];
if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
j -= (GC_signed_word)(hhdr != NULL ? ADDR(hhdr) : 1);
} else {
if (!HBLK_IS_FREE(hhdr)) {
GC_ASSERT(HBLK_ADDR(bi, j) == ADDR(hhdr->hb_block));
fn(hhdr->hb_block, client_data);
}
j--;
}
}
}
}
GC_INNER struct hblk *
GC_next_block(struct hblk *h, GC_bool allow_free)
{
REGISTER bottom_index *bi;
REGISTER size_t j = (size_t)(ADDR(h) >> LOG_HBLKSIZE) & (BOTTOM_SZ - 1);
GC_ASSERT(I_HOLD_READER_LOCK());
GET_BI(h, bi);
if (bi == GC_all_nils) {
REGISTER word hi = ADDR(h) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE);
bi = GC_all_bottom_indices;
while (bi != NULL && bi->key < hi)
bi = bi->asc_link;
j = 0;
}
for (; bi != NULL; bi = bi->asc_link) {
while (j < BOTTOM_SZ) {
hdr *hhdr = bi->index[j];
if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
j++;
} else {
if (allow_free || !HBLK_IS_FREE(hhdr)) {
GC_ASSERT(HBLK_ADDR(bi, j) == ADDR(hhdr->hb_block));
return hhdr->hb_block;
}
j += divHBLKSZ(hhdr->hb_sz);
}
}
j = 0;
}
return NULL;
}
GC_INNER struct hblk *
GC_prev_block(struct hblk *h)
{
bottom_index *bi;
GC_signed_word j = (ADDR(h) >> LOG_HBLKSIZE) & (BOTTOM_SZ - 1);
GC_ASSERT(I_HOLD_READER_LOCK());
GET_BI(h, bi);
if (bi == GC_all_nils) {
word hi = ADDR(h) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE);
bi = GC_all_bottom_indices_end;
while (bi != NULL && bi->key > hi)
bi = bi->desc_link;
j = BOTTOM_SZ - 1;
}
for (; bi != NULL; bi = bi->desc_link) {
while (j >= 0) {
hdr *hhdr = bi->index[j];
if (NULL == hhdr) {
--j;
} else if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
j -= (GC_signed_word)ADDR(hhdr);
} else {
GC_ASSERT(HBLK_ADDR(bi, j) == ADDR(hhdr->hb_block));
return hhdr->hb_block;
}
}
j = BOTTOM_SZ - 1;
}
return NULL;
}
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1999-2001 by Hewlett-Packard Company. All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#ifndef SMALL_CONFIG
/*
* Build a free list for two-pointer cleared objects inside the given block.
* Set the last link to be `ofl`. Return a pointer to the first free-list
* entry.
*/
STATIC ptr_t
GC_build_fl_clear2(struct hblk *h, ptr_t ofl)
{
ptr_t *p = (ptr_t *)h->hb_body;
ptr_t plim = (ptr_t)(h + 1);
p[0] = ofl;
p[1] = NULL;
p[2] = (ptr_t)p;
p[3] = NULL;
for (p += 4; ADDR_LT((ptr_t)p, plim); p += 4) {
p[0] = (ptr_t)(p - 2);
p[1] = NULL;
p[2] = (ptr_t)p;
p[3] = NULL;
}
return (ptr_t)(p - 2);
}
/* The same as above but uncleared objects. */
STATIC ptr_t
GC_build_fl2(struct hblk *h, ptr_t ofl)
{
ptr_t *p = (ptr_t *)h->hb_body;
ptr_t plim = (ptr_t)(h + 1);
p[0] = ofl;
p[2] = (ptr_t)p;
for (p += 4; ADDR_LT((ptr_t)p, plim); p += 4) {
p[0] = (ptr_t)(p - 2);
p[2] = (ptr_t)p;
}
return (ptr_t)(p - 2);
}
/* The same as above but for four-pointer cleared objects. */
STATIC ptr_t
GC_build_fl_clear4(struct hblk *h, ptr_t ofl)
{
ptr_t *p = (ptr_t *)h->hb_body;
ptr_t plim = (ptr_t)(h + 1);
p[0] = ofl;
p[1] = NULL;
p[2] = NULL;
p[3] = NULL;
for (p += 4; ADDR_LT((ptr_t)p, plim); p += 4) {
GC_PREFETCH_FOR_WRITE((ptr_t)(p + 64));
p[0] = (ptr_t)(p - 4);
p[1] = NULL;
CLEAR_DOUBLE(p + 2);
}
return (ptr_t)(p - 4);
}
/* The same as `GC_build_fl_clear4()` but uncleared objects. */
STATIC ptr_t
GC_build_fl4(struct hblk *h, ptr_t ofl)
{
ptr_t *p = (ptr_t *)h->hb_body;
ptr_t plim = (ptr_t)(h + 1);
p[0] = ofl;
p[4] = (ptr_t)p;
/* Unroll the loop by 2. */
for (p += 8; ADDR_LT((ptr_t)p, plim); p += 8) {
GC_PREFETCH_FOR_WRITE((ptr_t)(p + 64));
p[0] = (ptr_t)(p - 4);
p[4] = (ptr_t)p;
}
return (ptr_t)(p - 4);
}
#endif /* !SMALL_CONFIG */
GC_INNER ptr_t
GC_build_fl(struct hblk *h, ptr_t list, size_t lg, GC_bool clear)
{
ptr_t *p, *prev;
ptr_t plim; /*< points to last object in new `hblk` entity */
size_t lpw = GRANULES_TO_PTRS(lg);
/*
* Do a few prefetches here, just because it is cheap.
* If we were more serious about it, these should go inside the loops.
* But write prefetches usually do not seem to matter much.
*/
GC_PREFETCH_FOR_WRITE((ptr_t)h);
GC_PREFETCH_FOR_WRITE((ptr_t)h + 128);
GC_PREFETCH_FOR_WRITE((ptr_t)h + 256);
GC_PREFETCH_FOR_WRITE((ptr_t)h + 378);
#ifndef SMALL_CONFIG
/*
* Handle small objects sizes more efficiently. For larger objects
* the difference is less significant.
*/
switch (lpw) {
case 2:
if (clear) {
return GC_build_fl_clear2(h, list);
} else {
return GC_build_fl2(h, list);
}
case 4:
if (clear) {
return GC_build_fl_clear4(h, list);
} else {
return GC_build_fl4(h, list);
}
default:
break;
}
#endif /* !SMALL_CONFIG */
/* Clear the page if necessary. */
if (clear)
BZERO(h, HBLKSIZE);
/* Add objects to free list. */
prev = (ptr_t *)h->hb_body; /*< one object behind `p` */
/* The last place for the last object to start. */
plim = (ptr_t)h + HBLKSIZE - lpw * sizeof(ptr_t);
/* Make a list of all objects in `*h` with head as last object. */
for (p = prev + lpw; ADDR_GE(plim, (ptr_t)p); p += lpw) {
/* The current object's link points to last object. */
obj_link(p) = (ptr_t)prev;
prev = p;
}
p -= lpw;
/* `p` now points to the last object. */
/*
* Put `p` (which is now head of list of objects in `*h`) as first pointer
* in the appropriate free list for this size.
*/
*(ptr_t *)h = list;
return (ptr_t)p;
}
GC_INNER void
GC_new_hblk(size_t lg, int kind)
{
struct hblk *h; /*< the new heap block */
size_t lb_adjusted = GRANULES_TO_BYTES(lg);
GC_STATIC_ASSERT(sizeof(struct hblk) == HBLKSIZE);
GC_ASSERT(I_HOLD_LOCK());
/* Allocate a new heap block. */
h = GC_allochblk(lb_adjusted, kind, 0 /* `flags` */, 0 /* `align_m1` */);
if (UNLIKELY(NULL == h))
return; /*< out of memory */
/* Mark all objects if appropriate. */
if (IS_UNCOLLECTABLE(kind))
GC_set_hdr_marks(HDR(h));
/* Build the free list. */
GC_obj_kinds[kind].ok_freelist[lg]
= GC_build_fl(h, (ptr_t)GC_obj_kinds[kind].ok_freelist[lg], lg,
GC_debugging_started || GC_obj_kinds[kind].ok_init);
}
/*
* Routines for maintaining maps describing heap block layouts for various
* object sizes. Allows fast pointer validity checks and fast location of
* object start locations on machines (such as SPARC) with slow division.
*/
GC_API void GC_CALL
GC_register_displacement(size_t offset)
{
LOCK();
GC_register_displacement_inner(offset);
UNLOCK();
}
GC_INNER void
GC_register_displacement_inner(size_t offset)
{
GC_ASSERT(I_HOLD_LOCK());
if (offset >= VALID_OFFSET_SZ) {
ABORT("Bad argument to GC_register_displacement");
}
if (!GC_valid_offsets[offset]) {
GC_valid_offsets[offset] = TRUE;
GC_modws_valid_offsets[offset % sizeof(ptr_t)] = TRUE;
}
}
#ifndef MARK_BIT_PER_OBJ
GC_INNER GC_bool
GC_add_map_entry(size_t lg)
{
size_t displ;
hb_map_entry_t *new_map;
GC_ASSERT(I_HOLD_LOCK());
/*
* Ensure `displ % lg` fits into `hb_map_entry_t` type.
* Note: the maximum value is computed in this way to avoid compiler
* complains about constant truncation or expression overflow.
*/
GC_STATIC_ASSERT(
MAXOBJGRANULES - 1
<= (~(size_t)0 >> ((sizeof(size_t) - sizeof(hb_map_entry_t)) * 8)));
if (lg > MAXOBJGRANULES)
lg = 0;
if (LIKELY(GC_obj_map[lg] != NULL))
return TRUE;
new_map = (hb_map_entry_t *)GC_scratch_alloc(OBJ_MAP_LEN
* sizeof(hb_map_entry_t));
if (UNLIKELY(NULL == new_map))
return FALSE;
GC_COND_LOG_PRINTF("Adding block map for size of %u granules (%u bytes)\n",
(unsigned)lg, (unsigned)GRANULES_TO_BYTES(lg));
if (0 == lg) {
for (displ = 0; displ < OBJ_MAP_LEN; displ++) {
/* Set to a nonzero to get us out of the marker fast path. */
new_map[displ] = 1;
}
} else {
for (displ = 0; displ < OBJ_MAP_LEN; displ++) {
new_map[displ] = (hb_map_entry_t)(displ % lg);
}
}
GC_obj_map[lg] = new_map;
return TRUE;
}
#endif /* !MARK_BIT_PER_OBJ */
GC_INNER void
GC_initialize_offsets(void)
{
size_t i;
if (GC_all_interior_pointers) {
for (i = 0; i < VALID_OFFSET_SZ; ++i)
GC_valid_offsets[i] = TRUE;
} else {
BZERO(GC_valid_offsets, sizeof(GC_valid_offsets));
for (i = 0; i < sizeof(ptr_t); ++i)
GC_modws_valid_offsets[i] = FALSE;
}
}
/*
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* These are checking routines calls to which could be inserted by
* a preprocessor to validate C pointer arithmetic.
*/
STATIC void GC_CALLBACK
GC_default_same_obj_print_proc(void *p, void *q)
{
ABORT_ARG2("GC_same_obj test failed",
": %p and %p are not in the same object", p, q);
}
GC_same_obj_print_proc_t GC_same_obj_print_proc
= GC_default_same_obj_print_proc;
GC_API void *GC_CALL
GC_same_obj(void *p, void *q)
{
hdr *hhdr;
ptr_t base, limit;
size_t sz;
if (UNLIKELY(!GC_is_initialized))
GC_init();
hhdr = HDR(p);
if (NULL == hhdr) {
if (divHBLKSZ(ADDR(p)) != divHBLKSZ(ADDR(q)) && HDR(q) != NULL) {
GC_same_obj_print_proc((ptr_t)p, (ptr_t)q);
}
return p;
}
/*
* If it is a pointer to the middle of a large object, move it to
* the beginning.
*/
if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
struct hblk *h = GC_find_starting_hblk(HBLKPTR(p), &hhdr);
limit = (ptr_t)h + hhdr->hb_sz;
if (ADDR_GE((ptr_t)p, limit) || ADDR_GE((ptr_t)q, limit)
|| ADDR_LT((ptr_t)q, (ptr_t)h)) {
GC_same_obj_print_proc((ptr_t)p, (ptr_t)q);
}
return p;
}
sz = hhdr->hb_sz;
if (sz > MAXOBJBYTES) {
base = (ptr_t)HBLKPTR(p);
limit = base + sz;
if (ADDR_GE((ptr_t)p, limit)) {
GC_same_obj_print_proc((ptr_t)p, (ptr_t)q);
return p;
}
} else {
size_t offset;
if (HBLKPTR(p) != HBLKPTR(q)) {
/*
* Without this check, we might miss an error if `q` points to
* the first object on a page, and points just before the page.
*/
GC_same_obj_print_proc((ptr_t)p, (ptr_t)q);
return p;
}
offset = HBLKDISPL(p) % sz;
base = (ptr_t)p - offset;
limit = base + sz;
}
/*
* [`base`,`limit`) delimits the object containing `p`, if any.
* If `p` is not inside a valid object, then either `q` is also
* outside any valid object, or it is outside [`base`,`limit`).
*/
if (!ADDR_INSIDE((ptr_t)q, base, limit)) {
GC_same_obj_print_proc((ptr_t)p, (ptr_t)q);
}
return p;
}
STATIC void GC_CALLBACK
GC_default_is_valid_displacement_print_proc(void *p)
{
ABORT_ARG1("GC_is_valid_displacement test failed", ": %p not valid", p);
}
GC_valid_ptr_print_proc_t GC_is_valid_displacement_print_proc
= GC_default_is_valid_displacement_print_proc;
GC_API void *GC_CALL
GC_is_valid_displacement(void *p)
{
hdr *hhdr;
size_t offset;
struct hblk *h;
size_t sz;
if (UNLIKELY(!GC_is_initialized))
GC_init();
if (NULL == p)
return NULL;
hhdr = HDR(p);
if (NULL == hhdr)
return p;
h = HBLKPTR(p);
if (GC_all_interior_pointers) {
h = GC_find_starting_hblk(h, &hhdr);
} else if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
GC_is_valid_displacement_print_proc((ptr_t)p);
return p;
}
sz = hhdr->hb_sz;
offset = HBLKDISPL(p) % sz;
if ((sz > MAXOBJBYTES && ADDR_GE((ptr_t)p, (ptr_t)h + sz))
|| !GC_valid_offsets[offset]
|| (ADDR_LT((ptr_t)(h + 1), (ptr_t)p + sz - offset)
&& !IS_FORWARDING_ADDR_OR_NIL(HDR(h + 1)))) {
GC_is_valid_displacement_print_proc((ptr_t)p);
}
return p;
}
STATIC void GC_CALLBACK
GC_default_is_visible_print_proc(void *p)
{
ABORT_ARG1("GC_is_visible test failed", ": %p not GC-visible", p);
}
GC_valid_ptr_print_proc_t GC_is_visible_print_proc
= GC_default_is_visible_print_proc;
#ifndef THREADS
/* Could `p` be a stack address? */
STATIC GC_bool
GC_on_stack(ptr_t p)
{
return HOTTER_THAN(p, GC_stackbottom) && !HOTTER_THAN(p, GC_approx_sp());
}
/* Is the address `p` in one of the registered static root sections? */
STATIC GC_bool
GC_is_static_root(ptr_t p)
{
static size_t last_static_root_set = MAX_ROOT_SETS;
size_t i;
# if defined(CPPCHECK)
if (n_root_sets > MAX_ROOT_SETS)
ABORT("Bad n_root_sets");
# endif
if (last_static_root_set < n_root_sets
&& ADDR_INSIDE(p, GC_static_roots[last_static_root_set].r_start,
GC_static_roots[last_static_root_set].r_end))
return TRUE;
for (i = 0; i < n_root_sets; i++) {
if (ADDR_INSIDE(p, GC_static_roots[i].r_start, GC_static_roots[i].r_end)) {
last_static_root_set = i;
return TRUE;
}
}
return FALSE;
}
#endif /* !THREADS */
GC_API void *GC_CALL
GC_is_visible(void *p)
{
const hdr *hhdr;
if ((ADDR(p) & (ALIGNMENT - 1)) != 0)
goto fail;
if (UNLIKELY(!GC_is_initialized))
GC_init();
#ifdef THREADS
hhdr = HDR(p);
if (hhdr != NULL && NULL == GC_base(p)) {
goto fail;
} else {
/* May be inside thread stack. We cannot do much. */
return p;
}
#else
/* Check stack first. */
if (GC_on_stack((ptr_t)p))
return p;
hhdr = HDR(p);
if (NULL == hhdr) {
if (GC_is_static_root((ptr_t)p)) {
return p;
}
/* Else do it again correctly. */
# if defined(ANY_MSWIN) || defined(DYNAMIC_LOADING)
if (!GC_no_dls) {
GC_register_dynamic_libraries();
if (GC_is_static_root((ptr_t)p))
return p;
}
# endif
} else {
/* `p` points to the heap. */
word descr;
/* TODO: Should `GC_base` be manually inlined? */
ptr_t base = (ptr_t)GC_base(p);
if (NULL == base)
goto fail;
if (HBLKPTR(base) != HBLKPTR(p))
hhdr = HDR(base);
descr = hhdr->hb_descr;
retry:
switch (descr & GC_DS_TAGS) {
case GC_DS_LENGTH:
if ((word)((ptr_t)p - base) >= descr)
goto fail;
break;
case GC_DS_BITMAP:
if ((ptr_t)p - base >= (ptrdiff_t)PTRS_TO_BYTES(BITMAP_BITS))
goto fail;
# if ALIGNMENT != CPP_PTRSZ / 8
if ((ADDR(p) & (sizeof(ptr_t) - 1)) != 0)
goto fail;
# endif
if (!(((word)1 << (CPP_WORDSZ - 1 - (word)((ptr_t)p - base))) & descr))
goto fail;
break;
case GC_DS_PROC:
/* We could try to decipher this partially. For now we just punt. */
break;
case GC_DS_PER_OBJECT:
if (!(descr & SIGNB)) {
descr = *(word *)((ptr_t)base + (descr & ~(word)GC_DS_TAGS));
} else {
ptr_t type_descr = *(ptr_t *)base;
if (UNLIKELY(NULL == type_descr)) {
/* See the comment in `GC_mark_from`. */
goto fail;
}
descr = *(word *)(type_descr
- ((GC_signed_word)descr
+ (GC_INDIR_PER_OBJ_BIAS - GC_DS_PER_OBJECT)));
}
goto retry;
}
return p;
}
#endif
fail:
GC_is_visible_print_proc((ptr_t)p);
return p;
}
GC_API void *GC_CALL
GC_pre_incr(void **p, ptrdiff_t how_much)
{
void *initial = *p;
void *result = GC_same_obj((ptr_t)initial + how_much, initial);
if (!GC_all_interior_pointers) {
(void)GC_is_valid_displacement(result);
}
*p = result;
return result; /*< updated pointer */
}
GC_API void *GC_CALL
GC_post_incr(void **p, ptrdiff_t how_much)
{
void *initial = *p;
void *result = GC_same_obj((ptr_t)initial + how_much, initial);
if (!GC_all_interior_pointers) {
(void)GC_is_valid_displacement(result);
}
*p = result;
return initial; /*< original `*p` */
}
GC_API void GC_CALL
GC_set_same_obj_print_proc(GC_same_obj_print_proc_t fn)
{
GC_ASSERT(NONNULL_ARG_NOT_NULL(fn));
GC_same_obj_print_proc = fn;
}
GC_API GC_same_obj_print_proc_t GC_CALL
GC_get_same_obj_print_proc(void)
{
return GC_same_obj_print_proc;
}
GC_API void GC_CALL
GC_set_is_valid_displacement_print_proc(GC_valid_ptr_print_proc_t fn)
{
GC_ASSERT(NONNULL_ARG_NOT_NULL(fn));
GC_is_valid_displacement_print_proc = fn;
}
GC_API GC_valid_ptr_print_proc_t GC_CALL
GC_get_is_valid_displacement_print_proc(void)
{
return GC_is_valid_displacement_print_proc;
}
GC_API void GC_CALL
GC_set_is_visible_print_proc(GC_valid_ptr_print_proc_t fn)
{
GC_ASSERT(NONNULL_ARG_NOT_NULL(fn));
GC_is_visible_print_proc = fn;
}
GC_API GC_valid_ptr_print_proc_t GC_CALL
GC_get_is_visible_print_proc(void)
{
return GC_is_visible_print_proc;
}
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1998-1999 by Silicon Graphics. All rights reserved.
* Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#ifdef GC_USE_ENTIRE_HEAP
int GC_use_entire_heap = TRUE;
#else
int GC_use_entire_heap = FALSE;
#endif
/*
* Free heap blocks are kept on one of several free lists, depending on
* the size of the block. Each free list is doubly linked. Adjacent
* free blocks are coalesced.
*/
/*
* Largest block we will be allocated starting on a black-listed block.
* Must be not smaller than `HBLKSIZE`.
*/
#define MAX_BLACK_LIST_ALLOC (2 * HBLKSIZE)
/* Sizes up to this many `hblk` entities each have their own free list. */
#define UNIQUE_THRESHOLD 32
/*
* Sizes of at least this many heap blocks are mapped to a single free
* list.
*/
#define HUGE_THRESHOLD 256
/* In between sizes map this many distinct sizes to a single bin. */
#define FL_COMPRESSION 8
#define N_HBLK_FLS \
((HUGE_THRESHOLD - UNIQUE_THRESHOLD) / FL_COMPRESSION + UNIQUE_THRESHOLD)
/*
* List of completely empty heap blocks. Linked through `hb_next` field
* of header structure associated with block. Remains externally visible
* as used by GNU `gcj`.
*/
#ifndef GC_GCJ_SUPPORT
STATIC
#endif
struct hblk *GC_hblkfreelist[N_HBLK_FLS + 1] = { 0 };
GC_API void GC_CALL
GC_iterate_free_hblks(GC_walk_free_blk_fn fn, void *client_data)
{
int i;
for (i = 0; i <= N_HBLK_FLS; ++i) {
struct hblk *h;
for (h = GC_hblkfreelist[i]; h != NULL; h = HDR(h)->hb_next) {
fn(h, i, client_data);
}
}
}
/* Number of free bytes on each list. Remains visible to `gcj`. */
#ifndef GC_GCJ_SUPPORT
STATIC
#endif
word GC_free_bytes[N_HBLK_FLS + 1] = { 0 };
#ifndef GC_NO_DEINIT
GC_INNER void
GC_reset_freelist(void)
{
BZERO(GC_hblkfreelist, sizeof(GC_hblkfreelist));
BZERO(GC_free_bytes, sizeof(GC_free_bytes));
}
#endif
/*
* Return the largest `n` such that the number of free bytes on lists
* `n` .. `N_HBLK_FLS` is greater or equal to `GC_max_large_allocd_bytes`
* minus `GC_large_allocd_bytes`. If there is no such `n`, return 0.
*/
GC_INLINE size_t
GC_enough_large_bytes_left(void)
{
size_t n;
word bytes = GC_large_allocd_bytes;
GC_ASSERT(GC_max_large_allocd_bytes <= GC_heapsize);
for (n = N_HBLK_FLS; n > 0; n--) {
bytes += GC_free_bytes[n];
if (bytes >= GC_max_large_allocd_bytes)
break;
}
return n;
}
/* Map a number of blocks to the appropriate large block free-list index. */
STATIC size_t
GC_hblk_fl_from_blocks(size_t blocks_needed)
{
if (blocks_needed <= UNIQUE_THRESHOLD)
return blocks_needed;
if (blocks_needed >= HUGE_THRESHOLD)
return N_HBLK_FLS;
return (blocks_needed - UNIQUE_THRESHOLD) / FL_COMPRESSION
+ UNIQUE_THRESHOLD;
}
#define PHDR(hhdr) HDR((hhdr)->hb_prev)
#define NHDR(hhdr) HDR((hhdr)->hb_next)
#ifdef USE_MUNMAP
# define IS_MAPPED(hhdr) (((hhdr)->hb_flags & WAS_UNMAPPED) == 0)
#else
# define IS_MAPPED(hhdr) TRUE
#endif /* !USE_MUNMAP */
#if !defined(NO_DEBUGGING) || defined(GC_ASSERTIONS)
static void GC_CALLBACK
add_hb_sz(struct hblk *h, int i, void *total_free_ptr)
{
UNUSED_ARG(i);
*(word *)total_free_ptr += HDR(h)->hb_sz;
# if defined(CPPCHECK)
GC_noop1_ptr(h);
# endif
}
GC_INNER word
GC_compute_large_free_bytes(void)
{
word total_free = 0;
GC_iterate_free_hblks(add_hb_sz, &total_free);
return total_free;
}
#endif /* !NO_DEBUGGING || GC_ASSERTIONS */
#ifndef NO_DEBUGGING
static void GC_CALLBACK
print_hblkfreelist_item(struct hblk *h, int i, void *prev_index_ptr)
{
hdr *hhdr = HDR(h);
# if defined(CPPCHECK)
GC_noop1_ptr(h);
# endif
if (i != *(int *)prev_index_ptr) {
GC_printf("Free list %d (total size %lu):\n", i,
(unsigned long)GC_free_bytes[i]);
*(int *)prev_index_ptr = i;
}
# ifdef NO_BLACK_LISTING
GC_printf("\t%p size %lu\n", (void *)h, (unsigned long)hhdr->hb_sz);
# else
GC_printf("\t%p size %lu %s black listed\n", (void *)h,
(unsigned long)hhdr->hb_sz,
GC_is_black_listed(h, HBLKSIZE) != NULL ? "start"
: GC_is_black_listed(h, hhdr->hb_sz) != NULL ? "partially"
: "not");
# endif
}
void
GC_print_hblkfreelist(void)
{
word total;
int prev_index = -1;
GC_iterate_free_hblks(print_hblkfreelist_item, &prev_index);
GC_printf("GC_large_free_bytes: %lu\n", (unsigned long)GC_large_free_bytes);
total = GC_compute_large_free_bytes();
if (total != GC_large_free_bytes)
GC_err_printf("GC_large_free_bytes INCONSISTENT!! Should be: %lu\n",
(unsigned long)total);
}
/*
* Return the free-list index on which the block described by the header
* appears, or -1 if it appears nowhere.
*/
static int
free_list_index_of(const hdr *wanted)
{
int i;
for (i = 0; i <= N_HBLK_FLS; ++i) {
const struct hblk *h;
const hdr *hhdr;
for (h = GC_hblkfreelist[i]; h != NULL; h = hhdr->hb_next) {
hhdr = HDR(h);
if (hhdr == wanted)
return i;
}
}
return -1;
}
GC_API void GC_CALL
GC_foreach_heap_section_inner(GC_heap_section_proc fn, void *client_data)
{
size_t i;
/*
* The collector memory is organized in heap sections that are split in
* blocks. Each such block has a header (obtained by `HDR(p)`) and the
* block size is aligned to `HBLKSIZE`. The block headers are kept
* separately from the memory they point to.
*/
for (i = 0; i < GC_n_heap_sects; ++i) {
ptr_t start = GC_heap_sects[i].hs_start;
ptr_t finish = start + GC_heap_sects[i].hs_bytes;
ptr_t p;
/* Merge in contiguous sections. */
while (i + 1 < GC_n_heap_sects
&& GC_heap_sects[i + 1].hs_start == finish) {
++i;
finish = GC_heap_sects[i].hs_start + GC_heap_sects[i].hs_bytes;
}
fn(start, finish, GC_HEAP_SECTION_TYPE_WHOLE_SECT, client_data);
for (p = start; ADDR_LT(p, finish);) {
/*
* Lookup into 2-level tree data structure which uses address
* as a hash key to find the block header.
*/
hdr *hhdr = HDR(p);
if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
/*
* The pointer has no header registered in the headers cache.
* Skip one `HBLKSIZE` and retry. Might be mapped or not.
*/
fn(p, p + HBLKSIZE, GC_HEAP_SECTION_TYPE_FORWARDING, client_data);
p += HBLKSIZE;
continue;
}
if (HBLK_IS_FREE(hhdr)) {
/*
* The block is marked as free. Note: `hb_sz` is the size in bytes
* of the whole block.
*/
fn(p, p + hhdr->hb_sz,
IS_MAPPED(hhdr) ? GC_HEAP_SECTION_TYPE_FREE
: GC_HEAP_SECTION_TYPE_UNMAPPED,
client_data);
p += hhdr->hb_sz;
} else {
/*
* This heap block is used. Report also the padding, if any.
* Note: `hb_sz` is the size (in bytes) of objects in the block.
*/
ptr_t blockEnd = p + HBLKSIZE * OBJ_SZ_TO_BLOCKS(hhdr->hb_sz);
ptr_t usedBlockEnd = p + hhdr->hb_sz;
fn(p, usedBlockEnd, GC_HEAP_SECTION_TYPE_USED, client_data);
if (ADDR_LT(usedBlockEnd, blockEnd))
fn(usedBlockEnd, blockEnd, GC_HEAP_SECTION_TYPE_PADDING,
client_data);
p = blockEnd;
}
}
}
}
static void GC_CALLBACK
dump_regions_proc(void *start, void *finish, GC_heap_section_type type,
void *client_data)
{
hdr *hhdr;
int correct_index, actual_index;
UNUSED_ARG(client_data);
switch (type) {
case GC_HEAP_SECTION_TYPE_WHOLE_SECT:
GC_printf("***Section from %p to %p\n", start, finish);
break;
case GC_HEAP_SECTION_TYPE_FORWARDING:
GC_printf("\t%p Missing header!!(%p)\n", start, (void *)HDR(start));
break;
case GC_HEAP_SECTION_TYPE_FREE:
case GC_HEAP_SECTION_TYPE_UNMAPPED:
hhdr = HDR(start);
GC_printf("\t%p\tfree block of size 0x%lx bytes%s\n", start,
(unsigned long)hhdr->hb_sz,
type == GC_HEAP_SECTION_TYPE_UNMAPPED ? " (unmapped)" : "");
actual_index = free_list_index_of(hhdr);
correct_index = (int)GC_hblk_fl_from_blocks(divHBLKSZ(hhdr->hb_sz));
if (-1 == actual_index) {
GC_printf("\t\tBlock not on free list %d!!\n", correct_index);
} else if (correct_index != actual_index) {
GC_printf("\t\tBlock on list %d, should be on %d!!\n", actual_index,
correct_index);
}
break;
case GC_HEAP_SECTION_TYPE_USED:
GC_printf("\t%p\tused for blocks of size 0x%lx bytes\n", start,
(unsigned long)(ADDR(finish) - ADDR(start)));
break;
case GC_HEAP_SECTION_TYPE_PADDING:
/* Empty. */
break;
}
}
GC_API void GC_CALL
GC_dump_regions(void)
{
GC_foreach_heap_section_inner(dump_regions_proc, NULL);
}
#endif /* !NO_DEBUGGING */
/*
* Initialize `hhdr` for a `block` containing the indicated size
* `lb_adjusted` and `kind` of objects. Return `FALSE` on failure.
*/
static GC_bool
setup_header(hdr *hhdr, struct hblk *block, size_t lb_adjusted, int kind,
unsigned flags)
{
const struct obj_kind *ok;
word descr;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(lb_adjusted >= ALIGNMENT);
#ifndef MARK_BIT_PER_OBJ
if (lb_adjusted > MAXOBJBYTES)
flags |= LARGE_BLOCK;
#endif
ok = &GC_obj_kinds[kind];
#ifdef ENABLE_DISCLAIM
if (ok->ok_disclaim_proc)
flags |= HAS_DISCLAIM;
if (ok->ok_mark_unconditionally)
flags |= MARK_UNCONDITIONALLY;
#endif
/* Set size, kind and mark procedure fields. */
hhdr->hb_sz = lb_adjusted;
hhdr->hb_obj_kind = (unsigned char)kind;
hhdr->hb_flags = (unsigned char)flags;
hhdr->hb_block = block;
descr = ok->ok_descriptor;
#if ALIGNMENT > GC_DS_TAGS
/*
* An extra byte is not added in case of ignore-off-page allocated objects
* not smaller than `HBLKSIZE`.
*/
if (EXTRA_BYTES != 0 && (flags & IGNORE_OFF_PAGE) != 0 && kind == NORMAL
&& lb_adjusted >= HBLKSIZE)
descr += ALIGNMENT; /*< or set to 0 */
#endif
if (ok->ok_relocate_descr)
descr += lb_adjusted;
hhdr->hb_descr = descr;
#ifdef MARK_BIT_PER_OBJ
/*
* Set `hb_inv_sz` as portably as possible. We set it to the smallest
* value such that `lb_adjusted * inv_sz >= 2**32`.
* This may be more precision than necessary.
*/
if (lb_adjusted > MAXOBJBYTES) {
hhdr->hb_inv_sz = LARGE_INV_SZ;
} else {
unsigned32 inv_sz;
GC_ASSERT(lb_adjusted > 1);
# if CPP_WORDSZ > 32
inv_sz = (unsigned32)(((word)1 << 32) / lb_adjusted);
if (((inv_sz * (word)lb_adjusted) >> 32) == 0)
++inv_sz;
# else
inv_sz = (((unsigned32)1 << 31) / lb_adjusted) << 1;
while ((inv_sz * lb_adjusted) > lb_adjusted)
inv_sz++;
# endif
# if (CPP_WORDSZ == 32) && defined(__GNUC__)
GC_ASSERT(((1ULL << 32) + lb_adjusted - 1) / lb_adjusted == inv_sz);
# endif
hhdr->hb_inv_sz = inv_sz;
}
#else
{
size_t lg = BYTES_TO_GRANULES(lb_adjusted);
if (UNLIKELY(!GC_add_map_entry(lg))) {
/* Make it look like a valid block. */
hhdr->hb_sz = HBLKSIZE;
hhdr->hb_descr = 0;
hhdr->hb_flags |= LARGE_BLOCK;
hhdr->hb_map = NULL;
return FALSE;
}
hhdr->hb_map = GC_obj_map[(hhdr->hb_flags & LARGE_BLOCK) != 0 ? 0 : lg];
}
#endif
/* Clear mark bits. */
GC_clear_hdr_marks(hhdr);
hhdr->hb_last_reclaimed = (unsigned short)GC_gc_no;
return TRUE;
}
/*
* Remove `hhdr` from the free list (it is assumed to be specified by
* `index`).
*/
STATIC void
GC_remove_from_fl_at(hdr *hhdr, size_t index)
{
GC_ASSERT(modHBLKSZ(hhdr->hb_sz) == 0);
if (hhdr->hb_prev == 0) {
GC_ASSERT(HDR(GC_hblkfreelist[index]) == hhdr);
GC_hblkfreelist[index] = hhdr->hb_next;
} else {
hdr *phdr;
GET_HDR(hhdr->hb_prev, phdr);
phdr->hb_next = hhdr->hb_next;
}
/* We always need index to maintain free counts. */
GC_ASSERT(GC_free_bytes[index] >= hhdr->hb_sz);
GC_free_bytes[index] -= hhdr->hb_sz;
if (hhdr->hb_next != NULL) {
hdr *nhdr;
GC_ASSERT(!IS_FORWARDING_ADDR_OR_NIL(NHDR(hhdr)));
GET_HDR(hhdr->hb_next, nhdr);
nhdr->hb_prev = hhdr->hb_prev;
}
}
/*
* Remove `hhdr` from the appropriate free list (we assume it is on the
* size-appropriate free list).
*/
GC_INLINE void
GC_remove_from_fl(hdr *hhdr)
{
GC_remove_from_fl_at(hhdr, GC_hblk_fl_from_blocks(divHBLKSZ(hhdr->hb_sz)));
}
/* Return a pointer to the block ending just before `h`, if any. */
static struct hblk *
get_block_ending_at(struct hblk *h)
{
struct hblk *p = h - 1;
hdr *hhdr;
GET_HDR(p, hhdr);
if (hhdr != NULL) {
return GC_find_starting_hblk(p, &hhdr);
}
p = GC_prev_block(p);
if (p != NULL) {
hhdr = HDR(p);
if ((ptr_t)p + hhdr->hb_sz == (ptr_t)h) {
return p;
}
}
return NULL;
}
/* Return a pointer to the free block ending just before `h`, if any. */
STATIC struct hblk *
GC_free_block_ending_at(struct hblk *h)
{
struct hblk *p = get_block_ending_at(h);
if (p /* `!= NULL` */) { /*< CPPCHECK */
const hdr *hhdr = HDR(p);
if (HBLK_IS_FREE(hhdr)) {
return p;
}
}
return 0;
}
/*
* Add `hhdr` to the appropriate free list. We maintain individual
* free lists sorted by address.
*/
STATIC void
GC_add_to_fl(struct hblk *h, hdr *hhdr)
{
size_t index = GC_hblk_fl_from_blocks(divHBLKSZ(hhdr->hb_sz));
struct hblk *second = GC_hblkfreelist[index];
#if defined(GC_ASSERTIONS) && !defined(USE_MUNMAP) && !defined(CHERI_PURECAP)
{
struct hblk *next = (struct hblk *)((ptr_t)h + hhdr->hb_sz);
const hdr *nexthdr = HDR(next);
struct hblk *prev = GC_free_block_ending_at(h);
const hdr *prevhdr = HDR(prev);
GC_ASSERT(NULL == nexthdr || !HBLK_IS_FREE(nexthdr)
|| (GC_heapsize & SIGNB) != 0);
/* In the last case, blocks may be too large to be merged. */
GC_ASSERT(NULL == prev || !HBLK_IS_FREE(prevhdr)
|| (GC_heapsize & SIGNB) != 0);
}
#endif
GC_ASSERT(modHBLKSZ(hhdr->hb_sz) == 0);
GC_hblkfreelist[index] = h;
GC_free_bytes[index] += hhdr->hb_sz;
GC_ASSERT(GC_free_bytes[index] <= GC_large_free_bytes);
hhdr->hb_next = second;
hhdr->hb_prev = NULL;
if (second /* `!= NULL` */) { /*< CPPCHECK */
hdr *second_hdr;
GET_HDR(second, second_hdr);
second_hdr->hb_prev = h;
}
hhdr->hb_flags |= FREE_BLK;
}
#define BLOCKS_MERGE_OVERFLOW(hhdr, nexthdr) \
((((hhdr)->hb_sz + (nexthdr)->hb_sz) & SIZET_SIGNB) != 0)
#ifdef USE_MUNMAP
/*
* `GC_unmap_old` will avoid creating more than this many unmapped regions,
* but an unmapped region may be split again so exceeding the limit.
*/
# ifdef COUNT_UNMAPPED_REGIONS
/*
* Return the change in number of unmapped regions if the block `h` swaps
* from its current state of mapped/unmapped to the opposite state.
*/
static int
calc_num_unmapped_regions_delta(struct hblk *h, hdr *hhdr)
{
struct hblk *prev = get_block_ending_at(h);
struct hblk *next;
GC_bool prev_unmapped = FALSE;
GC_bool next_unmapped = FALSE;
next = GC_next_block((struct hblk *)((ptr_t)h + hhdr->hb_sz), TRUE);
/* Ensure next is contiguous with `h`. */
if (next != HBLK_PAGE_ALIGNED((ptr_t)h + hhdr->hb_sz)) {
next = NULL;
}
if (prev != NULL) {
const hdr *prevhdr = HDR(prev);
prev_unmapped = !IS_MAPPED(prevhdr);
}
if (next != NULL) {
const hdr *nexthdr = HDR(next);
next_unmapped = !IS_MAPPED(nexthdr);
}
if (prev_unmapped && next_unmapped) {
/*
* If `h` is unmapped, merge two unmapped regions into one.
* If `h` is remapped, split one unmapped region into two.
*/
return IS_MAPPED(hhdr) ? -1 : 1;
}
if (!prev_unmapped && !next_unmapped) {
/*
* If `h` is unmapped, create an isolated unmapped region.
* If `h` is remapped, remove it.
*/
return IS_MAPPED(hhdr) ? 1 : -1;
}
/*
* If `h` is unmapped, merge it with previous or next unmapped region.
* If `h` is remapped, reduce either previous or next unmapped region.
* In either way, no change to the number of unmapped regions.
*/
return 0;
}
# endif /* COUNT_UNMAPPED_REGIONS */
/*
* Update `GC_num_unmapped_regions` assuming the block `h` changes from
* its current state of mapped/unmapped to the opposite state.
*/
GC_INLINE void
GC_adjust_num_unmapped(struct hblk *h, hdr *hhdr)
{
# ifdef COUNT_UNMAPPED_REGIONS
GC_num_unmapped_regions += calc_num_unmapped_regions_delta(h, hhdr);
# else
UNUSED_ARG(h);
UNUSED_ARG(hhdr);
# endif
}
GC_INNER void
GC_unmap_old(unsigned threshold)
{
size_t i;
# ifdef COUNT_UNMAPPED_REGIONS
/*
* Skip unmapping if we have already exceeded the soft limit.
* This forgoes any opportunities to merge unmapped regions though.
*/
if (GC_num_unmapped_regions >= GC_UNMAPPED_REGIONS_SOFT_LIMIT)
return;
# endif
for (i = 0; i <= N_HBLK_FLS; ++i) {
struct hblk *h;
hdr *hhdr;
for (h = GC_hblkfreelist[i]; h != NULL; h = hhdr->hb_next) {
hhdr = HDR(h);
if (!IS_MAPPED(hhdr))
continue;
/*
* Check that the interval is not smaller than the `threshold`.
* The truncated counter value wrapping is handled correctly.
*/
if ((unsigned short)(GC_gc_no - hhdr->hb_last_reclaimed)
>= (unsigned short)threshold) {
# ifdef COUNT_UNMAPPED_REGIONS
/*
* Continue with unmapping the block only if it will not create
* too many unmapped regions, or if unmapping reduces the number
* of regions.
*/
int delta = calc_num_unmapped_regions_delta(h, hhdr);
GC_signed_word regions = GC_num_unmapped_regions + delta;
if (delta >= 0 && regions >= GC_UNMAPPED_REGIONS_SOFT_LIMIT) {
GC_COND_LOG_PRINTF("Unmapped regions limit reached!\n");
return;
}
GC_num_unmapped_regions = regions;
# endif
GC_unmap((ptr_t)h, hhdr->hb_sz);
hhdr->hb_flags |= WAS_UNMAPPED;
}
}
}
}
GC_INNER GC_bool
GC_merge_unmapped(void)
{
size_t i;
GC_bool merged = FALSE;
for (i = 0; i <= N_HBLK_FLS; ++i) {
struct hblk *h = GC_hblkfreelist[i];
while (h != NULL) {
struct hblk *next;
hdr *hhdr, *nexthdr;
size_t size, next_size;
GET_HDR(h, hhdr);
size = hhdr->hb_sz;
next = (struct hblk *)((ptr_t)h + size);
GET_HDR(next, nexthdr);
/* Coalesce with successor, if possible. */
if (NULL == nexthdr || !HBLK_IS_FREE(nexthdr)
|| BLOCKS_MERGE_OVERFLOW(hhdr, nexthdr)) {
/* Not mergeable with the successor. */
h = hhdr->hb_next;
continue;
}
next_size = nexthdr->hb_sz;
# ifdef CHERI_PURECAP
/* FIXME: Coalesce with super-capability. */
if (!CAPABILITY_COVERS_RANGE(h, ADDR(next), ADDR(next) + nextsize)) {
h = hhdr->hb_next;
continue;
}
# endif
/*
* Note that we usually try to avoid adjacent free blocks that are
* either both mapped or both unmapped. But that is not guaranteed
* to hold since we remap blocks when we split them, and do not merge
* at that point. It may also not hold if the merged block would be
* too big.
*/
if (IS_MAPPED(hhdr) && !IS_MAPPED(nexthdr)) {
/* Make both consistent, so that we can merge. */
if (size > next_size) {
GC_adjust_num_unmapped(next, nexthdr);
GC_remap((ptr_t)next, next_size);
} else {
GC_adjust_num_unmapped(h, hhdr);
GC_unmap((ptr_t)h, size);
GC_unmap_gap((ptr_t)h, size, (ptr_t)next, next_size);
hhdr->hb_flags |= WAS_UNMAPPED;
}
} else if (IS_MAPPED(nexthdr) && !IS_MAPPED(hhdr)) {
if (size > next_size) {
GC_adjust_num_unmapped(next, nexthdr);
GC_unmap((ptr_t)next, next_size);
GC_unmap_gap((ptr_t)h, size, (ptr_t)next, next_size);
} else {
GC_adjust_num_unmapped(h, hhdr);
GC_remap((ptr_t)h, size);
hhdr->hb_flags &= (unsigned char)~WAS_UNMAPPED;
hhdr->hb_last_reclaimed = nexthdr->hb_last_reclaimed;
}
} else if (!IS_MAPPED(hhdr) && !IS_MAPPED(nexthdr)) {
/* Unmap any gap in the middle. */
GC_unmap_gap((ptr_t)h, size, (ptr_t)next, next_size);
}
/* If they are both unmapped, we merge, but leave unmapped. */
GC_remove_from_fl_at(hhdr, i);
GC_remove_from_fl(nexthdr);
hhdr->hb_sz += nexthdr->hb_sz;
GC_remove_header(next);
GC_add_to_fl(h, hhdr);
merged = TRUE;
/* Start over at the beginning of list. */
h = GC_hblkfreelist[i];
}
}
return merged;
}
#endif /* USE_MUNMAP */
/*
* Return a pointer to a block starting at `h`. Memory for the block
* is mapped. Remove the block from its free list, and return the
* remainder (if any) to its appropriate free list. May fail by
* returning `NULL`. The header for the returned block must be set up
* by the caller. If the returned pointer is not `NULL`, then `hhdr`
* is the header for it.
*/
STATIC struct hblk *
GC_get_first_part(struct hblk *h, hdr *hhdr, size_t size_needed, size_t index)
{
size_t total_size;
struct hblk *rest;
hdr *rest_hdr;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(modHBLKSZ(size_needed) == 0);
total_size = hhdr->hb_sz;
GC_ASSERT(modHBLKSZ(total_size) == 0);
GC_remove_from_fl_at(hhdr, index);
if (total_size == size_needed)
return h;
rest = (struct hblk *)((ptr_t)h + size_needed);
rest_hdr = GC_install_header(rest);
if (UNLIKELY(NULL == rest_hdr)) {
/* FIXME: This is likely to be very bad news... */
WARN("Header allocation failed: dropping block\n", 0);
return NULL;
}
rest_hdr->hb_block = rest;
rest_hdr->hb_sz = total_size - size_needed;
rest_hdr->hb_flags = 0;
#ifdef GC_ASSERTIONS
/* Mark `h` as non-free, to avoid assertion about adjacent free blocks. */
hhdr->hb_flags &= (unsigned char)~FREE_BLK;
#endif
GC_add_to_fl(rest, rest_hdr);
return h;
}
/*
* Split the block. `hbp` is a free block; `last_hbp` points at address
* inside it; a new header for `last_hbp` is assumed to be already set up.
* Fix up the header of `hbp` to reflect the fact that it is being split,
* move it to the appropriate free list. `last_hbp` replaces `hbp` in the
* original free list. `last_hdr` is not completely filled in, since it
* is about to be allocated. It may, in fact, end up on the wrong free
* list for its size. That is not a disaster, since `last_hbp` is to be
* allocated by our caller. (Hence adding it to a free list is silly.
* But this path is hopefully rare enough that it does not matter.
* The code is cleaner this way.)
*/
STATIC void
GC_split_block(struct hblk *hbp, hdr *hhdr, struct hblk *last_hbp,
hdr *last_hdr, size_t index /* of free list */)
{
size_t h_size = (size_t)((ptr_t)last_hbp - (ptr_t)hbp);
struct hblk *prev = hhdr->hb_prev;
struct hblk *next = hhdr->hb_next;
/* Replace `hbp` with `last_hbp` on its free list. */
last_hdr->hb_prev = prev;
last_hdr->hb_next = next;
last_hdr->hb_block = last_hbp;
last_hdr->hb_sz = hhdr->hb_sz - h_size;
last_hdr->hb_flags = 0;
if (prev /* `!= NULL` */) { /*< CPPCHECK */
HDR(prev)->hb_next = last_hbp;
} else {
GC_hblkfreelist[index] = last_hbp;
}
if (next /* `!= NULL` */) {
HDR(next)->hb_prev = last_hbp;
}
GC_ASSERT(GC_free_bytes[index] > h_size);
GC_free_bytes[index] -= h_size;
#ifdef USE_MUNMAP
hhdr->hb_last_reclaimed = (unsigned short)GC_gc_no;
#endif
hhdr->hb_sz = h_size;
GC_add_to_fl(hbp, hhdr);
last_hdr->hb_flags |= FREE_BLK;
}
STATIC struct hblk *GC_allochblk_nth(size_t lb_adjusted, int kind,
unsigned flags, size_t index,
int may_split, size_t align_m1);
#ifdef USE_MUNMAP
# define AVOID_SPLIT_REMAPPED 2
#endif
GC_INNER struct hblk *
GC_allochblk(size_t lb_adjusted, int kind,
unsigned flags /* `IGNORE_OFF_PAGE` or 0 */, size_t align_m1)
{
size_t blocks, start_list;
struct hblk *result;
int may_split;
size_t split_limit; /* highest index of free list whose blocks we split */
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT((lb_adjusted & (GC_GRANULE_BYTES - 1)) == 0);
blocks = OBJ_SZ_TO_BLOCKS_CHECKED(lb_adjusted);
if (UNLIKELY(SIZET_SAT_ADD(blocks * HBLKSIZE, align_m1)
>= (GC_SIZE_MAX >> 1)))
return NULL; /* overflow */
start_list = GC_hblk_fl_from_blocks(blocks);
/* Try for an exact match first. */
result = GC_allochblk_nth(lb_adjusted, kind, flags, start_list, FALSE,
align_m1);
if (result != NULL)
return result;
may_split = TRUE;
if (GC_use_entire_heap || GC_dont_gc
|| GC_heapsize - GC_large_free_bytes < GC_requested_heapsize
|| GC_incremental || !GC_should_collect()) {
/* Should use more of the heap, even if it requires splitting. */
split_limit = N_HBLK_FLS;
} else if (GC_finalizer_bytes_freed > (GC_heapsize >> 4)) {
/*
* If we are deallocating lots of memory from finalizers, then fail
* and collect sooner rather than later.
*/
split_limit = 0;
} else {
/*
* If we have enough large blocks left to cover any previous request
* for large blocks, we go ahead and split. Assuming a steady state,
* that should be safe. It means that we can use the full heap
* if we allocate only small objects.
*/
split_limit = GC_enough_large_bytes_left();
#ifdef USE_MUNMAP
if (split_limit > 0)
may_split = AVOID_SPLIT_REMAPPED;
#endif
}
if (start_list < UNIQUE_THRESHOLD && 0 == align_m1) {
/*
* No reason to try `start_list` again, since all blocks are exact
* matches.
*/
++start_list;
}
for (; start_list <= split_limit; ++start_list) {
result = GC_allochblk_nth(lb_adjusted, kind, flags, start_list, may_split,
align_m1);
if (result != NULL)
break;
}
return result;
}
#define ALIGN_PAD_SZ(p, align_m1) \
(((align_m1) + 1 - (size_t)ADDR(p)) & (align_m1))
static GC_bool
next_hblk_fits_better(const hdr *hhdr, size_t size_avail, size_t size_needed,
size_t align_m1)
{
const hdr *nexthdr;
size_t next_size;
size_t next_ofs;
struct hblk *next_hbp = hhdr->hb_next;
if (NULL == next_hbp)
return FALSE; /*< no next block */
GET_HDR(next_hbp, nexthdr);
next_size = nexthdr->hb_sz;
if (size_avail <= next_size)
return FALSE; /*< not enough size */
next_ofs = ALIGN_PAD_SZ(next_hbp, align_m1);
return next_size >= size_needed + next_ofs
#ifndef NO_BLACK_LISTING
&& !GC_is_black_listed(next_hbp + divHBLKSZ(next_ofs), size_needed)
#endif
;
}
static struct hblk *
find_nonbl_hblk(struct hblk *last_hbp, size_t size_remain,
size_t eff_size_needed, size_t align_m1)
{
#ifdef NO_BLACK_LISTING
UNUSED_ARG(size_remain);
UNUSED_ARG(eff_size_needed);
return last_hbp + divHBLKSZ(ALIGN_PAD_SZ(last_hbp, align_m1));
#else
ptr_t search_end
= PTR_ALIGN_DOWN((ptr_t)last_hbp + size_remain, align_m1 + 1);
do {
struct hblk *next_hbp;
last_hbp += divHBLKSZ(ALIGN_PAD_SZ(last_hbp, align_m1));
next_hbp = GC_is_black_listed(last_hbp, eff_size_needed);
if (NULL == next_hbp)
return last_hbp; /*< not black-listed */
last_hbp = next_hbp;
} while (ADDR_GE(search_end, (ptr_t)last_hbp));
return NULL;
#endif
}
#ifndef NO_BLACK_LISTING
/*
* Allocate and drop the block in small chunks, to maximize the chance
* that we will recover some later. `hhdr` should correspond to `hbp`.
*/
static void
drop_hblk_in_chunks(size_t n, struct hblk *hbp, hdr *hhdr)
{
size_t total_size = hhdr->hb_sz;
const struct hblk *limit = hbp + divHBLKSZ(total_size);
GC_ASSERT(HDR(hbp) == hhdr);
GC_ASSERT(modHBLKSZ(total_size) == 0 && total_size > 0);
GC_large_free_bytes -= total_size;
GC_bytes_dropped += total_size;
GC_remove_from_fl_at(hhdr, n);
do {
(void)setup_header(hhdr, hbp, HBLKSIZE, PTRFREE, 0); /*< cannot fail */
if (GC_debugging_started)
BZERO(hbp, HBLKSIZE);
hbp++;
if (ADDR_GE(hbp, limit))
break;
hhdr = GC_install_header(hbp);
} while (LIKELY(hhdr != NULL)); /*< no header allocation failure? */
}
#endif /* !NO_BLACK_LISTING */
#if defined(MPROTECT_VDB) && defined(DONT_PROTECT_PTRFREE)
static GC_bool
is_hblks_mix_in_page(struct hblk *hbp, GC_bool is_ptrfree)
{
struct hblk *h = HBLK_PAGE_ALIGNED(hbp);
size_t i, cnt = divHBLKSZ(GC_page_size);
/*
* Iterate over blocks in the page to check if all the occupied blocks
* are pointer-free if we are going to allocate a pointer-free one,
* and vice versa.
*/
for (i = 0; i < cnt; i++) {
hdr *hhdr;
GET_HDR(&h[i], hhdr);
if (NULL == hhdr)
continue;
(void)GC_find_starting_hblk(&h[i], &hhdr);
if (!HBLK_IS_FREE(hhdr)) {
/* It is OK to check only the first found occupied block. */
return IS_PTRFREE(hhdr) != is_ptrfree;
}
}
/* All blocks are free. */
return FALSE;
}
#endif /* MPROTECT_VDB && DONT_PROTECT_PTRFREE */
/*
* The same as `GC_allochblk`, but with search restricted to the
* `index`-th free list. `flags` should be `IGNORE_OFF_PAGE` or zero;
* `may_split` indicates whether it is OK to split larger blocks; size
* `lb_adjusted` is in bytes. If `may_split` is set to
* `AVOID_SPLIT_REMAPPED`, then memory remapping followed by splitting
* should be generally avoided. Rounded-up `lb_adjusted` plus
* `align_m1` value should be less than `GC_SIZE_MAX / 2`.
*/
STATIC struct hblk *
GC_allochblk_nth(size_t lb_adjusted, int kind, unsigned flags, size_t index,
int may_split, size_t align_m1)
{
struct hblk *hbp, *last_hbp;
/* The header corresponding to `hbp`. */
hdr *hhdr;
/* Number of bytes in requested objects. */
size_t size_needed = (lb_adjusted + HBLKSIZE - 1) & ~(HBLKSIZE - 1);
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(((align_m1 + 1) & align_m1) == 0 && lb_adjusted > 0);
GC_ASSERT(0 == align_m1 || modHBLKSZ(align_m1 + 1) == 0);
#ifndef NO_BLACK_LISTING
retry:
#endif
/* Search for a big enough block in free list. */
for (hbp = GC_hblkfreelist[index];; hbp = hhdr->hb_next) {
size_t size_avail; /*< bytes available in this block */
size_t align_ofs;
if (hbp /* `!= NULL` */) {
/* CPPCHECK */
} else {
return NULL;
}
GET_HDR(hbp, hhdr); /*< set `hhdr` value */
size_avail = hhdr->hb_sz;
if (!may_split && size_avail != size_needed)
continue;
align_ofs = ALIGN_PAD_SZ(hbp, align_m1);
if (size_avail < size_needed + align_ofs)
continue; /*< the block is too small */
if (size_avail != size_needed) {
/*
* If the next heap block is obviously better, go on.
* This prevents us from disassembling a single large block to get
* tiny blocks.
*/
if (next_hblk_fits_better(hhdr, size_avail, size_needed, align_m1))
continue;
}
#if defined(MPROTECT_VDB) && defined(DONT_PROTECT_PTRFREE)
/*
* Avoid write-protecting pointer-free blocks (only the case
* if page size is larger than the block size).
*/
GC_ASSERT(GC_page_size != 0);
if (GC_page_size != HBLKSIZE
&& (!GC_incremental /*< not enabled yet */
|| GC_incremental_protection_needs() != GC_PROTECTS_NONE)
&& is_hblks_mix_in_page(hbp, kind == PTRFREE))
continue;
#endif
if (IS_UNCOLLECTABLE(kind)
|| (kind == PTRFREE && size_needed <= MAX_BLACK_LIST_ALLOC)) {
last_hbp = hbp + divHBLKSZ(align_ofs);
break;
}
last_hbp = find_nonbl_hblk(
hbp, size_avail - size_needed,
(flags & IGNORE_OFF_PAGE) != 0 ? HBLKSIZE : size_needed, align_m1);
/* Is non-black-listed part of enough size? */
if (last_hbp != NULL) {
#ifdef USE_MUNMAP
/* Avoid remapping followed by splitting. */
if (may_split == AVOID_SPLIT_REMAPPED && last_hbp != hbp
&& !IS_MAPPED(hhdr))
continue;
#endif
break;
}
#ifndef NO_BLACK_LISTING
/*
* The block is completely black-listed. If so, we need to
* drop some such blocks, since otherwise we spend all our
* time traversing them if pointer-free blocks are unpopular.
* A dropped block will be reconsidered at next collection.
*/
if (size_needed == HBLKSIZE && 0 == align_m1 && !GC_find_leak_inner
&& IS_MAPPED(hhdr) && (++GC_drop_blacklisted_count & 3) == 0) {
const struct hblk *prev = hhdr->hb_prev;
drop_hblk_in_chunks(index, hbp, hhdr);
if (NULL == prev)
goto retry;
/* Restore `hhdr` to point at free block. */
hhdr = HDR(prev);
continue;
}
if (size_needed > BL_LIMIT && size_avail - size_needed > BL_LIMIT) {
/* Punt, since anything else risks unreasonable heap growth. */
if (++GC_large_alloc_warn_suppressed >= GC_large_alloc_warn_interval) {
WARN("Repeated allocation of very large block"
" (appr. size %" WARN_PRIuPTR " KiB):\n"
"\tMay lead to memory leak and poor performance\n",
size_needed >> 10);
GC_large_alloc_warn_suppressed = 0;
}
last_hbp = hbp + divHBLKSZ(align_ofs);
break;
}
#endif
}
GC_ASSERT((ADDR(last_hbp) & align_m1) == 0);
if (last_hbp != hbp) {
hdr *last_hdr = GC_install_header(last_hbp);
if (UNLIKELY(NULL == last_hdr))
return NULL;
#ifdef USE_MUNMAP
/* Make sure it is mapped before we mangle it. */
if (!IS_MAPPED(hhdr)) {
GC_adjust_num_unmapped(hbp, hhdr);
GC_remap((ptr_t)hbp, hhdr->hb_sz);
hhdr->hb_flags &= (unsigned char)~WAS_UNMAPPED;
}
#endif
/* Split the block at `last_hbp`. */
GC_split_block(hbp, hhdr, last_hbp, last_hdr, index);
/*
* We must now allocate `last_hbp`, since it may be on the wrong
* free list.
*/
hbp = last_hbp;
hhdr = last_hdr;
}
GC_ASSERT(hhdr->hb_sz >= size_needed);
#ifdef USE_MUNMAP
if (!IS_MAPPED(hhdr)) {
GC_adjust_num_unmapped(hbp, hhdr);
GC_remap((ptr_t)hbp, hhdr->hb_sz);
hhdr->hb_flags &= (unsigned char)~WAS_UNMAPPED;
/* Note: this may leave adjacent, mapped free blocks. */
}
#endif
/*
* `hbp` may be on the wrong free list; the parameter `index` is
* important.
*/
hbp = GC_get_first_part(hbp, hhdr, size_needed, index);
if (UNLIKELY(NULL == hbp))
return NULL;
/* Add it to map of valid blocks. */
if (UNLIKELY(!GC_install_counts(hbp, size_needed)))
return NULL; /*< note: this leaks memory under very rare conditions */
/* Set up the header. */
GC_ASSERT(HDR(hbp) == hhdr);
#ifdef MARK_BIT_PER_OBJ
(void)setup_header(hhdr, hbp, lb_adjusted, kind, flags);
/* Result is always `TRUE`, not checked to avoid a cppcheck warning. */
#else
if (UNLIKELY(!setup_header(hhdr, hbp, lb_adjusted, kind, flags))) {
GC_remove_counts(hbp, size_needed);
return NULL; /*< ditto */
}
#endif
#ifndef GC_DISABLE_INCREMENTAL
/*
* Notify virtual dirty bit implementation that we are about to write.
* Ensure that pointer-free objects are not protected if it is avoidable.
* This also ensures that newly allocated blocks are treated as
* dirty - it is necessary since we do not protect free blocks.
*/
GC_ASSERT(modHBLKSZ(size_needed) == 0);
GC_remove_protection(hbp, divHBLKSZ(size_needed), IS_PTRFREE(hhdr));
#endif
/*
* We just successfully allocated a block. Restart count of consecutive
* failures.
*/
GC_alloc_fail_count = 0;
GC_large_free_bytes -= size_needed;
GC_ASSERT(IS_MAPPED(hhdr));
return hbp;
}
#ifdef VALGRIND_TRACKING
/*
* Note: this is intentionally defined in a file other than `malloc.c`
* and `reclaim.c` files.
*/
GC_ATTR_NOINLINE
GC_API void GC_CALLBACK
GC_free_profiler_hook(void *p)
{
# ifndef PARALLEL_MARK
GC_ASSERT(I_HOLD_LOCK());
# endif
/* Prevent treating this function by the compiler as a no-op one. */
GC_noop1_ptr(p);
}
#endif /* VALGRIND_TRACKING */
GC_INNER void
GC_freehblk(struct hblk *hbp)
{
struct hblk *next, *prev;
hdr *hhdr, *prevhdr, *nexthdr;
size_t size;
GET_HDR(hbp, hhdr);
size = HBLKSIZE * OBJ_SZ_TO_BLOCKS(hhdr->hb_sz);
if ((size & SIZET_SIGNB) != 0) {
/*
* Probably possible if we try to allocate more than half the address
* space at once. If we do not catch it here, strange things happen
* later.
*/
ABORT("Deallocating excessively large block. Too large an allocation?");
}
GC_remove_counts(hbp, size);
hhdr->hb_sz = size;
#ifdef USE_MUNMAP
hhdr->hb_last_reclaimed = (unsigned short)GC_gc_no;
#endif
/* Check for duplicate deallocation in the easy case. */
if (HBLK_IS_FREE(hhdr)) {
ABORT_ARG1("Duplicate large block deallocation", " of %p", (void *)hbp);
}
GC_ASSERT(IS_MAPPED(hhdr));
hhdr->hb_flags |= FREE_BLK;
next = (struct hblk *)((ptr_t)hbp + size);
GET_HDR(next, nexthdr);
prev = GC_free_block_ending_at(hbp);
/* Coalesce with successor, if possible. */
if (nexthdr != NULL && HBLK_IS_FREE(nexthdr)
&& IS_MAPPED(nexthdr)
#ifdef CHERI_PURECAP
/* FIXME: Coalesce with super-capability. */
/*
* Bounds of capability should span the entire coalesced memory;
* bounds being larger than the block size is OK; bounded by the
* imprecision of original capability obtained from system memory.
*/
&& CAPABILITY_COVERS_RANGE(hbp, ADDR(next), ADDR(next) + nexthdr->hb_sz)
#endif
&& !BLOCKS_MERGE_OVERFLOW(hhdr, nexthdr)) {
GC_remove_from_fl(nexthdr);
hhdr->hb_sz += nexthdr->hb_sz;
GC_remove_header(next);
}
/* Coalesce with predecessor, if possible. */
if (prev /* `!= NULL` */) { /*< CPPCHECK */
prevhdr = HDR(prev);
if (IS_MAPPED(prevhdr)
#ifdef CHERI_PURECAP
/* FIXME: Coalesce with super-capability. */
&& cheri_base_get(hbp) <= ADDR(prev)
#endif
&& !BLOCKS_MERGE_OVERFLOW(prevhdr, hhdr)) {
GC_remove_from_fl(prevhdr);
prevhdr->hb_sz += hhdr->hb_sz;
#ifdef USE_MUNMAP
prevhdr->hb_last_reclaimed = (unsigned short)GC_gc_no;
#endif
GC_remove_header(hbp);
hbp = prev;
hhdr = prevhdr;
}
}
/*
* FIXME: It is not clear if we really always want to do these merges
* with `USE_MUNMAP`, since it updates ages and hence prevents unmapping.
*/
GC_large_free_bytes += size;
GC_add_to_fl(hbp, hhdr);
}
/*
* Copyright (c) 1988-1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1996 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
* Copyright (c) 1999-2011 Hewlett-Packard Development Company, L.P.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* Separate free lists are maintained for different sized objects up
* to `MAXOBJBYTES`.
* The call `GC_allocobj(lg, kind)` ensures that the free list for the given
* kind of objects of the given size in granules is a nonempty one.
* It returns a pointer to the first entry on the free list.
* In a single-threaded world, `GC_allocobj` may be called to allocate
* an object of small size `lb` (and `NORMAL` kind) as follows
* (`GC_generic_malloc_inner` is a wrapper over `GC_allocobj` which also
* fills in `GC_size_map` if needed):
*
* ```
* lg = GC_size_map[lb];
* op = GC_objfreelist[lg];
* if (NULL == op) {
* op = GC_generic_malloc_inner(lb, NORMAL, 0);
* } else {
* GC_objfreelist[lg] = obj_link(op);
* GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
* }
* ```
*
* Note that this is very fast if the free list is not empty; it should
* only involve the execution of 4 or 5 simple instructions.
* All composite objects on freelists are cleared, except for
* their first "pointer-sized" word.
*/
/*
* The allocator uses `GC_allochblk` to allocate large chunks of objects.
* These chunks all start on addresses that are multiples of `HBLKSIZE`.
* Each allocated chunk has an associated header, which can be located
* quickly based on the address of the chunk. This makes it possible
* to check quickly whether an arbitrary address corresponds to an object
* administered by the allocator. (See `headers.c` file for details.)
*/
/* Number of bytes not intended to be collected. */
word GC_non_gc_bytes = 0;
word GC_gc_no = 0;
#ifndef NO_CLOCK
GC_API void GC_CALL
GC_start_performance_measurement(void)
{
GC_measure_performance = TRUE;
}
GC_API unsigned long GC_CALL
GC_get_full_gc_total_time(void)
{
return GC_full_gc_total_time;
}
GC_API unsigned long GC_CALL
GC_get_stopped_mark_total_time(void)
{
return GC_stopped_mark_total_time;
}
# ifndef MAX_TOTAL_TIME_DIVISOR
/*
* We shall not use big values here (so "outdated" delay time values would
* have less impact on "average" delay time value than newer ones).
*/
# define MAX_TOTAL_TIME_DIVISOR 1000
# endif
GC_API unsigned long GC_CALL
GC_get_avg_stopped_mark_time_ns(void)
{
unsigned long total_time;
unsigned divisor;
READER_LOCK();
total_time = (unsigned long)GC_world_stopped_total_time;
divisor = GC_world_stopped_total_divisor;
READER_UNLOCK();
if (0 == divisor) {
GC_ASSERT(0 == total_time);
/*
* No world-stopped collection has occurred since the start of
* performance measurements.
*/
return 0;
}
/* Halve values to prevent overflow during the multiplication. */
for (; total_time > ~0UL / (1000UL * 1000); total_time >>= 1) {
divisor >>= 1;
if (UNLIKELY(0 == divisor)) {
/* The actual result is larger than representable value. */
return ~0UL;
}
}
return total_time * (1000UL * 1000) / divisor;
}
#endif /* !NO_CLOCK */
#ifndef GC_DISABLE_INCREMENTAL
GC_INNER GC_bool GC_incremental = FALSE; /*< by default, stop the world */
#endif
GC_API int GC_CALL
GC_is_incremental_mode(void)
{
return (int)GC_incremental;
}
#ifdef THREADS
int GC_parallel = FALSE; /*< parallel collection is off by default */
#endif
#if defined(GC_FULL_FREQ) && !defined(CPPCHECK)
int GC_full_freq = GC_FULL_FREQ;
#else
/*
* Every `GC_full_freq + 1` collection is a full collection, whether we
* need it or not.
*/
int GC_full_freq = 19;
#endif
#ifdef THREAD_LOCAL_ALLOC
GC_INNER GC_bool GC_world_stopped = FALSE;
#endif
STATIC GC_bool GC_disable_automatic_collection = FALSE;
GC_API void GC_CALL
GC_set_disable_automatic_collection(int value)
{
LOCK();
GC_disable_automatic_collection = (GC_bool)value;
UNLOCK();
}
GC_API int GC_CALL
GC_get_disable_automatic_collection(void)
{
int value;
READER_LOCK();
value = (int)GC_disable_automatic_collection;
READER_UNLOCK();
return value;
}
/*
* The version macros are now defined in `gc_version.h` file, which is
* included by `gc.h` file, which, in turn, is included by `gc_priv.h` file.
*/
#ifndef GC_NO_VERSION_VAR
EXTERN_C_BEGIN
extern const GC_VERSION_VAL_T GC_version;
EXTERN_C_END
const GC_VERSION_VAL_T GC_version = ((GC_VERSION_VAL_T)GC_VERSION_MAJOR << 16)
| (GC_VERSION_MINOR << 8)
| GC_VERSION_MICRO;
#endif
GC_API GC_VERSION_VAL_T GC_CALL
GC_get_version(void)
{
return ((GC_VERSION_VAL_T)GC_VERSION_MAJOR << 16) | (GC_VERSION_MINOR << 8)
| GC_VERSION_MICRO;
}
GC_API int GC_CALL
GC_get_dont_add_byte_at_end(void)
{
#ifdef DONT_ADD_BYTE_AT_END
return 1;
#else
/* This is meaningful only if `GC_all_interior_pointers`. */
return 0;
#endif
}
/* Some more variables. */
#ifdef GC_DONT_EXPAND
int GC_dont_expand = TRUE;
#else
int GC_dont_expand = FALSE;
#endif
#if defined(GC_FREE_SPACE_DIVISOR) && !defined(CPPCHECK)
word GC_free_space_divisor = GC_FREE_SPACE_DIVISOR; /*< must be positive */
#else
word GC_free_space_divisor = 3;
#endif
GC_INNER int GC_CALLBACK
GC_never_stop_func(void)
{
return FALSE;
}
#if defined(GC_TIME_LIMIT) && !defined(CPPCHECK)
/*
* We try to keep pause times from exceeding this by much.
* In milliseconds.
*/
unsigned long GC_time_limit = GC_TIME_LIMIT;
#elif defined(PARALLEL_MARK)
/*
* The parallel marker cannot be interrupted for now, so the time limit
* is absent by default.
*/
unsigned long GC_time_limit = GC_TIME_UNLIMITED;
#else
unsigned long GC_time_limit = 15;
#endif
#ifndef NO_CLOCK
/*
* The nanoseconds add-on to `GC_time_limit` value.
* Not updated by `GC_set_time_limit()`.
* Ignored if the value of `GC_time_limit` is `GC_TIME_UNLIMITED`.
*/
STATIC unsigned long GC_time_lim_nsec = 0;
# define TV_NSEC_LIMIT (1000UL * 1000) /*< amount of nanoseconds in 1 ms */
GC_API void GC_CALL
GC_set_time_limit_tv(struct GC_timeval_s tv)
{
GC_ASSERT(tv.tv_ms <= GC_TIME_UNLIMITED);
GC_ASSERT(tv.tv_nsec < TV_NSEC_LIMIT);
GC_time_limit = tv.tv_ms;
GC_time_lim_nsec = tv.tv_nsec;
}
GC_API struct GC_timeval_s GC_CALL
GC_get_time_limit_tv(void)
{
struct GC_timeval_s tv;
tv.tv_ms = GC_time_limit;
tv.tv_nsec = GC_time_lim_nsec;
return tv;
}
/* Time at which we stopped world. Used only by `GC_timeout_stop_func()`. */
STATIC CLOCK_TYPE GC_start_time = CLOCK_TYPE_INITIALIZER;
#endif /* !NO_CLOCK */
/* Number of attempts at finishing collection within `GC_time_limit`. */
STATIC int GC_n_attempts = 0;
/* Note: accessed holding the allocator lock. */
STATIC GC_stop_func GC_default_stop_func = GC_never_stop_func;
GC_API void GC_CALL
GC_set_stop_func(GC_stop_func stop_func)
{
GC_ASSERT(NONNULL_ARG_NOT_NULL(stop_func));
LOCK();
GC_default_stop_func = stop_func;
UNLOCK();
}
GC_API GC_stop_func GC_CALL
GC_get_stop_func(void)
{
GC_stop_func stop_func;
READER_LOCK();
stop_func = GC_default_stop_func;
READER_UNLOCK();
return stop_func;
}
#if defined(GC_DISABLE_INCREMENTAL) || defined(NO_CLOCK)
# define GC_timeout_stop_func GC_default_stop_func
#else
STATIC int GC_CALLBACK
GC_timeout_stop_func(void)
{
CLOCK_TYPE current_time;
static unsigned count = 0;
unsigned long time_diff, nsec_diff;
GC_ASSERT(I_HOLD_LOCK());
if (GC_default_stop_func())
return TRUE;
if (GC_time_limit == GC_TIME_UNLIMITED || (count++ & 3) != 0)
return FALSE;
GET_TIME(current_time);
time_diff = MS_TIME_DIFF(current_time, GC_start_time);
nsec_diff = NS_FRAC_TIME_DIFF(current_time, GC_start_time);
# if defined(CPPCHECK)
GC_noop1_ptr(&nsec_diff);
# endif
if (time_diff >= GC_time_limit
&& (time_diff > GC_time_limit || nsec_diff >= GC_time_lim_nsec)) {
GC_COND_LOG_PRINTF("Abandoning stopped marking after %lu ms %lu ns"
" (attempt %d)\n",
time_diff, nsec_diff, GC_n_attempts);
return TRUE;
}
return FALSE;
}
#endif /* !GC_DISABLE_INCREMENTAL */
#ifdef THREADS
GC_INNER word GC_total_stacksize = 0;
#endif
/* The lowest value returned by `min_bytes_allocd()`. */
static size_t min_bytes_allocd_minimum = 1;
GC_API void GC_CALL
GC_set_min_bytes_allocd(size_t value)
{
GC_ASSERT(value > 0);
min_bytes_allocd_minimum = value;
}
GC_API size_t GC_CALL
GC_get_min_bytes_allocd(void)
{
return min_bytes_allocd_minimum;
}
/*
* Return the minimum number of bytes that must be allocated between
* collections to amortize the cost of the latter. Should be nonzero.
*/
static word
min_bytes_allocd(void)
{
word result;
word stack_size;
/*
* Total size of roots, it includes double stack size, since the stack
* is expensive to scan.
*/
word total_root_size;
/* Estimate of memory to be scanned during normal collection. */
word scan_size;
GC_ASSERT(I_HOLD_LOCK());
#ifdef THREADS
if (GC_need_to_lock) {
/* We are multi-threaded... */
stack_size = GC_total_stacksize;
/*
* For now, we just use the value computed during the latest garbage
* collection.
*/
# ifdef DEBUG_THREADS
GC_log_printf("Total stacks size: %lu\n", (unsigned long)stack_size);
# endif
} else
#endif
/* else */ {
#ifdef STACK_NOT_SCANNED
stack_size = 0;
#elif defined(STACK_GROWS_UP)
stack_size = (word)(GC_approx_sp() - GC_stackbottom);
#else
stack_size = (word)(GC_stackbottom - GC_approx_sp());
#endif
}
total_root_size = 2 * stack_size + GC_root_size;
scan_size = 2 * GC_composite_in_use + GC_atomic_in_use / 4 + total_root_size;
result = scan_size / GC_free_space_divisor;
if (GC_incremental) {
result /= 2;
}
return result > min_bytes_allocd_minimum ? result : min_bytes_allocd_minimum;
}
/*
* Return the number of bytes allocated, adjusted for explicit storage
* management, etc. This number is used in deciding when to trigger
* collections.
*/
STATIC word
GC_adj_bytes_allocd(void)
{
GC_signed_word result;
GC_signed_word expl_managed = (GC_signed_word)GC_non_gc_bytes
- (GC_signed_word)GC_non_gc_bytes_at_gc;
/*
* Do not count what was explicitly freed, or newly allocated for
* explicit management. Note that deallocating an explicitly managed
* object should not alter result, assuming the client is playing by
* the rules.
*/
result = (GC_signed_word)GC_bytes_allocd + (GC_signed_word)GC_bytes_dropped
- (GC_signed_word)GC_bytes_freed
+ (GC_signed_word)GC_finalizer_bytes_freed - expl_managed;
if (result > (GC_signed_word)GC_bytes_allocd) {
/* Probably a client bug or unfortunate scheduling. */
result = (GC_signed_word)GC_bytes_allocd;
}
/*
* We count objects enqueued for finalization as though they had been
* reallocated this round. Finalization is visible to user.
* And if we do not count this, we have stability problems for programs
* that finalize all objects.
*/
result += (GC_signed_word)GC_bytes_finalized;
if (result < (GC_signed_word)(GC_bytes_allocd >> 3)) {
/*
* Always count at least 1/8 of the allocations. We do not want to
* collect too infrequently, since that would inhibit coalescing of
* free storage blocks. This also makes us partially robust against
* client bugs.
*/
result = (GC_signed_word)(GC_bytes_allocd >> 3);
}
return (word)result;
}
/*
* Clear up a few frames worth of garbage left at the top of the stack.
* This is used to prevent us from accidentally treating garbage left
* on the stack by other parts of the collector as roots.
* This differs from the code in `misc.c` file, which actually tries
* to keep the stack clear of long-lived, client-generated garbage.
*/
STATIC void
GC_clear_a_few_frames(void)
{
#ifndef CLEAR_STACK_NPTRS
# define CLEAR_STACK_NPTRS 64 /*< pointers */
#endif
volatile ptr_t frames[CLEAR_STACK_NPTRS];
BZERO(CAST_AWAY_VOLATILE_PVOID(frames), sizeof(frames));
}
GC_API void GC_CALL
GC_start_incremental_collection(void)
{
#ifndef GC_DISABLE_INCREMENTAL
LOCK();
if (GC_incremental) {
GC_should_start_incremental_collection = TRUE;
if (!GC_dont_gc) {
GC_collect_a_little_inner(1);
}
}
UNLOCK();
#endif
}
GC_INNER GC_bool
GC_should_collect(void)
{
static word last_min_bytes_allocd, last_gc_no;
GC_ASSERT(I_HOLD_LOCK());
if (last_gc_no != GC_gc_no) {
last_min_bytes_allocd = min_bytes_allocd();
last_gc_no = GC_gc_no;
}
#ifndef GC_DISABLE_INCREMENTAL
if (GC_should_start_incremental_collection) {
GC_should_start_incremental_collection = FALSE;
return TRUE;
}
#endif
if (GC_disable_automatic_collection)
return FALSE;
if (GC_last_heap_growth_gc_no == GC_gc_no) {
/* Avoid expanding past limits used by black-listing. */
return TRUE;
}
return GC_adj_bytes_allocd() >= last_min_bytes_allocd;
}
/*
* Called at start of full collections. Not called if zero.
* Called with the allocator lock held. Not used by the collector itself.
*/
/* `STATIC` */ GC_start_callback_proc GC_start_call_back = 0;
GC_API void GC_CALL
GC_set_start_callback(GC_start_callback_proc fn)
{
LOCK();
GC_start_call_back = fn;
UNLOCK();
}
GC_API GC_start_callback_proc GC_CALL
GC_get_start_callback(void)
{
GC_start_callback_proc fn;
READER_LOCK();
fn = GC_start_call_back;
READER_UNLOCK();
return fn;
}
GC_INLINE void
GC_notify_full_gc(void)
{
if (GC_start_call_back != 0) {
(*GC_start_call_back)();
}
}
STATIC GC_bool GC_is_full_gc = FALSE;
STATIC GC_bool GC_stopped_mark(GC_stop_func stop_func);
STATIC void GC_finish_collection(void);
/*
* Initiate a garbage collection if appropriate. Choose judiciously
* between partial, full, and stop-world collections.
*/
STATIC void
GC_maybe_gc(void)
{
static int n_partial_gcs = 0;
GC_ASSERT(I_HOLD_LOCK());
ASSERT_CANCEL_DISABLED();
if (!GC_should_collect())
return;
if (!GC_incremental) {
GC_gcollect_inner();
return;
}
GC_ASSERT(!GC_collection_in_progress());
#ifdef PARALLEL_MARK
if (GC_parallel)
GC_wait_for_reclaim();
#endif
if (GC_need_full_gc || n_partial_gcs >= GC_full_freq) {
GC_COND_LOG_PRINTF(
"***>Full mark for collection #%lu after %lu allocd bytes\n",
(unsigned long)GC_gc_no + 1, (unsigned long)GC_bytes_allocd);
GC_notify_full_gc();
ENTER_GC();
GC_promote_black_lists();
(void)GC_reclaim_all((GC_stop_func)0, TRUE);
GC_clear_marks();
EXIT_GC();
n_partial_gcs = 0;
GC_is_full_gc = TRUE;
} else {
n_partial_gcs++;
}
/*
* Try to mark with the world stopped. If we run out of time, then this
* turns into an incremental marking.
*/
#ifndef NO_CLOCK
if (GC_time_limit != GC_TIME_UNLIMITED)
GET_TIME(GC_start_time);
#endif
if (GC_stopped_mark(GC_timeout_stop_func)) {
SAVE_CALLERS_TO_LAST_STACK();
GC_finish_collection();
} else if (!GC_is_full_gc) {
/* Count this as the first attempt. */
GC_n_attempts++;
}
}
STATIC GC_on_collection_event_proc GC_on_collection_event = 0;
GC_API void GC_CALL
GC_set_on_collection_event(GC_on_collection_event_proc fn)
{
/* `fn` may be 0 (means no event notifier). */
LOCK();
GC_on_collection_event = fn;
UNLOCK();
}
GC_API GC_on_collection_event_proc GC_CALL
GC_get_on_collection_event(void)
{
GC_on_collection_event_proc fn;
READER_LOCK();
fn = GC_on_collection_event;
READER_UNLOCK();
return fn;
}
GC_INNER GC_bool
GC_try_to_collect_inner(GC_stop_func stop_func)
{
#ifndef NO_CLOCK
CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;
GC_bool start_time_valid;
#endif
ASSERT_CANCEL_DISABLED();
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_is_initialized);
if (GC_dont_gc || (*stop_func)())
return FALSE;
if (GC_on_collection_event)
GC_on_collection_event(GC_EVENT_START);
if (GC_incremental && GC_collection_in_progress()) {
GC_COND_LOG_PRINTF(
"GC_try_to_collect_inner: finishing collection in progress\n");
/* Just finish collection already in progress. */
do {
if ((*stop_func)()) {
/* TODO: Notify `GC_EVENT_ABANDON`. */
return FALSE;
}
GC_collect_a_little_inner(1);
} while (GC_collection_in_progress());
}
GC_notify_full_gc();
#ifndef NO_CLOCK
start_time_valid = FALSE;
if ((GC_print_stats | (int)GC_measure_performance) != 0) {
if (GC_print_stats)
GC_log_printf("Initiating full world-stop collection!\n");
start_time_valid = TRUE;
GET_TIME(start_time);
}
#endif
GC_promote_black_lists();
/*
* Make sure all blocks have been reclaimed, so sweep routines do not
* see cleared mark bits. If we are guaranteed to finish, then this
* is unnecessary. In the find-leak case, we have to finish to
* guarantee that previously unmarked objects are not reported as leaks.
*/
#ifdef PARALLEL_MARK
if (GC_parallel)
GC_wait_for_reclaim();
#endif
ENTER_GC();
if ((GC_find_leak_inner || stop_func != GC_never_stop_func)
&& !GC_reclaim_all(stop_func, FALSE)) {
/* Aborted. So far everything is still consistent. */
EXIT_GC();
/* TODO: Notify `GC_EVENT_ABANDON`. */
return FALSE;
}
GC_invalidate_mark_state(); /*< flush mark stack */
GC_clear_marks();
SAVE_CALLERS_TO_LAST_STACK();
GC_is_full_gc = TRUE;
EXIT_GC();
if (!GC_stopped_mark(stop_func)) {
if (!GC_incremental) {
/*
* We are partially done and have no way to complete or use
* current work. Reestablish invariants as cheaply as possible.
*/
GC_invalidate_mark_state();
GC_unpromote_black_lists();
} else {
/*
* We claim the world is already (or still) consistent.
* We will finish incrementally.
*/
}
/* TODO: Notify `GC_EVENT_ABANDON`. */
return FALSE;
}
GC_finish_collection();
#ifndef NO_CLOCK
if (start_time_valid) {
CLOCK_TYPE current_time;
unsigned long time_diff, ns_frac_diff;
GET_TIME(current_time);
time_diff = MS_TIME_DIFF(current_time, start_time);
ns_frac_diff = NS_FRAC_TIME_DIFF(current_time, start_time);
if (GC_measure_performance) {
GC_full_gc_total_time += time_diff; /*< may wrap */
GC_full_gc_total_ns_frac += (unsigned32)ns_frac_diff;
if (GC_full_gc_total_ns_frac >= (unsigned32)1000000UL) {
/* Overflow of the nanoseconds part. */
GC_full_gc_total_ns_frac -= (unsigned32)1000000UL;
GC_full_gc_total_time++;
}
}
if (GC_print_stats)
GC_log_printf("Complete collection took %lu ms %lu ns\n", time_diff,
ns_frac_diff);
}
#endif
if (GC_on_collection_event)
GC_on_collection_event(GC_EVENT_END);
return TRUE;
}
/* The default value of `GC_rate`. */
#ifndef GC_RATE
# define GC_RATE 10
#endif
/*
* When `GC_collect_a_little_inner()` performs `n_blocks` units of garbage
* collection work, a unit is intended to touch roughly `GC_rate` pages.
* (But, every once in a while, we do more than that.) This needs to be
* a fairly large number with our current incremental collection strategy,
* since otherwise we allocate too much during garbage collection, and
* the cleanup gets expensive.
*/
STATIC unsigned GC_rate = GC_RATE;
GC_API void GC_CALL
GC_set_rate(int value)
{
GC_ASSERT(value > 0);
GC_rate = (unsigned)value;
}
GC_API int GC_CALL
GC_get_rate(void)
{
return (int)GC_rate;
}
/* The default maximum number of prior attempts at world stop marking. */
#ifndef MAX_PRIOR_ATTEMPTS
# define MAX_PRIOR_ATTEMPTS 3
#endif
/*
* The maximum number of prior attempts at world stop marking.
* A value of 1 means that we finish the second time, no matter how long
* it takes. Does not count the initial root scan for a full collection.
*/
static int max_prior_attempts = MAX_PRIOR_ATTEMPTS;
GC_API void GC_CALL
GC_set_max_prior_attempts(int value)
{
GC_ASSERT(value >= 0);
max_prior_attempts = value;
}
GC_API int GC_CALL
GC_get_max_prior_attempts(void)
{
return max_prior_attempts;
}
GC_INNER void
GC_collect_a_little_inner(size_t n_blocks)
{
IF_CANCEL(int cancel_state;)
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_is_initialized);
DISABLE_CANCEL(cancel_state);
if (GC_incremental && GC_collection_in_progress()) {
size_t i;
size_t max_deficit = GC_rate * n_blocks;
ENTER_GC();
#ifdef PARALLEL_MARK
if (GC_time_limit != GC_TIME_UNLIMITED)
GC_parallel_mark_disabled = TRUE;
#endif
for (i = GC_mark_deficit; i < max_deficit; i++) {
if (GC_mark_some(NULL))
break;
}
#ifdef PARALLEL_MARK
GC_parallel_mark_disabled = FALSE;
#endif
EXIT_GC();
if (i < max_deficit && !GC_dont_gc) {
GC_ASSERT(!GC_collection_in_progress());
/* Need to follow up with a full collection. */
SAVE_CALLERS_TO_LAST_STACK();
#ifdef PARALLEL_MARK
if (GC_parallel)
GC_wait_for_reclaim();
#endif
#ifndef NO_CLOCK
if (GC_time_limit != GC_TIME_UNLIMITED
&& GC_n_attempts < max_prior_attempts)
GET_TIME(GC_start_time);
#endif
if (GC_stopped_mark(GC_n_attempts < max_prior_attempts
? GC_timeout_stop_func
: GC_never_stop_func)) {
GC_finish_collection();
} else {
GC_n_attempts++;
}
}
if (GC_mark_deficit > 0) {
GC_mark_deficit
= GC_mark_deficit > max_deficit ? GC_mark_deficit - max_deficit : 0;
}
} else if (!GC_dont_gc) {
GC_maybe_gc();
}
RESTORE_CANCEL(cancel_state);
}
#if !defined(NO_FIND_LEAK) || !defined(SHORT_DBG_HDRS)
GC_INNER void (*GC_check_heap)(void) = 0;
GC_INNER void (*GC_print_all_smashed)(void) = 0;
#endif
GC_API int GC_CALL
GC_collect_a_little(void)
{
int result;
if (UNLIKELY(!GC_is_initialized))
GC_init();
LOCK();
/*
* Note: if the collection is in progress, this may do marking (not
* stopping the world) even in case of disabled garbage collection.
*/
GC_collect_a_little_inner(1);
result = (int)GC_collection_in_progress();
UNLOCK();
if (GC_debugging_started && !result)
GC_print_all_smashed();
return result;
}
#ifdef THREADS
GC_API void GC_CALL
GC_stop_world_external(void)
{
GC_ASSERT(GC_is_initialized);
LOCK();
# ifdef THREAD_LOCAL_ALLOC
GC_ASSERT(!GC_world_stopped);
# endif
ENTER_GC();
STOP_WORLD();
# ifdef THREAD_LOCAL_ALLOC
GC_world_stopped = TRUE;
# endif
}
GC_API void GC_CALL
GC_start_world_external(void)
{
# ifdef THREAD_LOCAL_ALLOC
GC_ASSERT(GC_world_stopped);
GC_world_stopped = FALSE;
# else
GC_ASSERT(GC_is_initialized);
# endif
START_WORLD();
EXIT_GC();
UNLOCK();
}
#endif /* THREADS */
#ifdef USE_MUNMAP
# ifndef MUNMAP_THRESHOLD
# define MUNMAP_THRESHOLD 7
# endif
GC_INNER unsigned GC_unmap_threshold = MUNMAP_THRESHOLD;
# define IF_USE_MUNMAP(x) x
# define COMMA_IF_USE_MUNMAP(x) /* comma */ , x
#else
# define IF_USE_MUNMAP(x)
# define COMMA_IF_USE_MUNMAP(x)
#endif /* !USE_MUNMAP */
/*
* We stop the world and mark from all roots. If `stop_func()` ever
* returns `TRUE`, we may fail and return `FALSE`. Increment `GC_gc_no`
* if we succeed.
*/
STATIC GC_bool
GC_stopped_mark(GC_stop_func stop_func)
{
ptr_t cold_gc_frame = GC_approx_sp();
unsigned abandoned_at;
#ifndef NO_CLOCK
CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;
GC_bool start_time_valid = FALSE;
#endif
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_is_initialized);
ENTER_GC();
#if !defined(REDIRECT_MALLOC) && defined(USE_WINALLOC)
GC_add_current_malloc_heap();
#endif
#if defined(REGISTER_LIBRARIES_EARLY)
GC_cond_register_dynamic_libraries();
#endif
#if !defined(GC_NO_FINALIZATION) && !defined(GC_TOGGLE_REFS_NOT_NEEDED)
GC_process_togglerefs();
#endif
/* Output blank line for convenience here. */
GC_COND_LOG_PRINTF(
"\n--> Marking for collection #%lu after %lu allocated bytes\n",
(unsigned long)GC_gc_no + 1, (unsigned long)GC_bytes_allocd);
#ifndef NO_CLOCK
if (GC_PRINT_STATS_FLAG || GC_measure_performance) {
GET_TIME(start_time);
start_time_valid = TRUE;
}
#endif
#ifdef THREADS
if (GC_on_collection_event)
GC_on_collection_event(GC_EVENT_PRE_STOP_WORLD);
#endif
STOP_WORLD();
#ifdef THREADS
if (GC_on_collection_event)
GC_on_collection_event(GC_EVENT_POST_STOP_WORLD);
# ifdef THREAD_LOCAL_ALLOC
GC_world_stopped = TRUE;
# elif defined(CPPCHECK)
/* Workaround a warning about adjacent same `if` condition. */
(void)0;
# endif
#endif
#ifdef MAKE_BACK_GRAPH
if (GC_print_back_height) {
GC_build_back_graph();
}
#endif
/* Notify about marking from all roots. */
if (GC_on_collection_event)
GC_on_collection_event(GC_EVENT_MARK_START);
/* Minimize junk left in my registers and on the stack. */
GC_clear_a_few_frames();
GC_noop6(0, 0, 0, 0, 0, 0);
GC_initiate_gc();
#ifdef PARALLEL_MARK
if (stop_func != GC_never_stop_func)
GC_parallel_mark_disabled = TRUE;
#endif
for (abandoned_at = 1; !(*stop_func)(); abandoned_at++) {
if (GC_mark_some(cold_gc_frame)) {
#ifdef PARALLEL_MARK
if (GC_parallel && GC_parallel_mark_disabled) {
GC_COND_LOG_PRINTF("Stopped marking done after %u iterations"
" with disabled parallel marker\n",
abandoned_at - 1);
}
#endif
abandoned_at = 0;
break;
}
}
#ifdef PARALLEL_MARK
GC_parallel_mark_disabled = FALSE;
#endif
if (abandoned_at > 0) {
/* Give the mutator a chance. */
GC_mark_deficit = abandoned_at - 1;
/* TODO: Notify `GC_EVENT_MARK_ABANDON`. */
} else {
GC_gc_no++;
/* Check all debugged objects for consistency. */
if (GC_debugging_started)
GC_check_heap();
if (GC_on_collection_event)
GC_on_collection_event(GC_EVENT_MARK_END);
}
#ifdef THREADS
if (GC_on_collection_event)
GC_on_collection_event(GC_EVENT_PRE_START_WORLD);
#endif
#ifdef THREAD_LOCAL_ALLOC
GC_world_stopped = FALSE;
#endif
START_WORLD();
#ifdef THREADS
if (GC_on_collection_event)
GC_on_collection_event(GC_EVENT_POST_START_WORLD);
#endif
#ifndef NO_CLOCK
if (start_time_valid) {
CLOCK_TYPE current_time;
unsigned long time_diff, ns_frac_diff;
/* TODO: Avoid code duplication from `GC_try_to_collect_inner`. */
GET_TIME(current_time);
time_diff = MS_TIME_DIFF(current_time, start_time);
ns_frac_diff = NS_FRAC_TIME_DIFF(current_time, start_time);
if (GC_measure_performance) {
GC_stopped_mark_total_time += time_diff; /*< may wrap */
GC_stopped_mark_total_ns_frac += (unsigned32)ns_frac_diff;
if (GC_stopped_mark_total_ns_frac >= (unsigned32)1000000UL) {
GC_stopped_mark_total_ns_frac -= (unsigned32)1000000UL;
GC_stopped_mark_total_time++;
}
}
if (GC_PRINT_STATS_FLAG || GC_measure_performance) {
unsigned total_time = GC_world_stopped_total_time;
unsigned divisor = GC_world_stopped_total_divisor;
/* Compute new world-stop delay total time. */
if (total_time > (((unsigned)-1) >> 1)
|| divisor >= MAX_TOTAL_TIME_DIVISOR) {
/* Halve values if overflow occurs. */
total_time >>= 1;
divisor >>= 1;
}
total_time += time_diff < (((unsigned)-1) >> 1) ? (unsigned)time_diff
: ((unsigned)-1) >> 1;
/* Update old `GC_world_stopped_total_time` and its divisor. */
GC_world_stopped_total_time = total_time;
GC_world_stopped_total_divisor = ++divisor;
if (GC_PRINT_STATS_FLAG && 0 == abandoned_at) {
GC_ASSERT(divisor != 0);
GC_log_printf("World-stopped marking took %lu ms %lu ns"
" (%u ms in average)\n",
time_diff, ns_frac_diff, total_time / divisor);
}
}
}
#endif
EXIT_GC();
if (0 == abandoned_at)
return TRUE;
GC_COND_LOG_PRINTF("Abandoned stopped marking after %u iterations\n",
abandoned_at - 1);
return FALSE;
}
GC_INNER void
GC_set_fl_marks(ptr_t q)
{
#ifdef GC_ASSERTIONS
ptr_t q2;
#endif
struct hblk *h = HBLKPTR(q);
const struct hblk *last_h = h;
hdr *hhdr;
#ifdef MARK_BIT_PER_OBJ
size_t sz;
#endif
GC_ASSERT(q != NULL);
hhdr = HDR(h);
#ifdef MARK_BIT_PER_OBJ
sz = hhdr->hb_sz;
#endif
#ifdef GC_ASSERTIONS
q2 = (ptr_t)obj_link(q);
#endif
for (;;) {
size_t bit_no = MARK_BIT_NO((size_t)((ptr_t)q - (ptr_t)h), sz);
if (!mark_bit_from_hdr(hhdr, bit_no)) {
set_mark_bit_from_hdr(hhdr, bit_no);
INCR_MARKS(hhdr);
}
q = (ptr_t)obj_link(q);
if (NULL == q)
break;
#ifdef GC_ASSERTIONS
/*
* Detect a cycle in the free list. The algorithm is to have
* a "twice faster" iterator over the list which meets the first
* one in case of a cycle existing in the list.
*/
if (q2 != NULL) {
q2 = (ptr_t)obj_link(q2);
GC_ASSERT(q2 != q);
if (q2 != NULL) {
q2 = (ptr_t)obj_link(q2);
GC_ASSERT(q2 != q);
}
}
#endif
h = HBLKPTR(q);
if (UNLIKELY(h != last_h)) {
last_h = h;
/* Update `hhdr` and `sz`. */
hhdr = HDR(h);
#ifdef MARK_BIT_PER_OBJ
sz = hhdr->hb_sz;
#endif
}
}
}
#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
/*
* Check that all mark bits for the free list, whose first entry is
* `*pfreelist`, are set. The check is skipped if `*pfreelist` points to
* a special value.
*/
void
GC_check_fl_marks(void **pfreelist)
{
/*
* TODO: There is a data race with `GC_FAST_MALLOC_GRANS` (which does
* not do atomic updates to the free-list). The race seems to be
* harmless, and for now we just skip this check in case of TSan.
*/
# if defined(AO_HAVE_load_acquire_read) && !defined(THREAD_SANITIZER)
ptr_t list = GC_cptr_load_acquire_read((volatile ptr_t *)pfreelist);
/* Atomic operations are used because the world is running. */
ptr_t p, prev, next;
if (ADDR(list) <= HBLKSIZE)
return;
prev = (ptr_t)pfreelist;
for (p = list; p != NULL; p = next) {
if (!GC_is_marked(p)) {
ABORT_ARG2("Unmarked local free-list entry", ": object %p on list %p",
(void *)p, (void *)list);
}
/*
* While traversing the free-list, it re-reads the pointer to the
* current node before accepting its next pointer and bails out
* if the latter has changed. That way, it will not try to follow
* the pointer which might be been modified after the object was
* returned to the client. It might perform the mark-check on the
* just allocated object but that should be harmless.
*/
next = GC_cptr_load_acquire_read((volatile ptr_t *)p);
if (GC_cptr_load((volatile ptr_t *)prev) != p)
break;
prev = p;
}
# else
/* FIXME: Not implemented (just skipped). */
(void)pfreelist;
# endif
}
#endif /* GC_ASSERTIONS && THREAD_LOCAL_ALLOC */
/*
* Clear all mark bits for the free list (specified by the first entry).
* Decrement `GC_bytes_found` by number of bytes on free list.
*/
STATIC void
GC_clear_fl_marks(ptr_t q)
{
struct hblk *h = HBLKPTR(q);
const struct hblk *last_h = h;
hdr *hhdr = HDR(h);
size_t sz = hhdr->hb_sz; /*< normally set only once */
for (;;) {
size_t bit_no = MARK_BIT_NO((size_t)((ptr_t)q - (ptr_t)h), sz);
if (mark_bit_from_hdr(hhdr, bit_no)) {
size_t n_marks = hhdr->hb_n_marks;
#ifdef LINT2
if (0 == n_marks)
ABORT("hhdr->hb_n_marks cannot be zero");
#else
GC_ASSERT(n_marks != 0);
#endif
clear_mark_bit_from_hdr(hhdr, bit_no);
n_marks--;
#ifdef PARALLEL_MARK
/* Approximate count, do not decrement to zero! */
if (n_marks != 0 || !GC_parallel) {
hhdr->hb_n_marks = n_marks;
}
#else
hhdr->hb_n_marks = n_marks;
#endif
}
GC_bytes_found -= (GC_signed_word)sz;
q = (ptr_t)obj_link(q);
if (NULL == q)
break;
h = HBLKPTR(q);
if (UNLIKELY(h != last_h)) {
last_h = h;
/* Update `hhdr` and `sz`. */
hhdr = HDR(h);
sz = hhdr->hb_sz;
}
}
}
/* Mark all objects on the free lists for every object kind. */
static void
set_all_fl_marks(void)
{
unsigned kind;
for (kind = 0; kind < GC_n_kinds; kind++) {
word size; /*< current object size */
for (size = 1; size <= MAXOBJGRANULES; size++) {
ptr_t q = (ptr_t)GC_obj_kinds[kind].ok_freelist[size];
if (q != NULL)
GC_set_fl_marks(q);
}
}
}
/*
* Clear free-list mark bits. Also subtract memory remaining from
* `GC_bytes_found` count.
*/
static void
clear_all_fl_marks(void)
{
unsigned kind;
for (kind = 0; kind < GC_n_kinds; kind++) {
word size; /*< current object size */
for (size = 1; size <= MAXOBJGRANULES; size++) {
ptr_t q = (ptr_t)GC_obj_kinds[kind].ok_freelist[size];
if (q != NULL)
GC_clear_fl_marks(q);
}
}
}
#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
void GC_check_tls(void);
#endif
GC_on_heap_resize_proc GC_on_heap_resize = 0;
/* Used for logging only. */
GC_INLINE int
GC_compute_heap_usage_percent(void)
{
word used = GC_composite_in_use + GC_atomic_in_use + GC_bytes_allocd;
word heap_sz = GC_heapsize - GC_unmapped_bytes;
#if defined(CPPCHECK)
word limit = (GC_WORD_MAX >> 1) / 50; /*< to avoid a false positive */
#else
const word limit = GC_WORD_MAX / 100;
#endif
return used >= heap_sz ? 0
: used < limit ? (int)((used * 100) / heap_sz)
: (int)(used / (heap_sz / 100));
}
#define GC_DBGLOG_PRINT_HEAP_IN_USE() \
GC_DBGLOG_PRINTF("In-use heap: %d%% (%lu KiB pointers + %lu KiB other)\n", \
GC_compute_heap_usage_percent(), \
TO_KiB_UL(GC_composite_in_use), \
TO_KiB_UL(GC_atomic_in_use + GC_bytes_allocd))
/*
* Finish up a collection. Assumes mark bits are consistent, but the
* world is otherwise running.
*/
STATIC void
GC_finish_collection(void)
{
#ifndef NO_CLOCK
CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;
CLOCK_TYPE finalize_time = CLOCK_TYPE_INITIALIZER;
#endif
GC_ASSERT(I_HOLD_LOCK());
#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC) \
&& !defined(DBG_HDRS_ALL)
/* Check that we marked some of our own data. */
GC_check_tls();
/* TODO: Add more checks. */
#endif
#ifndef NO_CLOCK
if (GC_print_stats)
GET_TIME(start_time);
#endif
if (GC_on_collection_event)
GC_on_collection_event(GC_EVENT_RECLAIM_START);
#ifndef GC_GET_HEAP_USAGE_NOT_NEEDED
if (GC_bytes_found > 0)
GC_reclaimed_bytes_before_gc += (word)GC_bytes_found;
#endif
GC_bytes_found = 0;
#if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG)
if (GETENV("GC_PRINT_ADDRESS_MAP") != NULL) {
GC_print_address_map();
}
#endif
COND_DUMP;
if (GC_find_leak_inner) {
set_all_fl_marks();
/* This just checks; it does not really reclaim anything. */
GC_start_reclaim(TRUE);
}
#ifndef GC_NO_FINALIZATION
GC_finalize();
#endif
#ifndef NO_CLOCK
if (GC_print_stats)
GET_TIME(finalize_time);
#endif
#ifdef MAKE_BACK_GRAPH
if (GC_print_back_height) {
GC_traverse_back_graph();
}
#endif
/*
* Clear free-list mark bits, in case they got accidentally marked
* (or `GC_find_leak` is set and they were intentionally marked).
* Note that composite objects on free list are cleared, thus
* accidentally marking a free list is not a problem; but some objects
* on the list itself might be marked, and the given function call
* fixes it.
*/
clear_all_fl_marks();
GC_VERBOSE_LOG_PRINTF("Bytes recovered before sweep - f.l. count = %ld\n",
(long)GC_bytes_found);
/* Reconstruct free lists to contain everything not marked. */
GC_start_reclaim(FALSE);
#ifdef USE_MUNMAP
if (GC_unmap_threshold > 0 /*< memory unmapping enabled? */
&& LIKELY(GC_gc_no != 1)) /*< do not unmap during `GC_init` */
GC_unmap_old(GC_unmap_threshold);
GC_ASSERT(GC_heapsize >= GC_unmapped_bytes);
#endif
GC_ASSERT(GC_our_mem_bytes >= GC_heapsize);
GC_DBGLOG_PRINTF(
"GC #%lu freed %ld bytes, heap %lu KiB (" IF_USE_MUNMAP(
"+ %lu KiB unmapped ") "+ %lu KiB internal)\n",
(unsigned long)GC_gc_no, (long)GC_bytes_found,
TO_KiB_UL(GC_heapsize - GC_unmapped_bytes) /*, */
COMMA_IF_USE_MUNMAP(TO_KiB_UL(GC_unmapped_bytes)),
TO_KiB_UL(GC_our_mem_bytes - GC_heapsize + sizeof(GC_arrays)));
GC_DBGLOG_PRINT_HEAP_IN_USE();
if (GC_is_full_gc) {
GC_used_heap_size_after_full = GC_heapsize - GC_large_free_bytes;
GC_need_full_gc = FALSE;
} else {
GC_need_full_gc = GC_heapsize - GC_used_heap_size_after_full
> min_bytes_allocd() + GC_large_free_bytes;
}
/* Reset or increment counters for next cycle. */
GC_n_attempts = 0;
GC_is_full_gc = FALSE;
GC_bytes_allocd_before_gc += GC_bytes_allocd;
GC_non_gc_bytes_at_gc = GC_non_gc_bytes;
GC_bytes_allocd = 0;
GC_bytes_dropped = 0;
GC_bytes_freed = 0;
GC_finalizer_bytes_freed = 0;
if (GC_on_collection_event)
GC_on_collection_event(GC_EVENT_RECLAIM_END);
#ifndef NO_CLOCK
if (GC_print_stats) {
CLOCK_TYPE done_time;
GET_TIME(done_time);
# if !defined(SMALL_CONFIG) && !defined(GC_NO_FINALIZATION)
/* A convenient place to output finalization statistics. */
GC_print_finalization_stats();
# endif
GC_log_printf("Finalize and initiate sweep took %lu ms %lu ns"
" + %lu ms %lu ns\n",
MS_TIME_DIFF(finalize_time, start_time),
NS_FRAC_TIME_DIFF(finalize_time, start_time),
MS_TIME_DIFF(done_time, finalize_time),
NS_FRAC_TIME_DIFF(done_time, finalize_time));
}
#elif !defined(SMALL_CONFIG) && !defined(GC_NO_FINALIZATION)
if (GC_print_stats)
GC_print_finalization_stats();
#endif
}
/* Note: if `stop_func` is 0, then `GC_default_stop_func` is used instead. */
STATIC GC_bool
GC_try_to_collect_general(GC_stop_func stop_func, GC_bool force_unmap)
{
GC_bool result;
#ifdef USE_MUNMAP
unsigned old_unmap_threshold;
#endif
IF_CANCEL(int cancel_state;)
if (UNLIKELY(!GC_is_initialized))
GC_init();
if (GC_debugging_started)
GC_print_all_smashed();
GC_notify_or_invoke_finalizers();
LOCK();
if (force_unmap) {
/*
* Record current heap size to make heap growth more conservative
* afterwards (as if the heap is growing from zero size again).
*/
GC_heapsize_at_forced_unmap = GC_heapsize;
}
DISABLE_CANCEL(cancel_state);
#ifdef USE_MUNMAP
old_unmap_threshold = GC_unmap_threshold;
if (force_unmap || (GC_force_unmap_on_gcollect && old_unmap_threshold > 0))
GC_unmap_threshold = 1; /*< unmap as much as possible */
#endif
/* Minimize junk left in my registers. */
GC_noop6(0, 0, 0, 0, 0, 0);
result = GC_try_to_collect_inner(stop_func != 0 ? stop_func
: GC_default_stop_func);
#ifdef USE_MUNMAP
/* Restore it. */
GC_unmap_threshold = old_unmap_threshold;
#endif
RESTORE_CANCEL(cancel_state);
UNLOCK();
if (result) {
if (GC_debugging_started)
GC_print_all_smashed();
GC_notify_or_invoke_finalizers();
}
return result;
}
/* Externally callable routines to invoke full, stop-the-world collection. */
GC_API int GC_CALL
GC_try_to_collect(GC_stop_func stop_func)
{
GC_ASSERT(NONNULL_ARG_NOT_NULL(stop_func));
return (int)GC_try_to_collect_general(stop_func, FALSE);
}
GC_API void GC_CALL
GC_gcollect(void)
{
/*
* Zero is passed as stop_func to get `GC_default_stop_func` value
* while holding the allocator lock (to prevent data race).
*/
(void)GC_try_to_collect_general(0, FALSE);
if (get_have_errors())
GC_print_all_errors();
}
GC_API void GC_CALL
GC_gcollect_and_unmap(void)
{
/* Collect and force memory unmapping to OS. */
(void)GC_try_to_collect_general(GC_never_stop_func, TRUE);
}
GC_INNER ptr_t
GC_os_get_mem(size_t bytes)
{
ptr_t space;
GC_ASSERT(I_HOLD_LOCK());
space = (ptr_t)GET_MEM(bytes); /*< `HBLKSIZE`-aligned */
if (UNLIKELY(NULL == space))
return NULL;
#ifdef USE_PROC_FOR_LIBRARIES
/* Add `HBLKSIZE`-aligned `GET_MEM`-generated block to `GC_our_memory`. */
if (GC_n_memory >= MAX_HEAP_SECTS)
ABORT("Too many GC-allocated memory sections: Increase MAX_HEAP_SECTS");
GC_our_memory[GC_n_memory].hs_start = space;
GC_our_memory[GC_n_memory].hs_bytes = bytes;
GC_n_memory++;
#endif
GC_our_mem_bytes += bytes;
GC_VERBOSE_LOG_PRINTF("Got %lu bytes from OS\n", (unsigned long)bytes);
return space;
}
/*
* Use the chunk of memory starting at `h` of size `sz` as part of the heap.
* Assumes `h` is `HBLKSIZE`-aligned, `sz` is a multiple of `HBLKSIZE`.
*/
STATIC void
GC_add_to_heap(struct hblk *h, size_t sz)
{
hdr *hhdr;
ptr_t endp;
size_t old_capacity = 0;
void *old_heap_sects = NULL;
#ifdef GC_ASSERTIONS
size_t i;
#endif
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(ADDR(h) % HBLKSIZE == 0);
GC_ASSERT(sz % HBLKSIZE == 0);
GC_ASSERT(sz > 0);
GC_ASSERT(GC_all_nils != NULL);
if (UNLIKELY(GC_n_heap_sects == GC_capacity_heap_sects)) {
/* Allocate new `GC_heap_sects` with sufficient capacity. */
#ifndef INITIAL_HEAP_SECTS
# define INITIAL_HEAP_SECTS 32
#endif
size_t new_capacity
= GC_n_heap_sects > 0 ? GC_n_heap_sects * 2 : INITIAL_HEAP_SECTS;
void *new_heap_sects
= GC_scratch_alloc(new_capacity * sizeof(struct HeapSect));
if (NULL == new_heap_sects) {
/* Retry with smaller yet sufficient capacity. */
new_capacity = GC_n_heap_sects + INITIAL_HEAP_SECTS;
new_heap_sects
= GC_scratch_alloc(new_capacity * sizeof(struct HeapSect));
if (NULL == new_heap_sects)
ABORT("Insufficient memory for heap sections");
}
old_capacity = GC_capacity_heap_sects;
old_heap_sects = GC_heap_sects;
/* Transfer `GC_heap_sects` contents to the newly allocated array. */
if (GC_n_heap_sects > 0)
BCOPY(old_heap_sects, new_heap_sects,
GC_n_heap_sects * sizeof(struct HeapSect));
GC_capacity_heap_sects = new_capacity;
GC_heap_sects = (struct HeapSect *)new_heap_sects;
GC_COND_LOG_PRINTF("Grew heap sections array to %lu elements\n",
(unsigned long)new_capacity);
}
while (UNLIKELY(ADDR(h) <= HBLKSIZE)) {
/* Cannot handle memory near address zero. */
++h;
sz -= HBLKSIZE;
if (0 == sz)
return;
}
while (UNLIKELY(ADDR(h) >= GC_WORD_MAX - sz)) {
/* Prevent overflow when calculating `endp`. */
sz -= HBLKSIZE;
if (0 == sz)
return;
}
endp = (ptr_t)h + sz;
hhdr = GC_install_header(h);
if (UNLIKELY(NULL == hhdr)) {
/*
* This is extremely unlikely. Cannot add it. This will almost
* certainly result in a `NULL` returned from the allocator, which
* is entirely appropriate.
*/
return;
}
#ifdef GC_ASSERTIONS
/* Ensure no intersection between sections. */
for (i = 0; i < GC_n_heap_sects; i++) {
ptr_t hs_start = GC_heap_sects[i].hs_start;
ptr_t hs_end = hs_start + GC_heap_sects[i].hs_bytes;
GC_ASSERT(!(ADDR_INSIDE((ptr_t)h, hs_start, hs_end)
|| (ADDR_LT(hs_start, endp) && ADDR_GE(hs_end, endp))
|| (ADDR_LT((ptr_t)h, hs_start) && ADDR_LT(hs_end, endp))));
}
#endif
GC_heap_sects[GC_n_heap_sects].hs_start = (ptr_t)h;
GC_heap_sects[GC_n_heap_sects].hs_bytes = sz;
GC_n_heap_sects++;
hhdr->hb_block = h;
hhdr->hb_sz = sz;
hhdr->hb_flags = 0;
GC_freehblk(h);
GC_heapsize += sz;
if (ADDR_GE((ptr_t)GC_least_plausible_heap_addr, (ptr_t)h)
|| UNLIKELY(NULL == GC_least_plausible_heap_addr)) {
/*
* Making it a little smaller than necessary prevents us from
* getting a false hit from the variable itself. There is some
* unintentional reflection here.
*/
GC_least_plausible_heap_addr = (ptr_t)h - sizeof(ptr_t);
}
if (ADDR_LT((ptr_t)GC_greatest_plausible_heap_addr, endp)) {
GC_greatest_plausible_heap_addr = endp;
}
#ifdef SET_REAL_HEAP_BOUNDS
if (ADDR(h) < GC_least_real_heap_addr
|| UNLIKELY(0 == GC_least_real_heap_addr))
GC_least_real_heap_addr = ADDR(h) - sizeof(ptr_t);
if (GC_greatest_real_heap_addr < ADDR(endp)) {
# ifdef INCLUDE_LINUX_THREAD_DESCR
/* Avoid heap intersection with the static data roots. */
GC_exclude_static_roots_inner((ptr_t)h, endp);
# endif
GC_greatest_real_heap_addr = ADDR(endp);
}
#endif
GC_handle_protected_regions_limit();
if (UNLIKELY(old_capacity > 0)) {
#ifndef GWW_VDB
/*
* Recycling may call `GC_add_to_heap()` again but should not cause
* resizing of `GC_heap_sects`.
*/
GC_scratch_recycle_no_gww(old_heap_sects,
old_capacity * sizeof(struct HeapSect));
#else
/* TODO: Implement GWW-aware recycling as in `alloc_mark_stack`. */
GC_noop1_ptr(old_heap_sects);
#endif
}
}
#ifndef NO_DEBUGGING
void
GC_print_heap_sects(void)
{
size_t i;
GC_printf("Total heap size: %lu" IF_USE_MUNMAP(" (%lu unmapped)") "\n",
(unsigned long)GC_heapsize /*, */
COMMA_IF_USE_MUNMAP((unsigned long)GC_unmapped_bytes));
for (i = 0; i < GC_n_heap_sects; i++) {
ptr_t start = GC_heap_sects[i].hs_start;
size_t len = GC_heap_sects[i].hs_bytes;
unsigned nbl = 0;
# ifndef NO_BLACK_LISTING
struct hblk *h;
for (h = (struct hblk *)start; ADDR_LT((ptr_t)h, start + len); h++) {
if (GC_is_black_listed(h, HBLKSIZE))
nbl++;
}
# endif
GC_printf("Section %u from %p to %p %u/%lu blacklisted\n", (unsigned)i,
(void *)start, (void *)&start[len], nbl,
(unsigned long)divHBLKSZ(len));
}
}
#endif /* !NO_DEBUGGING */
void *GC_least_plausible_heap_addr = MAKE_CPTR(GC_WORD_MAX);
void *GC_greatest_plausible_heap_addr = NULL;
STATIC word GC_max_heapsize = 0;
GC_API void GC_CALL
GC_set_max_heap_size(GC_word n)
{
GC_max_heapsize = n;
}
word GC_max_retries = 0;
GC_INNER void
GC_scratch_recycle_inner(void *ptr, size_t sz)
{
size_t page_offset;
size_t displ = 0;
size_t recycled_bytes;
GC_ASSERT(I_HOLD_LOCK());
if (NULL == ptr)
return;
GC_ASSERT(sz != 0);
GC_ASSERT(GC_page_size != 0);
/* TODO: Assert correct memory flags if `GWW_VDB`. */
page_offset = ADDR(ptr) & (GC_page_size - 1);
if (page_offset != 0)
displ = GC_page_size - page_offset;
recycled_bytes = sz > displ ? (sz - displ) & ~(GC_page_size - 1) : 0;
GC_COND_LOG_PRINTF("Recycle %lu/%lu scratch-allocated bytes at %p\n",
(unsigned long)recycled_bytes, (unsigned long)sz, ptr);
if (recycled_bytes > 0)
GC_add_to_heap((struct hblk *)((ptr_t)ptr + displ), recycled_bytes);
}
GC_INNER GC_bool
GC_expand_hp_inner(word n)
{
size_t sz;
struct hblk *space;
/* Number of bytes by which we expect the heap to expand soon. */
word expansion_slop;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_page_size != 0);
if (0 == n)
n = 1;
sz = ROUNDUP_PAGESIZE((size_t)n * HBLKSIZE);
GC_DBGLOG_PRINT_HEAP_IN_USE();
if (GC_max_heapsize != 0
&& (GC_max_heapsize < (word)sz
|| GC_heapsize > GC_max_heapsize - (word)sz)) {
/* Exceeded the self-imposed limit. */
return FALSE;
}
space = (struct hblk *)GC_os_get_mem(sz);
if (UNLIKELY(NULL == space)) {
WARN("Failed to expand heap by %" WARN_PRIuPTR " KiB\n", sz >> 10);
return FALSE;
}
GC_last_heap_growth_gc_no = GC_gc_no;
GC_INFOLOG_PRINTF("Grow heap to %lu KiB after %lu bytes allocated\n",
TO_KiB_UL(GC_heapsize + sz),
(unsigned long)GC_bytes_allocd);
/*
* Adjust heap limits generously for black-listing to work better.
* `GC_add_to_heap()` performs minimal adjustment needed for correctness.
*/
expansion_slop = min_bytes_allocd() + 4 * MAXHINCR * HBLKSIZE;
if ((0 == GC_last_heap_addr && (ADDR(space) & SIGNB) == 0)
|| (GC_last_heap_addr != 0 && GC_last_heap_addr < ADDR(space))) {
/* Assume the heap is growing up. */
if (LIKELY(ADDR(space) < GC_WORD_MAX - (sz + expansion_slop))) {
ptr_t new_limit = (ptr_t)space + sz + expansion_slop;
if (ADDR_LT((ptr_t)GC_greatest_plausible_heap_addr, new_limit))
GC_greatest_plausible_heap_addr = new_limit;
}
} else {
/* Heap is growing down. */
if (LIKELY(ADDR(space) > expansion_slop + sizeof(ptr_t))) {
ptr_t new_limit = (ptr_t)space - expansion_slop - sizeof(ptr_t);
if (ADDR_LT(new_limit, (ptr_t)GC_least_plausible_heap_addr))
GC_least_plausible_heap_addr = new_limit;
}
}
GC_last_heap_addr = ADDR(space);
GC_add_to_heap(space, sz);
if (GC_on_heap_resize)
(*GC_on_heap_resize)(GC_heapsize);
return TRUE;
}
GC_API int GC_CALL
GC_expand_hp(size_t bytes)
{
size_t n_blocks = OBJ_SZ_TO_BLOCKS_CHECKED(bytes);
word old_heapsize;
GC_bool result;
if (UNLIKELY(!GC_is_initialized))
GC_init();
LOCK();
old_heapsize = GC_heapsize;
result = GC_expand_hp_inner(n_blocks);
if (result) {
GC_requested_heapsize += bytes;
if (GC_dont_gc) {
/* Do not call `WARN()` if the heap growth is intentional. */
GC_ASSERT(GC_heapsize >= old_heapsize);
GC_heapsize_on_gc_disable += GC_heapsize - old_heapsize;
}
}
UNLOCK();
/*
* Really returns a `GC_bool` value, but the function is externally
* visible, so that is clumsy.
*/
return (int)result;
}
/*
* The minimum value of the ratio of allocated bytes since the latest
* collection to the amount of finalizers created since that collection
* which triggers the collection instead heap expansion. Has no effect
* in the incremental mode.
*/
#if defined(GC_ALLOCD_BYTES_PER_FINALIZER) && !defined(CPPCHECK)
STATIC word GC_allocd_bytes_per_finalizer = GC_ALLOCD_BYTES_PER_FINALIZER;
#else
STATIC word GC_allocd_bytes_per_finalizer = 10000;
#endif
GC_API void GC_CALL
GC_set_allocd_bytes_per_finalizer(GC_word value)
{
GC_allocd_bytes_per_finalizer = value;
}
GC_API GC_word GC_CALL
GC_get_allocd_bytes_per_finalizer(void)
{
return GC_allocd_bytes_per_finalizer;
}
GC_INNER GC_bool
GC_collect_or_expand(word needed_blocks, unsigned flags, GC_bool retry)
{
static word last_fo_entries, last_bytes_finalized;
GC_bool gc_not_stopped = TRUE;
word blocks_to_get;
IF_CANCEL(int cancel_state;)
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_is_initialized);
DISABLE_CANCEL(cancel_state);
if (!GC_incremental && !GC_dont_gc
&& ((GC_dont_expand && GC_bytes_allocd > 0)
|| (GC_fo_entries > last_fo_entries
&& (last_bytes_finalized | GC_bytes_finalized) != 0
&& (GC_fo_entries - last_fo_entries)
* GC_allocd_bytes_per_finalizer
> GC_bytes_allocd)
|| GC_should_collect())) {
/*
* Try to do a full collection using "default" `stop_func` (unless
* nothing has been allocated since the latest collection or heap
* expansion is disabled).
*/
gc_not_stopped = GC_try_to_collect_inner(
GC_bytes_allocd > 0 && (!GC_dont_expand || !retry)
? GC_default_stop_func
: GC_never_stop_func);
if (gc_not_stopped || !retry) {
/*
* Either the collection has not been aborted or this is the
* first attempt (in a loop).
*/
last_fo_entries = GC_fo_entries;
last_bytes_finalized = GC_bytes_finalized;
RESTORE_CANCEL(cancel_state);
return TRUE;
}
}
blocks_to_get = (GC_heapsize - GC_heapsize_at_forced_unmap)
/ (HBLKSIZE * GC_free_space_divisor)
+ needed_blocks;
if (blocks_to_get > MAXHINCR) {
#ifdef NO_BLACK_LISTING
UNUSED_ARG(flags);
blocks_to_get = needed_blocks > MAXHINCR ? needed_blocks : MAXHINCR;
#else
word slop;
/*
* Get the minimum required to make it likely that we can satisfy
* the current request in the presence of black-listing. This will
* probably be bigger than `MAXHINCR`.
*/
if ((flags & IGNORE_OFF_PAGE) != 0) {
slop = 4;
} else {
slop = 2 * divHBLKSZ(BL_LIMIT);
if (slop > needed_blocks)
slop = needed_blocks;
}
if (needed_blocks + slop > MAXHINCR) {
blocks_to_get = needed_blocks + slop;
} else {
blocks_to_get = MAXHINCR;
}
#endif
if (blocks_to_get > divHBLKSZ(GC_WORD_MAX))
blocks_to_get = divHBLKSZ(GC_WORD_MAX);
} else if (blocks_to_get < MINHINCR) {
blocks_to_get = MINHINCR;
}
if (GC_max_heapsize > GC_heapsize) {
word max_get_blocks = divHBLKSZ(GC_max_heapsize - GC_heapsize);
if (blocks_to_get > max_get_blocks)
blocks_to_get
= max_get_blocks > needed_blocks ? max_get_blocks : needed_blocks;
}
#ifdef USE_MUNMAP
if (GC_unmap_threshold > 1) {
/*
* Return as much memory to the OS as possible before trying to
* get memory from it.
*/
GC_unmap_old(0);
}
#endif
if (!GC_expand_hp_inner(blocks_to_get)
&& (blocks_to_get == needed_blocks
|| !GC_expand_hp_inner(needed_blocks))) {
if (!gc_not_stopped) {
/* Do not increment `GC_alloc_fail_count` here (and no warning). */
GC_gcollect_inner();
GC_ASSERT(0 == GC_bytes_allocd);
} else if (GC_alloc_fail_count++ < GC_max_retries) {
WARN("Out of Memory! Trying to continue...\n", 0);
GC_gcollect_inner();
} else {
#ifdef USE_MUNMAP
GC_ASSERT(GC_heapsize >= GC_unmapped_bytes);
#endif
#if !defined(SMALL_CONFIG) && (CPP_WORDSZ >= 32)
# define MAX_HEAPSIZE_WARNED_IN_BYTES (5 << 20) /*< 5 MB */
if (GC_heapsize > (word)MAX_HEAPSIZE_WARNED_IN_BYTES) {
WARN("Out of Memory! Heap size: %" WARN_PRIuPTR " MiB."
" Returning NULL!\n",
(GC_heapsize - GC_unmapped_bytes) >> 20);
} else
#endif
/* else */ {
WARN("Out of Memory! Heap size: %" WARN_PRIuPTR " bytes."
" Returning NULL!\n",
GC_heapsize - GC_unmapped_bytes);
}
RESTORE_CANCEL(cancel_state);
return FALSE;
}
} else if (GC_alloc_fail_count > 0) {
GC_COND_LOG_PRINTF("Memory available again...\n");
}
RESTORE_CANCEL(cancel_state);
return TRUE;
}
GC_INNER ptr_t
GC_allocobj(size_t lg, int kind)
{
#define MAX_ALLOCOBJ_RETRIES 3
int retry_cnt = 0;
void **flh = &GC_obj_kinds[kind].ok_freelist[lg];
#ifndef GC_DISABLE_INCREMENTAL
GC_bool tried_minor = FALSE;
#endif
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_is_initialized);
if (UNLIKELY(0 == lg))
return NULL;
while (NULL == *flh) {
/*
* Only a few iterations are expected at most, otherwise something
* is wrong in one of the functions called below.
*/
if (retry_cnt > MAX_ALLOCOBJ_RETRIES)
ABORT("Too many retries in GC_allocobj");
#ifndef GC_DISABLE_INCREMENTAL
if (GC_incremental && GC_time_limit != GC_TIME_UNLIMITED && !GC_dont_gc) {
/*
* True incremental mode, not just generational.
* Do our share of marking work.
*/
GC_collect_a_little_inner(1);
}
#endif
/* Sweep blocks for objects of this size. */
GC_ASSERT(!GC_is_full_gc || NULL == GC_obj_kinds[kind].ok_reclaim_list
|| NULL == GC_obj_kinds[kind].ok_reclaim_list[lg]);
GC_continue_reclaim(lg, kind);
#if defined(CPPCHECK)
GC_noop1_ptr(&flh);
#endif
if (*flh != NULL)
break;
GC_new_hblk(lg, kind);
#if defined(CPPCHECK)
GC_noop1_ptr(&flh);
#endif
if (*flh != NULL)
break;
#ifndef GC_DISABLE_INCREMENTAL
if (GC_incremental && GC_time_limit == GC_TIME_UNLIMITED && !tried_minor
&& !GC_dont_gc) {
GC_collect_a_little_inner(1);
tried_minor = TRUE;
continue;
}
#endif
if (UNLIKELY(!GC_collect_or_expand(1, 0 /* flags */, retry_cnt > 0)))
return NULL;
retry_cnt++;
}
/* Successful allocation; reset failure count. */
GC_alloc_fail_count = 0;
return (ptr_t)(*flh);
}
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1995 by Xerox Corporation. All rights reserved.
* Copyright (c) 1997 by Silicon Graphics. All rights reserved.
* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
* Copyright (c) 2007 Free Software Foundation, Inc.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#ifndef MSWINCE
# include <errno.h>
#endif
#include <string.h>
#ifdef KEEP_BACK_PTRS
/*
* Use a custom trivial `random()` implementation as the standard one might
* lead to crashes (if used from a multi-threaded code) or to a compiler
* warning about the deterministic result.
*/
static int
GC_rand(void)
{
static GC_RAND_STATE_T seed;
return GC_RAND_NEXT(&seed);
}
# define RANDOM() (long)GC_rand()
GC_INNER void
GC_store_back_pointer(ptr_t source, ptr_t dest)
{
if (GC_HAS_DEBUG_INFO(dest)) {
# ifdef PARALLEL_MARK
GC_cptr_store((volatile ptr_t *)&((oh *)dest)->oh_back_ptr,
(ptr_t)HIDE_BACK_PTR(source));
# else
((oh *)dest)->oh_back_ptr = HIDE_BACK_PTR(source);
# endif
}
}
GC_INNER void
GC_marked_for_finalization(ptr_t dest)
{
GC_store_back_pointer(MARKED_FOR_FINALIZATION, dest);
}
GC_API GC_ref_kind GC_CALL
GC_get_back_ptr_info(void *dest, void **base_p, size_t *offset_p)
{
oh *ohdr = (oh *)GC_base(dest);
ptr_t bp, bp_base;
# ifdef LINT2
/*
* Explicitly instruct the code analysis tool that `GC_get_back_ptr_info`
* is not expected to be called with an incorrect `dest` value.
*/
if (!ohdr)
ABORT("Invalid GC_get_back_ptr_info argument");
# endif
if (!GC_HAS_DEBUG_INFO((ptr_t)ohdr))
return GC_NO_SPACE;
bp = (ptr_t)GC_REVEAL_POINTER(ohdr->oh_back_ptr);
if (MARKED_FOR_FINALIZATION == bp)
return GC_FINALIZER_REFD;
if (MARKED_FROM_REGISTER == bp)
return GC_REFD_FROM_REG;
if (NOT_MARKED == bp)
return GC_UNREFERENCED;
# if ALIGNMENT == 1
/*
* Heuristically try to fix off-by-one errors we introduced by
* insisting on even addresses.
*/
{
ptr_t alternate_ptr = bp + 1;
ptr_t target = *(ptr_t *)bp;
ptr_t alternate_target = *(ptr_t *)alternate_ptr;
if (GC_least_real_heap_addr < ADDR(alternate_target)
&& ADDR(alternate_target) < GC_greatest_real_heap_addr
&& (GC_least_real_heap_addr >= ADDR(target)
|| ADDR(target) >= GC_greatest_real_heap_addr)) {
bp = alternate_ptr;
}
}
# endif
bp_base = (ptr_t)GC_base(bp);
if (NULL == bp_base) {
*base_p = bp;
*offset_p = 0;
return GC_REFD_FROM_ROOT;
} else {
if (GC_HAS_DEBUG_INFO(bp_base))
bp_base += sizeof(oh);
*base_p = bp_base;
*offset_p = (size_t)(bp - bp_base);
return GC_REFD_FROM_HEAP;
}
}
GC_API void *GC_CALL
GC_generate_random_heap_address(void)
{
size_t i;
word heap_offset = (word)RANDOM();
if (GC_heapsize > (word)GC_RAND_MAX) {
heap_offset *= GC_RAND_MAX;
heap_offset += (word)RANDOM();
}
/*
* This does not yield a uniform distribution, especially if e.g.
* `RAND_MAX` is `1.5 * GC_heapsize`. But for typical cases, it is
* not too bad.
*/
heap_offset %= GC_heapsize;
for (i = 0;; ++i) {
size_t size;
if (i >= GC_n_heap_sects)
ABORT("GC_generate_random_heap_address: size inconsistency");
size = GC_heap_sects[i].hs_bytes;
if (heap_offset < size)
break;
heap_offset -= size;
}
return GC_heap_sects[i].hs_start + heap_offset;
}
GC_API void *GC_CALL
GC_generate_random_valid_address(void)
{
ptr_t result;
ptr_t base;
do {
result = (ptr_t)GC_generate_random_heap_address();
base = (ptr_t)GC_base(result);
} while (NULL == base || !GC_is_marked(base));
return result;
}
GC_API void GC_CALL
GC_print_backtrace(void *p)
{
void *current = p;
int i;
GC_ASSERT(I_DONT_HOLD_LOCK());
GC_print_heap_obj((ptr_t)GC_base(current));
for (i = 0;; ++i) {
void *base;
size_t offset;
GC_ref_kind source = GC_get_back_ptr_info(current, &base, &offset);
if (GC_UNREFERENCED == source) {
GC_err_printf("Reference could not be found\n");
break;
}
if (GC_NO_SPACE == source) {
GC_err_printf("No debug info in object: Can't find reference\n");
break;
}
GC_err_printf("Reachable via %d levels of pointers from ", i);
switch (source) {
case GC_REFD_FROM_ROOT:
GC_err_printf("root at %p\n\n", base);
return;
case GC_REFD_FROM_REG:
GC_err_printf("root in register\n\n");
return;
case GC_FINALIZER_REFD:
GC_err_printf("list of finalizable objects\n\n");
return;
case GC_REFD_FROM_HEAP:
GC_err_printf("offset %ld in object:\n", (long)offset);
/* Take `GC_base(base)` to get real base, i.e. header. */
GC_print_heap_obj((ptr_t)GC_base(base));
break;
default:
GC_err_printf("INTERNAL ERROR: UNEXPECTED SOURCE!!!!\n");
return;
}
current = base;
}
}
GC_API void GC_CALL
GC_generate_random_backtrace(void)
{
void *current;
GC_ASSERT(I_DONT_HOLD_LOCK());
if (GC_try_to_collect(GC_never_stop_func) == 0) {
GC_err_printf("Cannot generate a backtrace: "
"garbage collection is disabled!\n");
return;
}
/* Generate/print a backtrace from a random heap address. */
LOCK();
current = GC_generate_random_valid_address();
UNLOCK();
GC_printf("\n***Chosen address %p in object\n", current);
GC_print_backtrace(current);
}
#endif /* KEEP_BACK_PTRS */
#define CROSSES_HBLK(p, sz) \
((ADDR((p) + (sizeof(oh) - 1) + (sz)) ^ ADDR(p)) >= HBLKSIZE)
/*
* Store debugging info into `p`. Return displaced pointer. Assume we hold
* the allocator lock.
*/
STATIC void *
GC_store_debug_info_inner(void *base, size_t sz, const char *string,
int linenum)
{
GC_uintptr_t *result = (GC_uintptr_t *)((oh *)base + 1);
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_size(base) >= sizeof(oh) + sz);
GC_ASSERT(!(SMALL_OBJ(sz) && CROSSES_HBLK((ptr_t)base, sz)));
#ifdef KEEP_BACK_PTRS
((oh *)base)->oh_back_ptr = HIDE_BACK_PTR(NOT_MARKED);
#endif
#ifdef MAKE_BACK_GRAPH
((oh *)base)->oh_bg_ptr = HIDE_BACK_PTR((ptr_t)0);
#endif
((oh *)base)->oh_string = string;
((oh *)base)->oh_int = linenum;
#ifdef SHORT_DBG_HDRS
UNUSED_ARG(sz);
#else
((oh *)base)->oh_sz = (GC_uintptr_t)sz;
((oh *)base)->oh_sf = START_FLAG ^ (GC_uintptr_t)result;
((GC_uintptr_t *)base)[BYTES_TO_PTRS(GC_size(base)) - 1]
= result[BYTES_TO_PTRS_ROUNDUP(sz)] = END_FLAG ^ (GC_uintptr_t)result;
#endif
return result;
}
#ifndef SHORT_DBG_HDRS
/*
* Check the object with debugging info at `ohdr`. Return `NULL` if it
* is OK. Else return clobbered address.
*/
STATIC ptr_t
GC_check_annotated_obj(oh *ohdr)
{
ptr_t body = (ptr_t)(ohdr + 1);
size_t gc_sz = GC_size(ohdr);
size_t lpw_up;
if (ohdr->oh_sz + DEBUG_BYTES > (GC_uintptr_t)gc_sz) {
return (ptr_t)(&ohdr->oh_sz);
}
if (ohdr->oh_sf != (START_FLAG ^ (GC_uintptr_t)body)) {
return (ptr_t)(&ohdr->oh_sf);
}
{
size_t lpw_m1 = BYTES_TO_PTRS(gc_sz) - 1;
if (((GC_uintptr_t *)ohdr)[lpw_m1] != (END_FLAG ^ (GC_uintptr_t)body)) {
return (ptr_t)(&((GC_uintptr_t *)ohdr)[lpw_m1]);
}
}
lpw_up = BYTES_TO_PTRS_ROUNDUP((size_t)ohdr->oh_sz);
if (((GC_uintptr_t *)body)[lpw_up] != (END_FLAG ^ (GC_uintptr_t)body)) {
return (ptr_t)(&((GC_uintptr_t *)body)[lpw_up]);
}
return NULL;
}
#endif /* !SHORT_DBG_HDRS */
STATIC GC_describe_type_fn GC_describe_type_fns[MAXOBJKINDS] = { 0 };
GC_API void GC_CALL
GC_register_describe_type_fn(int kind, GC_describe_type_fn fn)
{
GC_ASSERT((unsigned)kind < MAXOBJKINDS);
GC_describe_type_fns[kind] = fn;
}
#ifndef SHORT_DBG_HDRS
# define IF_NOT_SHORTDBG_HDRS(x) x
# define COMMA_IFNOT_SHORTDBG_HDRS(x) /* comma */ , x
#else
# define IF_NOT_SHORTDBG_HDRS(x)
# define COMMA_IFNOT_SHORTDBG_HDRS(x)
#endif
/*
* Print a human-readable description of the object to `stderr`.
* The object is assumed to have the debugging info.
*/
STATIC void
GC_print_obj(ptr_t base)
{
oh *ohdr = (oh *)base;
ptr_t q;
hdr *hhdr;
int kind;
const char *kind_str;
char buffer[GC_TYPE_DESCR_LEN + 1];
GC_ASSERT(I_DONT_HOLD_LOCK());
#ifdef LINT2
if (!ohdr)
ABORT("Invalid GC_print_obj argument");
#endif
q = (ptr_t)(ohdr + 1);
/*
* Print a type description for the object whose client-visible address
* is `q`.
*/
hhdr = GC_find_header(q);
kind = hhdr->hb_obj_kind;
if (GC_describe_type_fns[kind] != 0 && GC_is_marked(ohdr)) {
/*
* This should preclude free-list objects except with thread-local
* allocation.
*/
buffer[GC_TYPE_DESCR_LEN] = 0;
(*GC_describe_type_fns[kind])(q, buffer);
GC_ASSERT(buffer[GC_TYPE_DESCR_LEN] == 0);
kind_str = buffer;
} else {
switch (kind) {
case PTRFREE:
kind_str = "PTRFREE";
break;
case NORMAL:
kind_str = "NORMAL";
break;
case UNCOLLECTABLE:
kind_str = "UNCOLLECTABLE";
break;
#ifdef GC_ATOMIC_UNCOLLECTABLE
case AUNCOLLECTABLE:
kind_str = "ATOMIC_UNCOLLECTABLE";
break;
#endif
default:
kind_str = NULL;
/*
* The alternative is to use `snprintf(buffer)` but the latter is
* not quite portable (see `vsnprintf` in `misc.c` file).
*/
}
}
if (NULL != kind_str) {
GC_err_printf("%p (%s:%d," IF_NOT_SHORTDBG_HDRS(" sz= %lu,") " %s)\n",
(void *)((ptr_t)ohdr + sizeof(oh)), ohdr->oh_string,
GET_OH_LINENUM(ohdr) /*, */
COMMA_IFNOT_SHORTDBG_HDRS((unsigned long)ohdr->oh_sz),
kind_str);
} else {
GC_err_printf("%p (%s:%d," IF_NOT_SHORTDBG_HDRS(
" sz= %lu,") " kind= %d, descr= 0x%lx)\n",
(void *)((ptr_t)ohdr + sizeof(oh)), ohdr->oh_string,
GET_OH_LINENUM(ohdr) /*, */
COMMA_IFNOT_SHORTDBG_HDRS((unsigned long)ohdr->oh_sz),
kind, (unsigned long)hhdr->hb_descr);
}
PRINT_CALL_CHAIN(ohdr);
}
STATIC void
GC_debug_print_heap_obj_proc(ptr_t base)
{
GC_ASSERT(I_DONT_HOLD_LOCK());
if (GC_HAS_DEBUG_INFO(base)) {
GC_print_obj(base);
} else {
GC_default_print_heap_obj_proc(base);
}
}
#ifndef SHORT_DBG_HDRS
STATIC void GC_check_heap_proc(void);
#elif !defined(NO_FIND_LEAK)
static void
do_nothing(void)
{
}
#endif /* SHORT_DBG_HDRS */
/*
* Turn on the debugging mode. Should not be called if
* `GC_debugging_initialized` is already set.
*/
STATIC void
GC_start_debugging_inner(void)
{
GC_ASSERT(I_HOLD_LOCK());
#ifndef SHORT_DBG_HDRS
GC_check_heap = GC_check_heap_proc;
GC_print_all_smashed = GC_print_all_smashed_proc;
#elif !defined(NO_FIND_LEAK)
GC_check_heap = do_nothing;
GC_print_all_smashed = do_nothing;
#endif
GC_print_heap_obj = GC_debug_print_heap_obj_proc;
GC_debugging_initialized = TRUE;
GC_register_displacement_inner(sizeof(oh));
#if defined(CPPCHECK)
GC_noop1(GC_debug_header_size);
#endif
}
/*
* Check the allocation is successful, store debugging info into `base`,
* start the debugging mode (if not yet), and return displaced pointer.
*/
static void *
store_debug_info(void *base, size_t lb, const char *fn, GC_EXTRA_PARAMS)
{
void *result;
if (NULL == base) {
GC_err_printf("%s(%lu) returning NULL (%s:%d)\n", fn, (unsigned long)lb, s,
i);
return NULL;
}
LOCK();
if (!GC_debugging_initialized)
GC_start_debugging_inner();
result = GC_store_debug_info_inner(base, lb, s, i);
ADD_CALL_CHAIN(base, ra);
UNLOCK();
return result;
}
const size_t GC_debug_header_size = sizeof(oh);
GC_API size_t GC_CALL
GC_get_debug_header_size(void)
{
return sizeof(oh);
}
GC_API void GC_CALL
GC_debug_register_displacement(size_t offset)
{
LOCK();
GC_register_displacement_inner(offset);
GC_register_displacement_inner(sizeof(oh) + offset);
UNLOCK();
}
#ifdef GC_ADD_CALLER
# if defined(HAVE_DLADDR) && defined(GC_HAVE_RETURN_ADDR_PARENT) \
&& defined(FUNCPTR_IS_DATAPTR)
# include <dlfcn.h>
STATIC void
GC_caller_func_offset(GC_return_addr_t ra, const char **symp, int *offp)
{
Dl_info caller;
if (ra != 0 && dladdr((void *)ra, &caller) && caller.dli_sname != NULL) {
*symp = caller.dli_sname;
*offp = (int)((ptr_t)ra - (ptr_t)caller.dli_saddr);
}
if (NULL == *symp) {
*symp = "unknown";
/* Note: `*offp` is unchanged. */
}
}
# else
# define GC_caller_func_offset(ra, symp, offp) (void)(*(symp) = "unknown")
# endif
#endif /* GC_ADD_CALLER */
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_debug_malloc(size_t lb, GC_EXTRA_PARAMS)
{
void *base;
/*
* Note that according to `malloc()` specification, if size (`lb`) is
* zero, then `malloc()` returns either `NULL`, or a unique pointer
* value that can later be successfully passed to `free()`.
* We always do the latter.
*/
#if defined(_FORTIFY_SOURCE) && !defined(__clang__)
/* Workaround to avoid "exceeds maximum object size" gcc warning. */
base = GC_malloc(lb < GC_SIZE_MAX - DEBUG_BYTES ? lb + DEBUG_BYTES
: GC_SIZE_MAX >> 1);
#else
base = GC_malloc(SIZET_SAT_ADD(lb, DEBUG_BYTES));
#endif
#ifdef GC_ADD_CALLER
if (NULL == s) {
GC_caller_func_offset(ra, &s, &i);
}
#endif
return store_debug_info(base, lb, "GC_debug_malloc", OPT_RA s, i);
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_debug_malloc_ignore_off_page(size_t lb, GC_EXTRA_PARAMS)
{
void *base = GC_malloc_ignore_off_page(SIZET_SAT_ADD(lb, DEBUG_BYTES));
return store_debug_info(base, lb, "GC_debug_malloc_ignore_off_page",
OPT_RA s, i);
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_debug_malloc_atomic_ignore_off_page(size_t lb, GC_EXTRA_PARAMS)
{
void *base
= GC_malloc_atomic_ignore_off_page(SIZET_SAT_ADD(lb, DEBUG_BYTES));
return store_debug_info(base, lb, "GC_debug_malloc_atomic_ignore_off_page",
OPT_RA s, i);
}
STATIC void *
GC_debug_generic_malloc(size_t lb, int kind, GC_EXTRA_PARAMS)
{
void *base = GC_generic_malloc_aligned(SIZET_SAT_ADD(lb, DEBUG_BYTES), kind,
0 /* `flags` */, 0 /* `align_m1` */);
return store_debug_info(base, lb, "GC_debug_generic_malloc", OPT_RA s, i);
}
#ifdef DBG_HDRS_ALL
GC_INNER void *
GC_debug_generic_malloc_inner(size_t lb, int kind, unsigned flags)
{
void *base, *result;
GC_ASSERT(I_HOLD_LOCK());
base = GC_generic_malloc_inner(SIZET_SAT_ADD(lb, DEBUG_BYTES), kind, flags);
if (NULL == base) {
GC_err_printf("GC internal allocation (%lu bytes) returning NULL\n",
(unsigned long)lb);
return NULL;
}
if (!GC_debugging_initialized)
GC_start_debugging_inner();
result = GC_store_debug_info_inner(base, lb, "INTERNAL", 0);
ADD_CALL_CHAIN_INNER(base);
return result;
}
#endif /* DBG_HDRS_ALL */
#ifndef CPPCHECK
GC_API void *GC_CALL
GC_debug_malloc_stubborn(size_t lb, GC_EXTRA_PARAMS)
{
return GC_debug_malloc(lb, OPT_RA s, i);
}
GC_API void GC_CALL
GC_debug_change_stubborn(const void *p)
{
UNUSED_ARG(p);
}
#endif /* !CPPCHECK */
GC_API void GC_CALL
GC_debug_end_stubborn_change(const void *p)
{
const void *q = GC_base_C(p);
if (NULL == q) {
ABORT_ARG1("GC_debug_end_stubborn_change: bad arg", ": %p", p);
}
GC_end_stubborn_change(q);
}
GC_API void GC_CALL
GC_debug_ptr_store_and_dirty(void *p, const void *q)
{
*(void **)GC_is_visible(p)
= GC_is_valid_displacement(GC_CAST_AWAY_CONST_PVOID(q));
GC_debug_end_stubborn_change(p);
REACHABLE_AFTER_DIRTY(q);
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_debug_malloc_atomic(size_t lb, GC_EXTRA_PARAMS)
{
void *base = GC_malloc_atomic(SIZET_SAT_ADD(lb, DEBUG_BYTES));
return store_debug_info(base, lb, "GC_debug_malloc_atomic", OPT_RA s, i);
}
GC_API GC_ATTR_MALLOC char *GC_CALL
GC_debug_strdup(const char *str, GC_EXTRA_PARAMS)
{
char *copy;
size_t lb;
if (str == NULL) {
if (GC_find_leak_inner)
GC_err_printf("strdup(NULL) behavior is undefined\n");
return NULL;
}
lb = strlen(str) + 1;
copy = (char *)GC_debug_malloc_atomic(lb, OPT_RA s, i);
if (copy == NULL) {
#ifndef MSWINCE
errno = ENOMEM;
#endif
return NULL;
}
BCOPY(str, copy, lb);
return copy;
}
GC_API GC_ATTR_MALLOC char *GC_CALL
GC_debug_strndup(const char *str, size_t size, GC_EXTRA_PARAMS)
{
char *copy;
/* `str` is expected to be non-`NULL`. */
size_t len = strlen(str);
if (len > size)
len = size;
copy = (char *)GC_debug_malloc_atomic(len + 1, OPT_RA s, i);
if (copy == NULL) {
#ifndef MSWINCE
errno = ENOMEM;
#endif
return NULL;
}
if (len > 0)
BCOPY(str, copy, len);
copy[len] = '\0';
return copy;
}
#ifdef GC_REQUIRE_WCSDUP
# include <wchar.h> /*< for `wcslen()` */
GC_API GC_ATTR_MALLOC wchar_t *GC_CALL
GC_debug_wcsdup(const wchar_t *str, GC_EXTRA_PARAMS)
{
size_t lb = (wcslen(str) + 1) * sizeof(wchar_t);
wchar_t *copy = (wchar_t *)GC_debug_malloc_atomic(lb, OPT_RA s, i);
if (copy == NULL) {
# ifndef MSWINCE
errno = ENOMEM;
# endif
return NULL;
}
BCOPY(str, copy, lb);
return copy;
}
#endif /* GC_REQUIRE_WCSDUP */
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_debug_malloc_uncollectable(size_t lb, GC_EXTRA_PARAMS)
{
void *base
= GC_malloc_uncollectable(SIZET_SAT_ADD(lb, UNCOLLECTABLE_DEBUG_BYTES));
return store_debug_info(base, lb, "GC_debug_malloc_uncollectable", OPT_RA s,
i);
}
#ifdef GC_ATOMIC_UNCOLLECTABLE
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_debug_malloc_atomic_uncollectable(size_t lb, GC_EXTRA_PARAMS)
{
void *base = GC_malloc_atomic_uncollectable(
SIZET_SAT_ADD(lb, UNCOLLECTABLE_DEBUG_BYTES));
return store_debug_info(base, lb, "GC_debug_malloc_atomic_uncollectable",
OPT_RA s, i);
}
#endif /* GC_ATOMIC_UNCOLLECTABLE */
#ifdef LINT2
/*
* Copyright (c) 1996-1998 by Silicon Graphics. All rights reserved.
* Copyright (c) 2018-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/* This file is kept for a binary compatibility purpose only. */
#ifndef GC_ALLOC_PTRS_H
#define GC_ALLOC_PTRS_H
#ifdef __cplusplus
extern "C" {
#endif
#ifndef GC_API_PRIV
# define GC_API_PRIV GC_API
#endif
/*
* Some compilers do not accept `const` together with the `dllimport`
* attribute, so the symbols below are exported as non-constant ones.
*/
#ifndef GC_APIVAR_CONST
# if defined(GC_BUILD) || !defined(GC_DLL)
# define GC_APIVAR_CONST const
# else
# define GC_APIVAR_CONST /*< empty */
# endif
#endif
GC_API_PRIV void **GC_APIVAR_CONST GC_objfreelist_ptr;
GC_API_PRIV void **GC_APIVAR_CONST GC_aobjfreelist_ptr;
GC_API_PRIV void **GC_APIVAR_CONST GC_uobjfreelist_ptr;
#ifdef GC_ATOMIC_UNCOLLECTABLE
GC_API_PRIV void **GC_APIVAR_CONST GC_auobjfreelist_ptr;
#endif
/*
* Manually update the number of bytes allocated during the current
* collection cycle and the number of explicitly deallocated bytes of
* memory since the last collection, respectively. Both functions are
* unsynchronized, `GC_call_with_alloc_lock()` should be used to avoid
* data race.
*/
GC_API_PRIV void GC_CALL GC_incr_bytes_allocd(size_t /* `bytes` */);
GC_API_PRIV void GC_CALL GC_incr_bytes_freed(size_t /* `bytes` */);
#ifdef __cplusplus
} /* extern "C" */
#endif
#endif /* GC_ALLOC_PTRS_H */
#endif
GC_API void GC_CALL
GC_debug_free(void *p)
{
ptr_t base;
if (0 == p)
return;
base = (ptr_t)GC_base(p);
if (NULL == base) {
#if defined(REDIRECT_MALLOC) \
&& ((defined(NEED_CALLINFO) && defined(GC_HAVE_BUILTIN_BACKTRACE)) \
|| defined(REDIR_MALLOC_AND_LINUXTHREADS) \
|| (defined(SOLARIS) && defined(THREADS)) || defined(MSWIN32))
/*
* In some cases, we should ignore objects that do not belong to
* the collector heap. See the comment in `GC_free()`.
*/
if (!GC_is_heap_ptr(p))
return;
#endif
ABORT_ARG1("Invalid pointer passed to free()", ": %p", p);
}
if ((word)((ptr_t)p - base) != sizeof(oh)) {
#if defined(REDIRECT_FREE) && defined(USE_PROC_FOR_LIBRARIES)
/*
* TODO: Suppress the warning if `free()` caller is in `libpthread`
* or `libdl`.
*/
#endif
/*
* TODO: Suppress the warning for objects allocated by `GC_memalign`
* and friends (these ones do not have the debugging counterpart).
*/
GC_err_printf("GC_debug_free called on pointer %p w/o debugging info\n",
p);
} else {
#ifndef SHORT_DBG_HDRS
ptr_t clobbered = GC_check_annotated_obj((oh *)base);
size_t sz = GC_size(base);
if (clobbered != NULL) {
/* No "release" barrier is needed. */
GC_SET_HAVE_ERRORS();
if (((oh *)base)->oh_sz == (GC_uintptr_t)sz) {
GC_print_smashed_obj(
"GC_debug_free: found previously deallocated (?) object at", p,
clobbered);
/* Ignore double free. */
return;
} else {
GC_print_smashed_obj("GC_debug_free: found smashed location at", p,
clobbered);
}
}
/* Invalidate the size (mark the object as deallocated). */
((oh *)base)->oh_sz = (GC_uintptr_t)sz;
#endif /* !SHORT_DBG_HDRS */
}
#ifndef NO_FIND_LEAK
if (GC_find_leak_inner
# ifndef SHORT_DBG_HDRS
&& ((word)((ptr_t)p - base) != sizeof(oh) || !GC_findleak_delay_free)
# endif
) {
GC_free(base);
} else
#endif
/* else */ {
const hdr *hhdr = HDR(p);
if (hhdr->hb_obj_kind == UNCOLLECTABLE
#ifdef GC_ATOMIC_UNCOLLECTABLE
|| hhdr->hb_obj_kind == AUNCOLLECTABLE
#endif
) {
GC_free(base);
} else {
size_t sz = hhdr->hb_sz;
size_t i;
size_t lpw = BYTES_TO_PTRS(sz - sizeof(oh));
for (i = 0; i < lpw; ++i)
((GC_uintptr_t *)p)[i] = GC_FREED_MEM_MARKER;
GC_ASSERT((GC_uintptr_t *)p + i == (GC_uintptr_t *)(base + sz));
/*
* Update the counter even though the real deallocation
* is deferred.
*/
LOCK();
#ifdef LINT2
GC_incr_bytes_freed(sz);
#else
GC_bytes_freed += sz;
#endif
UNLOCK();
}
}
}
#if defined(THREADS) && defined(DBG_HDRS_ALL)
GC_INNER void
GC_debug_free_inner(void *p)
{
ptr_t base = (ptr_t)GC_base(p);
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT((word)((ptr_t)p - base) == sizeof(oh));
# ifdef LINT2
if (!base)
ABORT("Invalid GC_debug_free_inner argument");
# endif
# ifndef SHORT_DBG_HDRS
/* Invalidate the size. */
((oh *)base)->oh_sz = (GC_uintptr_t)GC_size(base);
# endif
GC_free_inner(base);
}
#endif
GC_API void *GC_CALL
GC_debug_realloc(void *p, size_t lb, GC_EXTRA_PARAMS)
{
ptr_t base;
void *result;
const hdr *hhdr;
if (NULL == p) {
return GC_debug_malloc(lb, OPT_RA s, i);
}
if (0 == lb) /* `&& p != NULL` */ {
GC_debug_free(p);
return NULL;
}
#ifdef GC_ADD_CALLER
if (NULL == s) {
GC_caller_func_offset(ra, &s, &i);
}
#endif
base = (ptr_t)GC_base(p);
if (NULL == base) {
ABORT_ARG1("Invalid pointer passed to realloc()", ": %p", p);
}
if ((word)((ptr_t)p - base) != sizeof(oh)) {
GC_err_printf("GC_debug_realloc called on pointer %p w/o debugging info\n",
p);
return GC_realloc(p, lb);
}
hhdr = HDR(base);
result
= GC_debug_generic_or_special_malloc(lb, hhdr->hb_obj_kind, OPT_RA s, i);
if (result != NULL) {
size_t old_sz;
#ifdef SHORT_DBG_HDRS
old_sz = GC_size(base) - sizeof(oh);
#else
old_sz = (size_t)(((oh *)base)->oh_sz);
#endif
if (old_sz > 0)
BCOPY(p, result, old_sz < lb ? old_sz : lb);
GC_debug_free(p);
}
return result;
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_debug_generic_or_special_malloc(size_t lb, int kind, GC_EXTRA_PARAMS)
{
switch (kind) {
case PTRFREE:
return GC_debug_malloc_atomic(lb, OPT_RA s, i);
case NORMAL:
return GC_debug_malloc(lb, OPT_RA s, i);
case UNCOLLECTABLE:
return GC_debug_malloc_uncollectable(lb, OPT_RA s, i);
#ifdef GC_ATOMIC_UNCOLLECTABLE
case AUNCOLLECTABLE:
return GC_debug_malloc_atomic_uncollectable(lb, OPT_RA s, i);
#endif
default:
return GC_debug_generic_malloc(lb, kind, OPT_RA s, i);
}
}
#ifndef SHORT_DBG_HDRS
/*
* Check all marked objects in the given block for validity.
* Note: avoid `GC_apply_to_each_object` for performance reasons.
*/
STATIC void GC_CALLBACK
GC_check_heap_block(struct hblk *hbp, void *dummy)
{
const hdr *hhdr = HDR(hbp);
ptr_t p = hbp->hb_body;
ptr_t plim;
size_t sz = hhdr->hb_sz;
size_t bit_no;
UNUSED_ARG(dummy);
GC_ASSERT((ptr_t)hhdr->hb_block == p);
plim = sz > MAXOBJBYTES ? p : p + HBLKSIZE - sz;
/* Go through all objects in block. */
for (bit_no = 0; ADDR_GE(plim, p); bit_no += MARK_BIT_OFFSET(sz), p += sz) {
if (mark_bit_from_hdr(hhdr, bit_no) && GC_HAS_DEBUG_INFO(p)) {
ptr_t clobbered = GC_check_annotated_obj((oh *)p);
if (clobbered != NULL)
GC_add_smashed(clobbered);
}
}
}
/*
* This assumes that all accessible objects are marked.
* Normally called by collector.
*/
STATIC void
GC_check_heap_proc(void)
{
GC_ASSERT(I_HOLD_LOCK());
GC_STATIC_ASSERT((sizeof(oh) & (GC_GRANULE_BYTES - 1)) == 0);
/* FIXME: Should we check for twice that alignment? */
GC_apply_to_all_blocks(GC_check_heap_block, NULL);
}
#endif /* !SHORT_DBG_HDRS */
#ifndef GC_NO_FINALIZATION
struct closure {
GC_finalization_proc cl_fn;
void *cl_data;
};
STATIC void *
GC_make_closure(GC_finalization_proc fn, void *data)
{
struct closure *result =
# ifdef DBG_HDRS_ALL
(struct closure *)GC_debug_malloc(sizeof(struct closure), GC_EXTRAS);
# else
(struct closure *)GC_malloc(sizeof(struct closure));
# endif
if (result != NULL) {
result->cl_fn = fn;
result->cl_data = data;
}
return result;
}
/*
* An auxiliary function to make finalization work correctly with
* displaced pointers introduced by the debugging allocators.
*/
STATIC void GC_CALLBACK
GC_debug_invoke_finalizer(void *obj, void *data)
{
struct closure *cl = (struct closure *)data;
cl->cl_fn((ptr_t)obj + sizeof(oh), cl->cl_data);
}
/* Special `finalizer_proc` value to detect `GC_register_finalizer` failure. */
# define OFN_UNSET ((GC_finalization_proc)(~(GC_funcptr_uint)0))
/* Set `ofn` and `ocd` to reflect the values we got back. */
static void
store_old(void *obj, GC_finalization_proc my_old_fn, struct closure *my_old_cd,
GC_finalization_proc *ofn, void **ocd)
{
if (my_old_fn != 0) {
if (my_old_fn == OFN_UNSET) {
/* `GC_register_finalizer()` failed; `*ofn` and `*ocd` are unchanged. */
return;
}
if (my_old_fn != GC_debug_invoke_finalizer) {
GC_err_printf("Debuggable object at %p had a non-debug finalizer\n",
obj);
/* This should probably be fatal. */
} else {
if (ofn)
*ofn = my_old_cd->cl_fn;
if (ocd)
*ocd = my_old_cd->cl_data;
}
} else {
if (ofn)
*ofn = 0;
if (ocd)
*ocd = NULL;
}
}
GC_API void GC_CALL
GC_debug_register_finalizer(void *obj, GC_finalization_proc fn, void *cd,
GC_finalization_proc *ofn, void **ocd)
{
GC_finalization_proc my_old_fn = OFN_UNSET;
void *my_old_cd = NULL; /*< to avoid "might be uninitialized" warning */
ptr_t base = (ptr_t)GC_base(obj);
if (NULL == base) {
/* We will not collect it, hence finalizer would not be run. */
if (ocd)
*ocd = NULL;
if (ofn)
*ofn = 0;
return;
}
if ((ptr_t)obj - base != sizeof(oh)) {
GC_err_printf("GC_debug_register_finalizer called with"
" non-base-pointer %p\n",
obj);
}
if (0 == fn) {
GC_register_finalizer(base, 0, NULL, &my_old_fn, &my_old_cd);
} else {
cd = GC_make_closure(fn, cd);
if (NULL == cd) {
/* Out of memory; `*ofn` and `*ocd` are unchanged. */
return;
}
GC_register_finalizer(base, GC_debug_invoke_finalizer, cd, &my_old_fn,
&my_old_cd);
}
store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd);
}
GC_API void GC_CALL
GC_debug_register_finalizer_no_order(void *obj, GC_finalization_proc fn,
void *cd, GC_finalization_proc *ofn,
void **ocd)
{
GC_finalization_proc my_old_fn = OFN_UNSET;
void *my_old_cd = NULL;
ptr_t base = (ptr_t)GC_base(obj);
if (NULL == base) {
if (ocd)
*ocd = NULL;
if (ofn)
*ofn = 0;
return;
}
if ((ptr_t)obj - base != sizeof(oh)) {
GC_err_printf("GC_debug_register_finalizer_no_order called with"
" non-base-pointer %p\n",
obj);
}
if (0 == fn) {
GC_register_finalizer_no_order(base, 0, NULL, &my_old_fn, &my_old_cd);
} else {
cd = GC_make_closure(fn, cd);
if (NULL == cd) {
/* Out of memory. */
return;
}
GC_register_finalizer_no_order(base, GC_debug_invoke_finalizer, cd,
&my_old_fn, &my_old_cd);
}
store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd);
}
GC_API void GC_CALL
GC_debug_register_finalizer_unreachable(void *obj, GC_finalization_proc fn,
void *cd, GC_finalization_proc *ofn,
void **ocd)
{
GC_finalization_proc my_old_fn = OFN_UNSET;
void *my_old_cd = NULL;
ptr_t base = (ptr_t)GC_base(obj);
if (NULL == base) {
if (ocd)
*ocd = NULL;
if (ofn)
*ofn = 0;
return;
}
if ((ptr_t)obj - base != sizeof(oh)) {
GC_err_printf("GC_debug_register_finalizer_unreachable called with"
" non-base-pointer %p\n",
obj);
}
if (0 == fn) {
GC_register_finalizer_unreachable(base, 0, NULL, &my_old_fn, &my_old_cd);
} else {
cd = GC_make_closure(fn, cd);
if (NULL == cd) {
/* Out of memory. */
return;
}
GC_register_finalizer_unreachable(base, GC_debug_invoke_finalizer, cd,
&my_old_fn, &my_old_cd);
}
store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd);
}
GC_API void GC_CALL
GC_debug_register_finalizer_ignore_self(void *obj, GC_finalization_proc fn,
void *cd, GC_finalization_proc *ofn,
void **ocd)
{
GC_finalization_proc my_old_fn = OFN_UNSET;
void *my_old_cd = NULL;
ptr_t base = (ptr_t)GC_base(obj);
if (NULL == base) {
if (ocd)
*ocd = NULL;
if (ofn)
*ofn = 0;
return;
}
if ((ptr_t)obj - base != sizeof(oh)) {
GC_err_printf("GC_debug_register_finalizer_ignore_self called with"
" non-base-pointer %p\n",
obj);
}
if (0 == fn) {
GC_register_finalizer_ignore_self(base, 0, NULL, &my_old_fn, &my_old_cd);
} else {
cd = GC_make_closure(fn, cd);
if (NULL == cd) {
/* Out of memory. */
return;
}
GC_register_finalizer_ignore_self(base, GC_debug_invoke_finalizer, cd,
&my_old_fn, &my_old_cd);
}
store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd);
}
# ifndef GC_TOGGLE_REFS_NOT_NEEDED
GC_API int GC_CALL
GC_debug_toggleref_add(void *obj, int is_strong_ref)
{
ptr_t base = (ptr_t)GC_base(obj);
if ((ptr_t)obj - base != sizeof(oh)) {
GC_err_printf("GC_debug_toggleref_add called with"
" non-base-pointer %p\n",
obj);
}
return GC_toggleref_add(base, is_strong_ref);
}
# endif /* !GC_TOGGLE_REFS_NOT_NEEDED */
#endif /* !GC_NO_FINALIZATION */
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_debug_malloc_replacement(size_t lb)
{
return GC_debug_malloc(lb, GC_DBG_EXTRAS);
}
GC_API void *GC_CALL
GC_debug_realloc_replacement(void *p, size_t lb)
{
return GC_debug_realloc(p, lb, GC_DBG_EXTRAS);
}
#ifdef GC_GCJ_SUPPORT
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_debug_gcj_malloc(size_t lb, const void *vtable_ptr, GC_EXTRA_PARAMS)
{
void *base, *result;
/* We are careful to avoid extra calls those could confuse the backtrace. */
LOCK();
/* A mechanism to invoke finalizers (same as in `GC_core_gcj_malloc`). */
if (GC_gc_no != GC_last_finalized_no) {
UNLOCK();
GC_notify_or_invoke_finalizers();
LOCK();
GC_last_finalized_no = GC_gc_no;
}
base = GC_generic_malloc_inner(SIZET_SAT_ADD(lb, DEBUG_BYTES),
GC_gcj_debug_kind, 0 /* `flags` */);
if (NULL == base) {
GC_oom_func oom_fn = GC_oom_fn;
UNLOCK();
GC_err_printf("GC_debug_gcj_malloc(%lu, %p) returning NULL (%s:%d)\n",
(unsigned long)lb, vtable_ptr, s, i);
return (*oom_fn)(lb);
}
*((const void **)((ptr_t)base + sizeof(oh))) = vtable_ptr;
if (!GC_debugging_initialized)
GC_start_debugging_inner();
result = GC_store_debug_info_inner(base, lb, s, i);
ADD_CALL_CHAIN(base, ra);
UNLOCK();
GC_dirty(result);
REACHABLE_AFTER_DIRTY(vtable_ptr);
return result;
}
#endif /* GC_GCJ_SUPPORT */
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1996 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
* Copyright (c) 2007 Free Software Foundation, Inc.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#ifndef GC_NO_FINALIZATION
# include "gc/javaxfc.h" /*< to get `GC_finalize_all()` as `extern "C"` */
/*
* Type of mark procedure used for marking from finalizable object.
* This procedure normally does not mark the object, only its descendants.
*/
typedef void (*finalization_mark_proc)(ptr_t /* `finalizable_obj_ptr` */);
# define HASH3(addr, size, log_size) \
((size_t)((ADDR(addr) >> 3) ^ (ADDR(addr) >> (3 + (log_size)))) \
& ((size) - (size_t)1))
# define HASH2(addr, log_size) HASH3(addr, (size_t)1 << (log_size), log_size)
struct hash_chain_entry {
GC_hidden_pointer hidden_key;
struct hash_chain_entry *next;
};
struct disappearing_link {
struct hash_chain_entry prolog;
# define dl_hidden_link prolog.hidden_key /*< field to be cleared */
# define dl_next(x) (struct disappearing_link *)((x)->prolog.next)
# define dl_set_next(x, y) \
(void)((x)->prolog.next = (struct hash_chain_entry *)(y))
GC_hidden_pointer dl_hidden_obj; /*< pointer to object base */
};
struct finalizable_object {
struct hash_chain_entry prolog;
/*
* Pointer to object base. No longer hidden once object is on
* `finalize_now` queue.
*/
# define fo_hidden_base prolog.hidden_key
# define fo_next(x) (struct finalizable_object *)((x)->prolog.next)
# define fo_set_next(x, y) ((x)->prolog.next = (struct hash_chain_entry *)(y))
GC_finalization_proc fo_fn; /*< the finalizer */
finalization_mark_proc fo_mark_proc; /*< mark-through procedure */
ptr_t fo_client_data;
size_t fo_object_sz; /*< in bytes */
};
# ifdef AO_HAVE_store
/*
* Update `finalize_now` atomically as `GC_should_invoke_finalizers`
* does not acquire the allocator lock.
*/
# define SET_FINALIZE_NOW(fo) \
GC_cptr_store((volatile ptr_t *)&GC_fnlz_roots.finalize_now, (ptr_t)(fo))
# else
# define SET_FINALIZE_NOW(fo) (void)(GC_fnlz_roots.finalize_now = (fo))
# endif /* !THREADS */
GC_API void GC_CALL
GC_push_finalizer_structures(void)
{
GC_ASSERT(ADDR(&GC_dl_hashtbl.head) % ALIGNMENT == 0);
GC_ASSERT(ADDR(&GC_fnlz_roots) % ALIGNMENT == 0);
# ifndef GC_LONG_REFS_NOT_NEEDED
GC_ASSERT(ADDR(&GC_ll_hashtbl.head) % ALIGNMENT == 0);
GC_PUSH_ALL_SYM(GC_ll_hashtbl.head);
# endif
GC_PUSH_ALL_SYM(GC_dl_hashtbl.head);
GC_PUSH_ALL_SYM(GC_fnlz_roots);
/* `GC_toggleref_arr` is pushed specially by `GC_mark_togglerefs`. */
}
/*
* Threshold of `log_size` to initiate full collection before growing
* a hash table.
*/
# ifndef GC_ON_GROW_LOG_SIZE_MIN
# define GC_ON_GROW_LOG_SIZE_MIN LOG_HBLKSIZE
# endif
/*
* Ensure the hash table has enough capacity. `*table_ptr` is a pointer
* to an array of hash headers. `*log_size_ptr` is the log of its current
* size. We update both `*table_ptr` and `*log_size_ptr` on success.
*/
STATIC void
GC_grow_table(struct hash_chain_entry ***table_ptr, unsigned *log_size_ptr,
const size_t *entries_ptr)
{
size_t i;
struct hash_chain_entry *p;
unsigned log_old_size = *log_size_ptr;
unsigned log_new_size = log_old_size + 1;
size_t old_size = NULL == *table_ptr ? 0 : (size_t)1 << log_old_size;
size_t new_size = (size_t)1 << log_new_size;
/* FIXME: Power-of-two size often gets rounded up to one more page. */
struct hash_chain_entry **new_table;
GC_ASSERT(I_HOLD_LOCK());
/*
* Avoid growing the table in case of at least 25% of entries can
* be deleted by enforcing a collection. Ignored for small tables.
* In the incremental mode we skip this optimization, as we want to
* avoid triggering a full collection whenever possible.
*/
if (log_old_size >= (unsigned)GC_ON_GROW_LOG_SIZE_MIN && !GC_incremental) {
IF_CANCEL(int cancel_state;)
DISABLE_CANCEL(cancel_state);
GC_gcollect_inner();
RESTORE_CANCEL(cancel_state);
/* `GC_finalize` might decrease entries value. */
if (*entries_ptr < ((size_t)1 << log_old_size) - (*entries_ptr >> 2))
return;
}
new_table = (struct hash_chain_entry **)GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(
new_size * sizeof(struct hash_chain_entry *), NORMAL);
if (NULL == new_table) {
if (NULL == *table_ptr) {
ABORT("Insufficient space for initial table allocation");
} else {
return;
}
}
for (i = 0; i < old_size; i++) {
for (p = (*table_ptr)[i]; p != NULL;) {
ptr_t real_key = (ptr_t)GC_REVEAL_POINTER(p->hidden_key);
struct hash_chain_entry *next = p->next;
size_t new_hash = HASH3(real_key, new_size, log_new_size);
p->next = new_table[new_hash];
GC_dirty(p);
new_table[new_hash] = p;
p = next;
}
}
*log_size_ptr = log_new_size;
*table_ptr = new_table;
GC_dirty(new_table); /*< entire object */
}
GC_API int GC_CALL
GC_register_disappearing_link(void **link)
{
ptr_t base;
base = (ptr_t)GC_base(link);
if (base == 0)
ABORT("Bad arg to GC_register_disappearing_link");
return GC_general_register_disappearing_link(link, base);
}
STATIC int
GC_register_disappearing_link_inner(struct dl_hashtbl_s *dl_hashtbl,
void **link, const void *obj,
const char *tbl_log_name)
{
struct disappearing_link *curr_dl;
size_t index;
struct disappearing_link *new_dl;
GC_ASSERT(GC_is_initialized);
if (UNLIKELY(GC_find_leak_inner))
return GC_UNIMPLEMENTED;
# ifdef GC_ASSERTIONS
/* Just check accessibility. */
GC_noop1_ptr(*link);
# endif
LOCK();
GC_ASSERT(obj != NULL && GC_base_C(obj) == obj);
if (UNLIKELY(NULL == dl_hashtbl->head)
|| UNLIKELY(dl_hashtbl->entries > ((size_t)1 << dl_hashtbl->log_size))) {
GC_grow_table((struct hash_chain_entry ***)&dl_hashtbl->head,
&dl_hashtbl->log_size, &dl_hashtbl->entries);
GC_COND_LOG_PRINTF("Grew %s table to %u entries\n", tbl_log_name,
1U << dl_hashtbl->log_size);
}
index = HASH2(link, dl_hashtbl->log_size);
for (curr_dl = dl_hashtbl->head[index]; curr_dl != 0;
curr_dl = dl_next(curr_dl)) {
if (curr_dl->dl_hidden_link == GC_HIDE_POINTER(link)) {
/* Alternatively, `GC_HIDE_NZ_POINTER()` could be used instead. */
curr_dl->dl_hidden_obj = GC_HIDE_POINTER(obj);
UNLOCK();
return GC_DUPLICATE;
}
}
new_dl = (struct disappearing_link *)GC_INTERNAL_MALLOC(
sizeof(struct disappearing_link), NORMAL);
if (UNLIKELY(NULL == new_dl)) {
GC_oom_func oom_fn = GC_oom_fn;
UNLOCK();
new_dl = (struct disappearing_link *)(*oom_fn)(
sizeof(struct disappearing_link));
if (0 == new_dl) {
return GC_NO_MEMORY;
}
/* It is not likely we will make it here, but... */
LOCK();
/* Recalculate `index` since the table may grow. */
index = HASH2(link, dl_hashtbl->log_size);
/* Check again that our disappearing link not in the table. */
for (curr_dl = dl_hashtbl->head[index]; curr_dl != 0;
curr_dl = dl_next(curr_dl)) {
if (curr_dl->dl_hidden_link == GC_HIDE_POINTER(link)) {
curr_dl->dl_hidden_obj = GC_HIDE_POINTER(obj);
UNLOCK();
# ifndef DBG_HDRS_ALL
/* Free unused `new_dl` returned by `GC_oom_fn()`. */
GC_free(new_dl);
# endif
return GC_DUPLICATE;
}
}
}
new_dl->dl_hidden_obj = GC_HIDE_POINTER(obj);
new_dl->dl_hidden_link = GC_HIDE_POINTER(link);
dl_set_next(new_dl, dl_hashtbl->head[index]);
GC_dirty(new_dl);
dl_hashtbl->head[index] = new_dl;
dl_hashtbl->entries++;
GC_dirty(dl_hashtbl->head + index);
UNLOCK();
return GC_SUCCESS;
}
GC_API int GC_CALL
GC_general_register_disappearing_link(void **link, const void *obj)
{
if ((ADDR(link) & (ALIGNMENT - 1)) != 0 || !NONNULL_ARG_NOT_NULL(link))
ABORT("Bad arg to GC_general_register_disappearing_link");
return GC_register_disappearing_link_inner(&GC_dl_hashtbl, link, obj, "dl");
}
# ifdef DBG_HDRS_ALL
# define FREE_DL_ENTRY(curr_dl) dl_set_next(curr_dl, NULL)
# else
# define FREE_DL_ENTRY(curr_dl) GC_free(curr_dl)
# endif
/* Unregisters given `link` and returns the link entry to free. */
GC_INLINE struct disappearing_link *
GC_unregister_disappearing_link_inner(struct dl_hashtbl_s *dl_hashtbl,
void **link)
{
struct disappearing_link *curr_dl;
struct disappearing_link *prev_dl = NULL;
size_t index;
GC_ASSERT(I_HOLD_LOCK());
if (UNLIKELY(NULL == dl_hashtbl->head))
return NULL;
index = HASH2(link, dl_hashtbl->log_size);
for (curr_dl = dl_hashtbl->head[index]; curr_dl;
curr_dl = dl_next(curr_dl)) {
if (curr_dl->dl_hidden_link == GC_HIDE_POINTER(link)) {
/* Remove found entry from the table. */
if (NULL == prev_dl) {
dl_hashtbl->head[index] = dl_next(curr_dl);
GC_dirty(dl_hashtbl->head + index);
} else {
dl_set_next(prev_dl, dl_next(curr_dl));
GC_dirty(prev_dl);
}
dl_hashtbl->entries--;
break;
}
prev_dl = curr_dl;
}
return curr_dl;
}
GC_API int GC_CALL
GC_unregister_disappearing_link(void **link)
{
struct disappearing_link *curr_dl;
if ((ADDR(link) & (ALIGNMENT - 1)) != 0) {
/* Nothing to do. */
return 0;
}
LOCK();
curr_dl = GC_unregister_disappearing_link_inner(&GC_dl_hashtbl, link);
UNLOCK();
if (NULL == curr_dl)
return 0;
FREE_DL_ENTRY(curr_dl);
return 1;
}
/*
* Mark from one finalizable object using the specified mark procedure.
* May not mark the object pointed to by `real_ptr` (i.e, it is the job
* of the caller, if appropriate). Note that this is called with the
* mutator running. This is safe only if the mutator (client) gets
* the allocator lock to reveal hidden pointers.
*/
GC_INLINE void
GC_mark_fo(ptr_t real_ptr, finalization_mark_proc fo_mark_proc)
{
GC_ASSERT(I_HOLD_LOCK());
fo_mark_proc(real_ptr);
/* Process objects pushed by the mark procedure. */
while (!GC_mark_stack_empty())
MARK_FROM_MARK_STACK();
}
/* Complete a collection in progress, if any. */
GC_INLINE void
GC_complete_ongoing_collection(void)
{
if (UNLIKELY(GC_collection_in_progress())) {
while (!GC_mark_some(NULL)) {
/* Empty. */
}
}
}
/* Toggle-refs support. */
# ifndef GC_TOGGLE_REFS_NOT_NEEDED
typedef union toggle_ref_u GCToggleRef;
STATIC GC_toggleref_func GC_toggleref_callback = 0;
GC_INNER void
GC_process_togglerefs(void)
{
size_t i;
size_t new_size = 0;
GC_bool needs_barrier = FALSE;
GC_ASSERT(I_HOLD_LOCK());
for (i = 0; i < GC_toggleref_array_size; ++i) {
GCToggleRef *r = &GC_toggleref_arr[i];
void *obj = r->strong_ref;
if ((ADDR(obj) & 1) != 0) {
obj = GC_REVEAL_POINTER(r->weak_ref);
GC_ASSERT((ADDR(obj) & 1) == 0);
}
if (NULL == obj)
continue;
switch (GC_toggleref_callback(obj)) {
case GC_TOGGLE_REF_DROP:
break;
case GC_TOGGLE_REF_STRONG:
GC_toggleref_arr[new_size++].strong_ref = obj;
needs_barrier = TRUE;
break;
case GC_TOGGLE_REF_WEAK:
GC_toggleref_arr[new_size++].weak_ref = GC_HIDE_POINTER(obj);
break;
default:
ABORT("Bad toggle-ref status returned by callback");
}
}
if (new_size < GC_toggleref_array_size) {
BZERO(&GC_toggleref_arr[new_size],
(GC_toggleref_array_size - new_size) * sizeof(GCToggleRef));
GC_toggleref_array_size = new_size;
}
if (needs_barrier)
GC_dirty(GC_toggleref_arr); /*< entire object */
}
STATIC void GC_normal_finalize_mark_proc(ptr_t);
STATIC void
GC_mark_togglerefs(void)
{
size_t i;
GC_ASSERT(I_HOLD_LOCK());
if (NULL == GC_toggleref_arr)
return;
GC_set_mark_bit(GC_toggleref_arr);
for (i = 0; i < GC_toggleref_array_size; ++i) {
void *obj = GC_toggleref_arr[i].strong_ref;
if (obj != NULL && (ADDR(obj) & 1) == 0) {
/* Push and mark the object. */
GC_mark_fo((ptr_t)obj, GC_normal_finalize_mark_proc);
GC_set_mark_bit(obj);
GC_complete_ongoing_collection();
}
}
}
STATIC void
GC_clear_togglerefs(void)
{
size_t i;
GC_ASSERT(I_HOLD_LOCK());
for (i = 0; i < GC_toggleref_array_size; ++i) {
GCToggleRef *r = &GC_toggleref_arr[i];
if ((ADDR(r->strong_ref) & 1) != 0) {
if (!GC_is_marked(GC_REVEAL_POINTER(r->weak_ref))) {
r->weak_ref = 0;
} else {
/* No need to copy, this garbage collector is a non-moving one. */
}
}
}
}
GC_API void GC_CALL
GC_set_toggleref_func(GC_toggleref_func fn)
{
LOCK();
GC_toggleref_callback = fn;
UNLOCK();
}
GC_API GC_toggleref_func GC_CALL
GC_get_toggleref_func(void)
{
GC_toggleref_func fn;
READER_LOCK();
fn = GC_toggleref_callback;
READER_UNLOCK();
return fn;
}
static GC_bool
ensure_toggleref_capacity(size_t capacity_inc)
{
GC_ASSERT(I_HOLD_LOCK());
if (NULL == GC_toggleref_arr) {
/* Set the initial capacity. */
GC_toggleref_array_capacity = 32;
GC_toggleref_arr = (GCToggleRef *)GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(
GC_toggleref_array_capacity * sizeof(GCToggleRef), NORMAL);
if (NULL == GC_toggleref_arr)
return FALSE;
}
if (GC_toggleref_array_size + capacity_inc >= GC_toggleref_array_capacity) {
GCToggleRef *new_array;
while (GC_toggleref_array_capacity
< GC_toggleref_array_size + capacity_inc) {
GC_toggleref_array_capacity *= 2;
if ((GC_toggleref_array_capacity
& ((size_t)1 << (sizeof(size_t) * 8 - 1)))
!= 0) {
/* An overflow. */
return FALSE;
}
}
new_array = (GCToggleRef *)GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(
GC_toggleref_array_capacity * sizeof(GCToggleRef), NORMAL);
if (UNLIKELY(NULL == new_array))
return FALSE;
if (LIKELY(GC_toggleref_array_size > 0))
BCOPY(GC_toggleref_arr, new_array,
GC_toggleref_array_size * sizeof(GCToggleRef));
GC_INTERNAL_FREE(GC_toggleref_arr);
GC_toggleref_arr = new_array;
}
return TRUE;
}
GC_API int GC_CALL
GC_toggleref_add(void *obj, int is_strong_ref)
{
int res = GC_SUCCESS;
GC_ASSERT(NONNULL_ARG_NOT_NULL(obj));
LOCK();
GC_ASSERT((ADDR(obj) & 1) == 0 && obj == GC_base(obj));
if (GC_toggleref_callback != 0) {
if (!ensure_toggleref_capacity(1)) {
res = GC_NO_MEMORY;
} else {
GCToggleRef *r = &GC_toggleref_arr[GC_toggleref_array_size];
if (is_strong_ref) {
r->strong_ref = obj;
GC_dirty(GC_toggleref_arr + GC_toggleref_array_size);
} else {
r->weak_ref = GC_HIDE_POINTER(obj);
GC_ASSERT((r->weak_ref & 1) != 0);
}
GC_toggleref_array_size++;
}
}
UNLOCK();
return res;
}
# endif /* !GC_TOGGLE_REFS_NOT_NEEDED */
/* Finalizer callback support. */
STATIC GC_await_finalize_proc GC_object_finalized_proc = 0;
GC_API void GC_CALL
GC_set_await_finalize_proc(GC_await_finalize_proc fn)
{
LOCK();
GC_object_finalized_proc = fn;
UNLOCK();
}
GC_API GC_await_finalize_proc GC_CALL
GC_get_await_finalize_proc(void)
{
GC_await_finalize_proc fn;
READER_LOCK();
fn = GC_object_finalized_proc;
READER_UNLOCK();
return fn;
}
# ifndef GC_LONG_REFS_NOT_NEEDED
GC_API int GC_CALL
GC_register_long_link(void **link, const void *obj)
{
if ((ADDR(link) & (ALIGNMENT - 1)) != 0 || !NONNULL_ARG_NOT_NULL(link))
ABORT("Bad arg to GC_register_long_link");
return GC_register_disappearing_link_inner(&GC_ll_hashtbl, link, obj,
"long dl");
}
GC_API int GC_CALL
GC_unregister_long_link(void **link)
{
struct disappearing_link *curr_dl;
if ((ADDR(link) & (ALIGNMENT - 1)) != 0) {
/* Nothing to do. */
return 0;
}
LOCK();
curr_dl = GC_unregister_disappearing_link_inner(&GC_ll_hashtbl, link);
UNLOCK();
if (NULL == curr_dl)
return 0;
FREE_DL_ENTRY(curr_dl);
return 1;
}
# endif /* !GC_LONG_REFS_NOT_NEEDED */
# ifndef GC_MOVE_DISAPPEARING_LINK_NOT_NEEDED
STATIC int
GC_move_disappearing_link_inner(struct dl_hashtbl_s *dl_hashtbl, void **link,
void **new_link)
{
struct disappearing_link *curr_dl, *new_dl;
struct disappearing_link *prev_dl = NULL;
size_t curr_index, new_index;
GC_hidden_pointer curr_hidden_link, new_hidden_link;
# ifdef GC_ASSERTIONS
GC_noop1_ptr(*new_link);
# endif
GC_ASSERT(I_HOLD_LOCK());
if (UNLIKELY(NULL == dl_hashtbl->head))
return GC_NOT_FOUND;
/* Find current link. */
curr_index = HASH2(link, dl_hashtbl->log_size);
curr_hidden_link = GC_HIDE_POINTER(link);
for (curr_dl = dl_hashtbl->head[curr_index]; curr_dl;
curr_dl = dl_next(curr_dl)) {
if (curr_dl->dl_hidden_link == curr_hidden_link)
break;
prev_dl = curr_dl;
}
if (UNLIKELY(NULL == curr_dl)) {
return GC_NOT_FOUND;
} else if (link == new_link) {
/* Nothing to do. */
return GC_SUCCESS;
}
/* `link` is found; now check `new_link` is not present. */
new_index = HASH2(new_link, dl_hashtbl->log_size);
new_hidden_link = GC_HIDE_POINTER(new_link);
for (new_dl = dl_hashtbl->head[new_index]; new_dl;
new_dl = dl_next(new_dl)) {
if (new_dl->dl_hidden_link == new_hidden_link) {
/* Target already registered; bail out. */
return GC_DUPLICATE;
}
}
/* Remove from old, add to new, update `link`. */
if (NULL == prev_dl) {
dl_hashtbl->head[curr_index] = dl_next(curr_dl);
} else {
dl_set_next(prev_dl, dl_next(curr_dl));
GC_dirty(prev_dl);
}
curr_dl->dl_hidden_link = new_hidden_link;
dl_set_next(curr_dl, dl_hashtbl->head[new_index]);
dl_hashtbl->head[new_index] = curr_dl;
GC_dirty(curr_dl);
GC_dirty(dl_hashtbl->head); /*< entire object */
return GC_SUCCESS;
}
GC_API int GC_CALL
GC_move_disappearing_link(void **link, void **new_link)
{
int result;
if ((ADDR(new_link) & (ALIGNMENT - 1)) != 0
|| !NONNULL_ARG_NOT_NULL(new_link))
ABORT("Bad new_link arg to GC_move_disappearing_link");
if ((ADDR(link) & (ALIGNMENT - 1)) != 0) {
/* Nothing to do. */
return GC_NOT_FOUND;
}
LOCK();
result = GC_move_disappearing_link_inner(&GC_dl_hashtbl, link, new_link);
UNLOCK();
return result;
}
# ifndef GC_LONG_REFS_NOT_NEEDED
GC_API int GC_CALL
GC_move_long_link(void **link, void **new_link)
{
int result;
if ((ADDR(new_link) & (ALIGNMENT - 1)) != 0
|| !NONNULL_ARG_NOT_NULL(new_link))
ABORT("Bad new_link arg to GC_move_long_link");
if ((ADDR(link) & (ALIGNMENT - 1)) != 0) {
/* Nothing to do. */
return GC_NOT_FOUND;
}
LOCK();
result = GC_move_disappearing_link_inner(&GC_ll_hashtbl, link, new_link);
UNLOCK();
return result;
}
# endif
# endif /* !GC_MOVE_DISAPPEARING_LINK_NOT_NEEDED */
/*
* Various finalization marker procedures. Note that mark stack overflow
* is handled by the caller, and is not a disaster.
*/
# if defined(_MSC_VER) && defined(I386)
GC_ATTR_NOINLINE
/* Otherwise some optimizer bug is tickled in VC for x86 (v19, at least). */
# endif
STATIC void
GC_normal_finalize_mark_proc(ptr_t p)
{
GC_mark_stack_top = GC_push_obj(p, HDR(p), GC_mark_stack_top,
GC_mark_stack + GC_mark_stack_size);
}
/*
* This only pays very partial attention to the mark descriptor.
* It does the right thing for normal and atomic objects, and treats
* most others as normal.
*/
STATIC void
GC_ignore_self_finalize_mark_proc(ptr_t p)
{
const hdr *hhdr = HDR(p);
word descr = hhdr->hb_descr;
ptr_t current_p;
ptr_t scan_limit;
ptr_t target_limit = p + hhdr->hb_sz - 1;
if ((descr & GC_DS_TAGS) == GC_DS_LENGTH) {
scan_limit = p + descr - sizeof(ptr_t);
} else {
scan_limit = target_limit + 1 - sizeof(ptr_t);
}
for (current_p = p; ADDR_GE(scan_limit, current_p); current_p += ALIGNMENT) {
ptr_t q;
LOAD_PTR_OR_CONTINUE(q, current_p);
if (ADDR_LT(q, p) || ADDR_LT(target_limit, q)) {
GC_PUSH_ONE_HEAP(q, current_p, GC_mark_stack_top);
}
}
}
STATIC void
GC_null_finalize_mark_proc(ptr_t p)
{
UNUSED_ARG(p);
}
/*
* `GC_unreachable_finalize_mark_proc` is an alias for normal marking,
* but it is explicitly tested for, and triggers different behavior.
* Objects registered in this way are not finalized if they are reachable
* by other finalizable objects, even if those other objects specify
* no ordering.
*/
STATIC void
GC_unreachable_finalize_mark_proc(ptr_t p)
{
/*
* A dummy comparison to ensure the compiler not to optimize two
* identical functions into a single one (thus, to ensure a unique
* address of each). Alternatively, `GC_noop1_ptr(p)` could be used.
*/
if (UNLIKELY(NULL == p))
return;
GC_normal_finalize_mark_proc(p);
}
/*
* Register a finalization function. See `gc.h` file for details.
* The last parameter is a procedure that determines marking for
* finalization ordering. Any objects marked by that procedure will be
* guaranteed to not have been finalized when this finalizer is invoked.
*/
STATIC void
GC_register_finalizer_inner(void *obj, GC_finalization_proc fn, void *cd,
GC_finalization_proc *ofn, void **ocd,
finalization_mark_proc mp)
{
struct finalizable_object *curr_fo;
size_t index;
struct finalizable_object *new_fo = 0;
const hdr *hhdr = NULL; /*< initialized to prevent warning */
GC_ASSERT(GC_is_initialized);
if (UNLIKELY(GC_find_leak_inner)) {
/* No-op. `*ocd` and `*ofn` remain unchanged. */
return;
}
LOCK();
GC_ASSERT(obj != NULL && GC_base_C(obj) == obj);
if (mp == GC_unreachable_finalize_mark_proc)
GC_need_unreachable_finalization = TRUE;
if (UNLIKELY(NULL == GC_fnlz_roots.fo_head)
|| UNLIKELY(GC_fo_entries > ((size_t)1 << GC_log_fo_table_size))) {
GC_grow_table((struct hash_chain_entry ***)&GC_fnlz_roots.fo_head,
&GC_log_fo_table_size, &GC_fo_entries);
GC_COND_LOG_PRINTF("Grew fo table to %u entries\n",
1U << GC_log_fo_table_size);
}
for (;;) {
struct finalizable_object *prev_fo = NULL;
GC_oom_func oom_fn;
index = HASH2(obj, GC_log_fo_table_size);
curr_fo = GC_fnlz_roots.fo_head[index];
while (curr_fo != NULL) {
GC_ASSERT(GC_size(curr_fo) >= sizeof(struct finalizable_object));
if (curr_fo->fo_hidden_base == GC_HIDE_POINTER(obj)) {
/*
* Interruption by a signal in the middle of this should be safe.
* The client may see only `*ocd` updated, but we will declare that
* to be his problem.
*/
if (ocd)
*ocd = curr_fo->fo_client_data;
if (ofn)
*ofn = curr_fo->fo_fn;
/* Delete the structure for `obj`. */
if (prev_fo == 0) {
GC_fnlz_roots.fo_head[index] = fo_next(curr_fo);
} else {
fo_set_next(prev_fo, fo_next(curr_fo));
GC_dirty(prev_fo);
}
if (fn == 0) {
GC_fo_entries--;
/*
* May not happen if we get a signal. But a high estimate will
* only make the table larger than necessary.
*/
# if !defined(THREADS) && !defined(DBG_HDRS_ALL)
GC_free(curr_fo);
# endif
} else {
curr_fo->fo_fn = fn;
curr_fo->fo_client_data = (ptr_t)cd;
curr_fo->fo_mark_proc = mp;
GC_dirty(curr_fo);
/*
* Reinsert it. We deleted it first to maintain consistency in
* the event of a signal.
*/
if (prev_fo == 0) {
GC_fnlz_roots.fo_head[index] = curr_fo;
} else {
fo_set_next(prev_fo, curr_fo);
GC_dirty(prev_fo);
}
}
if (NULL == prev_fo)
GC_dirty(GC_fnlz_roots.fo_head + index);
UNLOCK();
# ifndef DBG_HDRS_ALL
/* Free unused `new_fo` returned by `GC_oom_fn()`. */
GC_free(new_fo);
# endif
return;
}
prev_fo = curr_fo;
curr_fo = fo_next(curr_fo);
}
if (UNLIKELY(new_fo != 0)) {
/* `new_fo` is returned by `GC_oom_fn()`. */
GC_ASSERT(fn != 0);
# ifdef LINT2
if (NULL == hhdr)
ABORT("Bad hhdr in GC_register_finalizer_inner");
# endif
break;
}
if (fn == 0) {
if (ocd)
*ocd = 0;
if (ofn)
*ofn = 0;
UNLOCK();
return;
}
GET_HDR(obj, hhdr);
if (UNLIKELY(NULL == hhdr)) {
/* We will not collect it, hence finalizer would not be run. */
if (ocd)
*ocd = 0;
if (ofn)
*ofn = 0;
UNLOCK();
return;
}
new_fo = (struct finalizable_object *)GC_INTERNAL_MALLOC(
sizeof(struct finalizable_object), NORMAL);
if (LIKELY(new_fo != 0))
break;
oom_fn = GC_oom_fn;
UNLOCK();
new_fo = (struct finalizable_object *)(*oom_fn)(
sizeof(struct finalizable_object));
if (0 == new_fo) {
/* No enough memory. `*ocd` and `*ofn` remain unchanged. */
return;
}
/* It is not likely we will make it here, but... */
LOCK();
/*
* Recalculate index since the table may grow and check again that
* our finalizer is not in the table.
*/
}
GC_ASSERT(GC_size(new_fo) >= sizeof(struct finalizable_object));
if (ocd)
*ocd = 0;
if (ofn)
*ofn = 0;
new_fo->fo_hidden_base = GC_HIDE_POINTER(obj);
new_fo->fo_fn = fn;
new_fo->fo_client_data = (ptr_t)cd;
new_fo->fo_object_sz = hhdr->hb_sz;
new_fo->fo_mark_proc = mp;
fo_set_next(new_fo, GC_fnlz_roots.fo_head[index]);
GC_dirty(new_fo);
GC_fo_entries++;
GC_fnlz_roots.fo_head[index] = new_fo;
GC_dirty(GC_fnlz_roots.fo_head + index);
UNLOCK();
}
GC_API void GC_CALL
GC_register_finalizer(void *obj, GC_finalization_proc fn, void *cd,
GC_finalization_proc *ofn, void **ocd)
{
GC_register_finalizer_inner(obj, fn, cd, ofn, ocd,
GC_normal_finalize_mark_proc);
}
GC_API void GC_CALL
GC_register_finalizer_ignore_self(void *obj, GC_finalization_proc fn, void *cd,
GC_finalization_proc *ofn, void **ocd)
{
GC_register_finalizer_inner(obj, fn, cd, ofn, ocd,
GC_ignore_self_finalize_mark_proc);
}
GC_API void GC_CALL
GC_register_finalizer_no_order(void *obj, GC_finalization_proc fn, void *cd,
GC_finalization_proc *ofn, void **ocd)
{
GC_register_finalizer_inner(obj, fn, cd, ofn, ocd,
GC_null_finalize_mark_proc);
}
GC_API void GC_CALL
GC_register_finalizer_unreachable(void *obj, GC_finalization_proc fn, void *cd,
GC_finalization_proc *ofn, void **ocd)
{
GC_ASSERT(GC_java_finalization);
GC_register_finalizer_inner(obj, fn, cd, ofn, ocd,
GC_unreachable_finalize_mark_proc);
}
# ifndef NO_DEBUGGING
STATIC void
GC_dump_finalization_links(const struct dl_hashtbl_s *dl_hashtbl)
{
size_t dl_size = (size_t)1 << dl_hashtbl->log_size;
size_t i;
if (NULL == dl_hashtbl->head) {
/* The table is empty. */
return;
}
for (i = 0; i < dl_size; i++) {
struct disappearing_link *curr_dl;
for (curr_dl = dl_hashtbl->head[i]; curr_dl != 0;
curr_dl = dl_next(curr_dl)) {
ptr_t real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_dl->dl_hidden_obj);
ptr_t real_link = (ptr_t)GC_REVEAL_POINTER(curr_dl->dl_hidden_link);
GC_printf("Object: %p, link value: %p, link addr: %p\n",
(void *)real_ptr, *(void **)real_link, (void *)real_link);
}
}
}
GC_API void GC_CALL
GC_dump_finalization(void)
{
struct finalizable_object *curr_fo;
size_t i;
size_t fo_size
= GC_fnlz_roots.fo_head == NULL ? 0 : (size_t)1 << GC_log_fo_table_size;
GC_printf("\n***Disappearing (short) links:\n");
GC_dump_finalization_links(&GC_dl_hashtbl);
# ifndef GC_LONG_REFS_NOT_NEEDED
GC_printf("\n***Disappearing long links:\n");
GC_dump_finalization_links(&GC_ll_hashtbl);
# endif
GC_printf("\n***Finalizers:\n");
for (i = 0; i < fo_size; i++) {
for (curr_fo = GC_fnlz_roots.fo_head[i]; curr_fo != NULL;
curr_fo = fo_next(curr_fo)) {
ptr_t real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base);
GC_printf("Finalizable object: %p\n", (void *)real_ptr);
}
}
}
# endif /* !NO_DEBUGGING */
# ifndef THREADS
/*
* Checks and updates the level of finalizers recursion.
* Returns `NULL` if `GC_invoke_finalizers()` should not be called by
* the collector (to minimize the risk of a deep finalizers recursion),
* otherwise returns a pointer to `GC_finalizer_nested`.
*/
STATIC unsigned char *
GC_check_finalizer_nested(void)
{
unsigned nesting_level = GC_finalizer_nested;
if (nesting_level) {
/*
* We are inside another `GC_invoke_finalizers()`. Skip some
* implicitly-called `GC_invoke_finalizers()` depending on the
* nesting (recursion) level.
*/
if ((unsigned)(++GC_finalizer_skipped) < (1U << nesting_level))
return NULL;
GC_finalizer_skipped = 0;
}
GC_finalizer_nested = (unsigned char)(nesting_level + 1);
return &GC_finalizer_nested;
}
# endif /* !THREADS */
GC_INLINE void
GC_make_disappearing_links_disappear(struct dl_hashtbl_s *dl_hashtbl,
GC_bool is_remove_dangling)
{
size_t i;
size_t dl_size = (size_t)1 << dl_hashtbl->log_size;
GC_bool needs_barrier = FALSE;
GC_ASSERT(I_HOLD_LOCK());
if (NULL == dl_hashtbl->head) {
/* The table is empty. */
return;
}
for (i = 0; i < dl_size; i++) {
struct disappearing_link *curr_dl, *next_dl;
struct disappearing_link *prev_dl = NULL;
for (curr_dl = dl_hashtbl->head[i]; curr_dl != NULL; curr_dl = next_dl) {
next_dl = dl_next(curr_dl);
# if defined(GC_ASSERTIONS) && !defined(THREAD_SANITIZER)
/* Check accessibility of the location pointed by the link. */
GC_noop1_ptr(*(ptr_t *)GC_REVEAL_POINTER(curr_dl->dl_hidden_link));
# endif
if (is_remove_dangling) {
ptr_t real_link
= (ptr_t)GC_base(GC_REVEAL_POINTER(curr_dl->dl_hidden_link));
if (NULL == real_link || LIKELY(GC_is_marked(real_link))) {
prev_dl = curr_dl;
continue;
}
} else {
if (LIKELY(GC_is_marked(
(ptr_t)GC_REVEAL_POINTER(curr_dl->dl_hidden_obj)))) {
prev_dl = curr_dl;
continue;
}
*(ptr_t *)GC_REVEAL_POINTER(curr_dl->dl_hidden_link) = NULL;
}
/* Delete `curr_dl` entry from `dl_hashtbl`. */
if (NULL == prev_dl) {
dl_hashtbl->head[i] = next_dl;
needs_barrier = TRUE;
} else {
dl_set_next(prev_dl, next_dl);
GC_dirty(prev_dl);
}
GC_clear_mark_bit(curr_dl);
dl_hashtbl->entries--;
}
}
if (needs_barrier)
GC_dirty(dl_hashtbl->head); /*< entire object */
}
GC_INNER void
GC_finalize(void)
{
struct finalizable_object *curr_fo, *prev_fo, *next_fo;
ptr_t real_ptr;
size_t i;
size_t fo_size
= GC_fnlz_roots.fo_head == NULL ? 0 : (size_t)1 << GC_log_fo_table_size;
GC_bool needs_barrier = FALSE;
GC_ASSERT(I_HOLD_LOCK());
# ifndef SMALL_CONFIG
/* Save current `GC_dl_entries` value for stats printing. */
GC_old_dl_entries = GC_dl_hashtbl.entries;
# ifndef GC_LONG_REFS_NOT_NEEDED
/* Save current `GC_ll_entries` value for stats printing. */
GC_old_ll_entries = GC_ll_hashtbl.entries;
# endif
# endif
# ifndef GC_TOGGLE_REFS_NOT_NEEDED
GC_mark_togglerefs();
# endif
GC_make_disappearing_links_disappear(&GC_dl_hashtbl, FALSE);
/*
* Mark all objects reachable via chains of 1 or more pointers from
* finalizable objects.
*/
GC_ASSERT(!GC_collection_in_progress());
for (i = 0; i < fo_size; i++) {
for (curr_fo = GC_fnlz_roots.fo_head[i]; curr_fo != NULL;
curr_fo = fo_next(curr_fo)) {
GC_ASSERT(GC_size(curr_fo) >= sizeof(struct finalizable_object));
real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base);
if (!GC_is_marked(real_ptr)) {
GC_MARKED_FOR_FINALIZATION(real_ptr);
GC_mark_fo(real_ptr, curr_fo->fo_mark_proc);
if (GC_is_marked(real_ptr)) {
WARN("Finalization cycle involving %p\n", real_ptr);
}
}
}
}
/* Enqueue for finalization all objects that are still unreachable. */
GC_bytes_finalized = 0;
for (i = 0; i < fo_size; i++) {
curr_fo = GC_fnlz_roots.fo_head[i];
prev_fo = NULL;
while (curr_fo != NULL) {
real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base);
if (!GC_is_marked(real_ptr)) {
if (!GC_java_finalization) {
GC_set_mark_bit(real_ptr);
}
/* Delete from hash table. */
next_fo = fo_next(curr_fo);
if (NULL == prev_fo) {
GC_fnlz_roots.fo_head[i] = next_fo;
if (GC_object_finalized_proc) {
GC_dirty(GC_fnlz_roots.fo_head + i);
} else {
needs_barrier = TRUE;
}
} else {
fo_set_next(prev_fo, next_fo);
GC_dirty(prev_fo);
}
GC_fo_entries--;
if (GC_object_finalized_proc)
GC_object_finalized_proc(real_ptr);
/* Add to list of objects awaiting finalization. */
fo_set_next(curr_fo, GC_fnlz_roots.finalize_now);
GC_dirty(curr_fo);
SET_FINALIZE_NOW(curr_fo);
/* Unhide object pointer so any future collections will see it. */
curr_fo->fo_hidden_base
= (GC_hidden_pointer)GC_REVEAL_POINTER(curr_fo->fo_hidden_base);
GC_bytes_finalized
+= (word)curr_fo->fo_object_sz + sizeof(struct finalizable_object);
GC_ASSERT(GC_is_marked(GC_base(curr_fo)));
curr_fo = next_fo;
} else {
prev_fo = curr_fo;
curr_fo = fo_next(curr_fo);
}
}
}
if (GC_java_finalization) {
/*
* Make sure we mark everything reachable from objects finalized
* using the no-order `fo_mark_proc`.
*/
for (curr_fo = GC_fnlz_roots.finalize_now; curr_fo != NULL;
curr_fo = fo_next(curr_fo)) {
real_ptr = (ptr_t)curr_fo->fo_hidden_base; /*< revealed */
if (!GC_is_marked(real_ptr)) {
if (curr_fo->fo_mark_proc == GC_null_finalize_mark_proc) {
GC_mark_fo(real_ptr, GC_normal_finalize_mark_proc);
}
if (curr_fo->fo_mark_proc != GC_unreachable_finalize_mark_proc) {
GC_set_mark_bit(real_ptr);
}
}
}
/*
* Now revive finalize-when-unreachable objects reachable from other
* finalizable objects.
*/
if (GC_need_unreachable_finalization) {
curr_fo = GC_fnlz_roots.finalize_now;
# if defined(GC_ASSERTIONS) || defined(LINT2)
if (curr_fo != NULL && NULL == GC_fnlz_roots.fo_head)
ABORT("GC_fnlz_roots.fo_head is null");
# endif
for (prev_fo = NULL; curr_fo != NULL;
prev_fo = curr_fo, curr_fo = next_fo) {
next_fo = fo_next(curr_fo);
if (curr_fo->fo_mark_proc != GC_unreachable_finalize_mark_proc)
continue;
real_ptr = (ptr_t)curr_fo->fo_hidden_base; /*< revealed */
if (!GC_is_marked(real_ptr)) {
GC_set_mark_bit(real_ptr);
continue;
}
if (NULL == prev_fo) {
SET_FINALIZE_NOW(next_fo);
} else {
fo_set_next(prev_fo, next_fo);
GC_dirty(prev_fo);
}
curr_fo->fo_hidden_base = GC_HIDE_POINTER(real_ptr);
GC_bytes_finalized
-= (word)curr_fo->fo_object_sz + sizeof(struct finalizable_object);
i = HASH2(real_ptr, GC_log_fo_table_size);
fo_set_next(curr_fo, GC_fnlz_roots.fo_head[i]);
GC_dirty(curr_fo);
GC_fo_entries++;
GC_fnlz_roots.fo_head[i] = curr_fo;
curr_fo = prev_fo;
needs_barrier = TRUE;
}
}
}
if (needs_barrier)
GC_dirty(GC_fnlz_roots.fo_head); /*< entire object */
/* Remove dangling disappearing links. */
GC_make_disappearing_links_disappear(&GC_dl_hashtbl, TRUE);
# ifndef GC_TOGGLE_REFS_NOT_NEEDED
GC_clear_togglerefs();
# endif
# ifndef GC_LONG_REFS_NOT_NEEDED
GC_make_disappearing_links_disappear(&GC_ll_hashtbl, FALSE);
GC_make_disappearing_links_disappear(&GC_ll_hashtbl, TRUE);
# endif
if (GC_alloc_fail_count > 0) {
/*
* Do not prevent running finalizers if there has been an allocation
* failure recently.
*/
# ifdef THREADS
GC_reset_finalizer_nested();
# else
GC_finalizer_nested = 0;
# endif
}
}
/*
* Count of finalizers to run, at most, during a single invocation
* of `GC_invoke_finalizers()`; zero means no limit. Accessed with the
* allocator lock held.
*/
STATIC unsigned GC_interrupt_finalizers = 0;
# ifndef JAVA_FINALIZATION_NOT_NEEDED
/*
* Enqueue all remaining finalizers to be run. A collection in progress,
* if any, is completed when the first finalizer is enqueued.
*/
STATIC void
GC_enqueue_all_finalizers(void)
{
size_t i;
size_t fo_size
= GC_fnlz_roots.fo_head == NULL ? 0 : (size_t)1 << GC_log_fo_table_size;
GC_ASSERT(I_HOLD_LOCK());
GC_bytes_finalized = 0;
for (i = 0; i < fo_size; i++) {
struct finalizable_object *curr_fo = GC_fnlz_roots.fo_head[i];
GC_fnlz_roots.fo_head[i] = NULL;
while (curr_fo != NULL) {
struct finalizable_object *next_fo;
ptr_t real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base);
GC_mark_fo(real_ptr, GC_normal_finalize_mark_proc);
GC_set_mark_bit(real_ptr);
GC_complete_ongoing_collection();
next_fo = fo_next(curr_fo);
/* Add to list of objects awaiting finalization. */
fo_set_next(curr_fo, GC_fnlz_roots.finalize_now);
GC_dirty(curr_fo);
SET_FINALIZE_NOW(curr_fo);
/* Unhide object pointer so any future collections will see it. */
curr_fo->fo_hidden_base
= (GC_hidden_pointer)GC_REVEAL_POINTER(curr_fo->fo_hidden_base);
GC_bytes_finalized
+= curr_fo->fo_object_sz + sizeof(struct finalizable_object);
curr_fo = next_fo;
}
}
/* All entries are deleted from the hash table. */
GC_fo_entries = 0;
}
GC_API void GC_CALL
GC_finalize_all(void)
{
LOCK();
while (GC_fo_entries > 0) {
unsigned saved_interrupt_finalizers;
GC_enqueue_all_finalizers();
/* Reset temporarily. */
saved_interrupt_finalizers = GC_interrupt_finalizers;
GC_interrupt_finalizers = 0;
UNLOCK();
GC_invoke_finalizers();
/*
* Running the finalizers in this thread is arguably not a good idea
* when we should be notifying another thread to run them.
* But otherwise we do not have a great way to wait for them to run.
*/
LOCK();
GC_interrupt_finalizers = saved_interrupt_finalizers;
}
UNLOCK();
}
# endif /* !JAVA_FINALIZATION_NOT_NEEDED */
GC_API void GC_CALL
GC_set_interrupt_finalizers(unsigned value)
{
LOCK();
GC_interrupt_finalizers = value;
UNLOCK();
}
GC_API unsigned GC_CALL
GC_get_interrupt_finalizers(void)
{
unsigned value;
READER_LOCK();
value = GC_interrupt_finalizers;
READER_UNLOCK();
return value;
}
GC_API int GC_CALL
GC_should_invoke_finalizers(void)
{
# ifdef AO_HAVE_load
return GC_cptr_load((volatile ptr_t *)&GC_fnlz_roots.finalize_now) != NULL;
# else
return GC_fnlz_roots.finalize_now != NULL;
# endif /* !THREADS */
}
GC_API int GC_CALL
GC_invoke_finalizers(void)
{
int count = 0;
word bytes_freed_before = 0; /*< initialized to prevent warning */
GC_ASSERT(I_DONT_HOLD_LOCK());
while (GC_should_invoke_finalizers()) {
struct finalizable_object *curr_fo;
ptr_t real_ptr;
LOCK();
if (count == 0) {
/* Note: we hold the allocator lock here. */
bytes_freed_before = GC_bytes_freed;
} else if (UNLIKELY(GC_interrupt_finalizers != 0)
&& (unsigned)count >= GC_interrupt_finalizers) {
UNLOCK();
break;
}
curr_fo = GC_fnlz_roots.finalize_now;
# ifdef THREADS
if (UNLIKELY(NULL == curr_fo)) {
UNLOCK();
break;
}
# endif
SET_FINALIZE_NOW(fo_next(curr_fo));
UNLOCK();
fo_set_next(curr_fo, 0);
real_ptr = (ptr_t)curr_fo->fo_hidden_base; /*< revealed */
curr_fo->fo_fn(real_ptr, curr_fo->fo_client_data);
curr_fo->fo_client_data = NULL;
++count;
/*
* Explicit freeing of `curr_fo` is probably a bad idea.
* It throws off accounting if nearly all objects are finalizable.
* Otherwise it should not matter.
*/
}
/* `bytes_freed_before` is initialized whenever `count` is nonzero. */
if (count != 0
# if defined(THREADS) && !defined(THREAD_SANITIZER)
/*
* A quick check whether some memory was freed. The race with
* `GC_free()` is safe to be ignored because we only need to know
* if the current thread has deallocated something.
*/
&& bytes_freed_before != GC_bytes_freed
# endif
) {
LOCK();
GC_finalizer_bytes_freed += (GC_bytes_freed - bytes_freed_before);
UNLOCK();
}
return count;
}
static word last_finalizer_notification = 0;
GC_INNER void
GC_notify_or_invoke_finalizers(void)
{
# if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
static word last_back_trace_gc_no = 0;
# endif
GC_finalizer_notifier_proc notifier_fn = 0;
# if defined(THREADS) && !defined(KEEP_BACK_PTRS) && !defined(MAKE_BACK_GRAPH)
/* Quick check (while unlocked) for an empty finalization queue. */
if (!GC_should_invoke_finalizers())
return;
# endif
LOCK();
# if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
/*
* This is a convenient place to generate backtraces if appropriate,
* since that code is not callable with the allocator lock.
*/
if (GC_gc_no != last_back_trace_gc_no
&& LIKELY(GC_gc_no > 1) /*< skip initial collection */) {
# ifdef KEEP_BACK_PTRS
static GC_bool bt_in_progress = FALSE;
if (!bt_in_progress) {
long i;
/* Prevent a recursion or parallel usage. */
bt_in_progress = TRUE;
for (i = 0; i < GC_backtraces; ++i) {
/*
* FIXME: This tolerates concurrent heap mutation, which may
* cause occasional mysterious results. We need to release
* the allocator lock, since `GC_print_callers()` acquires it.
* It probably should not.
*/
void *current = GC_generate_random_valid_address();
UNLOCK();
GC_printf("\n***Chosen address %p in object\n", current);
GC_print_backtrace(current);
LOCK();
}
bt_in_progress = FALSE;
}
# endif
last_back_trace_gc_no = GC_gc_no;
# ifdef MAKE_BACK_GRAPH
if (GC_print_back_height) {
GC_print_back_graph_stats();
}
# endif
}
# endif
if (NULL == GC_fnlz_roots.finalize_now) {
UNLOCK();
return;
}
if (!GC_finalize_on_demand) {
unsigned char *pnested;
# ifdef THREADS
if (UNLIKELY(GC_in_thread_creation)) {
UNLOCK();
return;
}
# endif
pnested = GC_check_finalizer_nested();
UNLOCK();
/* Skip `GC_invoke_finalizers()` if nested. */
if (pnested != NULL) {
(void)GC_invoke_finalizers();
/* Reset since no more finalizers or interrupted. */
*pnested = 0;
# ifndef THREADS
GC_ASSERT(NULL == GC_fnlz_roots.finalize_now
|| GC_interrupt_finalizers > 0);
# else
/*
* Note: in the multi-threaded case GC can run concurrently and
* add more finalizers to run.
*/
# endif
}
return;
}
/* These variables require synchronization to avoid data race. */
if (last_finalizer_notification != GC_gc_no) {
notifier_fn = GC_finalizer_notifier;
last_finalizer_notification = GC_gc_no;
}
UNLOCK();
if (notifier_fn != 0) {
/* Invoke the notifier. */
(*notifier_fn)();
}
}
# ifndef SMALL_CONFIG
# ifndef GC_LONG_REFS_NOT_NEEDED
# define IF_LONG_REFS_PRESENT_ELSE(x, y) (x)
# else
# define IF_LONG_REFS_PRESENT_ELSE(x, y) (y)
# endif
GC_INNER void
GC_print_finalization_stats(void)
{
const struct finalizable_object *fo;
unsigned long ready = 0;
GC_log_printf(
"%lu finalization entries;"
" %lu/%lu short/long disappearing links alive\n",
(unsigned long)GC_fo_entries, (unsigned long)GC_dl_hashtbl.entries,
(unsigned long)IF_LONG_REFS_PRESENT_ELSE(GC_ll_hashtbl.entries, 0));
for (fo = GC_fnlz_roots.finalize_now; fo != NULL; fo = fo_next(fo))
++ready;
GC_log_printf("%lu finalization-ready objects;"
" %ld/%ld short/long links cleared\n",
ready, (long)GC_old_dl_entries - (long)GC_dl_hashtbl.entries,
(long)IF_LONG_REFS_PRESENT_ELSE(
GC_old_ll_entries - GC_ll_hashtbl.entries, 0));
}
# endif /* !SMALL_CONFIG */
#endif /* !GC_NO_FINALIZATION */
/*
* Copyright (c) 2011 by Hewlett-Packard Company. All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#ifdef ENABLE_DISCLAIM
# include "gc/gc_disclaim.h"
# if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
/* The first bit is already used for a debug purpose. */
# define FINALIZER_CLOSURE_FLAG 0x2
# else
# define FINALIZER_CLOSURE_FLAG 0x1
# endif
STATIC int GC_CALLBACK
GC_finalized_disclaim(void *obj)
{
# ifdef AO_HAVE_load
ptr_t fc_p = GC_cptr_load((volatile ptr_t *)obj);
# else
ptr_t fc_p = *(ptr_t *)obj;
# endif
if ((ADDR(fc_p) & FINALIZER_CLOSURE_FLAG) != 0) {
/*
* The disclaim function may be passed fragments from the free-list,
* on which it should not run finalization. To recognize this case,
* we use the fact that the value of the first pointer of such
* fragments is always, at least, multiple of a pointer size (a link
* to the next fragment, or `NULL`).
*
* Note: if it is desirable to have a finalizer which does not use
* the first pointer for storing the finalization information,
* `GC_disclaim_and_reclaim()` must be extended to clear fragments
* so that the assumption holds for the selected pointer location.
*/
const struct GC_finalizer_closure *fc
= (struct GC_finalizer_closure *)CPTR_CLEAR_FLAGS(
fc_p, FINALIZER_CLOSURE_FLAG);
GC_ASSERT(!GC_find_leak_inner);
fc->proc((ptr_t *)obj + 1, fc->cd);
}
return 0;
}
STATIC void
GC_register_disclaim_proc_inner(unsigned kind, GC_disclaim_proc proc,
GC_bool mark_unconditionally)
{
GC_ASSERT(kind < MAXOBJKINDS);
if (UNLIKELY(GC_find_leak_inner))
return;
GC_obj_kinds[kind].ok_disclaim_proc = proc;
GC_obj_kinds[kind].ok_mark_unconditionally = mark_unconditionally;
}
GC_API void GC_CALL
GC_init_finalized_malloc(void)
{
/* Initialize the collector just in case it is not done yet. */
GC_init();
LOCK();
if (GC_finalized_kind != 0) {
UNLOCK();
return;
}
/*
* The finalizer closure is placed in the first pointer of the
* object in order to use the lower bits to distinguish live
* objects from objects on the free list. The downside of this is
* that we need one-pointer offset interior pointers, and that
* `GC_base()` does not return the start of the user region.
*/
GC_register_displacement_inner(sizeof(ptr_t));
/*
* And, the pointer to the finalizer closure object itself is displaced
* due to baking in this indicator.
*/
GC_register_displacement_inner(FINALIZER_CLOSURE_FLAG);
GC_register_displacement_inner(sizeof(oh) | FINALIZER_CLOSURE_FLAG);
GC_finalized_kind
= GC_new_kind_inner(GC_new_free_list_inner(), GC_DS_LENGTH, TRUE, TRUE);
GC_ASSERT(GC_finalized_kind != 0);
GC_register_disclaim_proc_inner(GC_finalized_kind, GC_finalized_disclaim,
TRUE);
UNLOCK();
}
GC_API void GC_CALL
GC_register_disclaim_proc(int kind, GC_disclaim_proc proc,
int mark_unconditionally)
{
LOCK();
GC_register_disclaim_proc_inner((unsigned)kind, proc,
(GC_bool)mark_unconditionally);
UNLOCK();
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_finalized_malloc(size_t lb, const struct GC_finalizer_closure *fclos)
{
void *op;
ptr_t fc_p;
# ifndef LINT2
/* Actually, there is no data race because the variable is set once. */
GC_ASSERT(GC_finalized_kind != 0);
# endif
GC_ASSERT(NONNULL_ARG_NOT_NULL(fclos));
GC_ASSERT((ADDR(fclos) & FINALIZER_CLOSURE_FLAG) == 0);
op = GC_malloc_kind(SIZET_SAT_ADD(lb, sizeof(ptr_t)),
(int)GC_finalized_kind);
if (UNLIKELY(NULL == op))
return NULL;
/*
* Set the flag (w/o conversion to a numeric type) and store
* the finalizer closure.
*/
fc_p = CPTR_SET_FLAGS(GC_CAST_AWAY_CONST_PVOID(fclos),
FINALIZER_CLOSURE_FLAG);
# ifdef AO_HAVE_store
GC_cptr_store((volatile ptr_t *)op, fc_p);
# else
*(ptr_t *)op = fc_p;
# endif
GC_dirty(op);
REACHABLE_AFTER_DIRTY(fc_p);
return (ptr_t *)op + 1;
}
#endif /* ENABLE_DISCLAIM */
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#include <string.h>
/* Allocate reclaim list for the kind. Returns `TRUE` on success. */
STATIC GC_bool
GC_alloc_reclaim_list(struct obj_kind *ok)
{
struct hblk **result;
GC_ASSERT(I_HOLD_LOCK());
result = (struct hblk **)GC_scratch_alloc((MAXOBJGRANULES + 1)
* sizeof(struct hblk *));
if (UNLIKELY(NULL == result))
return FALSE;
BZERO(result, (MAXOBJGRANULES + 1) * sizeof(struct hblk *));
ok->ok_reclaim_list = result;
return TRUE;
}
/*
* Allocate a large block of size `lb_adjusted` bytes with the requested
* alignment (`align_m1 + 1`). The block is not cleared. We assume that
* the size is nonzero and a multiple of `GC_GRANULE_BYTES`, and that
* it already includes `EXTRA_BYTES` value. The `flags` argument should
* be `IGNORE_OFF_PAGE` or 0. Calls `GC_allochblk()` to do the actual
* allocation, but also triggers collection and/or heap expansion
* as appropriate. Updates value of `GC_bytes_allocd`; does also other
* accounting.
*/
STATIC ptr_t
GC_alloc_large(size_t lb_adjusted, int kind, unsigned flags, size_t align_m1)
{
/*
* TODO: It is unclear which retries limit is sufficient (value of 3 leads
* to fail in some 32-bit applications, 10 is a kind of arbitrary value).
*/
#define MAX_ALLOCLARGE_RETRIES 10
int retry_cnt;
size_t n_blocks; /*< includes alignment */
struct hblk *h;
ptr_t result;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(lb_adjusted != 0 && (lb_adjusted & (GC_GRANULE_BYTES - 1)) == 0);
n_blocks = OBJ_SZ_TO_BLOCKS_CHECKED(SIZET_SAT_ADD(lb_adjusted, align_m1));
if (UNLIKELY(!GC_is_initialized)) {
UNLOCK(); /*< just to unset `GC_lock_holder` */
GC_init();
LOCK();
}
/* Do our share of marking work. */
if (GC_incremental && !GC_dont_gc) {
GC_collect_a_little_inner(n_blocks);
}
h = GC_allochblk(lb_adjusted, kind, flags, align_m1);
#ifdef USE_MUNMAP
if (NULL == h && GC_merge_unmapped()) {
h = GC_allochblk(lb_adjusted, kind, flags, align_m1);
}
#endif
for (retry_cnt = 0; NULL == h; retry_cnt++) {
/*
* Only a few iterations are expected at most, otherwise something
* is wrong in one of the functions called below.
*/
if (retry_cnt > MAX_ALLOCLARGE_RETRIES)
ABORT("Too many retries in GC_alloc_large");
if (UNLIKELY(!GC_collect_or_expand(n_blocks, flags, retry_cnt > 0)))
return NULL;
h = GC_allochblk(lb_adjusted, kind, flags, align_m1);
}
GC_bytes_allocd += lb_adjusted;
if (lb_adjusted > HBLKSIZE) {
GC_large_allocd_bytes += HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb_adjusted);
if (GC_large_allocd_bytes > GC_max_large_allocd_bytes)
GC_max_large_allocd_bytes = GC_large_allocd_bytes;
}
/* FIXME: Do we need some way to reset `GC_max_large_allocd_bytes`? */
result = h->hb_body;
GC_ASSERT((ADDR(result) & align_m1) == 0);
return result;
}
/*
* Allocate a large block of given size in bytes, clear it if appropriate.
* We assume that the size is nonzero and a multiple of `GC_GRANULE_BYTES`,
* and that it already includes `EXTRA_BYTES` value. Update value of
* `GC_bytes_allocd`.
*/
STATIC ptr_t
GC_alloc_large_and_clear(size_t lb_adjusted, int kind, unsigned flags)
{
ptr_t result;
GC_ASSERT(I_HOLD_LOCK());
result = GC_alloc_large(lb_adjusted, kind, flags, 0 /* `align_m1` */);
if (LIKELY(result != NULL)
&& (GC_debugging_started || GC_obj_kinds[kind].ok_init)) {
/* Clear the whole block, in case of `GC_realloc` call. */
BZERO(result, HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb_adjusted));
}
return result;
}
/*
* Fill in additional entries in `GC_size_map`, including the `i`-th one.
* Note that a filled in section of the array ending at `n` always has
* the length of at least `n / 4`.
*/
STATIC void
GC_extend_size_map(size_t i)
{
size_t original_lg = ALLOC_REQUEST_GRANS(i);
size_t lg;
/*
* The size we try to preserve. Close to `i`, unless this would
* introduce too many distinct sizes.
*/
size_t byte_sz = GRANULES_TO_BYTES(original_lg);
size_t smaller_than_i = byte_sz - (byte_sz >> 3);
/* The lowest indexed entry we initialize. */
size_t low_limit;
size_t number_of_objs;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(0 == GC_size_map[i]);
if (0 == GC_size_map[smaller_than_i]) {
low_limit = byte_sz - (byte_sz >> 2); /*< much smaller than `i` */
lg = original_lg;
while (GC_size_map[low_limit] != 0)
low_limit++;
} else {
low_limit = smaller_than_i + 1;
while (GC_size_map[low_limit] != 0)
low_limit++;
lg = ALLOC_REQUEST_GRANS(low_limit);
lg += lg >> 3;
if (lg < original_lg)
lg = original_lg;
}
/*
* For these larger sizes, we use an even number of granules.
* This makes it easier to, e.g., construct a 16-byte-aligned
* allocator even if `GC_GRANULE_BYTES` is 8.
*/
lg = (lg + 1) & ~(size_t)1;
if (lg > MAXOBJGRANULES)
lg = MAXOBJGRANULES;
/* If we can fit the same number of larger objects in a block, do so. */
GC_ASSERT(lg != 0);
number_of_objs = HBLK_GRANULES / lg;
GC_ASSERT(number_of_objs != 0);
lg = (HBLK_GRANULES / number_of_objs) & ~(size_t)1;
/*
* We may need one extra byte; do not always fill in
* `GC_size_map[byte_sz]`.
*/
byte_sz = GRANULES_TO_BYTES(lg) - EXTRA_BYTES;
for (; low_limit <= byte_sz; low_limit++)
GC_size_map[low_limit] = lg;
}
STATIC void *
GC_generic_malloc_inner_small(size_t lb, int kind)
{
struct obj_kind *ok = &GC_obj_kinds[kind];
size_t lg = GC_size_map[lb];
void **opp = &ok->ok_freelist[lg];
void *op = *opp;
GC_ASSERT(I_HOLD_LOCK());
if (UNLIKELY(NULL == op)) {
if (0 == lg) {
if (UNLIKELY(!GC_is_initialized)) {
UNLOCK(); /*< just to unset `GC_lock_holder` */
GC_init();
LOCK();
lg = GC_size_map[lb];
}
if (0 == lg) {
GC_extend_size_map(lb);
lg = GC_size_map[lb];
GC_ASSERT(lg != 0);
}
/* Retry. */
opp = &ok->ok_freelist[lg];
op = *opp;
}
if (NULL == op) {
if (NULL == ok->ok_reclaim_list && !GC_alloc_reclaim_list(ok))
return NULL;
op = GC_allocobj(lg, kind);
if (NULL == op)
return NULL;
}
}
*opp = obj_link(op);
obj_link(op) = NULL;
GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
return op;
}
GC_INNER void *
GC_generic_malloc_inner(size_t lb, int kind, unsigned flags)
{
size_t lb_adjusted;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(kind < MAXOBJKINDS);
if (SMALL_OBJ(lb)) {
return GC_generic_malloc_inner_small(lb, kind);
}
#if MAX_EXTRA_BYTES > 0
if ((flags & IGNORE_OFF_PAGE) != 0 && lb >= HBLKSIZE) {
/* No need to add `EXTRA_BYTES`. */
lb_adjusted = lb;
} else
#endif
/* else */ {
lb_adjusted = ADD_EXTRA_BYTES(lb);
}
return GC_alloc_large_and_clear(ROUNDUP_GRANULE_SIZE(lb_adjusted), kind,
flags);
}
#ifdef GC_COLLECT_AT_MALLOC
# if defined(CPPCHECK)
size_t GC_dbg_collect_at_malloc_min_lb = 16 * 1024; /*< some value */
# else
size_t GC_dbg_collect_at_malloc_min_lb = (GC_COLLECT_AT_MALLOC);
# endif
#endif
GC_INNER void *
GC_generic_malloc_aligned(size_t lb, int kind, unsigned flags, size_t align_m1)
{
void *result;
GC_ASSERT(kind < MAXOBJKINDS);
if (UNLIKELY(get_have_errors()))
GC_print_all_errors();
GC_notify_or_invoke_finalizers();
GC_DBG_COLLECT_AT_MALLOC(lb);
if (SMALL_OBJ(lb) && LIKELY(align_m1 < GC_GRANULE_BYTES)) {
LOCK();
result = GC_generic_malloc_inner_small(lb, kind);
UNLOCK();
} else {
#ifdef THREADS
size_t lg;
#endif
size_t lb_adjusted;
GC_bool init;
#if MAX_EXTRA_BYTES > 0
if ((flags & IGNORE_OFF_PAGE) != 0 && lb >= HBLKSIZE) {
/* No need to add `EXTRA_BYTES`. */
lb_adjusted = ROUNDUP_GRANULE_SIZE(lb);
# ifdef THREADS
lg = BYTES_TO_GRANULES(lb_adjusted);
# endif
} else
#endif
/* else */ {
#ifndef THREADS
size_t lg; /*< CPPCHECK */
#endif
if (UNLIKELY(0 == lb))
lb = 1;
lg = ALLOC_REQUEST_GRANS(lb);
lb_adjusted = GRANULES_TO_BYTES(lg);
}
init = GC_obj_kinds[kind].ok_init;
if (LIKELY(align_m1 < GC_GRANULE_BYTES)) {
align_m1 = 0;
} else if (align_m1 < HBLKSIZE) {
align_m1 = HBLKSIZE - 1;
}
LOCK();
result = GC_alloc_large(lb_adjusted, kind, flags, align_m1);
if (LIKELY(result != NULL)) {
if (GC_debugging_started
#ifndef THREADS
|| init
#endif
) {
BZERO(result, HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb_adjusted));
} else {
#ifdef THREADS
GC_ASSERT(GRANULES_TO_PTRS(lg) >= 2);
/*
* Clear any memory that might be used for the GC descriptors
* before we release the allocator lock.
*/
((ptr_t *)result)[0] = NULL;
((ptr_t *)result)[1] = NULL;
((ptr_t *)result)[GRANULES_TO_PTRS(lg) - 1] = NULL;
((ptr_t *)result)[GRANULES_TO_PTRS(lg) - 2] = NULL;
#endif
}
}
UNLOCK();
#ifdef THREADS
if (init && !GC_debugging_started && result != NULL) {
/* Clear the rest (i.e. excluding the initial 2 words). */
BZERO((ptr_t *)result + 2,
HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb_adjusted) - 2 * sizeof(ptr_t));
}
#endif
}
if (UNLIKELY(NULL == result)) {
result = (*GC_get_oom_fn())(lb);
/* Note: result might be misaligned. */
}
return result;
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_generic_malloc(size_t lb, int kind)
{
return GC_generic_malloc_aligned(lb, kind, 0 /* `flags` */,
0 /* `align_m1` */);
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_malloc_kind_global(size_t lb, int kind)
{
return GC_malloc_kind_aligned_global(lb, kind, 0 /* `align_m1` */);
}
GC_INNER void *
GC_malloc_kind_aligned_global(size_t lb, int kind, size_t align_m1)
{
GC_ASSERT(kind < MAXOBJKINDS);
if (SMALL_OBJ(lb) && LIKELY(align_m1 < HBLKSIZE / 2)) {
void *op;
void **opp;
size_t lg;
GC_DBG_COLLECT_AT_MALLOC(lb);
LOCK();
lg = GC_size_map[lb];
opp = &GC_obj_kinds[kind].ok_freelist[lg];
op = *opp;
if (UNLIKELY(align_m1 >= GC_GRANULE_BYTES)) {
/* TODO: Avoid linear search. */
for (; (ADDR(op) & align_m1) != 0; op = *opp) {
opp = &obj_link(op);
}
}
if (LIKELY(op != NULL)) {
GC_ASSERT(PTRFREE == kind || NULL == obj_link(op)
|| (ADDR(obj_link(op)) < GC_greatest_real_heap_addr
&& GC_least_real_heap_addr < ADDR(obj_link(op))));
*opp = obj_link(op);
if (kind != PTRFREE)
obj_link(op) = NULL;
GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
UNLOCK();
GC_ASSERT((ADDR(op) & align_m1) == 0);
return op;
}
UNLOCK();
}
/*
* We make the `GC_clear_stack()` call a tail one, hoping to get more
* of the stack.
*/
return GC_clear_stack(
GC_generic_malloc_aligned(lb, kind, 0 /* `flags` */, align_m1));
}
#if defined(THREADS) && !defined(THREAD_LOCAL_ALLOC)
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_malloc_kind(size_t lb, int kind)
{
return GC_malloc_kind_global(lb, kind);
}
#endif
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_malloc_atomic(size_t lb)
{
/* Allocate `lb` bytes of atomic (pointer-free) data. */
return GC_malloc_kind(lb, PTRFREE);
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_malloc(size_t lb)
{
/* Allocate `lb` bytes of composite (pointerful) data. */
return GC_malloc_kind(lb, NORMAL);
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_generic_malloc_uncollectable(size_t lb, int kind)
{
void *op;
size_t lb_orig = lb;
GC_ASSERT(kind < MAXOBJKINDS);
if (EXTRA_BYTES != 0 && LIKELY(lb != 0)) {
/*
* We do not need the extra byte, since this will not be collected
* anyway.
*/
lb--;
}
if (SMALL_OBJ(lb)) {
void **opp;
size_t lg;
if (UNLIKELY(get_have_errors()))
GC_print_all_errors();
GC_notify_or_invoke_finalizers();
GC_DBG_COLLECT_AT_MALLOC(lb_orig);
LOCK();
lg = GC_size_map[lb];
opp = &GC_obj_kinds[kind].ok_freelist[lg];
op = *opp;
if (LIKELY(op != NULL)) {
*opp = obj_link(op);
obj_link(op) = 0;
GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
/*
* Mark bit was already set on free list. It will be cleared only
* temporarily during a collection, as a result of the normal
* free-list mark bit clearing.
*/
GC_non_gc_bytes += GRANULES_TO_BYTES((word)lg);
} else {
op = GC_generic_malloc_inner_small(lb, kind);
if (NULL == op) {
GC_oom_func oom_fn = GC_oom_fn;
UNLOCK();
return (*oom_fn)(lb_orig);
}
/* For small objects, the free lists are completely marked. */
}
GC_ASSERT(GC_is_marked(op));
UNLOCK();
} else {
op = GC_generic_malloc_aligned(lb, kind, 0 /* `flags` */,
0 /* `align_m1` */);
if (op /* `!= NULL` */) { /*< CPPCHECK */
hdr *hhdr = HDR(op);
GC_ASSERT(HBLKDISPL(op) == 0); /*< large block */
/*
* We do not need to acquire the allocator lock before `HDR(op)`,
* since we have an undisguised pointer, but we need it while we
* adjust the mark bits.
*/
LOCK();
set_mark_bit_from_hdr(hhdr, 0); /*< the only object */
#ifndef THREADS
/*
* This is not guaranteed in the multi-threaded case because the
* counter could be updated before locking.
*/
GC_ASSERT(hhdr->hb_n_marks == 0);
#endif
hhdr->hb_n_marks = 1;
UNLOCK();
}
}
return op;
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_malloc_uncollectable(size_t lb)
{
/* Allocate `lb` bytes of pointerful, traced, but not collectible data. */
return GC_generic_malloc_uncollectable(lb, UNCOLLECTABLE);
}
#ifdef GC_ATOMIC_UNCOLLECTABLE
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_malloc_atomic_uncollectable(size_t lb)
{
return GC_generic_malloc_uncollectable(lb, AUNCOLLECTABLE);
}
#endif /* GC_ATOMIC_UNCOLLECTABLE */
#if defined(REDIRECT_MALLOC) && !defined(REDIRECT_MALLOC_IN_HEADER)
# ifndef MSWINCE
# include <errno.h>
# endif
/*
* Avoid unnecessary nested procedure calls here, by `#define` some `malloc`
* replacements. Otherwise we end up saving a meaningless return address in
* the object. It also speeds things up, but it is admittedly quite ugly.
*/
# define GC_debug_malloc_replacement(lb) GC_debug_malloc(lb, GC_DBG_EXTRAS)
# if defined(CPPCHECK)
# define REDIRECT_MALLOC_F GC_malloc /*< e.g. */
# else
# define REDIRECT_MALLOC_F REDIRECT_MALLOC
# endif
void *
malloc(size_t lb)
{
/*
* It might help to manually inline the `GC_malloc` call here.
* But any decent compiler should reduce the extra procedure call
* to at most a jump instruction in this case.
*/
# if defined(SOLARIS) && defined(THREADS) && defined(I386)
/*
* Thread initialization can call `malloc` before we are ready for.
* It is not clear that this is enough to help matters. The thread
* implementation may well call `malloc` at other inopportune times.
*/
if (UNLIKELY(!GC_is_initialized))
return sbrk(lb);
# endif
return (void *)REDIRECT_MALLOC_F(lb);
}
# ifdef REDIR_MALLOC_AND_LINUXTHREADS
# ifdef HAVE_LIBPTHREAD_SO
STATIC ptr_t GC_libpthread_start = NULL;
STATIC ptr_t GC_libpthread_end = NULL;
# endif
STATIC ptr_t GC_libld_start = NULL;
STATIC ptr_t GC_libld_end = NULL;
static GC_bool lib_bounds_set = FALSE;
GC_INNER void
GC_init_lib_bounds(void)
{
IF_CANCEL(int cancel_state;)
/*
* This test does not need to ensure memory visibility, since the bounds
* will be set when/if we create another thread.
*/
if (LIKELY(lib_bounds_set))
return;
DISABLE_CANCEL(cancel_state);
GC_init(); /*< if not called yet */
# if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)
LOCK(); /*< just to set `GC_lock_holder` */
# endif
# ifdef HAVE_LIBPTHREAD_SO
if (!GC_text_mapping("libpthread-", &GC_libpthread_start,
&GC_libpthread_end)) {
WARN("Failed to find libpthread.so text mapping: Expect crash\n", 0);
/*
* This might still work with some versions of `libpthread`,
* so we do not `abort`.
*/
}
# endif
if (!GC_text_mapping("ld-", &GC_libld_start, &GC_libld_end)) {
WARN("Failed to find ld.so text mapping: Expect crash\n", 0);
}
# if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)
UNLOCK();
# endif
RESTORE_CANCEL(cancel_state);
lib_bounds_set = TRUE;
}
# endif /* REDIR_MALLOC_AND_LINUXTHREADS */
void *
calloc(size_t n, size_t lb)
{
if (UNLIKELY((lb | n) > GC_SQRT_SIZE_MAX) /*< fast initial test */
&& lb && n > GC_SIZE_MAX / lb)
return (*GC_get_oom_fn())(GC_SIZE_MAX); /*< `n * lb` overflow */
# ifdef REDIR_MALLOC_AND_LINUXTHREADS
/*
* The linker may allocate some memory that is only pointed to by
* memory-mapped thread stacks. Make sure it is not collectible.
*/
{
ptr_t caller = (ptr_t)__builtin_return_address(0);
GC_init_lib_bounds();
if (ADDR_INSIDE(caller, GC_libld_start, GC_libld_end)
# ifdef HAVE_LIBPTHREAD_SO
/*
* Note: the two ranges are actually usually adjacent, so there
* may be a way to speed this up.
*/
|| ADDR_INSIDE(caller, GC_libpthread_start, GC_libpthread_end)
# endif
) {
return GC_generic_malloc_uncollectable(n * lb, UNCOLLECTABLE);
}
}
# endif
return (void *)REDIRECT_MALLOC_F(n * lb);
}
# ifndef strdup
char *
strdup(const char *s)
{
size_t lb = strlen(s) + 1;
char *result = (char *)REDIRECT_MALLOC_F(lb);
if (UNLIKELY(NULL == result)) {
errno = ENOMEM;
return NULL;
}
BCOPY(s, result, lb);
return result;
}
# else
/*
* If `strdup` is macro defined, we assume that it actually calls `malloc`,
* and thus the right thing will happen even without overriding it.
* This seems to be true on most Linux systems.
*/
# endif /* strdup */
# ifndef strndup
/* This is similar to `strdup()`. */
char *
strndup(const char *str, size_t size)
{
char *copy;
size_t len = strlen(str);
if (UNLIKELY(len > size))
len = size;
copy = (char *)REDIRECT_MALLOC_F(len + 1);
if (UNLIKELY(NULL == copy)) {
errno = ENOMEM;
return NULL;
}
if (LIKELY(len > 0))
BCOPY(str, copy, len);
copy[len] = '\0';
return copy;
}
# endif /* !strndup */
# undef GC_debug_malloc_replacement
#endif /* REDIRECT_MALLOC */
/* Explicitly deallocate the object. `hhdr` should correspond to `p`. */
static void
free_internal(void *p, const hdr *hhdr)
{
size_t lb = hhdr->hb_sz; /*< size in bytes */
size_t lg = BYTES_TO_GRANULES(lb); /*< size in granules */
int kind = hhdr->hb_obj_kind;
GC_bytes_freed += lb;
if (IS_UNCOLLECTABLE(kind))
GC_non_gc_bytes -= lb;
if (LIKELY(lg <= MAXOBJGRANULES)) {
struct obj_kind *ok = &GC_obj_kinds[kind];
void **flh;
/*
* It is unnecessary to clear the mark bit. If the object is
* reallocated, it does not matter. Otherwise, the collector will
* do it, since it is on a free list.
*/
if (ok->ok_init && LIKELY(lb > sizeof(ptr_t))) {
BZERO((ptr_t *)p + 1, lb - sizeof(ptr_t));
}
flh = &ok->ok_freelist[lg];
obj_link(p) = *flh;
*flh = (ptr_t)p;
} else {
if (lb > HBLKSIZE) {
GC_large_allocd_bytes -= HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb);
}
GC_ASSERT(ADDR(HBLKPTR(p)) == ADDR(hhdr->hb_block));
GC_freehblk(hhdr->hb_block);
}
}
GC_API void GC_CALL
GC_free(void *p)
{
const hdr *hhdr;
if (p /* `!= NULL` */) {
/* CPPCHECK */
} else {
/* Required by ANSI. It is not my fault... */
return;
}
#ifdef LOG_ALLOCS
GC_log_printf("GC_free(%p) after GC #%lu\n", p, (unsigned long)GC_gc_no);
#endif
hhdr = HDR(p);
#if defined(REDIRECT_MALLOC) \
&& ((defined(NEED_CALLINFO) && defined(GC_HAVE_BUILTIN_BACKTRACE)) \
|| defined(REDIR_MALLOC_AND_LINUXTHREADS) \
|| (defined(SOLARIS) && defined(THREADS)) || defined(MSWIN32))
/*
* This might be called indirectly by `GC_print_callers` to free the
* result of `backtrace_symbols()`. For Solaris, we have to redirect
* `malloc` calls during initialization. For the others, this seems
* to happen implicitly. Do not try to deallocate that memory.
*/
if (UNLIKELY(NULL == hhdr))
return;
#endif
GC_ASSERT(GC_base(p) == p);
LOCK();
free_internal(p, hhdr);
FREE_PROFILER_HOOK(p);
UNLOCK();
}
#ifdef THREADS
GC_INNER void
GC_free_inner(void *p)
{
GC_ASSERT(I_HOLD_LOCK());
free_internal(p, HDR(p));
}
#endif /* THREADS */
#if defined(REDIRECT_MALLOC) && !defined(REDIRECT_FREE)
# define REDIRECT_FREE GC_free
#endif
#if defined(REDIRECT_FREE) && !defined(REDIRECT_MALLOC_IN_HEADER)
# if defined(CPPCHECK)
# define REDIRECT_FREE_F GC_free /*< e.g. */
# else
# define REDIRECT_FREE_F REDIRECT_FREE
# endif
void
free(void *p)
{
# ifdef IGNORE_FREE
UNUSED_ARG(p);
# else
# if defined(REDIR_MALLOC_AND_LINUXTHREADS) \
&& !defined(USE_PROC_FOR_LIBRARIES)
/*
* Do not bother with initialization checks. If nothing has been
* initialized, then the check fails, and that is safe, since we have
* not allocated uncollectible objects neither.
*/
ptr_t caller = (ptr_t)__builtin_return_address(0);
/*
* This test does not need to ensure memory visibility, since the bounds
* will be set when/if we create another thread.
*/
if (ADDR_INSIDE(caller, GC_libld_start, GC_libld_end)
# ifdef HAVE_LIBPTHREAD_SO
|| ADDR_INSIDE(caller, GC_libpthread_start, GC_libpthread_end)
# endif
) {
GC_free(p);
return;
}
# endif
REDIRECT_FREE_F(p);
# endif
}
#endif /* REDIRECT_FREE */
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996 by Silicon Graphics. All rights reserved.
* Copyright (c) 2000 by Hewlett-Packard Company. All rights reserved.
* Copyright (c) 2009-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* These are extra allocation routines that are likely to be less
* frequently used than those in `malloc.c` file. They are separate in
* the hope that the `.o` file will be excluded from statically linked
* executables. We should probably break this up further.
*/
#include <string.h>
#ifndef MSWINCE
# include <errno.h>
#endif
/*
* Some externally visible but unadvertised variables to allow access
* to free lists from inlined allocators without include `gc_priv.h` file
* or introducing dependencies on internal data structure layouts.
*/
void **const GC_objfreelist_ptr = GC_objfreelist;
void **const GC_aobjfreelist_ptr = GC_aobjfreelist;
void **const GC_uobjfreelist_ptr = GC_uobjfreelist;
#ifdef GC_ATOMIC_UNCOLLECTABLE
void **const GC_auobjfreelist_ptr = GC_auobjfreelist;
#endif
GC_API int GC_CALL
GC_get_kind_and_size(const void *p, size_t *psize)
{
const hdr *hhdr = HDR(p);
if (psize != NULL) {
*psize = hhdr->hb_sz;
}
return hhdr->hb_obj_kind;
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_generic_or_special_malloc(size_t lb, int kind)
{
switch (kind) {
case PTRFREE:
case NORMAL:
return GC_malloc_kind(lb, kind);
case UNCOLLECTABLE:
#ifdef GC_ATOMIC_UNCOLLECTABLE
case AUNCOLLECTABLE:
#endif
return GC_generic_malloc_uncollectable(lb, kind);
default:
return GC_generic_malloc_aligned(lb, kind, 0 /* `flags` */, 0);
}
}
GC_API void *GC_CALL
GC_realloc(void *p, size_t lb)
{
hdr *hhdr;
void *result;
#if defined(_FORTIFY_SOURCE) && defined(__GNUC__) && !defined(__clang__)
/*
* Use `cleared_p` instead of `p` as a workaround to avoid passing
* `alloc_size(lb)` attribute associated with `p` to `memset`
* (including a `memset` call inside `GC_free`).
*/
volatile GC_uintptr_t cleared_p = (GC_uintptr_t)p;
#else
# define cleared_p p
#endif
size_t sz; /*< current size in bytes */
size_t orig_sz; /*< original `sz` (in bytes) */
int obj_kind;
if (NULL == p) {
/* Required by ANSI. */
return GC_malloc(lb);
}
if (0 == lb) /* `&& p != NULL` */ {
#ifndef IGNORE_FREE
GC_free(p);
#endif
return NULL;
}
hhdr = HDR(HBLKPTR(p));
sz = hhdr->hb_sz;
obj_kind = hhdr->hb_obj_kind;
orig_sz = sz;
if (sz > MAXOBJBYTES) {
const struct obj_kind *ok = &GC_obj_kinds[obj_kind];
word descr = ok->ok_descriptor;
/* Round it up to the next whole heap block. */
sz = (sz + HBLKSIZE - 1) & ~(HBLKSIZE - 1);
#if ALIGNMENT > GC_DS_TAGS
/*
* An extra byte is not added in case of ignore-off-page allocated
* objects not smaller than `HBLKSIZE`.
*/
GC_ASSERT(sz >= HBLKSIZE);
if (EXTRA_BYTES != 0 && (hhdr->hb_flags & IGNORE_OFF_PAGE) != 0
&& obj_kind == NORMAL)
descr += ALIGNMENT; /*< or set to 0 */
#endif
if (ok->ok_relocate_descr) {
descr += sz;
}
/*
* `GC_realloc` might be changing the block size while
* `GC_reclaim_block` or `GC_clear_hdr_marks` is examining it.
* The change to the size field is benign, in that `GC_reclaim`
* (and `GC_clear_hdr_marks`) would work correctly with either
* value, since we are not changing the number of objects in
* the block. But seeing a half-updated value (though unlikely
* to occur in practice) could be probably bad.
* Using unordered atomic accesses on `hb_sz` and `hb_descr`
* fields would solve the issue. (The alternate solution might
* be to initially overallocate large objects, so we do not
* have to adjust the size in `GC_realloc`, if they still fit.
* But that is probably more expensive, since we may end up
* scanning a bunch of zeros during the collection.)
*/
#ifdef AO_HAVE_store
AO_store(&hhdr->hb_sz, sz);
AO_store((AO_t *)&hhdr->hb_descr, descr);
#else
{
LOCK();
hhdr->hb_sz = sz;
hhdr->hb_descr = descr;
UNLOCK();
}
#endif
#ifdef MARK_BIT_PER_OBJ
GC_ASSERT(hhdr->hb_inv_sz == LARGE_INV_SZ);
#else
GC_ASSERT((hhdr->hb_flags & LARGE_BLOCK) != 0
&& hhdr->hb_map[ANY_INDEX] == 1);
#endif
if (IS_UNCOLLECTABLE(obj_kind))
GC_non_gc_bytes += (sz - orig_sz);
/* Extra area is already cleared by `GC_alloc_large_and_clear`. */
}
if (ADD_EXTRA_BYTES(lb) <= sz) {
if (lb >= (sz >> 1)) {
if (orig_sz > lb) {
/* Clear unneeded part of object to avoid bogus pointer tracing. */
BZERO((ptr_t)cleared_p + lb, orig_sz - lb);
}
return p;
}
/*
* Shrink it. Note: shrinking of large blocks is not implemented
* efficiently.
*/
sz = lb;
}
result = GC_generic_or_special_malloc((word)lb, obj_kind);
if (LIKELY(result != NULL)) {
/*
* In case of shrink, it could also return original object.
* But this gives the client warning of imminent disaster.
*/
BCOPY(p, result, sz);
#ifndef IGNORE_FREE
GC_free((ptr_t)cleared_p);
#endif
}
return result;
#undef cleared_p
}
#if defined(REDIRECT_MALLOC) && !defined(REDIRECT_REALLOC)
# define REDIRECT_REALLOC GC_realloc
#endif
#ifdef REDIRECT_REALLOC
/* As with `malloc`, avoid two levels of extra calls here. */
# define GC_debug_realloc_replacement(p, lb) \
GC_debug_realloc(p, lb, GC_DBG_EXTRAS)
# if !defined(REDIRECT_MALLOC_IN_HEADER)
void *
realloc(void *p, size_t lb)
{
return REDIRECT_REALLOC(p, lb);
}
# endif
# undef GC_debug_realloc_replacement
#endif /* REDIRECT_REALLOC */
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_generic_malloc_ignore_off_page(size_t lb, int kind)
{
return GC_generic_malloc_aligned(lb, kind, IGNORE_OFF_PAGE,
0 /* `align_m1` */);
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_malloc_ignore_off_page(size_t lb)
{
return GC_generic_malloc_aligned(lb, NORMAL, IGNORE_OFF_PAGE, 0);
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_malloc_atomic_ignore_off_page(size_t lb)
{
return GC_generic_malloc_aligned(lb, PTRFREE, IGNORE_OFF_PAGE, 0);
}
/*
* Increment `GC_bytes_allocd` from code that does not have direct access
* to `GC_arrays`.
*/
void GC_CALL
GC_incr_bytes_allocd(size_t n)
{
GC_bytes_allocd += n;
}
/* The same as `GC_incr_bytes_allocd` but for `GC_bytes_freed`. */
void GC_CALL
GC_incr_bytes_freed(size_t n)
{
GC_bytes_freed += n;
}
GC_API size_t GC_CALL
GC_get_expl_freed_bytes_since_gc(void)
{
return (size_t)GC_bytes_freed;
}
GC_API void GC_CALL
GC_generic_malloc_many(size_t lb_adjusted, int kind, void **result)
{
void *op;
void *p;
void **opp;
/* The value of `lb_adjusted` converted to granules. */
size_t lg;
word my_bytes_allocd = 0;
struct obj_kind *ok;
struct hblk **rlh;
GC_ASSERT(lb_adjusted != 0 && (lb_adjusted & (GC_GRANULE_BYTES - 1)) == 0);
/* Currently a single object is always allocated if manual VDB. */
/*
* TODO: `GC_dirty` should be called for each linked object (but the
* last one) to support multiple objects allocation.
*/
if (UNLIKELY(lb_adjusted > MAXOBJBYTES) || GC_manual_vdb) {
op = GC_generic_malloc_aligned(lb_adjusted - EXTRA_BYTES, kind,
0 /* `flags` */, 0 /* `align_m1` */);
if (LIKELY(op != NULL))
obj_link(op) = NULL;
*result = op;
#ifndef NO_MANUAL_VDB
if (GC_manual_vdb && GC_is_heap_ptr(result)) {
GC_dirty_inner(result);
REACHABLE_AFTER_DIRTY(op);
}
#endif
return;
}
GC_ASSERT(kind < MAXOBJKINDS);
lg = BYTES_TO_GRANULES(lb_adjusted);
if (UNLIKELY(get_have_errors()))
GC_print_all_errors();
GC_notify_or_invoke_finalizers();
GC_DBG_COLLECT_AT_MALLOC(lb_adjusted - EXTRA_BYTES);
if (UNLIKELY(!GC_is_initialized))
GC_init();
LOCK();
/* Do our share of marking work. */
if (GC_incremental && !GC_dont_gc) {
GC_collect_a_little_inner(1);
}
/* First see if we can reclaim a page of objects waiting to be reclaimed. */
ok = &GC_obj_kinds[kind];
rlh = ok->ok_reclaim_list;
if (rlh != NULL) {
struct hblk *hbp;
hdr *hhdr;
while ((hbp = rlh[lg]) != NULL) {
hhdr = HDR(hbp);
rlh[lg] = hhdr->hb_next;
GC_ASSERT(hhdr->hb_sz == lb_adjusted);
hhdr->hb_last_reclaimed = (unsigned short)GC_gc_no;
#ifdef PARALLEL_MARK
if (GC_parallel) {
GC_signed_word my_bytes_allocd_tmp
= (GC_signed_word)AO_load(&GC_bytes_allocd_tmp);
GC_ASSERT(my_bytes_allocd_tmp >= 0);
/*
* We only decrement it while holding the allocator lock.
* Thus, we cannot accidentally adjust it down in more than
* one thread simultaneously.
*/
if (my_bytes_allocd_tmp != 0) {
(void)AO_fetch_and_add(&GC_bytes_allocd_tmp,
(AO_t)(-my_bytes_allocd_tmp));
GC_bytes_allocd += (word)my_bytes_allocd_tmp;
}
GC_acquire_mark_lock();
++GC_fl_builder_count;
UNLOCK();
GC_release_mark_lock();
}
#endif
op = GC_reclaim_generic(hbp, hhdr, lb_adjusted, ok->ok_init, 0,
&my_bytes_allocd);
if (op != 0) {
#ifdef PARALLEL_MARK
if (GC_parallel) {
*result = op;
(void)AO_fetch_and_add(&GC_bytes_allocd_tmp, (AO_t)my_bytes_allocd);
GC_acquire_mark_lock();
--GC_fl_builder_count;
if (GC_fl_builder_count == 0)
GC_notify_all_builder();
# ifdef THREAD_SANITIZER
GC_release_mark_lock();
LOCK();
GC_bytes_found += (GC_signed_word)my_bytes_allocd;
UNLOCK();
# else
/* The resulting `GC_bytes_found` may be inaccurate. */
GC_bytes_found += (GC_signed_word)my_bytes_allocd;
GC_release_mark_lock();
# endif
(void)GC_clear_stack(0);
return;
}
#endif
/* We also reclaimed memory, so we need to adjust that count. */
GC_bytes_found += (GC_signed_word)my_bytes_allocd;
GC_bytes_allocd += my_bytes_allocd;
goto out;
}
#ifdef PARALLEL_MARK
if (GC_parallel) {
GC_acquire_mark_lock();
--GC_fl_builder_count;
if (GC_fl_builder_count == 0)
GC_notify_all_builder();
GC_release_mark_lock();
/*
* The allocator lock is needed for access to the reclaim list.
* We must decrement `GC_fl_builder_count` before reacquiring
* the allocator lock. Hopefully this path is rare.
*/
LOCK();
/* Reload `rlh` after locking. */
rlh = ok->ok_reclaim_list;
if (NULL == rlh)
break;
}
#endif
}
}
/*
* Next try to use prefix of global free list if there is one.
* We do not refill it, but we need to use it up before allocating
* a new block ourselves.
*/
opp = &ok->ok_freelist[lg];
if ((op = *opp) != NULL) {
*opp = NULL;
my_bytes_allocd = 0;
for (p = op; p != NULL; p = obj_link(p)) {
my_bytes_allocd += lb_adjusted;
if ((word)my_bytes_allocd >= HBLKSIZE) {
*opp = obj_link(p);
obj_link(p) = NULL;
break;
}
}
GC_bytes_allocd += my_bytes_allocd;
goto out;
}
/* Next try to allocate a new block worth of objects of this size. */
{
struct hblk *h
= GC_allochblk(lb_adjusted, kind, 0 /* `flags` */, 0 /* `align_m1` */);
if (h /* `!= NULL` */) { /*< CPPCHECK */
if (IS_UNCOLLECTABLE(kind))
GC_set_hdr_marks(HDR(h));
GC_bytes_allocd += HBLKSIZE - (HBLKSIZE % lb_adjusted);
#ifdef PARALLEL_MARK
if (GC_parallel) {
GC_acquire_mark_lock();
++GC_fl_builder_count;
UNLOCK();
GC_release_mark_lock();
op = GC_build_fl(h, NULL, lg, ok->ok_init || GC_debugging_started);
*result = op;
GC_acquire_mark_lock();
--GC_fl_builder_count;
if (GC_fl_builder_count == 0)
GC_notify_all_builder();
GC_release_mark_lock();
(void)GC_clear_stack(0);
return;
}
#endif
op = GC_build_fl(h, NULL, lg, ok->ok_init || GC_debugging_started);
goto out;
}
}
/*
* As a last attempt, try allocating a single object.
* Note that this may trigger a collection or expand the heap.
*/
op = GC_generic_malloc_inner(lb_adjusted - EXTRA_BYTES, kind,
0 /* `flags` */);
if (op != NULL)
obj_link(op) = NULL;
out:
*result = op;
UNLOCK();
(void)GC_clear_stack(0);
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_malloc_many(size_t lb)
{
void *result;
size_t lg, lb_adjusted;
if (UNLIKELY(0 == lb))
lb = 1;
lg = ALLOC_REQUEST_GRANS(lb);
lb_adjusted = GRANULES_TO_BYTES(lg);
GC_generic_malloc_many(lb_adjusted, NORMAL, &result);
return result;
}
/*
* TODO: The debugging variant of `GC_memalign` and friends is tricky
* and currently missing. The major difficulty is: `store_debug_info`
* should return the pointer of the object with the requested alignment
* (unlike the object header).
*/
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_memalign(size_t align, size_t lb)
{
size_t align_m1 = align - 1;
/* Check the alignment argument. */
if (UNLIKELY(0 == align || (align & align_m1) != 0))
return NULL;
/* TODO: Use thread-local allocation. */
if (align <= GC_GRANULE_BYTES)
return GC_malloc(lb);
return GC_malloc_kind_aligned_global(lb, NORMAL, align_m1);
}
GC_API int GC_CALL
GC_posix_memalign(void **memptr, size_t align, size_t lb)
{
void *p;
size_t align_minus_one = align - 1; /*< to workaround a cppcheck warning */
/* Check alignment properly. */
if (UNLIKELY(align < sizeof(void *) || (align_minus_one & align) != 0)) {
#ifdef MSWINCE
return ERROR_INVALID_PARAMETER;
#else
return EINVAL;
#endif
}
p = GC_memalign(align, lb);
if (UNLIKELY(NULL == p)) {
#ifdef MSWINCE
return ERROR_NOT_ENOUGH_MEMORY;
#else
return ENOMEM;
#endif
}
*memptr = p;
return 0; /*< success */
}
#ifndef GC_NO_VALLOC
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_valloc(size_t lb)
{
if (UNLIKELY(!GC_is_initialized))
GC_init();
GC_ASSERT(GC_real_page_size != 0);
return GC_memalign(GC_real_page_size, lb);
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_pvalloc(size_t lb)
{
if (UNLIKELY(!GC_is_initialized))
GC_init();
GC_ASSERT(GC_real_page_size != 0);
lb = SIZET_SAT_ADD(lb, GC_real_page_size - 1) & ~(GC_real_page_size - 1);
return GC_memalign(GC_real_page_size, lb);
}
#endif /* !GC_NO_VALLOC */
GC_API GC_ATTR_MALLOC char *GC_CALL
GC_strdup(const char *s)
{
/*
* Implementation of a variant of `strdup()` that uses the collector
* to allocate a copy of the string.
*/
char *copy;
size_t lb;
if (s == NULL)
return NULL;
lb = strlen(s) + 1;
copy = (char *)GC_malloc_atomic(lb);
if (UNLIKELY(NULL == copy)) {
#ifndef MSWINCE
errno = ENOMEM;
#endif
return NULL;
}
BCOPY(s, copy, lb);
return copy;
}
GC_API GC_ATTR_MALLOC char *GC_CALL
GC_strndup(const char *str, size_t size)
{
char *copy;
/* Note: `str` is expected to be non-`NULL`. */
size_t len = strlen(str);
if (UNLIKELY(len > size))
len = size;
copy = (char *)GC_malloc_atomic(len + 1);
if (UNLIKELY(NULL == copy)) {
#ifndef MSWINCE
errno = ENOMEM;
#endif
return NULL;
}
if (LIKELY(len > 0))
BCOPY(str, copy, len);
copy[len] = '\0';
return copy;
}
#ifdef GC_REQUIRE_WCSDUP
# include <wchar.h> /*< for `wcslen()` */
GC_API GC_ATTR_MALLOC wchar_t *GC_CALL
GC_wcsdup(const wchar_t *str)
{
size_t lb = (wcslen(str) + 1) * sizeof(wchar_t);
wchar_t *copy = (wchar_t *)GC_malloc_atomic(lb);
if (UNLIKELY(NULL == copy)) {
# ifndef MSWINCE
errno = ENOMEM;
# endif
return NULL;
}
BCOPY(str, copy, lb);
return copy;
}
# if !defined(wcsdup) && defined(REDIRECT_MALLOC) \
&& !defined(REDIRECT_MALLOC_IN_HEADER)
wchar_t *
wcsdup(const wchar_t *str)
{
return GC_wcsdup(str);
}
# endif
#endif /* GC_REQUIRE_WCSDUP */
#ifndef CPPCHECK
GC_API void *GC_CALL
GC_malloc_stubborn(size_t lb)
{
return GC_malloc(lb);
}
GC_API void GC_CALL
GC_change_stubborn(const void *p)
{
UNUSED_ARG(p);
}
#endif /* !CPPCHECK */
GC_API void GC_CALL
GC_end_stubborn_change(const void *p)
{
GC_dirty(p); /*< entire object */
}
GC_API void GC_CALL
GC_ptr_store_and_dirty(void *p, const void *q)
{
*(const void **)p = q;
GC_dirty(p);
REACHABLE_AFTER_DIRTY(q);
}
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1995 by Xerox Corporation. All rights reserved.
* Copyright (c) 2000 by Hewlett-Packard Company. All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* Make arguments appear live to compiler. Put here to minimize the
* risk of inlining. Used to minimize junk left in registers.
*/
GC_ATTR_NOINLINE
void
GC_noop6(word arg1, word arg2, word arg3, word arg4, word arg5, word arg6)
{
UNUSED_ARG(arg1);
UNUSED_ARG(arg2);
UNUSED_ARG(arg3);
UNUSED_ARG(arg4);
UNUSED_ARG(arg5);
UNUSED_ARG(arg6);
/* Avoid `GC_noop6` calls to be optimized away. */
#if defined(AO_HAVE_compiler_barrier) && !defined(BASE_ATOMIC_OPS_EMULATED)
AO_compiler_barrier(); /*< to serve as a special side-effect */
#else
GC_noop1(0);
#endif
}
GC_API void GC_CALL
GC_noop1(GC_word x)
{
#if defined(AO_HAVE_store) && defined(THREAD_SANITIZER)
AO_store(&GC_noop_sink, (AO_t)x);
#else
GC_noop_sink = x;
#endif
}
GC_API void GC_CALL
GC_noop1_ptr(volatile void *p)
{
#if CPP_PTRSZ > CPP_WORDSZ
# if defined(AO_HAVE_store) && defined(THREAD_SANITIZER)
GC_cptr_store(&GC_noop_sink_ptr, (ptr_t)CAST_AWAY_VOLATILE_PVOID(p));
# else
GC_noop_sink_ptr = (ptr_t)CAST_AWAY_VOLATILE_PVOID(p);
# endif
#else
GC_noop1(ADDR(p));
#endif
}
/*
* Initialize `GC_obj_kinds` properly and standard free lists properly.
* This must be done statically since they may be accessed before
* `GC_init` is called. It is done here, since we need to deal with
* mark descriptors. Note: `GC_obj_kinds[NORMAL].ok_descriptor` is
* adjusted in `GC_init()` for `EXTRA_BYTES`.
*/
GC_INNER struct obj_kind GC_obj_kinds[MAXOBJKINDS] = {
/* `PTRFREE` */
{ &GC_aobjfreelist[0], 0 /*< filled in dynamically */,
/* `0 |` */ GC_DS_LENGTH, FALSE,
FALSE
/*, */ OK_DISCLAIM_INITZ },
/* `NORMAL` */
{ &GC_objfreelist[0], 0,
/* `0 |` */ GC_DS_LENGTH, TRUE /*< add length to descriptor template */,
TRUE
/*, */ OK_DISCLAIM_INITZ },
/* `UNCOLLECTABLE` */
{ &GC_uobjfreelist[0], 0,
/* `0 |` */ GC_DS_LENGTH, TRUE /*< add length to descriptor template */,
TRUE
/*, */ OK_DISCLAIM_INITZ },
#ifdef GC_ATOMIC_UNCOLLECTABLE
/* `AUNCOLLECTABLE` */
{ &GC_auobjfreelist[0], 0,
/* `0 |` */ GC_DS_LENGTH, FALSE,
FALSE
/*, */ OK_DISCLAIM_INITZ },
#endif
};
#ifndef GC_NO_DEINIT
/* Note: keep this close to `GC_obj_kinds` definition. */
GC_INNER void
GC_reset_obj_kinds(void)
{
unsigned i;
for (i = 0; i < GC_N_KINDS_INITIAL_VALUE; i++)
GC_obj_kinds[i].ok_reclaim_list = NULL;
GC_obj_kinds[PTRFREE].ok_freelist = &GC_aobjfreelist[0];
GC_obj_kinds[NORMAL].ok_freelist = &GC_objfreelist[0];
GC_obj_kinds[UNCOLLECTABLE].ok_freelist = &GC_uobjfreelist[0];
# ifdef GC_ATOMIC_UNCOLLECTABLE
GC_obj_kinds[AUNCOLLECTABLE].ok_freelist = &GC_auobjfreelist[0];
# endif
GC_obj_kinds[NORMAL].ok_descriptor = GC_DS_LENGTH;
GC_n_kinds = GC_N_KINDS_INITIAL_VALUE;
}
#endif
#ifndef INITIAL_MARK_STACK_SIZE
/*
* `INITIAL_MARK_STACK_SIZE * sizeof(mse)` should be a multiple of `HBLKSIZE`.
* The incremental collector actually likes a larger size, since it wants to
* push all marked dirty objects before marking anything new.
* Currently we let it grow dynamically.
*/
# define INITIAL_MARK_STACK_SIZE (1 * HBLKSIZE)
#endif
#if !defined(GC_DISABLE_INCREMENTAL)
/*
* The number of dirty pages we marked from, excluding pointer-free pages,
* etc. Used for logging only.
*/
STATIC word GC_n_rescuing_pages = 0;
#endif
GC_API void GC_CALL
GC_set_pointer_mask(GC_word value)
{
#ifdef DYNAMIC_POINTER_MASK
GC_ASSERT(value >= 0xff); /*< a simple sanity check */
GC_pointer_mask = value;
#else
if (value
# ifdef POINTER_MASK
!= (word)(POINTER_MASK)
# else
!= GC_WORD_MAX
# endif
) {
ABORT("Dynamic pointer mask/shift is unsupported");
}
#endif
}
GC_API GC_word GC_CALL
GC_get_pointer_mask(void)
{
#ifdef DYNAMIC_POINTER_MASK
GC_word value = GC_pointer_mask;
if (0 == value) {
GC_ASSERT(!GC_is_initialized);
value = GC_WORD_MAX;
}
return value;
#elif defined(POINTER_MASK)
return POINTER_MASK;
#else
return GC_WORD_MAX;
#endif
}
GC_API void GC_CALL
GC_set_pointer_shift(unsigned value)
{
#ifdef DYNAMIC_POINTER_MASK
GC_ASSERT(value < CPP_WORDSZ);
GC_pointer_shift = (unsigned char)value;
#else
if (value
# ifdef POINTER_SHIFT
!= (unsigned)(POINTER_SHIFT)
# endif
) {
ABORT("Dynamic pointer mask/shift is unsupported");
}
#endif
}
GC_API unsigned GC_CALL
GC_get_pointer_shift(void)
{
#ifdef DYNAMIC_POINTER_MASK
return GC_pointer_shift;
#elif defined(POINTER_SHIFT)
GC_STATIC_ASSERT((unsigned)(POINTER_SHIFT) < CPP_WORDSZ);
return POINTER_SHIFT;
#else
return 0;
#endif
}
GC_INNER GC_bool
GC_collection_in_progress(void)
{
return GC_mark_state != MS_NONE;
}
GC_INNER void
GC_clear_hdr_marks(hdr *hhdr)
{
size_t last_bit;
#ifdef AO_HAVE_load
/* Atomic access is used to avoid racing with `GC_realloc`. */
last_bit = FINAL_MARK_BIT(AO_load((volatile AO_t *)&hhdr->hb_sz));
#else
/*
* No race as `GC_realloc` holds the allocator lock while updating
* `hb_sz` field.
*/
last_bit = FINAL_MARK_BIT(hhdr->hb_sz);
#endif
BZERO(CAST_AWAY_VOLATILE_PVOID(hhdr->hb_marks), sizeof(hhdr->hb_marks));
set_mark_bit_from_hdr(hhdr, last_bit);
hhdr->hb_n_marks = 0;
}
GC_INNER void
GC_set_hdr_marks(hdr *hhdr)
{
size_t i;
size_t sz = hhdr->hb_sz;
size_t n_marks = FINAL_MARK_BIT(sz);
#ifdef USE_MARK_BYTES
for (i = 0; i <= n_marks; i += MARK_BIT_OFFSET(sz)) {
hhdr->hb_marks[i] = 1;
}
#else
/*
* Note that all bits are set even in case of not `MARK_BIT_PER_OBJ`,
* instead of setting every `n`-th bit where `n` is `MARK_BIT_OFFSET(sz)`.
* This is done for a performance reason.
*/
for (i = 0; i < divWORDSZ(n_marks); ++i) {
hhdr->hb_marks[i] = GC_WORD_MAX;
}
/* Set the remaining bits near the end (plus one bit past the end). */
hhdr->hb_marks[i] = ((((word)1 << modWORDSZ(n_marks)) - 1) << 1) | 1;
#endif
#ifdef MARK_BIT_PER_OBJ
hhdr->hb_n_marks = n_marks;
#else
hhdr->hb_n_marks = HBLK_OBJS(sz);
#endif
}
/* Clear all mark bits associated with block `h`. */
static void GC_CALLBACK
clear_marks_for_block(struct hblk *h, void *dummy)
{
hdr *hhdr = HDR(h);
UNUSED_ARG(dummy);
if (IS_UNCOLLECTABLE(hhdr->hb_obj_kind)) {
/*
* Mark bit for these is cleared only once the object is deallocated
* explicitly. This either frees the block, or the bit is cleared
* once the object is on the free list.
*/
return;
}
GC_clear_hdr_marks(hhdr);
#if defined(CPPCHECK)
GC_noop1_ptr(h);
#endif
}
/* Slow but general routines for setting/clearing/getting mark bits. */
GC_API void GC_CALL
GC_set_mark_bit(const void *p)
{
struct hblk *h = HBLKPTR(p);
hdr *hhdr = HDR(h);
size_t bit_no = MARK_BIT_NO((size_t)((ptr_t)p - (ptr_t)h), hhdr->hb_sz);
if (!mark_bit_from_hdr(hhdr, bit_no)) {
set_mark_bit_from_hdr(hhdr, bit_no);
INCR_MARKS(hhdr);
}
}
GC_API void GC_CALL
GC_clear_mark_bit(const void *p)
{
struct hblk *h = HBLKPTR(p);
hdr *hhdr = HDR(h);
size_t bit_no = MARK_BIT_NO((size_t)((ptr_t)p - (ptr_t)h), hhdr->hb_sz);
if (mark_bit_from_hdr(hhdr, bit_no)) {
size_t n_marks = hhdr->hb_n_marks;
GC_ASSERT(n_marks != 0);
clear_mark_bit_from_hdr(hhdr, bit_no);
n_marks--;
#ifdef PARALLEL_MARK
/*
* Do not decrement to zero. The counts are approximate due to
* concurrency issues, but we need to ensure that a count of zero
* implies an empty block.
*/
if (n_marks != 0 || !GC_parallel)
hhdr->hb_n_marks = n_marks;
#else
hhdr->hb_n_marks = n_marks;
#endif
}
}
GC_API int GC_CALL
GC_is_marked(const void *p)
{
struct hblk *h = HBLKPTR(p);
hdr *hhdr = HDR(h);
size_t bit_no = MARK_BIT_NO((size_t)((ptr_t)p - (ptr_t)h), hhdr->hb_sz);
return (int)mark_bit_from_hdr(hhdr, bit_no); /*< 0 or 1 */
}
GC_INNER void
GC_clear_marks(void)
{
/* The initialization is needed for `GC_push_roots()`. */
GC_ASSERT(GC_is_initialized);
GC_apply_to_all_blocks(clear_marks_for_block, NULL);
GC_objects_are_marked = FALSE;
GC_mark_state = MS_INVALID;
GC_scan_ptr = NULL;
}
GC_INNER void
GC_initiate_gc(void)
{
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_is_initialized);
#ifndef GC_DISABLE_INCREMENTAL
if (GC_incremental) {
# ifdef CHECKSUMS
GC_read_dirty(FALSE);
GC_check_dirty();
# else
GC_read_dirty(GC_mark_state == MS_INVALID);
# endif
}
GC_n_rescuing_pages = 0;
#endif
if (GC_mark_state == MS_NONE) {
GC_mark_state = MS_PUSH_RESCUERS;
} else {
/* This is really a full collection, and mark bits are invalid. */
GC_ASSERT(GC_mark_state == MS_INVALID);
}
GC_scan_ptr = NULL;
}
#ifdef PARALLEL_MARK
/* Initiate parallel marking. */
STATIC void GC_do_parallel_mark(void);
#endif
#ifdef GC_DISABLE_INCREMENTAL
# define GC_push_next_marked_dirty(h) GC_push_next_marked(h)
#else
STATIC struct hblk *GC_push_next_marked_dirty(struct hblk *h);
#endif /* !GC_DISABLE_INCREMENTAL */
STATIC struct hblk *GC_push_next_marked(struct hblk *h);
STATIC struct hblk *GC_push_next_marked_uncollectable(struct hblk *h);
static void alloc_mark_stack(size_t);
static void
push_roots_and_advance(GC_bool push_all, ptr_t cold_gc_frame)
{
if (GC_scan_ptr != NULL) {
/* Not ready to push. */
return;
}
GC_push_roots(push_all, cold_gc_frame);
GC_objects_are_marked = TRUE;
if (GC_mark_state != MS_INVALID)
GC_mark_state = MS_ROOTS_PUSHED;
}
STATIC GC_on_mark_stack_empty_proc GC_on_mark_stack_empty = 0;
GC_API void GC_CALL
GC_set_on_mark_stack_empty(GC_on_mark_stack_empty_proc fn)
{
LOCK();
GC_on_mark_stack_empty = fn;
UNLOCK();
}
GC_API GC_on_mark_stack_empty_proc GC_CALL
GC_get_on_mark_stack_empty(void)
{
GC_on_mark_stack_empty_proc fn;
READER_LOCK();
fn = GC_on_mark_stack_empty;
READER_UNLOCK();
return fn;
}
#ifdef WRAP_MARK_SOME
/*
* For Win32, this is called after we establish a structured exception
* (or signal) handler, in case Windows unmaps one of our root segments.
* Note that this code should never generate an incremental GC write fault.
*/
STATIC GC_bool
GC_mark_some_inner(ptr_t cold_gc_frame)
#else
GC_INNER GC_bool
GC_mark_some(ptr_t cold_gc_frame)
#endif
{
GC_ASSERT(I_HOLD_LOCK());
switch (GC_mark_state) {
case MS_NONE:
return TRUE;
case MS_PUSH_RESCUERS:
if (ADDR_GE((ptr_t)GC_mark_stack_top,
(ptr_t)(GC_mark_stack_limit - INITIAL_MARK_STACK_SIZE / 2))) {
/*
* Go ahead and mark, even though that might cause us to see more
* marked dirty objects later on. Avoid this in the future.
*/
GC_mark_stack_too_small = TRUE;
MARK_FROM_MARK_STACK();
} else {
GC_scan_ptr = GC_push_next_marked_dirty(GC_scan_ptr);
#ifndef GC_DISABLE_INCREMENTAL
if (NULL == GC_scan_ptr) {
GC_COND_LOG_PRINTF("Marked from %lu dirty pages\n",
(unsigned long)GC_n_rescuing_pages);
}
#endif
push_roots_and_advance(FALSE, cold_gc_frame);
}
GC_ASSERT(GC_mark_state == MS_PUSH_RESCUERS
|| GC_mark_state == MS_ROOTS_PUSHED
|| GC_mark_state == MS_INVALID);
break;
case MS_PUSH_UNCOLLECTABLE:
if (ADDR_GE((ptr_t)GC_mark_stack_top,
(ptr_t)(GC_mark_stack + GC_mark_stack_size / 4))) {
#ifdef PARALLEL_MARK
/* Avoid this, since we do not parallelize the marker here. */
if (GC_parallel)
GC_mark_stack_too_small = TRUE;
#endif
MARK_FROM_MARK_STACK();
} else {
GC_scan_ptr = GC_push_next_marked_uncollectable(GC_scan_ptr);
push_roots_and_advance(TRUE, cold_gc_frame);
}
GC_ASSERT(GC_mark_state == MS_PUSH_UNCOLLECTABLE
|| GC_mark_state == MS_ROOTS_PUSHED
|| GC_mark_state == MS_INVALID);
break;
case MS_ROOTS_PUSHED:
#ifdef PARALLEL_MARK
/*
* Eventually, incremental marking should run asynchronously
* in multiple threads, without acquiring the allocator lock.
* For now, parallel marker is disabled if there is a chance that
* marking could be interrupted by a client-supplied time limit
* or custom stop function.
*/
if (GC_parallel && !GC_parallel_mark_disabled) {
GC_do_parallel_mark();
GC_ASSERT(ADDR_LT((ptr_t)GC_mark_stack_top, GC_first_nonempty));
GC_mark_stack_top = GC_mark_stack - 1;
if (GC_mark_stack_too_small) {
alloc_mark_stack(2 * GC_mark_stack_size);
}
if (GC_mark_state == MS_ROOTS_PUSHED) {
GC_mark_state = MS_NONE;
return TRUE;
}
GC_ASSERT(GC_mark_state == MS_INVALID);
break;
}
#endif
if (ADDR_GE((ptr_t)GC_mark_stack_top, (ptr_t)GC_mark_stack)) {
MARK_FROM_MARK_STACK();
} else {
GC_on_mark_stack_empty_proc on_ms_empty = GC_on_mark_stack_empty;
if (on_ms_empty != 0) {
GC_mark_stack_top
= on_ms_empty(GC_mark_stack_top, GC_mark_stack_limit);
/* If we pushed new items, we need to continue processing. */
if (ADDR_GE((ptr_t)GC_mark_stack_top, (ptr_t)GC_mark_stack))
break;
}
if (GC_mark_stack_too_small) {
alloc_mark_stack(2 * GC_mark_stack_size);
}
GC_mark_state = MS_NONE;
return TRUE;
}
GC_ASSERT(GC_mark_state == MS_ROOTS_PUSHED || GC_mark_state == MS_INVALID);
break;
case MS_INVALID:
case MS_PARTIALLY_INVALID:
if (!GC_objects_are_marked) {
GC_mark_state = MS_PUSH_UNCOLLECTABLE;
break;
}
if (ADDR_GE((ptr_t)GC_mark_stack_top, (ptr_t)GC_mark_stack)) {
MARK_FROM_MARK_STACK();
GC_ASSERT(GC_mark_state == MS_PARTIALLY_INVALID
|| GC_mark_state == MS_INVALID);
break;
}
if (NULL == GC_scan_ptr && GC_mark_state == MS_INVALID) {
/*
* About to start a heap scan for marked objects.
* Mark stack is empty. OK to reallocate.
*/
if (GC_mark_stack_too_small) {
alloc_mark_stack(2 * GC_mark_stack_size);
}
GC_mark_state = MS_PARTIALLY_INVALID;
}
GC_scan_ptr = GC_push_next_marked(GC_scan_ptr);
if (GC_mark_state == MS_PARTIALLY_INVALID)
push_roots_and_advance(TRUE, cold_gc_frame);
GC_ASSERT(GC_mark_state == MS_ROOTS_PUSHED
|| GC_mark_state == MS_PARTIALLY_INVALID
|| GC_mark_state == MS_INVALID);
break;
default:
ABORT("GC_mark_some: bad state");
}
return FALSE;
}
#ifdef PARALLEL_MARK
GC_INNER GC_bool GC_parallel_mark_disabled = FALSE;
#endif
#ifdef WRAP_MARK_SOME
GC_INNER GC_bool
GC_mark_some(ptr_t cold_gc_frame)
{
GC_bool ret_val;
if (GC_no_dls) {
ret_val = GC_mark_some_inner(cold_gc_frame);
} else {
/*
* Windows appears to asynchronously create and remove writable
* memory mappings, for reasons we have not yet understood.
* Since we look for writable regions to determine the root set, we
* may try to mark from an address range that disappeared since we
* started the collection. Thus we have to recover from faults here.
* This code seems to be necessary for WinCE (at least in the case
* we would decide to add `MEM_PRIVATE` sections to data roots in
* `GC_register_dynamic_libraries`). It is conceivable that this is
* the same issue as with terminating threads that we see with Linux
* and `USE_PROC_FOR_LIBRARIES`.
*/
# ifndef NO_SEH_AVAILABLE
__try {
ret_val = GC_mark_some_inner(cold_gc_frame);
} __except (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION
? EXCEPTION_EXECUTE_HANDLER
: EXCEPTION_CONTINUE_SEARCH) {
goto handle_ex;
}
# else
# if defined(USE_PROC_FOR_LIBRARIES) && !defined(DEFAULT_VDB)
if (GC_auto_incremental) {
static GC_bool is_warned = FALSE;
if (!is_warned) {
is_warned = TRUE;
WARN("Incremental GC incompatible with /proc roots\n", 0);
}
/* Unclear if this could still work... */
}
# endif
/*
* If `USE_PROC_FOR_LIBRARIES`, then we are handling the case in
* which `/proc` is used for root finding, and we have threads.
* We may find a stack for a thread that is in the process of
* exiting, and disappears while we are marking it.
* This seems extremely difficult to avoid otherwise.
*/
GC_setup_temporary_fault_handler();
if (SETJMP(GC_jmp_buf) != 0)
goto handle_ex;
ret_val = GC_mark_some_inner(cold_gc_frame);
GC_reset_fault_handler();
# endif
}
# if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS)
/*
* With `DllMain`-based thread tracking, a thread may have started
* while we were marking. This is logically equivalent to the
* exception case; our results are invalid and we have to start over.
* This cannot be prevented since we cannot block in `DllMain()`.
*/
if (GC_started_thread_while_stopped())
goto handle_thr_start;
# endif
return ret_val;
handle_ex:
/* Exception handler starts here for all cases. */
# if defined(NO_SEH_AVAILABLE)
GC_reset_fault_handler();
# endif
{
static word warned_gc_no;
/* Report caught `ACCESS_VIOLATION`, once per collection. */
if (warned_gc_no != GC_gc_no) {
GC_COND_LOG_PRINTF("Memory mapping disappeared at collection #%lu\n",
(unsigned long)GC_gc_no + 1);
warned_gc_no = GC_gc_no;
}
}
# if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS)
handle_thr_start:
# endif
/*
* We have bad roots on the mark stack - discard it.
* Rescan from the marked objects; redetermine the roots.
*/
# ifdef REGISTER_LIBRARIES_EARLY
START_WORLD();
GC_cond_register_dynamic_libraries();
STOP_WORLD();
# endif
GC_invalidate_mark_state();
GC_scan_ptr = NULL;
return FALSE;
}
#endif /* WRAP_MARK_SOME */
GC_INNER void
GC_invalidate_mark_state(void)
{
GC_mark_state = MS_INVALID;
GC_mark_stack_top = GC_mark_stack - 1;
}
STATIC mse *
GC_signal_mark_stack_overflow(mse *msp)
{
GC_mark_state = MS_INVALID;
#ifdef PARALLEL_MARK
/*
* We are using a `local_mark_stack` in parallel mode, so do
* not signal the global mark stack to be resized.
* That will be done in `GC_return_mark_stack` if required.
*/
if (!GC_parallel)
GC_mark_stack_too_small = TRUE;
#else
GC_mark_stack_too_small = TRUE;
#endif
GC_COND_LOG_PRINTF("Mark stack overflow; current size: %lu entries\n",
(unsigned long)GC_mark_stack_size);
#if defined(CPPCHECK)
GC_noop1_ptr(msp);
#endif
return msp - GC_MARK_STACK_DISCARDS;
}
GC_ATTR_NO_SANITIZE_ADDR_MEM_THREAD
GC_INNER mse *
GC_mark_from(mse *mark_stack_top, mse *mark_stack, mse *mark_stack_limit)
{
GC_signed_word credit = HBLKSIZE; /*< remaining credit for marking work */
word descr;
ptr_t current_p; /*< pointer to the current candidate pointer */
ptr_t q; /*< the candidate pointer itself */
ptr_t limit = NULL; /*< the limit (incl.) of the current candidate range */
ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
DECLARE_HDR_CACHE;
#define SPLIT_RANGE_PTRS 128 /*< must be power of 2 */
GC_objects_are_marked = TRUE;
INIT_HDR_CACHE;
#if defined(OS2) || CPP_PTRSZ > CPP_WORDSZ
/* OS/2: use untweaked variant to circumvent a compiler problem. */
while (ADDR_GE((ptr_t)mark_stack_top, (ptr_t)mark_stack) && credit >= 0)
#else
while (((((word)mark_stack_top - (word)mark_stack) | (word)credit) & SIGNB)
== 0)
#endif
{
current_p = mark_stack_top->mse_start;
descr = mark_stack_top->mse_descr;
retry:
/*
* `current_p` and `descr` describe the current object.
* `*mark_stack_top` is vacant.
* The following is zero only for small objects described by a simple
* length descriptor. For many applications this is the common case,
* so we try to detect it quickly.
*/
if (descr & (~(word)(PTRS_TO_BYTES(SPLIT_RANGE_PTRS) - 1) | GC_DS_TAGS)) {
word tag = descr & GC_DS_TAGS;
GC_STATIC_ASSERT(GC_DS_TAGS == 0x3);
switch (tag) {
case GC_DS_LENGTH:
/*
* Large length. Process part of the range to avoid pushing
* too much on the stack.
*/
/* Either it is a heap object or a region outside the heap. */
GC_ASSERT(descr < GC_greatest_real_heap_addr - GC_least_real_heap_addr
|| GC_least_real_heap_addr + sizeof(ptr_t)
>= ADDR(current_p) + descr
|| ADDR(current_p) >= GC_greatest_real_heap_addr);
#ifdef PARALLEL_MARK
# define SHARE_BYTES 2048
if (descr > SHARE_BYTES && GC_parallel
&& ADDR_LT((ptr_t)mark_stack_top, (ptr_t)(mark_stack_limit - 1))) {
word new_size = (descr >> 1) & ~(word)(sizeof(ptr_t) - 1);
mark_stack_top->mse_start = current_p;
/* This makes sure we handle misaligned pointers. */
mark_stack_top->mse_descr
= (new_size + sizeof(ptr_t)) | GC_DS_LENGTH;
mark_stack_top++;
# ifdef ENABLE_TRACE
if (ADDR_INSIDE(GC_trace_ptr, current_p, current_p + descr)) {
GC_log_printf("GC #%lu: large section; start %p, len %lu,"
" splitting (parallel) at %p\n",
(unsigned long)GC_gc_no, (void *)current_p,
(unsigned long)descr,
(void *)(current_p + new_size));
}
# endif
current_p += new_size;
descr -= new_size;
goto retry;
}
#endif /* PARALLEL_MARK */
limit = current_p + PTRS_TO_BYTES(SPLIT_RANGE_PTRS - 1);
mark_stack_top->mse_start = limit;
mark_stack_top->mse_descr
= descr - PTRS_TO_BYTES(SPLIT_RANGE_PTRS - 1);
#ifdef ENABLE_TRACE
if (ADDR_INSIDE(GC_trace_ptr, current_p, current_p + descr)) {
GC_log_printf("GC #%lu: large section; start %p, len %lu,"
" splitting at %p\n",
(unsigned long)GC_gc_no, (void *)current_p,
(unsigned long)descr, (void *)limit);
}
#endif
/*
* Make sure that pointers overlapping the two ranges are
* considered.
*/
limit += sizeof(ptr_t) - ALIGNMENT;
break;
case GC_DS_BITMAP:
mark_stack_top--;
#ifdef ENABLE_TRACE
if (ADDR_INSIDE(GC_trace_ptr, current_p,
current_p + PTRS_TO_BYTES(BITMAP_BITS))) {
GC_log_printf("GC #%lu: tracing from %p bitmap descr 0x%lx\n",
(unsigned long)GC_gc_no, (void *)current_p,
(unsigned long)descr);
}
#endif
descr &= ~(word)GC_DS_TAGS;
credit -= (GC_signed_word)PTRS_TO_BYTES(CPP_PTRSZ / 2); /*< guess */
for (; descr != 0;
descr <<= 1, current_p += sizeof(ptr_t)) { /*< not `ALIGNMENT` */
if ((descr & SIGNB) == 0)
continue;
LOAD_PTR_OR_CONTINUE(q, current_p);
FIXUP_POINTER(q);
if (ADDR_LT(least_ha, q) && ADDR_LT(q, greatest_ha)) {
PREFETCH(q);
#ifdef ENABLE_TRACE
if (GC_trace_ptr == current_p) {
GC_log_printf("GC #%lu: considering(3) %p -> %p\n",
(unsigned long)GC_gc_no, (void *)current_p,
(void *)q);
}
#endif
PUSH_CONTENTS(q, mark_stack_top, mark_stack_limit, current_p);
}
}
continue;
case GC_DS_PROC:
mark_stack_top--;
#ifdef ENABLE_TRACE
if (ADDR_GE(GC_trace_ptr, current_p)) {
const void *base = GC_base(current_p);
if (base != NULL && GC_base(GC_trace_ptr) == base) {
GC_log_printf("GC #%lu: tracing from %p, proc descr 0x%lx\n",
(unsigned long)GC_gc_no, (void *)current_p,
(unsigned long)descr);
}
}
#endif
credit -= GC_PROC_BYTES;
mark_stack_top = (*PROC(descr))((word *)current_p, mark_stack_top,
mark_stack_limit, ENV(descr));
continue;
case GC_DS_PER_OBJECT:
if (!(descr & SIGNB)) {
/* Descriptor is in the object. */
descr = *(word *)(current_p + descr - GC_DS_PER_OBJECT);
} else {
/*
* Descriptor is in the type descriptor pointed to by the first
* "pointer-sized" word of the object.
*/
ptr_t type_descr = *(ptr_t *)current_p;
/*
* `type_descr` is either a valid pointer to the descriptor
* structure, or this object was on a free list.
* If it was anything but the last object on the free list,
* we will misinterpret the next object on the free list as
* the type descriptor, and get a zero GC descriptor, which
* is ideal. Unfortunately, we need to check for the last
* object case explicitly.
*/
if (UNLIKELY(NULL == type_descr)) {
mark_stack_top--;
continue;
}
descr = *(word *)(type_descr
- ((GC_signed_word)descr
+ (GC_INDIR_PER_OBJ_BIAS - GC_DS_PER_OBJECT)));
}
if (0 == descr) {
/*
* Can happen either because we generated a zero GC descriptor
* or we saw a pointer to a free object.
*/
mark_stack_top--;
continue;
}
goto retry;
}
} else {
/* Small object with length descriptor. */
mark_stack_top--;
#ifndef SMALL_CONFIG
if (descr < sizeof(ptr_t))
continue;
#endif
#ifdef ENABLE_TRACE
if (ADDR_INSIDE(GC_trace_ptr, current_p, current_p + descr)) {
GC_log_printf("GC #%lu: small object; start %p, len %lu\n",
(unsigned long)GC_gc_no, (void *)current_p,
(unsigned long)descr);
}
#endif
limit = current_p + descr;
}
/* The simple case in which we are scanning a range. */
GC_ASSERT((ADDR(current_p) & (ALIGNMENT - 1)) == 0);
credit -= limit - current_p;
limit -= sizeof(ptr_t);
{
#define PREF_DIST 4
#if !defined(SMALL_CONFIG) && !(defined(E2K) && defined(USE_PTR_HWTAG))
ptr_t deferred;
# ifdef CHERI_PURECAP
/*
* Check each pointer for validity before dereferencing to prevent
* capability exceptions. Utilize the pointer meta-data to speed-up
* the loop. If the loop is below the pointer bounds, skip the rest
* of marking for that chunk. If the limit capability restricts us to
* reading fewer than size of a pointer, then there cannot possibly be
* a pointer at `limit`'s pointer, and reading at that location will
* raise a capability exception.
*/
{
word cap_limit = cheri_base_get(limit) + cheri_length_get(limit);
if (ADDR(limit) + sizeof(ptr_t) > cap_limit) {
/* Decrement limit so that it to be within bounds of `current_p`. */
GC_ASSERT(cap_limit > sizeof(ptr_t));
limit = (ptr_t)cheri_address_set(
current_p, (cap_limit - sizeof(ptr_t)) & ~(sizeof(ptr_t) - 1));
goto check_limit;
}
}
# endif
/*
* Try to prefetch the next pointer to be examined as soon as possible.
* Empirically, this also seems to help slightly without prefetches,
* at least on Linux/i686. Presumably this loop ends up with less
* register pressure, and gcc thus ends up generating slightly better
* code. Overall gcc code quality for this loop is still not great.
*/
for (;;) {
PREFETCH(limit - PREF_DIST * CACHE_LINE_SIZE);
GC_ASSERT(ADDR_GE(limit, current_p));
# ifdef CHERI_PURECAP
if (ADDR(limit) < cheri_base_get(limit))
goto next_object;
if (!HAS_TAG_AND_PERM_LOAD(limit)) {
limit -= ALIGNMENT;
goto check_limit;
}
# endif
deferred = *(ptr_t *)limit;
FIXUP_POINTER(deferred);
limit -= ALIGNMENT;
# ifdef CHERI_PURECAP
if (!HAS_TAG_AND_PERM_LOAD(deferred))
goto check_limit;
# endif
if (ADDR_LT(least_ha, deferred) && ADDR_LT(deferred, greatest_ha)) {
PREFETCH(deferred);
break;
}
# ifndef CHERI_PURECAP
if (ADDR_LT(limit, current_p))
goto next_object;
/*
* Unroll once, so we do not do too many of the prefetches based
* on `limit`.
*/
deferred = *(ptr_t *)limit;
FIXUP_POINTER(deferred);
limit -= ALIGNMENT;
if (ADDR_LT(least_ha, deferred) && ADDR_LT(deferred, greatest_ha)) {
PREFETCH(deferred);
break;
}
# else
check_limit:
# endif
if (ADDR_LT(limit, current_p))
goto next_object;
}
#endif
for (; ADDR_GE(limit, current_p); current_p += ALIGNMENT) {
/*
* Empirically, unrolling this loop does not help a lot.
* Since `PUSH_CONTENTS` expands to a lot of code, we do not.
*/
LOAD_PTR_OR_CONTINUE(q, current_p);
FIXUP_POINTER(q);
PREFETCH(current_p + PREF_DIST * CACHE_LINE_SIZE);
if (ADDR_LT(least_ha, q) && ADDR_LT(q, greatest_ha)) {
/*
* Prefetch the content of the object we just pushed.
* It is likely we will need them soon.
*/
PREFETCH(q);
#ifdef ENABLE_TRACE
if (GC_trace_ptr == current_p) {
GC_log_printf("GC #%lu: considering(1) %p -> %p\n",
(unsigned long)GC_gc_no, (void *)current_p,
(void *)q);
}
#endif
PUSH_CONTENTS(q, mark_stack_top, mark_stack_limit, current_p);
}
}
#if !defined(SMALL_CONFIG) && !(defined(E2K) && defined(USE_PTR_HWTAG))
/*
* We still need to mark the entry we previously prefetched.
* We already know that it passes the preliminary pointer validity test.
*/
# ifdef ENABLE_TRACE
if (GC_trace_ptr == current_p) {
GC_log_printf("GC #%lu: considering(2) %p -> %p\n",
(unsigned long)GC_gc_no, (void *)current_p,
(void *)deferred);
}
# endif
PUSH_CONTENTS(deferred, mark_stack_top, mark_stack_limit, current_p);
next_object:;
#endif
}
}
return mark_stack_top;
}
#ifdef PARALLEL_MARK
/* Note: this is protected by the mark lock. */
STATIC GC_bool GC_help_wanted = FALSE;
/* Number of running helpers. Protected by the mark lock. */
STATIC unsigned GC_helper_count = 0;
/*
* Number of active helpers. May increase and decrease within each
* mark cycle; but once it returns to zero, it stays for the cycle.
* Protected by the mark lock.
*/
STATIC unsigned GC_active_count = 0;
# ifdef LINT2
# define LOCAL_MARK_STACK_SIZE (HBLKSIZE / 8)
# else
/*
* Under normal circumstances, this is big enough to guarantee we do not
* overflow half of it in a single call to `GC_mark_from`.
*/
# define LOCAL_MARK_STACK_SIZE HBLKSIZE
# endif
GC_INNER void
GC_wait_for_markers_init(void)
{
GC_signed_word count;
GC_ASSERT(I_HOLD_LOCK());
if (0 == GC_markers_m1)
return;
# ifndef CAN_HANDLE_FORK
GC_ASSERT(NULL == GC_main_local_mark_stack);
# else
if (NULL == GC_main_local_mark_stack)
# endif
{
size_t bytes_to_get
= ROUNDUP_PAGESIZE_IF_MMAP(LOCAL_MARK_STACK_SIZE * sizeof(mse));
/*
* Allocate the local mark stack for the thread that holds the
* allocator lock.
*/
GC_ASSERT(GC_page_size != 0);
GC_main_local_mark_stack = (mse *)GC_os_get_mem(bytes_to_get);
if (NULL == GC_main_local_mark_stack)
ABORT("Insufficient memory for main local_mark_stack");
}
/*
* Reuse the mark lock and builders count to synchronize marker threads
* startup.
*/
GC_acquire_mark_lock();
GC_fl_builder_count += GC_markers_m1;
count = GC_fl_builder_count;
GC_release_mark_lock();
if (count != 0) {
GC_ASSERT(count > 0);
GC_wait_for_reclaim();
}
}
/*
* Steal mark stack entries starting at `mse` `low` into mark stack `local`
* until we either steal `mse` `high`, or we have `n_to_get` entries.
* Return a pointer to the top of the local mark stack. `*next` is replaced
* by a pointer to the next unscanned mark stack entry.
*/
STATIC mse *
GC_steal_mark_stack(mse *low, mse *high, mse *local, size_t n_to_get,
mse **next)
{
mse *p;
mse *top = local - 1;
size_t i = 0;
GC_ASSERT(ADDR_GE((ptr_t)high, (ptr_t)(low - 1))
&& (word)(high - low + 1) <= GC_mark_stack_size);
for (p = low; ADDR_GE((ptr_t)high, (ptr_t)p) && i <= n_to_get; ++p) {
word descr = AO_load(&p->mse_descr);
if (descr != 0) {
/* Must be ordered after read of `mse_descr`. */
AO_store_release_write(&p->mse_descr, 0);
/*
* More than one thread may get this entry, but that is only
* a minor performance problem.
*/
++top;
top->mse_start = p->mse_start;
top->mse_descr = descr;
GC_ASSERT((descr & GC_DS_TAGS) != GC_DS_LENGTH /* 0 */
|| descr < GC_greatest_real_heap_addr - GC_least_real_heap_addr
|| GC_least_real_heap_addr + sizeof(ptr_t)
>= ADDR(p->mse_start) + descr
|| ADDR(p->mse_start) >= GC_greatest_real_heap_addr);
/* If this is a big object, count it as `descr / 256 + 1` objects. */
++i;
if ((descr & GC_DS_TAGS) == GC_DS_LENGTH)
i += (size_t)(descr >> 8);
}
}
*next = p;
# if defined(CPPCHECK)
GC_noop1_ptr(local);
# endif
return top;
}
/* Copy back a local mark stack. `low` and `high` are inclusive bounds. */
STATIC void
GC_return_mark_stack(mse *low, mse *high)
{
mse *my_top;
mse *my_start;
size_t stack_size;
if (ADDR_LT((ptr_t)high, (ptr_t)low))
return;
stack_size = high - low + 1;
GC_acquire_mark_lock();
/* Note: the concurrent modification is impossible. */
my_top = GC_mark_stack_top;
my_start = my_top + 1;
if ((word)(my_start - GC_mark_stack + stack_size)
> (word)GC_mark_stack_size) {
GC_COND_LOG_PRINTF("No room to copy back mark stack\n");
GC_mark_state = MS_INVALID;
GC_mark_stack_too_small = TRUE;
/* We drop the local mark stack. We will fix things later. */
} else {
BCOPY(low, my_start, stack_size * sizeof(mse));
GC_ASSERT((mse *)GC_cptr_load((volatile ptr_t *)&GC_mark_stack_top)
== my_top);
/* Ensures visibility of previously written stack contents. */
GC_cptr_store_release_write((volatile ptr_t *)&GC_mark_stack_top,
(ptr_t)(my_top + stack_size));
}
GC_release_mark_lock();
GC_notify_all_marker();
}
# ifndef N_LOCAL_ITERS
# define N_LOCAL_ITERS 1
# endif
/*
* Note: called only when the local and the main mark stacks are both
* empty.
*/
static GC_bool
has_inactive_helpers(void)
{
GC_bool res;
GC_acquire_mark_lock();
res = GC_active_count < GC_helper_count;
GC_release_mark_lock();
return res;
}
/*
* Mark from the local mark stack. On return, the local mark stack
* is empty. But this may be achieved by copying the local mark stack
* back into the global one. We do not hold the mark lock.
*/
STATIC void
GC_do_local_mark(mse *local_mark_stack, mse *local_top)
{
unsigned n;
for (;;) {
for (n = 0; n < N_LOCAL_ITERS; ++n) {
local_top = GC_mark_from(local_top, local_mark_stack,
local_mark_stack + LOCAL_MARK_STACK_SIZE);
if (ADDR_LT((ptr_t)local_top, (ptr_t)local_mark_stack))
return;
if ((word)(local_top - local_mark_stack) >= LOCAL_MARK_STACK_SIZE / 2) {
GC_return_mark_stack(local_mark_stack, local_top);
return;
}
}
if (ADDR_LT(GC_cptr_load((volatile ptr_t *)&GC_mark_stack_top),
GC_cptr_load(&GC_first_nonempty))
&& ADDR_LT((ptr_t)(local_mark_stack + 1), (ptr_t)local_top)
&& has_inactive_helpers()) {
/*
* Try to share the load, since the main stack is empty, and the helper
* threads are waiting for a refill. The entries near the bottom of
* the stack are likely to require more work. Thus we return those,
* even though it is harder.
*/
mse *new_bottom = local_mark_stack + (local_top - local_mark_stack) / 2;
GC_ASSERT(ADDR_LT((ptr_t)local_mark_stack, (ptr_t)new_bottom)
&& ADDR_LT((ptr_t)new_bottom, (ptr_t)local_top));
GC_return_mark_stack(local_mark_stack, new_bottom - 1);
memmove(local_mark_stack, new_bottom,
(local_top - new_bottom + 1) * sizeof(mse));
local_top -= new_bottom - local_mark_stack;
}
}
}
# ifndef ENTRIES_TO_GET
# define ENTRIES_TO_GET 5
# endif
/*
* Mark using the local mark stack until the global mark stack is empty and
* there are no active workers. Update `GC_first_nonempty` to reflect the
* progress. Caller holds the mark lock. Caller has already incremented
* `GC_helper_count`; we decrement it, and maintain `GC_active_count`.
*/
STATIC void
GC_mark_local(mse *local_mark_stack, int id)
{
mse *my_first_nonempty;
GC_active_count++;
my_first_nonempty = (mse *)GC_cptr_load(&GC_first_nonempty);
GC_ASSERT(ADDR_GE((ptr_t)my_first_nonempty, (ptr_t)GC_mark_stack));
GC_ASSERT(
ADDR_GE(GC_cptr_load((volatile ptr_t *)&GC_mark_stack_top) + sizeof(mse),
(ptr_t)my_first_nonempty));
GC_VERBOSE_LOG_PRINTF("Starting mark helper %d\n", id);
GC_release_mark_lock();
for (;;) {
size_t n_on_stack, n_to_get;
mse *my_top, *local_top;
mse *global_first_nonempty = (mse *)GC_cptr_load(&GC_first_nonempty);
GC_ASSERT(ADDR_GE((ptr_t)my_first_nonempty, (ptr_t)GC_mark_stack)
&& ADDR_GE(GC_cptr_load((volatile ptr_t *)&GC_mark_stack_top)
+ sizeof(mse),
(ptr_t)my_first_nonempty));
GC_ASSERT(ADDR_GE((ptr_t)global_first_nonempty, (ptr_t)GC_mark_stack));
if (ADDR_LT((ptr_t)my_first_nonempty, (ptr_t)global_first_nonempty)) {
my_first_nonempty = global_first_nonempty;
} else if (ADDR_LT((ptr_t)global_first_nonempty,
(ptr_t)my_first_nonempty)) {
(void)GC_cptr_compare_and_swap(&GC_first_nonempty,
(ptr_t)global_first_nonempty,
(ptr_t)my_first_nonempty);
/*
* If this fails, then we just go ahead, without updating
* `GC_first_nonempty`.
*/
}
/*
* Perhaps we should also update `GC_first_nonempty`, if it is less.
* But that would require usage of the atomic updates.
*/
my_top = (mse *)GC_cptr_load_acquire((volatile ptr_t *)&GC_mark_stack_top);
if (ADDR_LT((ptr_t)my_top, (ptr_t)my_first_nonempty)) {
GC_acquire_mark_lock();
/*
* Note: asynchronous modification is impossible here, since
* we hold the mark lock.
*/
my_top = GC_mark_stack_top;
n_on_stack = my_top - my_first_nonempty + 1;
if (0 == n_on_stack) {
GC_active_count--;
GC_ASSERT(GC_active_count <= GC_helper_count);
/* Other markers may redeposit objects on the stack. */
if (0 == GC_active_count)
GC_notify_all_marker();
while (GC_active_count > 0
&& ADDR_LT((ptr_t)GC_mark_stack_top,
GC_cptr_load(&GC_first_nonempty))) {
/*
* We will be notified if either `GC_active_count` reaches zero,
* or if more objects are pushed on the global mark stack.
*/
GC_wait_marker();
}
if (0 == GC_active_count
&& ADDR_LT((ptr_t)GC_mark_stack_top,
GC_cptr_load(&GC_first_nonempty))) {
GC_bool need_to_notify = FALSE;
/*
* The above conditions cannot be falsified while we hold
* the mark lock, since neither `GC_active_count` nor
* `GC_mark_stack_top` can change. `GC_first_nonempty` can
* only be incremented asynchronously. Thus we know that
* both conditions are actually held simultaneously.
*/
GC_helper_count--;
if (0 == GC_helper_count)
need_to_notify = TRUE;
GC_VERBOSE_LOG_PRINTF("Finished mark helper %d\n", id);
if (need_to_notify)
GC_notify_all_marker();
return;
}
/*
* Else there is something on the stack again, or another helper
* may push something.
*/
GC_active_count++;
GC_ASSERT(GC_active_count > 0);
GC_release_mark_lock();
continue;
} else {
GC_release_mark_lock();
}
} else {
n_on_stack = my_top - my_first_nonempty + 1;
}
n_to_get = ENTRIES_TO_GET;
if (n_on_stack < 2 * ENTRIES_TO_GET)
n_to_get = 1;
local_top
= GC_steal_mark_stack(my_first_nonempty, my_top, local_mark_stack,
n_to_get, &my_first_nonempty);
GC_ASSERT(ADDR_GE((ptr_t)my_first_nonempty, (ptr_t)GC_mark_stack)
&& ADDR_GE(GC_cptr_load((volatile ptr_t *)&GC_mark_stack_top)
+ sizeof(mse),
(ptr_t)my_first_nonempty));
GC_do_local_mark(local_mark_stack, local_top);
}
}
/*
* Perform parallel mark. We hold the allocator lock, but not the mark lock.
* Currently runs until the mark stack is empty.
*/
STATIC void
GC_do_parallel_mark(void)
{
GC_ASSERT(I_HOLD_LOCK());
GC_acquire_mark_lock();
GC_ASSERT(!GC_help_wanted);
GC_ASSERT(0 == GC_active_count && 0 == GC_helper_count);
GC_VERBOSE_LOG_PRINTF("Starting marking for mark phase number %lu\n",
(unsigned long)GC_mark_no);
GC_cptr_store(&GC_first_nonempty, (ptr_t)GC_mark_stack);
GC_active_count = 0;
GC_helper_count = 1;
GC_help_wanted = TRUE;
/* Wake up potential helpers. */
GC_notify_all_marker();
GC_mark_local(GC_main_local_mark_stack, 0);
GC_help_wanted = FALSE;
/* Done; clean up. */
while (GC_helper_count > 0) {
GC_wait_marker();
}
/* `GC_helper_count` cannot be incremented while not `GC_help_wanted`. */
GC_VERBOSE_LOG_PRINTF("Finished marking for mark phase number %lu\n",
(unsigned long)GC_mark_no);
GC_mark_no++;
GC_release_mark_lock();
GC_notify_all_marker();
}
GC_INNER void
GC_help_marker(word my_mark_no)
{
# define my_id my_id_mse.mse_descr
/*
* Put `my_id` inside the structure to keep `local_mark_stack` aligned
* explicitly.
*/
mse my_id_mse;
mse local_mark_stack[LOCAL_MARK_STACK_SIZE];
/* Note: `local_mark_stack` is quite big (up to 128 KiB). */
GC_ASSERT(I_DONT_HOLD_LOCK());
GC_ASSERT(GC_parallel);
while (GC_mark_no < my_mark_no
|| (!GC_help_wanted && GC_mark_no == my_mark_no)) {
GC_wait_marker();
}
my_id = GC_helper_count;
if (GC_mark_no != my_mark_no || my_id > (unsigned)GC_markers_m1) {
/*
* The second test is useful only if the original threads can also
* act as helpers. Under Linux they cannot.
*/
return;
}
GC_helper_count = (unsigned)my_id + 1;
GC_mark_local(local_mark_stack, (int)my_id);
/* `GC_mark_local` decrements `GC_helper_count`. */
# undef my_id
}
#endif /* PARALLEL_MARK */
/*
* Allocate or reallocate space for mark stack of size `n` entries.
* May silently fail.
*/
static void
alloc_mark_stack(size_t n)
{
#ifdef GWW_VDB
static GC_bool GC_incremental_at_stack_alloc = FALSE;
GC_bool recycle_old;
#endif
mse *new_stack;
GC_ASSERT(I_HOLD_LOCK());
new_stack = (mse *)GC_scratch_alloc(n * sizeof(struct GC_ms_entry));
#ifdef GWW_VDB
/*
* Do not recycle a stack segment obtained with the wrong flags.
* Win32 `GetWriteWatch` requires the right kind of memory.
*/
recycle_old = !GC_auto_incremental || GC_incremental_at_stack_alloc;
GC_incremental_at_stack_alloc = GC_auto_incremental;
#endif
GC_mark_stack_too_small = FALSE;
if (GC_mark_stack != NULL) {
if (new_stack != 0) {
#ifdef GWW_VDB
if (recycle_old)
#endif
{
/* Recycle old space. */
GC_scratch_recycle_inner(
GC_mark_stack, GC_mark_stack_size * sizeof(struct GC_ms_entry));
}
GC_mark_stack = new_stack;
GC_mark_stack_size = n;
/* FIXME: Do we need some way to reset `GC_mark_stack_size`? */
GC_mark_stack_limit = new_stack + n;
GC_COND_LOG_PRINTF("Grew mark stack to %lu frames\n",
(unsigned long)GC_mark_stack_size);
} else {
WARN("Failed to grow mark stack to %" WARN_PRIuPTR " frames\n", n);
}
} else if (NULL == new_stack) {
GC_err_printf("No space for mark stack\n");
EXIT();
} else {
GC_mark_stack = new_stack;
GC_mark_stack_size = n;
GC_mark_stack_limit = new_stack + n;
}
GC_mark_stack_top = GC_mark_stack - 1;
}
GC_INNER void
GC_mark_init(void)
{
alloc_mark_stack(INITIAL_MARK_STACK_SIZE);
}
GC_API void GC_CALL
GC_push_all(void *bottom, void *top)
{
mse *mark_stack_top;
word length;
bottom = PTR_ALIGN_UP((ptr_t)bottom, ALIGNMENT);
top = PTR_ALIGN_DOWN((ptr_t)top, ALIGNMENT);
if (ADDR_GE((ptr_t)bottom, (ptr_t)top))
return;
mark_stack_top = GC_mark_stack_top + 1;
if (ADDR_GE((ptr_t)mark_stack_top, (ptr_t)GC_mark_stack_limit)) {
ABORT("Unexpected mark stack overflow");
}
length = (word)((ptr_t)top - (ptr_t)bottom);
#if GC_DS_TAGS > ALIGNMENT - 1
length = (length + GC_DS_TAGS) & ~(word)GC_DS_TAGS; /*< round up */
#endif
mark_stack_top->mse_start = (ptr_t)bottom;
mark_stack_top->mse_descr = length | GC_DS_LENGTH;
GC_mark_stack_top = mark_stack_top;
}
GC_API struct GC_ms_entry *GC_CALL
GC_custom_push_range(void *bottom, void *top,
struct GC_ms_entry *mark_stack_top,
struct GC_ms_entry *mark_stack_limit)
{
word length;
bottom = PTR_ALIGN_UP((ptr_t)bottom, ALIGNMENT);
top = PTR_ALIGN_DOWN((ptr_t)top, ALIGNMENT);
if (ADDR_GE((ptr_t)bottom, (ptr_t)top))
return mark_stack_top;
length = (word)((ptr_t)top - (ptr_t)bottom);
#if GC_DS_TAGS > ALIGNMENT - 1
length = (length + GC_DS_TAGS) & ~(word)GC_DS_TAGS; /*< round up */
#endif
return GC_custom_push_proc(length | GC_DS_LENGTH, bottom, mark_stack_top,
mark_stack_limit);
}
GC_API struct GC_ms_entry *GC_CALL
GC_custom_push_proc(GC_word descr, void *obj,
struct GC_ms_entry *mark_stack_top,
struct GC_ms_entry *mark_stack_limit)
{
mark_stack_top++;
if (ADDR_GE((ptr_t)mark_stack_top, (ptr_t)mark_stack_limit)) {
mark_stack_top = GC_signal_mark_stack_overflow(mark_stack_top);
}
mark_stack_top->mse_start = (ptr_t)obj;
mark_stack_top->mse_descr = descr;
return mark_stack_top;
}
GC_API void GC_CALL
GC_push_proc(GC_word descr, void *obj)
{
GC_mark_stack_top = GC_custom_push_proc(descr, obj, GC_mark_stack_top,
GC_mark_stack_limit);
}
#ifndef GC_DISABLE_INCREMENTAL
/*
* Analogous to `GC_push_all`, but push only those pages `h` with
* `dirty_fn(h) != 0`. We use `GC_push_all` to actually push the block.
* Used both to selectively push dirty pages, or to push a block in
* a piecemeal fashion, to allow for more marking concurrency.
* Will not overflow mark stack if `GC_push_all` pushes a small fixed
* number of entries. (This is invoked only if `GC_push_all` pushes
* a single entry, or if it marks each object before pushing it, thus
* ensuring progress in the event of a stack overflow.)
*/
STATIC void
GC_push_selected(ptr_t bottom, ptr_t top, GC_bool (*dirty_fn)(struct hblk *))
{
struct hblk *h;
bottom = PTR_ALIGN_UP(bottom, ALIGNMENT);
top = PTR_ALIGN_DOWN(top, ALIGNMENT);
if (ADDR_GE(bottom, top))
return;
h = HBLKPTR(bottom + HBLKSIZE);
if (ADDR_GE((ptr_t)h, top)) {
if ((*dirty_fn)(h - 1)) {
GC_push_all(bottom, top);
}
return;
}
if ((*dirty_fn)(h - 1)) {
if ((word)(GC_mark_stack_top - GC_mark_stack)
> 3 * GC_mark_stack_size / 4) {
GC_push_all(bottom, top);
return;
}
GC_push_all(bottom, h);
}
while (ADDR_GE(top, (ptr_t)(h + 1))) {
if ((*dirty_fn)(h)) {
if ((word)(GC_mark_stack_top - GC_mark_stack)
> 3 * GC_mark_stack_size / 4) {
/* Danger of mark stack overflow. */
GC_push_all(h, top);
return;
} else {
GC_push_all(h, h + 1);
}
}
h++;
}
if ((ptr_t)h != top && (*dirty_fn)(h)) {
GC_push_all(h, top);
}
}
GC_API void GC_CALL
GC_push_conditional(void *bottom, void *top, int all)
{
if (!all) {
GC_push_selected((ptr_t)bottom, (ptr_t)top, GC_page_was_dirty);
} else {
# ifdef PROC_VDB
if (GC_auto_incremental) {
/* Pages that were never dirtied cannot contain pointers. */
GC_push_selected((ptr_t)bottom, (ptr_t)top, GC_page_was_ever_dirty);
} else
# endif
/* else */ {
GC_push_all(bottom, top);
}
}
}
# ifndef NO_VDB_FOR_STATIC_ROOTS
# ifndef PROC_VDB
/*
* Same as `GC_page_was_dirty` but `h` is allowed to point to some page
* in the registered static roots only. Not used if the manual VDB is on.
*/
STATIC GC_bool
GC_static_page_was_dirty(struct hblk *h)
{
return get_pht_entry_from_index(GC_grungy_pages, PHT_HASH(h));
}
# endif
GC_INNER void
GC_push_conditional_static(void *bottom, void *top, GC_bool all)
{
# ifdef PROC_VDB
/*
* Just redirect to the generic routine because `PROC_VDB`
* implementation gets the dirty bits map for the whole process memory.
*/
GC_push_conditional(bottom, top, all);
# else
if (all || !GC_is_vdb_for_static_roots()) {
GC_push_all(bottom, top);
} else {
GC_push_selected((ptr_t)bottom, (ptr_t)top, GC_static_page_was_dirty);
}
# endif
}
# endif /* !NO_VDB_FOR_STATIC_ROOTS */
#else
GC_API void GC_CALL
GC_push_conditional(void *bottom, void *top, int all)
{
UNUSED_ARG(all);
GC_push_all(bottom, top);
}
#endif /* GC_DISABLE_INCREMENTAL */
#if defined(DARWIN) && defined(THREADS)
void
GC_push_one(word p)
{
GC_PUSH_ONE_STACK((ptr_t)p, MARKED_FROM_REGISTER);
}
#endif /* DARWIN && THREADS */
#if defined(GC_WIN32_THREADS)
GC_INNER void
GC_push_many_regs(const word *regs, unsigned count)
{
unsigned i;
for (i = 0; i < count; i++)
GC_PUSH_ONE_STACK((ptr_t)regs[i], MARKED_FROM_REGISTER);
}
#endif /* GC_WIN32_THREADS */
GC_API struct GC_ms_entry *GC_CALL
GC_mark_and_push(void *obj, mse *mark_stack_top, mse *mark_stack_limit,
void **src)
{
hdr *hhdr;
PREFETCH(obj);
GET_HDR(obj, hhdr);
if ((UNLIKELY(IS_FORWARDING_ADDR_OR_NIL(hhdr))
&& (!GC_all_interior_pointers
|| NULL == (hhdr = GC_find_header(GC_base(obj)))))
|| UNLIKELY(HBLK_IS_FREE(hhdr))) {
GC_ADD_TO_BLACK_LIST_NORMAL((ptr_t)obj, (ptr_t)src);
return mark_stack_top;
}
return GC_push_contents_hdr((ptr_t)obj, mark_stack_top, mark_stack_limit,
(ptr_t)src, hhdr, TRUE);
}
GC_ATTR_NO_SANITIZE_ADDR
GC_INNER void
#if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS)
GC_mark_and_push_stack(ptr_t p, ptr_t source)
#else
GC_mark_and_push_stack(ptr_t p)
# define source ((ptr_t)0)
#endif
{
hdr *hhdr;
ptr_t r = p;
PREFETCH(p);
GET_HDR(p, hhdr);
if (UNLIKELY(IS_FORWARDING_ADDR_OR_NIL(hhdr))) {
if (NULL == hhdr || (r = (ptr_t)GC_base(p)) == NULL
|| (hhdr = HDR(r)) == NULL) {
GC_ADD_TO_BLACK_LIST_STACK(p, source);
return;
}
}
if (UNLIKELY(HBLK_IS_FREE(hhdr))) {
GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
return;
}
#ifdef THREADS
/*
* Pointer is on the stack. We may have dirtied the object it points to,
* but have not called `GC_dirty` yet.
*/
GC_dirty(p); /*< entire object */
#endif
GC_mark_stack_top = GC_push_contents_hdr(
r, GC_mark_stack_top, GC_mark_stack_limit, source, hhdr, FALSE);
/*
* We silently ignore pointers to near the end of a block, which is
* very mildly suboptimal.
*/
/* FIXME: We should probably add a header word to address this. */
#undef source
}
#ifdef TRACE_BUF
# ifndef TRACE_ENTRIES
# define TRACE_ENTRIES 1000
# endif
struct trace_entry {
const char *caller_fn_name;
word gc_no;
word bytes_allocd;
GC_hidden_pointer arg1;
GC_hidden_pointer arg2;
} GC_trace_buf[TRACE_ENTRIES] = { { (const char *)NULL, 0, 0, 0, 0 } };
void
GC_add_trace_entry(const char *caller_fn_name, ptr_t arg1, ptr_t arg2)
{
size_t i = GC_trace_buf_pos;
GC_trace_buf[i].caller_fn_name = caller_fn_name;
GC_trace_buf[i].gc_no = GC_gc_no;
GC_trace_buf[i].bytes_allocd = GC_bytes_allocd;
GC_trace_buf[i].arg1 = GC_HIDE_POINTER(arg1);
GC_trace_buf[i].arg2 = GC_HIDE_POINTER(arg2);
i++;
if (i >= TRACE_ENTRIES)
i = 0;
GC_trace_buf_pos = i;
}
GC_API void GC_CALL
GC_print_trace_inner(GC_word gc_no)
{
size_t i;
for (i = GC_trace_buf_pos;; i--) {
struct trace_entry *p;
if (0 == i)
i = TRACE_ENTRIES;
p = &GC_trace_buf[i - 1];
/*
* Compare `gc_no` values (`p->gc_no` is less than given `gc_no`)
* taking into account that the counter may overflow.
*/
if (((p->gc_no - gc_no) & SIGNB) != 0 || NULL == p->caller_fn_name) {
return;
}
GC_printf("Trace:%s (gc:%lu, bytes:%lu) %p, %p\n", p->caller_fn_name,
(unsigned long)p->gc_no, (unsigned long)p->bytes_allocd,
GC_REVEAL_POINTER(p->arg1), GC_REVEAL_POINTER(p->arg2));
if (i == GC_trace_buf_pos + 1)
break;
}
GC_printf("Trace incomplete\n");
}
GC_API void GC_CALL
GC_print_trace(GC_word gc_no)
{
READER_LOCK();
GC_print_trace_inner(gc_no);
READER_UNLOCK();
}
#endif /* TRACE_BUF */
GC_ATTR_NO_SANITIZE_ADDR_MEM_THREAD
GC_API void GC_CALL
GC_push_all_eager(void *bottom, void *top)
{
REGISTER ptr_t current_p;
REGISTER word lim_addr;
REGISTER ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
REGISTER ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
#define GC_greatest_plausible_heap_addr greatest_ha
#define GC_least_plausible_heap_addr least_ha
if (NULL == top)
return;
/* Check all pointers in range and push if they appear to be valid. */
current_p = PTR_ALIGN_UP((ptr_t)bottom, ALIGNMENT);
lim_addr = ADDR(PTR_ALIGN_DOWN((ptr_t)top, ALIGNMENT)) - sizeof(ptr_t);
#ifdef CHERI_PURECAP
{
word cap_limit = cheri_base_get(current_p) + cheri_length_get(current_p);
if (lim_addr >= cap_limit)
lim_addr = cap_limit - sizeof(ptr_t);
}
#endif
for (; ADDR(current_p) <= lim_addr; current_p += ALIGNMENT) {
REGISTER ptr_t q;
LOAD_PTR_OR_CONTINUE(q, current_p);
GC_PUSH_ONE_STACK(q, current_p);
}
#undef GC_greatest_plausible_heap_addr
#undef GC_least_plausible_heap_addr
}
GC_INNER void
GC_push_all_stack(ptr_t bottom, ptr_t top)
{
GC_ASSERT(I_HOLD_LOCK());
#ifndef NEED_FIXUP_POINTER
if (GC_all_interior_pointers
# if defined(THREADS) && defined(MPROTECT_VDB)
&& !GC_auto_incremental
# endif
&& ADDR_LT((ptr_t)GC_mark_stack_top,
(ptr_t)(GC_mark_stack_limit - INITIAL_MARK_STACK_SIZE / 8))) {
GC_push_all(bottom, top);
} else
#endif
/* else */ {
GC_push_all_eager(bottom, top);
}
}
#if defined(WRAP_MARK_SOME) && defined(PARALLEL_MARK)
GC_ATTR_NO_SANITIZE_ADDR_MEM_THREAD
GC_INNER void
GC_push_conditional_eager(void *bottom, void *top, GC_bool all)
{
REGISTER ptr_t current_p;
REGISTER ptr_t lim;
REGISTER ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
REGISTER ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
# define GC_greatest_plausible_heap_addr greatest_ha
# define GC_least_plausible_heap_addr least_ha
if (NULL == top)
return;
/* TODO: If not `all`, then scan only dirty pages. */
(void)all;
current_p = PTR_ALIGN_UP((ptr_t)bottom, ALIGNMENT);
lim = PTR_ALIGN_DOWN((ptr_t)top, ALIGNMENT) - sizeof(ptr_t);
for (; ADDR_GE(lim, current_p); current_p += ALIGNMENT) {
REGISTER ptr_t q;
LOAD_PTR_OR_CONTINUE(q, current_p);
GC_PUSH_ONE_HEAP(q, current_p, GC_mark_stack_top);
}
# undef GC_greatest_plausible_heap_addr
# undef GC_least_plausible_heap_addr
}
#endif /* WRAP_MARK_SOME && PARALLEL_MARK */
#if !defined(SMALL_CONFIG) && !defined(USE_MARK_BYTES) \
&& !defined(MARK_BIT_PER_OBJ) && GC_GRANULE_PTRS <= 4
# define USE_PUSH_MARKED_ACCELERATORS
# if GC_GRANULE_PTRS == 1
# define PUSH_GRANULE(q) \
do { \
ptr_t qcontents = (q)[0]; \
GC_PUSH_ONE_HEAP(qcontents, q, GC_mark_stack_top); \
} while (0)
# elif GC_GRANULE_PTRS == 2
# define PUSH_GRANULE(q) \
do { \
ptr_t qcontents = (q)[0]; \
GC_PUSH_ONE_HEAP(qcontents, q, GC_mark_stack_top); \
qcontents = (q)[1]; \
GC_PUSH_ONE_HEAP(qcontents, (q) + 1, GC_mark_stack_top); \
} while (0)
# else
# define PUSH_GRANULE(q) \
do { \
ptr_t qcontents = (q)[0]; \
GC_PUSH_ONE_HEAP(qcontents, q, GC_mark_stack_top); \
qcontents = (q)[1]; \
GC_PUSH_ONE_HEAP(qcontents, (q) + 1, GC_mark_stack_top); \
qcontents = (q)[2]; \
GC_PUSH_ONE_HEAP(qcontents, (q) + 2, GC_mark_stack_top); \
qcontents = (q)[3]; \
GC_PUSH_ONE_HEAP(qcontents, (q) + 3, GC_mark_stack_top); \
} while (0)
# endif
/*
* Push all objects reachable from marked objects in the given block
* containing objects of size 1 granule.
*/
GC_ATTR_NO_SANITIZE_THREAD
STATIC void
GC_push_marked1(struct hblk *h, const hdr *hhdr)
{
const word *mark_word_addr
= (word *)CAST_AWAY_VOLATILE_PVOID(hhdr->hb_marks);
ptr_t *p;
ptr_t plim;
/*
* Allow registers to be used for some frequently accessed global variables.
* Otherwise aliasing issues are likely to prevent that.
*/
ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
mse *mark_stack_top = GC_mark_stack_top;
mse *mark_stack_limit = GC_mark_stack_limit;
# undef GC_mark_stack_top
# undef GC_mark_stack_limit
# define GC_mark_stack_top mark_stack_top
# define GC_mark_stack_limit mark_stack_limit
# define GC_greatest_plausible_heap_addr greatest_ha
# define GC_least_plausible_heap_addr least_ha
p = (ptr_t *)h->hb_body;
plim = (ptr_t)h + HBLKSIZE;
/* Go through all granules in block. */
while (ADDR_LT((ptr_t)p, plim)) {
word mark_word = *mark_word_addr++;
ptr_t *q;
for (q = p; mark_word != 0; mark_word >>= 1) {
if ((mark_word & 1) != 0)
PUSH_GRANULE(q);
q += GC_GRANULE_PTRS;
}
p += CPP_WORDSZ * GC_GRANULE_PTRS;
}
# undef GC_greatest_plausible_heap_addr
# undef GC_least_plausible_heap_addr
# undef GC_mark_stack_top
# undef GC_mark_stack_limit
# define GC_mark_stack_limit GC_arrays._mark_stack_limit
# define GC_mark_stack_top GC_arrays._mark_stack_top
GC_mark_stack_top = mark_stack_top;
}
# ifndef UNALIGNED_PTRS
/*
* Push all objects reachable from marked objects in the given block
* of two-granule objects.
*/
GC_ATTR_NO_SANITIZE_THREAD
STATIC void
GC_push_marked2(struct hblk *h, const hdr *hhdr)
{
const word *mark_word_addr
= (word *)CAST_AWAY_VOLATILE_PVOID(hhdr->hb_marks);
ptr_t *p;
ptr_t plim;
ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
mse *mark_stack_top = GC_mark_stack_top;
mse *mark_stack_limit = GC_mark_stack_limit;
# undef GC_mark_stack_top
# undef GC_mark_stack_limit
# define GC_mark_stack_top mark_stack_top
# define GC_mark_stack_limit mark_stack_limit
# define GC_greatest_plausible_heap_addr greatest_ha
# define GC_least_plausible_heap_addr least_ha
p = (ptr_t *)h->hb_body;
plim = (ptr_t)h + HBLKSIZE;
/* Go through all granules in block. */
while (ADDR_LT((ptr_t)p, plim)) {
word mark_word = *mark_word_addr++;
ptr_t *q;
for (q = p; mark_word != 0; mark_word >>= 2) {
if (mark_word & 1) {
PUSH_GRANULE(q);
PUSH_GRANULE(q + GC_GRANULE_PTRS);
}
q += 2 * GC_GRANULE_PTRS;
}
p += CPP_WORDSZ * GC_GRANULE_PTRS;
}
# undef GC_greatest_plausible_heap_addr
# undef GC_least_plausible_heap_addr
# undef GC_mark_stack_top
# undef GC_mark_stack_limit
# define GC_mark_stack_limit GC_arrays._mark_stack_limit
# define GC_mark_stack_top GC_arrays._mark_stack_top
GC_mark_stack_top = mark_stack_top;
}
# if GC_GRANULE_PTRS < 4
/*
* Push all objects reachable from marked objects in the given block of
* four-granule objects. There is a risk of mark stack overflow here.
* But we handle that. And only unmarked objects get pushed, so it is
* not very likely.
*/
GC_ATTR_NO_SANITIZE_THREAD
STATIC void
GC_push_marked4(struct hblk *h, const hdr *hhdr)
{
const word *mark_word_addr
= (word *)CAST_AWAY_VOLATILE_PVOID(hhdr->hb_marks);
ptr_t *p;
ptr_t plim;
ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
mse *mark_stack_top = GC_mark_stack_top;
mse *mark_stack_limit = GC_mark_stack_limit;
# undef GC_mark_stack_top
# undef GC_mark_stack_limit
# define GC_mark_stack_top mark_stack_top
# define GC_mark_stack_limit mark_stack_limit
# define GC_greatest_plausible_heap_addr greatest_ha
# define GC_least_plausible_heap_addr least_ha
p = (ptr_t *)h->hb_body;
plim = (ptr_t)h + HBLKSIZE;
/* Go through all granules in block. */
while (ADDR_LT((ptr_t)p, plim)) {
word mark_word = *mark_word_addr++;
ptr_t *q;
for (q = p; mark_word != 0; mark_word >>= 4) {
if (mark_word & 1) {
PUSH_GRANULE(q);
PUSH_GRANULE(q + GC_GRANULE_PTRS);
PUSH_GRANULE(q + 2 * GC_GRANULE_PTRS);
PUSH_GRANULE(q + 3 * GC_GRANULE_PTRS);
}
q += 4 * GC_GRANULE_PTRS;
}
p += CPP_WORDSZ * GC_GRANULE_PTRS;
}
# undef GC_greatest_plausible_heap_addr
# undef GC_least_plausible_heap_addr
# undef GC_mark_stack_top
# undef GC_mark_stack_limit
# define GC_mark_stack_limit GC_arrays._mark_stack_limit
# define GC_mark_stack_top GC_arrays._mark_stack_top
GC_mark_stack_top = mark_stack_top;
}
# endif
# endif
#endif /* !USE_MARK_BYTES && !MARK_BIT_PER_OBJ && !SMALL_CONFIG */
/* Push all objects reachable from marked objects in the given block. */
STATIC void
GC_push_marked(struct hblk *h, const hdr *hhdr)
{
size_t sz = hhdr->hb_sz;
ptr_t p;
size_t bit_no;
ptr_t plim;
mse *mark_stack_top;
mse *mark_stack_limit = GC_mark_stack_limit;
/* Some quick shortcuts: */
if ((/* `0 |` */ GC_DS_LENGTH) == hhdr->hb_descr)
return;
if (GC_block_empty(hhdr))
return; /*< nothing marked */
#if !defined(GC_DISABLE_INCREMENTAL)
GC_n_rescuing_pages++;
#endif
GC_objects_are_marked = TRUE;
switch (BYTES_TO_GRANULES(sz)) {
#ifdef USE_PUSH_MARKED_ACCELERATORS
case 1:
GC_push_marked1(h, hhdr);
break;
# ifndef UNALIGNED_PTRS
case 2:
GC_push_marked2(h, hhdr);
break;
# if GC_GRANULE_PTRS < 4
case 4:
GC_push_marked4(h, hhdr);
break;
# endif
# endif /* !UNALIGNED_PTRS */
#else
case 1: /*< to suppress "switch statement contains no case" warning */
#endif
default:
plim = sz > MAXOBJBYTES ? h->hb_body
: CAST_THRU_UINTPTR(ptr_t, (h + 1)->hb_body) - sz;
mark_stack_top = GC_mark_stack_top;
for (p = h->hb_body, bit_no = 0; ADDR_GE(plim, p);
p += sz, bit_no += MARK_BIT_OFFSET(sz)) {
/* Mark from fields inside the object. */
if (mark_bit_from_hdr(hhdr, bit_no)) {
mark_stack_top
= GC_push_obj(p, hhdr, mark_stack_top, mark_stack_limit);
}
}
GC_mark_stack_top = mark_stack_top;
}
}
#ifdef ENABLE_DISCLAIM
/*
* Unconditionally mark from all objects that have not been reclaimed.
* This is useful in order to retain pointers reachable from the disclaim
* notifiers. To determine whether an object has been reclaimed, we
* require that any live object has a nonzero as one of the two least
* significant bits of the first "pointer-sized" word. On the other hand,
* the reclaimed object is a member of free lists, and thus contains
* a pointer-aligned next-pointer as the first "pointer-sized" word.
*/
GC_ATTR_NO_SANITIZE_THREAD
STATIC void
GC_push_unconditionally(struct hblk *h, const hdr *hhdr)
{
size_t sz = hhdr->hb_sz;
ptr_t p;
ptr_t plim;
mse *mark_stack_top;
mse *mark_stack_limit = GC_mark_stack_limit;
if ((/* `0 |` */ GC_DS_LENGTH) == hhdr->hb_descr)
return;
# if !defined(GC_DISABLE_INCREMENTAL)
GC_n_rescuing_pages++;
# endif
GC_objects_are_marked = TRUE;
plim = sz > MAXOBJBYTES ? h->hb_body
: CAST_THRU_UINTPTR(ptr_t, (h + 1)->hb_body) - sz;
mark_stack_top = GC_mark_stack_top;
for (p = h->hb_body; ADDR_GE(plim, p); p += sz) {
if ((ADDR(*(ptr_t *)p) & 0x3) != 0) {
mark_stack_top = GC_push_obj(p, hhdr, mark_stack_top, mark_stack_limit);
}
}
GC_mark_stack_top = mark_stack_top;
}
#endif /* ENABLE_DISCLAIM */
#ifndef GC_DISABLE_INCREMENTAL
/* Test whether any page in the given block is dirty. */
STATIC GC_bool
GC_block_was_dirty(struct hblk *h, const hdr *hhdr)
{
size_t sz;
ptr_t p;
# ifdef AO_HAVE_load
/* Atomic access is used to avoid racing with `GC_realloc`. */
sz = AO_load((volatile AO_t *)&hhdr->hb_sz);
# else
sz = hhdr->hb_sz;
# endif
if (sz <= MAXOBJBYTES) {
return GC_page_was_dirty(h);
}
for (p = (ptr_t)h; ADDR_LT(p, (ptr_t)h + sz); p += HBLKSIZE) {
if (GC_page_was_dirty((struct hblk *)p))
return TRUE;
}
return FALSE;
}
#endif /* GC_DISABLE_INCREMENTAL */
/*
* Similar to `GC_push_marked`, but skip over unallocated blocks and
* return address of next plausible block.
*/
STATIC struct hblk *
GC_push_next_marked(struct hblk *h)
{
hdr *hhdr = HDR(h);
if (UNLIKELY(IS_FORWARDING_ADDR_OR_NIL(hhdr) || HBLK_IS_FREE(hhdr))) {
h = GC_next_block(h, FALSE);
if (NULL == h)
return NULL;
hhdr = GC_find_header(h);
} else {
#ifdef LINT2
if (NULL == h)
ABORT("Bad HDR() definition");
#endif
}
GC_push_marked(h, hhdr);
return h + OBJ_SZ_TO_BLOCKS(hhdr->hb_sz);
}
#ifndef GC_DISABLE_INCREMENTAL
/* Identical to `GC_push_next_marked`, but mark only from dirty pages. */
STATIC struct hblk *
GC_push_next_marked_dirty(struct hblk *h)
{
hdr *hhdr;
GC_ASSERT(I_HOLD_LOCK());
if (!GC_incremental)
ABORT("Dirty bits not set up");
for (;; h += OBJ_SZ_TO_BLOCKS(hhdr->hb_sz)) {
hhdr = HDR(h);
if (UNLIKELY(IS_FORWARDING_ADDR_OR_NIL(hhdr) || HBLK_IS_FREE(hhdr))) {
h = GC_next_block(h, FALSE);
if (NULL == h)
return NULL;
hhdr = GC_find_header(h);
} else {
# ifdef LINT2
if (NULL == h)
ABORT("Bad HDR() definition");
# endif
}
if (GC_block_was_dirty(h, hhdr))
break;
}
# ifdef ENABLE_DISCLAIM
if ((hhdr->hb_flags & MARK_UNCONDITIONALLY) != 0) {
GC_push_unconditionally(h, hhdr);
/*
* Then we may ask, why not also add the `MARK_UNCONDITIONALLY`
* case to `GC_push_next_marked`, which is also applied to
* uncollectible blocks? But it seems to me that the function
* does not need to scan uncollectible (and unconditionally
* marked) blocks since those are already handled in the
* `MS_PUSH_UNCOLLECTABLE` phase.
*/
} else
# endif
/* else */ {
GC_push_marked(h, hhdr);
}
return h + OBJ_SZ_TO_BLOCKS(hhdr->hb_sz);
}
#endif /* !GC_DISABLE_INCREMENTAL */
/*
* Similar to `GC_push_next_marked`, but for uncollectible pages.
* Needed since we do not clear marks for such pages, even for full
* collections.
*/
STATIC struct hblk *
GC_push_next_marked_uncollectable(struct hblk *h)
{
hdr *hhdr = HDR(h);
for (;;) {
if (UNLIKELY(IS_FORWARDING_ADDR_OR_NIL(hhdr) || HBLK_IS_FREE(hhdr))) {
h = GC_next_block(h, FALSE);
if (NULL == h)
return NULL;
hhdr = GC_find_header(h);
} else {
#ifdef LINT2
if (NULL == h)
ABORT("Bad HDR() definition");
#endif
}
if (hhdr->hb_obj_kind == UNCOLLECTABLE) {
GC_push_marked(h, hhdr);
break;
}
#ifdef ENABLE_DISCLAIM
if ((hhdr->hb_flags & MARK_UNCONDITIONALLY) != 0) {
GC_push_unconditionally(h, hhdr);
break;
}
#endif
h += OBJ_SZ_TO_BLOCKS(hhdr->hb_sz);
hhdr = HDR(h);
}
return h + OBJ_SZ_TO_BLOCKS(hhdr->hb_sz);
}
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 2009-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#if defined(E2K) && !defined(THREADS)
# include <alloca.h>
#endif
/*
* Data structure for list of root sets.
* We keep a hash table, so that we can filter out duplicate additions.
* Under Win32, we need to do a better job of filtering overlaps, so
* we resort to sequential search, and pay the price.
*/
/* Register dynamic library data segments. */
int GC_no_dls = 0;
#if !defined(NO_DEBUGGING) || defined(GC_ASSERTIONS)
GC_INNER word
GC_compute_root_size(void)
{
size_t i;
word size = 0;
for (i = 0; i < n_root_sets; i++) {
size += (word)(GC_static_roots[i].r_end - GC_static_roots[i].r_start);
}
return size;
}
#endif /* !NO_DEBUGGING || GC_ASSERTIONS */
#if !defined(NO_DEBUGGING)
/* For the debugging purpose. */
void
GC_print_static_roots(void)
{
size_t i;
word size;
for (i = 0; i < n_root_sets; i++) {
GC_printf("From %p to %p%s\n", (void *)GC_static_roots[i].r_start,
(void *)GC_static_roots[i].r_end,
GC_static_roots[i].r_tmp ? " (temporary)" : "");
}
GC_printf("GC_root_size= %lu\n", (unsigned long)GC_root_size);
if ((size = GC_compute_root_size()) != GC_root_size)
GC_err_printf("GC_root_size incorrect!! Should be: %lu\n",
(unsigned long)size);
}
#endif /* !NO_DEBUGGING */
#ifndef ANY_MSWIN
GC_INLINE size_t
rt_hash(ptr_t addr)
{
word val = ADDR(addr);
# if CPP_WORDSZ > 4 * LOG_RT_SIZE
# if CPP_WORDSZ > 8 * LOG_RT_SIZE
val ^= val >> (8 * LOG_RT_SIZE);
# endif
val ^= val >> (4 * LOG_RT_SIZE);
# endif
val ^= val >> (2 * LOG_RT_SIZE);
return (size_t)((val >> LOG_RT_SIZE) ^ val) & (RT_SIZE - 1);
}
GC_INNER void *
GC_roots_present(ptr_t b)
{
size_t h;
struct roots *p;
GC_ASSERT(I_HOLD_READER_LOCK());
h = rt_hash(b);
for (p = GC_root_index[h]; p != NULL; p = p->r_next) {
if (p->r_start == (ptr_t)b)
break;
}
return p;
}
/* Add the given root structure to the index. */
GC_INLINE void
add_roots_to_index(struct roots *p)
{
size_t h = rt_hash(p->r_start);
p->r_next = GC_root_index[h];
GC_root_index[h] = p;
}
#endif /* !ANY_MSWIN */
GC_API void GC_CALL
GC_add_roots(void *b, void *e)
{
if (UNLIKELY(!GC_is_initialized))
GC_init();
LOCK();
GC_add_roots_inner((ptr_t)b, (ptr_t)e, FALSE);
UNLOCK();
}
GC_INNER void
GC_add_roots_inner(ptr_t b, ptr_t e, GC_bool tmp)
{
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(ADDR_GE(e, b));
b = PTR_ALIGN_UP(b, ALIGNMENT);
e = PTR_ALIGN_DOWN(e, ALIGNMENT);
if (ADDR_GE(b, e)) {
/* Nothing to do. */
return;
}
#ifdef ANY_MSWIN
/*
* Spend the time to ensure that there are no overlapping or adjacent
* intervals. This could be done faster with e.g. a balanced tree.
* But the execution time here is virtually guaranteed to be dominated
* by the time it takes to scan the roots.
*/
{
size_t i;
struct roots *old = NULL; /*< initialized to prevent warning */
for (i = 0; i < n_root_sets; i++) {
old = GC_static_roots + i;
if (ADDR_GE(old->r_end, b) && ADDR_GE(e, old->r_start)) {
if (ADDR_LT(b, old->r_start)) {
GC_root_size += (word)(old->r_start - b);
old->r_start = b;
}
if (ADDR_LT(old->r_end, e)) {
GC_root_size += (word)(e - old->r_end);
old->r_end = e;
}
old->r_tmp &= tmp;
break;
}
}
if (i < n_root_sets) {
/* Merge other overlapping intervals. */
struct roots *other;
for (i++; i < n_root_sets; i++) {
other = GC_static_roots + i;
b = other->r_start;
e = other->r_end;
if (ADDR_GE(old->r_end, b) && ADDR_GE(e, old->r_start)) {
if (ADDR_LT(b, old->r_start)) {
GC_root_size += (word)(old->r_start - b);
old->r_start = b;
}
if (ADDR_LT(old->r_end, e)) {
GC_root_size += (word)(e - old->r_end);
old->r_end = e;
}
old->r_tmp &= other->r_tmp;
/* Delete this entry. */
GC_root_size -= (word)(other->r_end - other->r_start);
other->r_start = GC_static_roots[n_root_sets - 1].r_start;
other->r_end = GC_static_roots[n_root_sets - 1].r_end;
n_root_sets--;
}
}
return;
}
}
#else
{
struct roots *old = (struct roots *)GC_roots_present(b);
if (old != NULL) {
if (ADDR_GE(old->r_end, e)) {
old->r_tmp &= tmp;
/* Already there. */
return;
}
if (old->r_tmp == tmp || !tmp) {
/* Extend the existing root. */
GC_root_size += (word)(e - old->r_end);
old->r_end = e;
old->r_tmp = tmp;
return;
}
b = old->r_end;
}
}
#endif
if (n_root_sets == MAX_ROOT_SETS) {
ABORT("Too many root sets");
}
#ifdef DEBUG_ADD_DEL_ROOTS
GC_log_printf("Adding data root section %u: %p .. %p%s\n",
(unsigned)n_root_sets, (void *)b, (void *)e,
tmp ? " (temporary)" : "");
#endif
GC_static_roots[n_root_sets].r_start = (ptr_t)b;
GC_static_roots[n_root_sets].r_end = (ptr_t)e;
GC_static_roots[n_root_sets].r_tmp = tmp;
#ifndef ANY_MSWIN
GC_static_roots[n_root_sets].r_next = 0;
add_roots_to_index(GC_static_roots + n_root_sets);
#endif
GC_root_size += (word)(e - b);
n_root_sets++;
}
GC_API void GC_CALL
GC_clear_roots(void)
{
if (UNLIKELY(!GC_is_initialized))
GC_init();
LOCK();
#ifdef THREADS
GC_roots_were_cleared = TRUE;
#endif
n_root_sets = 0;
GC_root_size = 0;
#ifndef ANY_MSWIN
BZERO(GC_root_index, sizeof(GC_root_index));
#endif
#ifdef DEBUG_ADD_DEL_ROOTS
GC_log_printf("Clear all data root sections\n");
#endif
UNLOCK();
}
STATIC void
GC_remove_root_at_pos(size_t i)
{
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(i < n_root_sets);
#ifdef DEBUG_ADD_DEL_ROOTS
GC_log_printf("Remove data root section at %u: %p .. %p%s\n", (unsigned)i,
(void *)GC_static_roots[i].r_start,
(void *)GC_static_roots[i].r_end,
GC_static_roots[i].r_tmp ? " (temporary)" : "");
#endif
GC_root_size
-= (word)(GC_static_roots[i].r_end - GC_static_roots[i].r_start);
GC_static_roots[i].r_start = GC_static_roots[n_root_sets - 1].r_start;
GC_static_roots[i].r_end = GC_static_roots[n_root_sets - 1].r_end;
GC_static_roots[i].r_tmp = GC_static_roots[n_root_sets - 1].r_tmp;
n_root_sets--;
}
#ifndef ANY_MSWIN
STATIC void
GC_rebuild_root_index(void)
{
size_t i;
BZERO(GC_root_index, sizeof(GC_root_index));
for (i = 0; i < n_root_sets; i++)
add_roots_to_index(GC_static_roots + i);
}
#endif /* !ANY_MSWIN */
#if defined(ANY_MSWIN) || defined(DYNAMIC_LOADING)
STATIC void
GC_remove_tmp_roots(void)
{
size_t i;
# ifndef ANY_MSWIN
size_t old_n_roots = n_root_sets;
# endif
GC_ASSERT(I_HOLD_LOCK());
for (i = 0; i < n_root_sets;) {
if (GC_static_roots[i].r_tmp) {
GC_remove_root_at_pos(i);
} else {
i++;
}
}
# ifndef ANY_MSWIN
if (n_root_sets < old_n_roots)
GC_rebuild_root_index();
# endif
}
#endif /* ANY_MSWIN || DYNAMIC_LOADING */
STATIC void GC_remove_roots_inner(ptr_t b, ptr_t e);
GC_API void GC_CALL
GC_remove_roots(void *b, void *e)
{
/* A quick check whether has nothing to do. */
if (ADDR_GE(PTR_ALIGN_UP((ptr_t)b, ALIGNMENT),
PTR_ALIGN_DOWN((ptr_t)e, ALIGNMENT)))
return;
LOCK();
GC_remove_roots_inner((ptr_t)b, (ptr_t)e);
UNLOCK();
}
STATIC void
GC_remove_roots_inner(ptr_t b, ptr_t e)
{
size_t i;
#ifndef ANY_MSWIN
size_t old_n_roots = n_root_sets;
#endif
GC_ASSERT(I_HOLD_LOCK());
for (i = 0; i < n_root_sets;) {
if (ADDR_GE(GC_static_roots[i].r_start, b)
&& ADDR_GE(e, GC_static_roots[i].r_end)) {
GC_remove_root_at_pos(i);
} else {
i++;
}
}
#ifndef ANY_MSWIN
if (n_root_sets < old_n_roots)
GC_rebuild_root_index();
#endif
}
#ifdef USE_PROC_FOR_LIBRARIES
/*
* Exchange the elements of the roots table. Requires rebuild of the roots
* index table after the swap.
*/
GC_INLINE void
swap_static_roots(size_t i, size_t j)
{
ptr_t r_start = GC_static_roots[i].r_start;
ptr_t r_end = GC_static_roots[i].r_end;
GC_bool r_tmp = GC_static_roots[i].r_tmp;
GC_static_roots[i].r_start = GC_static_roots[j].r_start;
GC_static_roots[i].r_end = GC_static_roots[j].r_end;
GC_static_roots[i].r_tmp = GC_static_roots[j].r_tmp;
/* No need to swap `r_next` values. */
GC_static_roots[j].r_start = r_start;
GC_static_roots[j].r_end = r_end;
GC_static_roots[j].r_tmp = r_tmp;
}
GC_INNER void
GC_remove_roots_subregion(ptr_t b, ptr_t e)
{
size_t i;
GC_bool rebuild = FALSE;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(ADDR(b) % ALIGNMENT == 0 && ADDR(e) % ALIGNMENT == 0);
for (i = 0; i < n_root_sets; i++) {
ptr_t r_start, r_end;
if (GC_static_roots[i].r_tmp) {
/* The remaining roots are skipped as they are all temporary. */
# ifdef GC_ASSERTIONS
size_t j;
for (j = i + 1; j < n_root_sets; j++) {
GC_ASSERT(GC_static_roots[j].r_tmp);
}
# endif
break;
}
r_start = GC_static_roots[i].r_start;
r_end = GC_static_roots[i].r_end;
if (ADDR_GE(r_start, e) || LIKELY(ADDR_GE(b, r_end)))
continue;
# ifdef DEBUG_ADD_DEL_ROOTS
GC_log_printf("Removing %p .. %p from root section %u (%p .. %p)\n",
(void *)b, (void *)e, (unsigned)i, (void *)r_start,
(void *)r_end);
# endif
if (ADDR_LT(r_start, b)) {
GC_root_size -= (word)(r_end - b);
GC_static_roots[i].r_end = b;
/* No need to rebuild as hash does not use `r_end` value. */
if (ADDR_LT(e, r_end)) {
size_t j;
if (rebuild) {
GC_rebuild_root_index();
rebuild = FALSE;
}
/* Note: updates `n_root_sets` as well. */
GC_add_roots_inner(e, r_end, FALSE);
for (j = i + 1; j < n_root_sets; j++)
if (GC_static_roots[j].r_tmp)
break;
if (j < n_root_sets - 1 && !GC_static_roots[n_root_sets - 1].r_tmp) {
/* Exchange the roots to have all temporary ones at the end. */
swap_static_roots(j, n_root_sets - 1);
rebuild = TRUE;
}
}
} else {
if (ADDR_LT(e, r_end)) {
GC_root_size -= (word)(e - r_start);
GC_static_roots[i].r_start = e;
} else {
GC_remove_root_at_pos(i);
if (i + 1 < n_root_sets && GC_static_roots[i].r_tmp
&& !GC_static_roots[i + 1].r_tmp) {
size_t j;
for (j = i + 2; j < n_root_sets; j++)
if (GC_static_roots[j].r_tmp)
break;
/* Exchange the roots to have all temporary ones at the end. */
swap_static_roots(i, j - 1);
}
i--;
}
rebuild = TRUE;
}
}
if (rebuild)
GC_rebuild_root_index();
}
#endif /* USE_PROC_FOR_LIBRARIES */
#if !defined(NO_DEBUGGING)
GC_API int GC_CALL
GC_is_tmp_root(void *p)
{
# ifndef HAS_REAL_READER_LOCK
static size_t last_root_set; /*< initialized to 0; no shared access */
# elif defined(AO_HAVE_load) || defined(AO_HAVE_store)
static volatile AO_t last_root_set;
# else
/* Note: a race is acceptable, it is just a cached index. */
static volatile size_t last_root_set;
# endif
size_t i;
int res;
READER_LOCK();
/* First try the cached root. */
# if defined(AO_HAVE_load) && defined(HAS_REAL_READER_LOCK)
i = AO_load(&last_root_set);
# else
i = last_root_set;
# endif
if (i < n_root_sets
&& ADDR_INSIDE((ptr_t)p, GC_static_roots[i].r_start,
GC_static_roots[i].r_end)) {
res = (int)GC_static_roots[i].r_tmp;
} else {
res = 0;
for (i = 0; i < n_root_sets; i++) {
if (ADDR_INSIDE((ptr_t)p, GC_static_roots[i].r_start,
GC_static_roots[i].r_end)) {
res = (int)GC_static_roots[i].r_tmp;
# if defined(AO_HAVE_store) && defined(HAS_REAL_READER_LOCK)
AO_store(&last_root_set, i);
# else
last_root_set = i;
# endif
break;
}
}
}
READER_UNLOCK();
return res;
}
#endif /* !NO_DEBUGGING */
GC_INNER ptr_t
GC_approx_sp(void)
{
volatile ptr_t sp;
/*
* This also forces stack to grow if necessary. Otherwise the later
* accesses might cause the kernel to think we are doing something wrong.
*/
STORE_APPROX_SP_TO(sp);
return (/* no volatile */ ptr_t)sp;
}
GC_API void GC_CALL
GC_clear_exclusion_table(void)
{
#ifdef DEBUG_ADD_DEL_ROOTS
GC_log_printf("Clear static root exclusions (%u elements)\n",
(unsigned)GC_excl_table_entries);
#endif
GC_excl_table_entries = 0;
}
/*
* Return the first exclusion range that includes an address not lower
* than `start_addr`.
*/
STATIC struct exclusion *
GC_next_exclusion(ptr_t start_addr)
{
size_t low = 0;
size_t high;
if (UNLIKELY(0 == GC_excl_table_entries))
return NULL;
high = GC_excl_table_entries - 1;
while (high > low) {
size_t mid = (low + high) >> 1;
/* `low` <= `mid` < `high`. */
if (ADDR_GE(start_addr, GC_excl_table[mid].e_end)) {
low = mid + 1;
} else {
high = mid;
}
}
if (ADDR_GE(start_addr, GC_excl_table[low].e_end))
return NULL;
return GC_excl_table + low;
}
GC_INNER void
GC_exclude_static_roots_inner(ptr_t start, ptr_t finish)
{
struct exclusion *next;
size_t next_index;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(ADDR(start) % ALIGNMENT == 0);
GC_ASSERT(ADDR_LT(start, finish));
next = GC_next_exclusion(start);
if (next != NULL) {
if (ADDR_LT(next->e_start, finish)) {
/* Incomplete error check. */
ABORT("Exclusion ranges overlap");
}
if (ADDR(next->e_start) == ADDR(finish)) {
/* Extend old range backwards. */
next->e_start = start;
#ifdef DEBUG_ADD_DEL_ROOTS
GC_log_printf("Updating static root exclusion to %p .. %p\n",
(void *)start, (void *)next->e_end);
#endif
return;
}
}
next_index = GC_excl_table_entries;
if (next_index >= MAX_EXCLUSIONS)
ABORT("Too many exclusions");
if (next != NULL) {
size_t i;
next_index = (size_t)(next - GC_excl_table);
for (i = GC_excl_table_entries; i > next_index; --i) {
GC_excl_table[i] = GC_excl_table[i - 1];
}
}
#ifdef DEBUG_ADD_DEL_ROOTS
GC_log_printf("Adding static root exclusion at %u: %p .. %p\n",
(unsigned)next_index, (void *)start, (void *)finish);
#endif
GC_excl_table[next_index].e_start = start;
GC_excl_table[next_index].e_end = finish;
++GC_excl_table_entries;
}
GC_API void GC_CALL
GC_exclude_static_roots(void *b, void *e)
{
if (b == e) {
/* Nothing to exclude. */
return;
}
/* Round boundaries in direction reverse to that of `GC_add_roots`. */
#if ALIGNMENT > 1
b = PTR_ALIGN_DOWN((ptr_t)b, ALIGNMENT);
e = UNLIKELY(ADDR(e) > ~(word)(ALIGNMENT - 1))
? PTR_ALIGN_DOWN((ptr_t)e, ALIGNMENT) /*< overflow */
: PTR_ALIGN_UP((ptr_t)e, ALIGNMENT);
#endif
LOCK();
GC_exclude_static_roots_inner((ptr_t)b, (ptr_t)e);
UNLOCK();
}
#if defined(WRAP_MARK_SOME) && defined(PARALLEL_MARK)
# define GC_PUSH_CONDITIONAL(b, t, all) \
(GC_parallel ? GC_push_conditional_eager(b, t, all) \
: GC_push_conditional_static(b, t, all))
#else
# define GC_PUSH_CONDITIONAL(b, t, all) GC_push_conditional_static(b, t, all)
#endif
/* Invoke `GC_push_conditional` on ranges that are not excluded. */
STATIC void
GC_push_conditional_with_exclusions(ptr_t bottom, ptr_t top, GC_bool all)
{
while (ADDR_LT(bottom, top)) {
struct exclusion *next = GC_next_exclusion(bottom);
ptr_t excl_start = top;
if (next != NULL) {
if (ADDR_GE(next->e_start, top)) {
next = NULL;
} else {
excl_start = next->e_start;
}
}
if (ADDR_LT(bottom, excl_start))
GC_PUSH_CONDITIONAL(bottom, excl_start, all);
if (NULL == next)
break;
bottom = next->e_end;
}
}
#ifdef IA64
GC_INNER void
GC_push_all_register_sections(ptr_t bs_lo, ptr_t bs_hi, GC_bool eager,
struct GC_traced_stack_sect_s *traced_stack_sect)
{
GC_ASSERT(I_HOLD_LOCK());
while (traced_stack_sect != NULL) {
ptr_t frame_bs_lo = traced_stack_sect->backing_store_end;
GC_ASSERT(ADDR_GE(bs_hi, frame_bs_lo));
if (eager) {
GC_push_all_eager(frame_bs_lo, bs_hi);
} else {
GC_push_all_stack(frame_bs_lo, bs_hi);
}
bs_hi = traced_stack_sect->saved_backing_store_ptr;
traced_stack_sect = traced_stack_sect->prev;
}
GC_ASSERT(ADDR_GE(bs_hi, bs_lo));
if (eager) {
GC_push_all_eager(bs_lo, bs_hi);
} else {
GC_push_all_stack(bs_lo, bs_hi);
}
}
#endif /* IA64 */
#ifdef THREADS
GC_INNER void
GC_push_all_stack_sections(ptr_t lo /* top */, ptr_t hi /* bottom */,
struct GC_traced_stack_sect_s *traced_stack_sect)
{
GC_ASSERT(I_HOLD_LOCK());
while (traced_stack_sect != NULL) {
GC_ASSERT(HOTTER_THAN(lo, (ptr_t)traced_stack_sect));
# ifdef STACK_GROWS_UP
GC_push_all_stack((ptr_t)traced_stack_sect, lo);
# else
GC_push_all_stack(lo, (ptr_t)traced_stack_sect);
# endif
lo = traced_stack_sect->saved_stack_ptr;
GC_ASSERT(lo != NULL);
traced_stack_sect = traced_stack_sect->prev;
}
GC_ASSERT(!HOTTER_THAN(hi, lo));
# ifdef STACK_GROWS_UP
/* We got them backwards! */
GC_push_all_stack(hi, lo);
# else
GC_push_all_stack(lo, hi);
# endif
}
#else /* !THREADS */
/*
* Similar to `GC_push_all_eager`, but only the part hotter than
* `cold_gc_frame` is scanned immediately. Needed to ensure that
* callee-save registers are not missed. Treats all interior pointers
* as valid and scans part of the area immediately, to make sure that
* saved register values are not lost. `cold_gc_frame` delimits the
* stack section that must be scanned eagerly. A zero value indicates
* that no eager scanning is needed. We do not need to worry about
* the manual VDB case here, since this is only called in the
* single-threaded case. We assume that we cannot collect between
* an assignment and the corresponding `GC_dirty()` call.
*/
STATIC void
GC_push_all_stack_partially_eager(ptr_t bottom, ptr_t top, ptr_t cold_gc_frame)
{
# ifndef NEED_FIXUP_POINTER
if (GC_all_interior_pointers) {
/*
* Push the hot end of the stack eagerly, so that register values saved
* inside GC frames are marked before they disappear. The rest of the
* marking can be deferred until later.
*/
if (0 == cold_gc_frame) {
GC_push_all_stack(bottom, top);
return;
}
GC_ASSERT(ADDR_GE(cold_gc_frame, bottom) && ADDR_GE(top, cold_gc_frame));
# ifdef STACK_GROWS_UP
GC_push_all(bottom, cold_gc_frame + sizeof(ptr_t));
GC_push_all_eager(cold_gc_frame, top);
# else
GC_push_all(cold_gc_frame - sizeof(ptr_t), top);
GC_push_all_eager(bottom, cold_gc_frame);
# endif
} else
# endif
/* else */ {
GC_push_all_eager(bottom, top);
}
# ifdef TRACE_BUF
GC_add_trace_entry("GC_push_all_stack", bottom, top);
# endif
}
/* Similar to `GC_push_all_stack_sections()` but also uses `cold_gc_frame`. */
STATIC void
GC_push_all_stack_part_eager_sections(
ptr_t lo /* top */, ptr_t hi /* bottom */, ptr_t cold_gc_frame,
struct GC_traced_stack_sect_s *traced_stack_sect)
{
GC_ASSERT(traced_stack_sect == NULL || cold_gc_frame == NULL
|| HOTTER_THAN(cold_gc_frame, (ptr_t)traced_stack_sect));
while (traced_stack_sect != NULL) {
GC_ASSERT(HOTTER_THAN(lo, (ptr_t)traced_stack_sect));
# ifdef STACK_GROWS_UP
GC_push_all_stack_partially_eager((ptr_t)traced_stack_sect, lo,
cold_gc_frame);
# else
GC_push_all_stack_partially_eager(lo, (ptr_t)traced_stack_sect,
cold_gc_frame);
# endif
lo = traced_stack_sect->saved_stack_ptr;
GC_ASSERT(lo != NULL);
traced_stack_sect = traced_stack_sect->prev;
/* Note: use at most once. */
cold_gc_frame = NULL;
}
GC_ASSERT(!HOTTER_THAN(hi, lo));
# ifdef STACK_GROWS_UP
/* We got them backwards! */
GC_push_all_stack_partially_eager(hi, lo, cold_gc_frame);
# else
GC_push_all_stack_partially_eager(lo, hi, cold_gc_frame);
# endif
}
#endif /* !THREADS */
/*
* Push enough of the current stack eagerly to ensure that callee-save
* registers saved in GC frames are scanned. In the single-threaded case,
* schedule the entire stack for scanning. The 2nd argument (`context`)
* is a pointer to the (possibly `NULL`) thread context, for (currently
* hypothetical) more precise stack scanning. In the presence of threads,
* push enough of the current stack to ensure that callee-save registers
* saved in collector frames have been seen.
*/
/* TODO: Merge it with per-thread stuff. */
STATIC void
GC_push_current_stack(ptr_t cold_gc_frame, void *context)
{
UNUSED_ARG(context);
GC_ASSERT(I_HOLD_LOCK());
#if defined(THREADS)
/* `cold_gc_frame` is non-`NULL`. */
# ifdef STACK_GROWS_UP
GC_push_all_eager(cold_gc_frame, GC_approx_sp());
# else
GC_push_all_eager(GC_approx_sp(), cold_gc_frame);
/*
* For IA-64, the register stack backing store is handled in the
* thread-specific code.
*/
# endif
#else
GC_push_all_stack_part_eager_sections(GC_approx_sp(), GC_stackbottom,
cold_gc_frame, GC_traced_stack_sect);
# ifdef IA64
/*
* We also need to push the register stack backing store.
* This should really be done in the same way as the regular stack.
* For now we fudge it a bit. Note that the backing store grows up,
* so we cannot use `GC_push_all_stack_partially_eager`.
*/
{
ptr_t bsp = GC_save_regs_ret_val;
ptr_t cold_gc_bs_pointer = bsp - 2048;
if (GC_all_interior_pointers
&& ADDR_LT(GC_register_stackbottom, cold_gc_bs_pointer)) {
/*
* Adjust `cold_gc_bs_pointer` if below our innermost
* "traced stack section" in backing store.
*/
if (GC_traced_stack_sect != NULL
&& ADDR_LT(cold_gc_bs_pointer,
GC_traced_stack_sect->backing_store_end)) {
cold_gc_bs_pointer = GC_traced_stack_sect->backing_store_end;
}
GC_push_all_register_sections(GC_register_stackbottom,
cold_gc_bs_pointer, FALSE,
GC_traced_stack_sect);
GC_push_all_eager(cold_gc_bs_pointer, bsp);
} else {
GC_push_all_register_sections(GC_register_stackbottom, bsp,
TRUE /* `eager` */, GC_traced_stack_sect);
}
/*
* All values should be sufficiently aligned that we do not have to
* worry about the boundary.
*/
}
# elif defined(E2K)
/* We also need to push procedure stack store. Procedure stack grows up. */
{
ptr_t bs_lo;
size_t stack_size;
/* TODO: Support `ps_ofs` here and in `GC_do_blocking_inner`. */
GET_PROCEDURE_STACK_LOCAL(0, &bs_lo, &stack_size);
GC_push_all_eager(bs_lo, bs_lo + stack_size);
}
# endif
#endif /* !THREADS */
}
GC_INNER void (*GC_push_typed_structures)(void) = 0;
GC_INNER void
GC_cond_register_dynamic_libraries(void)
{
GC_ASSERT(I_HOLD_LOCK());
#if defined(DYNAMIC_LOADING) && !defined(MSWIN_XBOX1) || defined(ANY_MSWIN)
GC_remove_tmp_roots();
if (!GC_no_dls)
GC_register_dynamic_libraries();
#else
GC_no_dls = TRUE;
#endif
}
STATIC void
GC_push_regs_and_stack(ptr_t cold_gc_frame)
{
GC_ASSERT(I_HOLD_LOCK());
#ifdef THREADS
if (NULL == cold_gc_frame) {
/* `GC_push_all_stacks` should push registers and stack. */
return;
}
#endif
GC_with_callee_saves_pushed(GC_push_current_stack, cold_gc_frame);
}
GC_INNER void
GC_push_roots(GC_bool all, ptr_t cold_gc_frame)
{
size_t i;
unsigned kind;
GC_ASSERT(I_HOLD_LOCK());
/* The initialization is needed for `GC_push_all_stacks`. */
GC_ASSERT(GC_is_initialized);
/*
* Next push static data. This must happen early on, since it is not
* robust against mark stack overflow. Re-register dynamic libraries,
* in case one got added. There is some argument for doing this as late
* as possible, especially on Win32, where it can change asynchronously.
* In those cases, we do it here. But on other platforms, it is not safe
* with the world stopped, so we do it earlier.
*/
#if !defined(REGISTER_LIBRARIES_EARLY)
GC_cond_register_dynamic_libraries();
#endif
/* Mark everything in static data areas. */
for (i = 0; i < n_root_sets; i++) {
GC_push_conditional_with_exclusions(GC_static_roots[i].r_start,
GC_static_roots[i].r_end, all);
}
/*
* Mark all free-list header blocks, if those were allocated from
* the garbage-collected heap. This makes sure they do not disappear
* if we are not marking from static data. It also saves us the trouble
* of scanning them, and possibly that of marking the free lists.
*/
for (kind = 0; kind < GC_n_kinds; kind++) {
const void *base = GC_base(GC_obj_kinds[kind].ok_freelist);
if (base != NULL) {
GC_set_mark_bit(base);
}
}
/*
* Mark from the collector internal roots if those might otherwise
* have been excluded.
*/
#ifndef GC_NO_FINALIZATION
GC_push_finalizer_structures();
#endif
#ifdef THREADS
if (GC_no_dls || GC_roots_were_cleared)
GC_push_thread_structures();
#endif
if (GC_push_typed_structures) {
GC_push_typed_structures();
}
#if defined(THREAD_LOCAL_ALLOC)
/*
* Mark thread-local free lists, even if their mark descriptor excludes
* the link field. If the world is not stopped, this is unsafe.
* It is also unnecessary, since we will do this again with the world
* stopped.
*/
if (GC_world_stopped) {
GC_mark_thread_local_free_lists();
}
#endif
/*
* Now traverse stacks, and mark from register contents.
* These must be done last, since they can legitimately overflow
* the mark stack. This is usually done by saving the current
* context on the stack, and then just tracing from the stack.
*/
#ifdef STACK_NOT_SCANNED
UNUSED_ARG(cold_gc_frame);
#else
GC_push_regs_and_stack(cold_gc_frame);
#endif
if (GC_push_other_roots != 0) {
/*
* In the multi-threaded case, this also pushes thread stacks.
* Note that without the interior pointers recognition lots of stuff
* may have already been pushed, and this should be careful about
* mark stack overflows.
*/
(*GC_push_other_roots)();
}
}
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1996 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
* Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
* Copyright (c) 2009-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#ifdef ENABLE_DISCLAIM
#endif
#ifdef PARALLEL_MARK
GC_INNER GC_signed_word GC_fl_builder_count = 0;
#endif
/*
* We defer printing of leaked objects until we are done with the
* collection cycle, since the routine for printing objects needs
* to run outside the collector, e.g. without the allocator lock.
*/
#ifndef NO_FIND_LEAK
# ifndef MAX_LEAKED
# define MAX_LEAKED 40
# endif
STATIC ptr_t GC_leaked[MAX_LEAKED] = { NULL };
#endif
#if !defined(EAGER_SWEEP) && defined(ENABLE_DISCLAIM)
STATIC void GC_reclaim_unconditionally_marked(void);
#endif
#ifndef SHORT_DBG_HDRS
# ifndef MAX_SMASHED
# define MAX_SMASHED 20
# endif
/*
* List of smashed (clobbered) locations. We defer printing these,
* since we cannot always print them nicely with the allocator lock held.
* We put them here instead of in `GC_arrays`, since it may be useful to
* be able to look at them with the debugger.
*/
STATIC ptr_t GC_smashed[MAX_SMASHED] = { NULL };
GC_INNER void
GC_add_smashed(ptr_t smashed)
{
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_is_marked(GC_base(smashed)));
/* FIXME: Prevent adding an object while printing smashed list. */
GC_smashed[GC_n_smashed] = smashed;
/*
* In case of overflow, we keep the first `MAX_SMASHED - 1` entries
* plus the last one.
*/
if (GC_n_smashed < MAX_SMASHED - 1)
++GC_n_smashed;
GC_SET_HAVE_ERRORS();
}
GC_INNER void
GC_print_smashed_obj(const char *msg, void *p, ptr_t clobbered)
{
oh *ohdr = (oh *)GC_base(p);
GC_ASSERT(I_DONT_HOLD_LOCK());
# ifdef LINT2
if (!ohdr)
ABORT("Invalid GC_print_smashed_obj argument");
# endif
if (ADDR_GE((ptr_t)(&ohdr->oh_sz), clobbered) || NULL == ohdr->oh_string) {
GC_err_printf("%s %p in or near object at %p(<smashed>, appr. sz= %lu)\n",
msg, (void *)clobbered, p,
(unsigned long)(GC_size(ohdr) - DEBUG_BYTES));
} else {
GC_err_printf("%s %p in or near object at %p (%s:%d, sz= %lu)\n", msg,
(void *)clobbered, p,
ADDR(ohdr->oh_string) < HBLKSIZE ? "(smashed string)"
: ohdr->oh_string[0] == '\0' ? "EMPTY(smashed?)"
: ohdr->oh_string,
GET_OH_LINENUM(ohdr), (unsigned long)ohdr->oh_sz);
PRINT_CALL_CHAIN(ohdr);
}
}
GC_INNER void
GC_print_all_smashed_proc(void)
{
unsigned i;
GC_ASSERT(I_DONT_HOLD_LOCK());
if (GC_n_smashed == 0)
return;
GC_err_printf("GC_check_heap_block: found %u smashed heap objects:\n",
GC_n_smashed);
for (i = 0; i < GC_n_smashed; ++i) {
ptr_t base = (ptr_t)GC_base(GC_smashed[i]);
# ifdef LINT2
if (!base)
ABORT("Invalid GC_smashed element");
# endif
GC_print_smashed_obj("", base + sizeof(oh), GC_smashed[i]);
GC_smashed[i] = 0;
}
GC_n_smashed = 0;
}
GC_INNER int
GC_has_other_debug_info(ptr_t base)
{
ptr_t body = (ptr_t)((oh *)base + 1);
size_t sz = GC_size(base);
if (HBLKPTR(base) != HBLKPTR(body) || sz < DEBUG_BYTES + EXTRA_BYTES) {
return 0;
}
if (((oh *)base)->oh_sf != (START_FLAG ^ (GC_uintptr_t)body)
&& ((GC_uintptr_t *)base)[BYTES_TO_PTRS(sz) - 1]
!= (END_FLAG ^ (GC_uintptr_t)body)) {
return 0;
}
if (((oh *)base)->oh_sz == (GC_uintptr_t)sz) {
/* Object may have had debug info, but has been deallocated. */
return -1;
}
return 1;
}
#endif /* !SHORT_DBG_HDRS */
GC_INNER void
GC_default_print_heap_obj_proc(ptr_t p)
{
ptr_t base = (ptr_t)GC_base(p);
int kind = HDR(base)->hb_obj_kind;
GC_err_printf("object at %p of appr. %lu bytes (%s)\n", (void *)base,
(unsigned long)GC_size(base),
kind == PTRFREE ? "atomic"
: IS_UNCOLLECTABLE(kind) ? "uncollectable"
: "composite");
}
GC_INNER void (*GC_print_heap_obj)(ptr_t p) = GC_default_print_heap_obj_proc;
#if !defined(NO_FIND_LEAK) || !defined(SHORT_DBG_HDRS)
GC_INNER void
GC_print_all_errors(void)
{
static GC_bool printing_errors = FALSE;
GC_bool have_errors;
# ifndef NO_FIND_LEAK
unsigned i, n_leaked;
ptr_t leaked[MAX_LEAKED];
# endif
LOCK();
if (printing_errors) {
UNLOCK();
return;
}
have_errors = get_have_errors();
printing_errors = TRUE;
# ifndef NO_FIND_LEAK
n_leaked = GC_n_leaked;
if (n_leaked > 0) {
GC_ASSERT(n_leaked <= MAX_LEAKED);
BCOPY(GC_leaked, leaked, n_leaked * sizeof(ptr_t));
GC_n_leaked = 0;
BZERO(GC_leaked, n_leaked * sizeof(ptr_t));
}
# endif
UNLOCK();
if (GC_debugging_started) {
GC_print_all_smashed();
} else {
have_errors = FALSE;
}
# ifndef NO_FIND_LEAK
if (n_leaked > 0) {
GC_err_printf("Found %u leaked objects:\n", n_leaked);
have_errors = TRUE;
}
for (i = 0; i < n_leaked; i++) {
ptr_t p = leaked[i];
# ifndef SKIP_LEAKED_OBJECTS_PRINTING
GC_print_heap_obj(p);
# endif
GC_free(p);
}
# endif
if (have_errors
# ifndef GC_ABORT_ON_LEAK
&& GETENV("GC_ABORT_ON_LEAK") != NULL
# endif
) {
ABORT("Leaked or smashed objects encountered");
}
LOCK();
printing_errors = FALSE;
UNLOCK();
}
#endif
/* The reclaim phase. */
GC_INNER GC_bool
GC_block_empty(const hdr *hhdr)
{
return 0 == hhdr->hb_n_marks;
}
STATIC GC_bool
GC_block_nearly_full(const hdr *hhdr, size_t sz)
{
return hhdr->hb_n_marks > HBLK_OBJS(sz) * 7 / 8;
}
/*
* TODO: This should perhaps again be specialized for `USE_MARK_BYTES`
* and `USE_MARK_BITS` cases.
*/
GC_INLINE ptr_t
GC_clear_block(ptr_t q, size_t sz, word *pcount)
{
ptr_t *p = (ptr_t *)q;
ptr_t plim = q + sz;
/* Clear object, advance `p` to next object in the process. */
#ifdef USE_MARK_BYTES
GC_ASSERT((sz & 1) == 0);
GC_ASSERT((ADDR(p) & (2 * sizeof(ptr_t) - 1)) == 0);
p[1] = NULL; /*< but do not clear link field */
for (p += 2; ADDR_LT((ptr_t)p, plim); p += 2) {
CLEAR_DOUBLE(p);
}
#else
/* Skip link field. */
p++;
while (ADDR_LT((ptr_t)p, plim)) {
*p++ = NULL;
}
#endif
*pcount += sz;
return (ptr_t)p;
}
/*
* Restore unmarked small objects in `h` of size `sz` (in bytes) to the
* object free list. Returns the new list. Clears unmarked objects.
*/
STATIC ptr_t
GC_reclaim_clear(struct hblk *hbp, const hdr *hhdr, size_t sz, ptr_t list,
word *pcount)
{
size_t bit_no;
ptr_t p, plim;
GC_ASSERT(hhdr == GC_find_header(hbp));
#ifndef THREADS
GC_ASSERT(sz == hhdr->hb_sz);
#else
/* Skip the assertion because of a potential race with `GC_realloc`. */
#endif
GC_ASSERT((sz & (sizeof(ptr_t) - 1)) == 0);
/* Go through all objects in the block. */
p = hbp->hb_body;
plim = p + HBLKSIZE - sz;
for (bit_no = 0; ADDR_GE(plim, p); bit_no += MARK_BIT_OFFSET(sz)) {
if (mark_bit_from_hdr(hhdr, bit_no)) {
p += sz;
} else {
/* The object is available - put it on list. */
obj_link(p) = list;
list = p;
FREE_PROFILER_HOOK(p);
p = GC_clear_block(p, sz, pcount);
}
}
return list;
}
/* The same thing as `GC_reclaim_clear`, but do not clear objects. */
STATIC ptr_t
GC_reclaim_uninit(struct hblk *hbp, const hdr *hhdr, size_t sz, ptr_t list,
word *pcount)
{
size_t bit_no;
word n_bytes_found = 0;
ptr_t p, plim;
#ifndef THREADS
GC_ASSERT(sz == hhdr->hb_sz);
#endif
/* Go through all objects in the block. */
p = hbp->hb_body;
plim = (ptr_t)hbp + HBLKSIZE - sz;
for (bit_no = 0; ADDR_GE(plim, p); bit_no += MARK_BIT_OFFSET(sz), p += sz) {
if (!mark_bit_from_hdr(hhdr, bit_no)) {
n_bytes_found += sz;
/* The object is available - put it on list. */
obj_link(p) = list;
list = p;
FREE_PROFILER_HOOK(p);
}
}
*pcount += n_bytes_found;
return list;
}
#ifdef ENABLE_DISCLAIM
/*
* Call reclaim notifier for block's kind on each unmarked object in block,
* all within a pair of corresponding enter/leave callbacks.
*/
STATIC ptr_t
GC_disclaim_and_reclaim(struct hblk *hbp, hdr *hhdr, size_t sz, ptr_t list,
word *pcount)
{
size_t bit_no;
ptr_t p, plim;
int(GC_CALLBACK * disclaim)(void *)
= GC_obj_kinds[hhdr->hb_obj_kind].ok_disclaim_proc;
GC_ASSERT(disclaim != 0);
# ifndef THREADS
GC_ASSERT(sz == hhdr->hb_sz);
# endif
p = hbp->hb_body;
plim = p + HBLKSIZE - sz;
for (bit_no = 0; ADDR_GE(plim, p); bit_no += MARK_BIT_OFFSET(sz)) {
if (mark_bit_from_hdr(hhdr, bit_no)) {
p += sz;
} else if (disclaim(p)) {
set_mark_bit_from_hdr(hhdr, bit_no);
INCR_MARKS(hhdr);
p += sz;
} else {
obj_link(p) = list;
list = p;
FREE_PROFILER_HOOK(p);
p = GC_clear_block(p, sz, pcount);
}
}
return list;
}
#endif /* ENABLE_DISCLAIM */
#ifndef NO_FIND_LEAK
# ifndef SHORT_DBG_HDRS
STATIC GC_bool
GC_check_leaked(ptr_t base)
{
size_t i;
size_t lpw;
ptr_t *p;
if (
# if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
(*(GC_uintptr_t *)base & 1) != 0 &&
# endif
GC_has_other_debug_info(base) >= 0)
return TRUE; /*< object has leaked */
/* Validate freed object's content. */
p = (ptr_t *)(base + sizeof(oh));
lpw = BYTES_TO_PTRS(HDR(base)->hb_sz - sizeof(oh));
for (i = 0; i < lpw; ++i)
if ((GC_uintptr_t)p[i] != GC_FREED_MEM_MARKER) {
/* Do not reclaim it in this cycle. */
GC_set_mark_bit(base);
/* Alter-after-free has been detected. */
GC_add_smashed((ptr_t)(&p[i]));
/* Do not report any other smashed locations in the object. */
break;
}
return FALSE; /*< `GC_debug_free()` has been called */
}
# endif /* !SHORT_DBG_HDRS */
GC_INLINE void
GC_add_leaked(ptr_t leaked)
{
GC_ASSERT(I_HOLD_LOCK());
# ifndef SHORT_DBG_HDRS
if (GC_findleak_delay_free && !GC_check_leaked(leaked))
return;
# endif
GC_SET_HAVE_ERRORS();
if (GC_n_leaked < MAX_LEAKED) {
GC_leaked[GC_n_leaked++] = leaked;
/* Make sure it is not reclaimed this cycle. */
GC_set_mark_bit(leaked);
}
}
/* Do not really reclaim objects, just check for unmarked ones. */
STATIC void
GC_reclaim_check(struct hblk *hbp, const hdr *hhdr, size_t sz)
{
size_t bit_no;
ptr_t p, plim;
# ifndef THREADS
GC_ASSERT(sz == hhdr->hb_sz);
# endif
/* Go through all objects in the block. */
p = hbp->hb_body;
plim = p + HBLKSIZE - sz;
for (bit_no = 0; ADDR_GE(plim, p); bit_no += MARK_BIT_OFFSET(sz), p += sz) {
if (!mark_bit_from_hdr(hhdr, bit_no))
GC_add_leaked(p);
}
}
#endif /* !NO_FIND_LEAK */
/*
* Is a pointer-free block? Same as `IS_PTRFREE()` macro but uses
* unordered atomic access to avoid racing with `GC_realloc`.
*/
#ifdef AO_HAVE_load
# define IS_PTRFREE_SAFE(hhdr) (AO_load((AO_t *)&(hhdr)->hb_descr) == 0)
#else
/*
* No race as `GC_realloc` holds the allocator lock when updating
* `hb_descr` field.
*/
# define IS_PTRFREE_SAFE(hhdr) IS_PTRFREE(hhdr)
#endif
GC_INNER ptr_t
GC_reclaim_generic(struct hblk *hbp, hdr *hhdr, size_t sz, GC_bool init,
ptr_t list, word *pcount)
{
ptr_t result;
#ifndef PARALLEL_MARK
GC_ASSERT(I_HOLD_LOCK());
#endif
GC_ASSERT(GC_find_header(hbp) == hhdr);
#ifndef GC_DISABLE_INCREMENTAL
GC_remove_protection(hbp, 1, IS_PTRFREE_SAFE(hhdr));
#endif
#ifdef ENABLE_DISCLAIM
if ((hhdr->hb_flags & HAS_DISCLAIM) != 0) {
result = GC_disclaim_and_reclaim(hbp, hhdr, sz, list, pcount);
} else
#endif
/* else */ {
if (init || GC_debugging_started) {
result = GC_reclaim_clear(hbp, hhdr, sz, list, pcount);
} else {
#ifndef AO_HAVE_load
GC_ASSERT(IS_PTRFREE(hhdr));
#endif
result = GC_reclaim_uninit(hbp, hhdr, sz, list, pcount);
}
}
if (IS_UNCOLLECTABLE(hhdr->hb_obj_kind))
GC_set_hdr_marks(hhdr);
return result;
}
/*
* Restore unmarked small objects in the block pointed to by `hbp` to
* the appropriate object free list. If entirely empty blocks are to
* be completely deallocated, then caller should perform that check.
*/
STATIC void
GC_reclaim_small_nonempty_block(struct hblk *hbp, size_t sz,
GC_bool report_if_found)
{
hdr *hhdr;
GC_ASSERT(I_HOLD_LOCK());
hhdr = HDR(hbp);
hhdr->hb_last_reclaimed = (unsigned short)GC_gc_no;
if (report_if_found) {
#ifndef NO_FIND_LEAK
GC_reclaim_check(hbp, hhdr, sz);
#endif
} else {
struct obj_kind *ok = &GC_obj_kinds[hhdr->hb_obj_kind];
void **flh = &ok->ok_freelist[BYTES_TO_GRANULES(sz)];
*flh = GC_reclaim_generic(hbp, hhdr, sz, ok->ok_init, (ptr_t)(*flh),
(/* unsigned */ word *)&GC_bytes_found);
}
}
#ifdef ENABLE_DISCLAIM
STATIC void
GC_disclaim_and_reclaim_or_free_small_block(struct hblk *hbp)
{
hdr *hhdr;
size_t sz;
struct obj_kind *ok;
void **flh;
void *flh_next;
GC_ASSERT(I_HOLD_LOCK());
hhdr = HDR(hbp);
sz = hhdr->hb_sz;
ok = &GC_obj_kinds[hhdr->hb_obj_kind];
flh = &ok->ok_freelist[BYTES_TO_GRANULES(sz)];
hhdr->hb_last_reclaimed = (unsigned short)GC_gc_no;
flh_next = GC_reclaim_generic(hbp, hhdr, sz, ok->ok_init, (ptr_t)(*flh),
(/* unsigned */ word *)&GC_bytes_found);
if (hhdr->hb_n_marks) {
*flh = flh_next;
} else {
GC_ASSERT(hbp == hhdr->hb_block);
GC_bytes_found += (GC_signed_word)HBLKSIZE;
GC_freehblk(hbp);
}
}
#endif /* ENABLE_DISCLAIM */
/*
* Restore an unmarked large object or an entirely empty block of
* small objects to the heap block free list. Otherwise enqueue the
* block for later processing by `GC_reclaim_small_nonempty_block()`.
* If `report_if_found` is `TRUE`, then process any block immediately,
* and simply report free objects; do not actually reclaim them.
*/
STATIC void GC_CALLBACK
GC_reclaim_block(struct hblk *hbp, void *report_if_found)
{
hdr *hhdr;
size_t sz; /*< size of objects in current block */
struct obj_kind *ok;
GC_ASSERT(I_HOLD_LOCK());
#if defined(CPPCHECK)
GC_noop1_ptr(report_if_found);
#endif
hhdr = HDR(hbp);
ok = &GC_obj_kinds[hhdr->hb_obj_kind];
#ifdef AO_HAVE_load
/* Atomic access is used to avoid racing with `GC_realloc`. */
sz = AO_load((volatile AO_t *)&hhdr->hb_sz);
#else
/*
* No race as `GC_realloc` holds the allocator lock while
* updating `hb_sz`.
*/
sz = hhdr->hb_sz;
#endif
if (sz > MAXOBJBYTES) {
/* The case of 1 big object. */
if (!mark_bit_from_hdr(hhdr, 0)) {
if (report_if_found) {
GC_ASSERT(hbp == hhdr->hb_block);
#ifndef NO_FIND_LEAK
GC_add_leaked((ptr_t)hbp);
#endif
} else {
#ifdef ENABLE_DISCLAIM
if (UNLIKELY((hhdr->hb_flags & HAS_DISCLAIM) != 0)) {
if (ok->ok_disclaim_proc(hbp)) {
/* Not disclaimed, thus resurrect the object. */
set_mark_bit_from_hdr(hhdr, 0);
goto in_use;
}
}
#endif
GC_ASSERT(hbp == hhdr->hb_block);
if (sz > HBLKSIZE) {
GC_large_allocd_bytes -= HBLKSIZE * OBJ_SZ_TO_BLOCKS(sz);
}
GC_bytes_found += (GC_signed_word)sz;
GC_freehblk(hbp);
FREE_PROFILER_HOOK(hbp);
}
} else {
#ifdef ENABLE_DISCLAIM
in_use:
#endif
if (IS_PTRFREE_SAFE(hhdr)) {
GC_atomic_in_use += sz;
} else {
GC_composite_in_use += sz;
}
}
} else {
GC_bool empty = GC_block_empty(hhdr);
#ifdef PARALLEL_MARK
/*
* Count can be low or one too high because we sometimes have to
* ignore decrements. Objects can also potentially be repeatedly
* marked by each marker. Here we assume 3 markers at most, but
* this is extremely unlikely to fail spuriously with more.
* And if it does, it should be looked at.
*/
GC_ASSERT(sz != 0
&& (GC_markers_m1 > 1 ? 3 : GC_markers_m1 + 1)
* (HBLKSIZE / sz + 1)
+ 16
>= hhdr->hb_n_marks);
#else
GC_ASSERT(sz * hhdr->hb_n_marks <= HBLKSIZE);
#endif
#ifdef VALGRIND_TRACKING
/*
* Call `GC_free_profiler_hook()` on freed objects so that
* a profiling tool could track the allocations.
*/
{
ptr_t p = hbp->hb_body;
ptr_t plim = p + HBLKSIZE - sz;
size_t bit_no;
for (bit_no = 0; ADDR_GE(plim, p);
bit_no += MARK_BIT_OFFSET(sz), p += sz) {
if (!mark_bit_from_hdr(hhdr, bit_no))
FREE_PROFILER_HOOK(p);
}
}
#endif
GC_ASSERT(hbp == hhdr->hb_block);
if (report_if_found) {
GC_reclaim_small_nonempty_block(hbp, sz, TRUE /* `report_if_found` */);
} else if (empty) {
#ifdef ENABLE_DISCLAIM
if ((hhdr->hb_flags & HAS_DISCLAIM) != 0) {
GC_disclaim_and_reclaim_or_free_small_block(hbp);
} else
#endif
/* else */ {
GC_bytes_found += (GC_signed_word)HBLKSIZE;
GC_freehblk(hbp);
FREE_PROFILER_HOOK(hbp);
}
} else if (GC_find_leak_inner || !GC_block_nearly_full(hhdr, sz)) {
/* Group of smaller objects, enqueue the real work. */
struct hblk **rlh = ok->ok_reclaim_list;
if (rlh != NULL) {
rlh += BYTES_TO_GRANULES(sz);
hhdr->hb_next = *rlh;
*rlh = hbp;
}
} else {
/* Not worth salvaging. */
}
/*
* We used to do the `GC_block_nearly_full` check later, but we
* already have the right cache context here. Also doing it here
* avoids some silly lock contention in `GC_malloc_many()`.
*/
if (IS_PTRFREE_SAFE(hhdr)) {
GC_atomic_in_use += (word)sz * hhdr->hb_n_marks;
} else {
GC_composite_in_use += (word)sz * hhdr->hb_n_marks;
}
}
}
#if !defined(NO_DEBUGGING)
/*
* Routines to gather and print heap block info intended for debugging.
* Otherwise should be called with the allocator lock held.
*/
struct Print_stats {
size_t number_of_blocks;
size_t total_bytes;
};
EXTERN_C_BEGIN /*< to avoid "no previous prototype" clang warning */
unsigned
GC_n_set_marks(const hdr *);
EXTERN_C_END
# ifdef USE_MARK_BYTES
/*
* Return the number of set mark bits in the given header.
* Remains externally visible as used by GNU `gcj` currently.
* There could be a race between `GC_clear_hdr_marks` and this
* function but the latter is for a debug purpose.
*/
GC_ATTR_NO_SANITIZE_THREAD
unsigned
GC_n_set_marks(const hdr *hhdr)
{
unsigned result = 0;
size_t i;
size_t offset = MARK_BIT_OFFSET(hhdr->hb_sz);
size_t limit = FINAL_MARK_BIT(hhdr->hb_sz);
for (i = 0; i < limit; i += offset) {
result += (unsigned)hhdr->hb_marks[i];
}
/* The one should be set past the end. */
GC_ASSERT(hhdr->hb_marks[limit]);
return result;
}
# else
/* Number of set bits in a word. Not performance critical. */
static unsigned
count_ones(word v)
{
unsigned result = 0;
for (; v > 0; v >>= 1) {
if (v & 1)
result++;
}
return result;
}
unsigned
GC_n_set_marks(const hdr *hhdr)
{
unsigned result = 0;
size_t i;
# ifdef MARK_BIT_PER_OBJ
size_t n_objs = HBLK_OBJS(hhdr->hb_sz);
size_t n_mark_words = divWORDSZ(n_objs > 0 ? n_objs : 1); /*< round down */
for (i = 0; i <= n_mark_words; i++) {
result += count_ones(hhdr->hb_marks[i]);
}
# else
for (i = 0; i < HB_MARKS_SZ; i++) {
result += count_ones(hhdr->hb_marks[i]);
}
# endif
GC_ASSERT(result > 0);
/* Exclude the one bit set past the end. */
result--;
# ifndef MARK_BIT_PER_OBJ
if (IS_UNCOLLECTABLE(hhdr->hb_obj_kind)) {
size_t lg = BYTES_TO_GRANULES(hhdr->hb_sz);
/*
* As mentioned in `GC_set_hdr_marks`, all the bits are set instead of
* every `n`-th, thus the result should be adjusted.
*/
GC_ASSERT((unsigned)lg != 0 && result % lg == 0);
result /= (unsigned)lg;
}
# endif
return result;
}
# endif /* !USE_MARK_BYTES */
GC_API unsigned GC_CALL
GC_count_set_marks_in_hblk(const void *p)
{
return GC_n_set_marks(HDR(p));
}
STATIC void GC_CALLBACK
GC_print_block_descr(struct hblk *h, void *raw_ps)
{
const hdr *hhdr = HDR(h);
size_t sz = hhdr->hb_sz;
struct Print_stats *ps = (struct Print_stats *)raw_ps;
size_t n_marks = (size_t)GC_n_set_marks(hhdr);
size_t n_objs = HBLK_OBJS(sz);
# ifndef PARALLEL_MARK
GC_ASSERT(hhdr->hb_n_marks == n_marks);
# endif
# if defined(CPPCHECK)
GC_noop1_ptr(h);
# endif
GC_ASSERT((n_objs > 0 ? n_objs : 1) >= n_marks);
GC_printf("%u,%u,%u,%u\n", hhdr->hb_obj_kind, (unsigned)sz,
(unsigned)n_marks, (unsigned)n_objs);
ps->number_of_blocks++;
ps->total_bytes += (sz + HBLKSIZE - 1) & ~(HBLKSIZE - 1); /*< round up */
}
void
GC_print_block_list(void)
{
struct Print_stats pstats;
GC_printf("kind(0=ptrfree/1=normal/2=unc.),"
"obj_sz,#marks_set,#objs_in_block\n");
BZERO(&pstats, sizeof(pstats));
GC_apply_to_all_blocks(GC_print_block_descr, &pstats);
GC_printf("blocks= %lu, total_bytes= %lu\n",
(unsigned long)pstats.number_of_blocks,
(unsigned long)pstats.total_bytes);
if (pstats.total_bytes + GC_large_free_bytes != GC_heapsize)
GC_err_printf("LOST SOME BLOCKS!! Total bytes should be: %lu\n",
(unsigned long)(GC_heapsize - GC_large_free_bytes));
}
GC_API void GC_CALL
GC_print_free_list(int kind, size_t lg)
{
void *flh_next;
int n;
GC_ASSERT(kind < MAXOBJKINDS);
GC_ASSERT(lg <= MAXOBJGRANULES);
flh_next = GC_obj_kinds[kind].ok_freelist[lg];
for (n = 0; flh_next != NULL; n++) {
GC_printf("Free object in heap block %p [%d]: %p\n",
(void *)HBLKPTR(flh_next), n, flh_next);
flh_next = obj_link(flh_next);
}
}
#endif /* !NO_DEBUGGING */
/*
* Clear all `obj_link` pointers in the list of free objects `*flp`.
* Clear `*flp`. This must be done before dropping a list of free
* `gcj`-style objects, since may otherwise end up with dangling
* "descriptor" pointers. It may help for other pointer-containing
* objects.
*/
STATIC void
GC_clear_fl_links(void **flp)
{
void *next;
for (next = *flp; next != NULL; next = *flp) {
*flp = NULL;
flp = &obj_link(next);
}
}
GC_INNER void
GC_start_reclaim(GC_bool report_if_found)
{
int kind;
GC_ASSERT(I_HOLD_LOCK());
#if defined(PARALLEL_MARK)
GC_ASSERT(0 == GC_fl_builder_count);
#endif
/* Reset in-use counters. `GC_reclaim_block` recomputes them. */
GC_composite_in_use = 0;
GC_atomic_in_use = 0;
/* Clear reclaim- and free-lists. */
for (kind = 0; kind < (int)GC_n_kinds; kind++) {
struct hblk **rlist = GC_obj_kinds[kind].ok_reclaim_list;
GC_bool should_clobber = GC_obj_kinds[kind].ok_descriptor != 0;
if (NULL == rlist) {
/* Means this object kind is not used. */
continue;
}
if (!report_if_found) {
void **fop;
void **lim = &GC_obj_kinds[kind].ok_freelist[MAXOBJGRANULES + 1];
for (fop = GC_obj_kinds[kind].ok_freelist;
ADDR_LT((ptr_t)fop, (ptr_t)lim); fop++) {
if (*fop != NULL) {
if (should_clobber) {
GC_clear_fl_links(fop);
} else {
*fop = NULL;
}
}
}
} else {
/* Free-list objects are marked, and it is safe to leave them. */
}
BZERO(rlist, (MAXOBJGRANULES + 1) * sizeof(void *));
}
/*
* Go through all heap blocks, and reclaim unmarked objects or enqueue
* the block for later processing.
*/
GC_apply_to_all_blocks(GC_reclaim_block, NUMERIC_TO_VPTR(report_if_found));
#ifdef EAGER_SWEEP
/*
* This is a very stupid thing to do. We make it possible anyway.
*/
GC_reclaim_all((GC_stop_func)0, FALSE);
#elif defined(ENABLE_DISCLAIM)
/*
* However, make sure to clear reclaimable objects of kinds with
* unconditional marking enabled before we do any significant
* marking work.
*/
GC_reclaim_unconditionally_marked();
#endif
#if defined(PARALLEL_MARK)
GC_ASSERT(0 == GC_fl_builder_count);
#endif
}
GC_INNER void
GC_continue_reclaim(size_t lg, int kind)
{
struct hblk *hbp;
struct obj_kind *ok = &GC_obj_kinds[kind];
struct hblk **rlh = ok->ok_reclaim_list;
void **flh;
GC_ASSERT(I_HOLD_LOCK());
if (NULL == rlh) {
/* No blocks of this kind. */
return;
}
flh = &ok->ok_freelist[lg];
for (rlh += lg; (hbp = *rlh) != NULL;) {
const hdr *hhdr = HDR(hbp);
*rlh = hhdr->hb_next;
GC_reclaim_small_nonempty_block(hbp, hhdr->hb_sz, FALSE);
if (*flh != NULL) {
/* The appropriate free list is nonempty. */
break;
}
}
}
GC_INNER GC_bool
GC_reclaim_all(GC_stop_func stop_func, GC_bool ignore_old)
{
size_t lg;
int kind;
const hdr *hhdr;
struct hblk *hbp;
struct hblk **rlp;
struct hblk **rlh;
#ifndef NO_CLOCK
CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;
if (GC_print_stats == VERBOSE)
GET_TIME(start_time);
#endif
GC_ASSERT(I_HOLD_LOCK());
for (kind = 0; kind < (int)GC_n_kinds; kind++) {
rlp = GC_obj_kinds[kind].ok_reclaim_list;
if (NULL == rlp)
continue;
for (lg = 1; lg <= MAXOBJGRANULES; lg++) {
for (rlh = rlp + lg; (hbp = *rlh) != NULL;) {
if (stop_func != (GC_stop_func)0 && (*stop_func)()) {
return FALSE;
}
hhdr = HDR(hbp);
*rlh = hhdr->hb_next;
if (!ignore_old || (word)hhdr->hb_last_reclaimed == GC_gc_no - 1) {
/*
* It is likely we will need it this time, too. It has been
* touched recently, so this should not trigger paging.
*/
GC_reclaim_small_nonempty_block(hbp, hhdr->hb_sz, FALSE);
}
}
}
}
#ifndef NO_CLOCK
if (GC_print_stats == VERBOSE) {
CLOCK_TYPE done_time;
GET_TIME(done_time);
GC_verbose_log_printf("Disposing of reclaim lists took %lu ms %lu ns\n",
MS_TIME_DIFF(done_time, start_time),
NS_FRAC_TIME_DIFF(done_time, start_time));
}
#endif
return TRUE;
}
#if !defined(EAGER_SWEEP) && defined(ENABLE_DISCLAIM)
/*
* We do an eager sweep on heap blocks where unconditional marking has
* been enabled, so that any reclaimable objects have been reclaimed
* before we start marking. This is a simplified `GC_reclaim_all`
* restricted to kinds where `ok_mark_unconditionally` is `TRUE`.
*/
STATIC void
GC_reclaim_unconditionally_marked(void)
{
int kind;
GC_ASSERT(I_HOLD_LOCK());
for (kind = 0; kind < (int)GC_n_kinds; kind++) {
size_t lg;
struct obj_kind *ok = &GC_obj_kinds[kind];
struct hblk **rlp = ok->ok_reclaim_list;
if (NULL == rlp || !ok->ok_mark_unconditionally)
continue;
for (lg = 1; lg <= MAXOBJGRANULES; lg++) {
struct hblk **rlh = rlp + lg;
struct hblk *hbp;
while ((hbp = *rlh) != NULL) {
const hdr *hhdr = HDR(hbp);
*rlh = hhdr->hb_next;
GC_reclaim_small_nonempty_block(hbp, hhdr->hb_sz, FALSE);
}
}
}
}
#endif /* !EAGER_SWEEP && ENABLE_DISCLAIM */
struct enumerate_reachable_s {
GC_reachable_object_proc proc;
void *client_data;
};
STATIC void GC_CALLBACK
GC_do_enumerate_reachable_objects(struct hblk *hbp, void *ed_ptr)
{
const hdr *hhdr = HDR(hbp);
ptr_t p, plim;
const struct enumerate_reachable_s *ped
= (struct enumerate_reachable_s *)ed_ptr;
size_t sz = hhdr->hb_sz;
size_t bit_no;
if (GC_block_empty(hhdr))
return;
p = hbp->hb_body;
if (sz > MAXOBJBYTES) {
/* The case of 1 big object. */
plim = p;
} else {
plim = p + HBLKSIZE - sz;
}
/* Go through all objects in the block. */
for (bit_no = 0; ADDR_GE(plim, p); bit_no += MARK_BIT_OFFSET(sz), p += sz) {
if (mark_bit_from_hdr(hhdr, bit_no)) {
ped->proc(p, sz, ped->client_data);
}
}
}
GC_API void GC_CALL
GC_enumerate_reachable_objects_inner(GC_reachable_object_proc proc,
void *client_data)
{
struct enumerate_reachable_s ed;
GC_ASSERT(I_HOLD_READER_LOCK());
ed.proc = proc;
ed.client_data = client_data;
GC_apply_to_all_blocks(GC_do_enumerate_reachable_objects, &ed);
}
/*
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1999-2000 by Hewlett-Packard Company. All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* Some simple primitives for allocation with explicit type information.
* Simple objects are allocated such that they contain a `GC_descr` at the
* end (in the last allocated word). This descriptor may be a procedure
* which then examines an extended descriptor passed as its environment.
*
* Arrays are treated as simple objects if they have sufficiently simple
* structure. Otherwise they are allocated from an array kind that supplies
* a special mark procedure. These arrays contain a pointer to a
* `complex_descriptor` as their last "pointer-sized" word.
* This is done because the environment field is too small, and the collector
* must trace the `complex_descriptor`.
*
* Note that descriptors inside objects may appear cleared, if we encounter
* a false reference to an object on a free list. In the case of a simple
* object, this is OK, since a zero descriptor corresponds to examining no
* fields. In the `complex_descriptor` case, we explicitly check for that
* case.
*
* Note: major parts of this code have not been tested at all and are not
* testable, since they are not accessible through the current interface.
*/
#include "gc/gc_typed.h"
/* Object kind for objects with indirect (possibly extended) descriptors. */
STATIC int GC_explicit_kind = 0;
/*
* Object kind for objects with complex descriptors and
* `GC_array_mark_proc`.
*/
STATIC int GC_array_kind = 0;
#define ED_INITIAL_SIZE 100
/* Indices of the typed mark procedures. */
STATIC unsigned GC_typed_mark_proc_index = 0;
STATIC unsigned GC_array_mark_proc_index = 0;
STATIC void
GC_push_typed_structures_proc(void)
{
GC_PUSH_ALL_SYM(GC_ext_descriptors);
}
/*
* Add a multi-word bitmap to `GC_ext_descriptors` arrays.
* Returns starting index on success, -1 otherwise.
*/
STATIC GC_signed_word
GC_add_ext_descriptor(const word *bm, size_t nbits)
{
GC_signed_word result;
size_t i;
size_t nwords = divWORDSZ(nbits + CPP_WORDSZ - 1);
LOCK();
while (UNLIKELY(GC_avail_descr + nwords >= GC_ed_size)) {
typed_ext_descr_t *newExtD;
size_t new_size;
size_t ed_size = GC_ed_size;
if (0 == ed_size) {
GC_ASSERT(ADDR(&GC_ext_descriptors) % ALIGNMENT == 0);
GC_push_typed_structures = GC_push_typed_structures_proc;
UNLOCK();
new_size = ED_INITIAL_SIZE;
} else {
UNLOCK();
new_size = 2 * ed_size;
if (new_size > MAX_ENV)
return -1;
}
newExtD = (typed_ext_descr_t *)GC_malloc_atomic(
new_size * sizeof(typed_ext_descr_t));
if (NULL == newExtD)
return -1;
LOCK();
if (ed_size == GC_ed_size) {
if (GC_avail_descr != 0) {
BCOPY(GC_ext_descriptors, newExtD,
GC_avail_descr * sizeof(typed_ext_descr_t));
}
GC_ed_size = new_size;
GC_ext_descriptors = newExtD;
} else {
/* Another thread is already resized it in the meantime. */
}
}
result = (GC_signed_word)GC_avail_descr;
for (i = 0; i < nwords - 1; i++) {
GC_ext_descriptors[(size_t)result + i].ed_bitmap = bm[i];
GC_ext_descriptors[(size_t)result + i].ed_continued = TRUE;
}
/* Clear irrelevant (highest) bits for the last element. */
GC_ext_descriptors[(size_t)result + i].ed_bitmap
= bm[i] & (GC_WORD_MAX >> (nwords * CPP_WORDSZ - nbits));
GC_ext_descriptors[(size_t)result + i].ed_continued = FALSE;
GC_avail_descr += nwords;
GC_ASSERT(result >= 0);
UNLOCK();
return result;
}
/* Table of bitmap descriptors for `n` pointer-long all-pointer objects. */
STATIC GC_descr GC_bm_table[CPP_WORDSZ / 2] = { 0 };
/*
* Return a descriptor for the concatenation of 2 objects, each one is
* `lpw` pointers long and described by descriptor `d`. The result is
* known to be short enough to fit into a bitmap descriptor.
* `d` is a `GC_DS_LENGTH` or `GC_DS_BITMAP` descriptor.
*/
STATIC GC_descr
GC_double_descr(GC_descr d, size_t lpw)
{
GC_ASSERT(GC_bm_table[0] == GC_DS_BITMAP); /*< `bm` table is initialized */
if ((d & GC_DS_TAGS) == GC_DS_LENGTH) {
d = GC_bm_table[BYTES_TO_PTRS(d)];
}
d |= (d & ~(GC_descr)GC_DS_TAGS) >> lpw;
return d;
}
STATIC mse *GC_CALLBACK GC_typed_mark_proc(word *addr, mse *mark_stack_top,
mse *mark_stack_limit, word env);
STATIC mse *GC_CALLBACK GC_array_mark_proc(word *addr, mse *mark_stack_top,
mse *mark_stack_limit, word env);
STATIC void
GC_init_explicit_typing(void)
{
unsigned i;
/*
* Set up object kind with simple indirect descriptor.
* Descriptor is in the last `word` of the object.
*/
GC_typed_mark_proc_index = GC_new_proc_inner(GC_typed_mark_proc);
GC_explicit_kind = (int)GC_new_kind_inner(
GC_new_free_list_inner(),
(PTRS_TO_BYTES(GC_WORD_MAX) | GC_DS_PER_OBJECT), TRUE, TRUE);
/* Set up object kind with array descriptor. */
GC_array_mark_proc_index = GC_new_proc_inner(GC_array_mark_proc);
GC_array_kind = (int)GC_new_kind_inner(
GC_new_free_list_inner(), GC_MAKE_PROC(GC_array_mark_proc_index, 0),
FALSE, TRUE);
GC_bm_table[0] = GC_DS_BITMAP;
for (i = 1; i < CPP_WORDSZ / 2; i++) {
GC_bm_table[i] = (GC_WORD_MAX << (CPP_WORDSZ - i)) | GC_DS_BITMAP;
}
}
STATIC mse *GC_CALLBACK
GC_typed_mark_proc(word *addr, mse *mark_stack_top, mse *mark_stack_limit,
word env)
{
word bm;
ptr_t current_p = (ptr_t)addr;
ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
DECLARE_HDR_CACHE;
/* The allocator lock is held by the collection initiating thread. */
GC_ASSERT(GC_get_parallel() || I_HOLD_LOCK());
bm = GC_ext_descriptors[env].ed_bitmap;
INIT_HDR_CACHE;
for (; bm != 0; bm >>= 1, current_p += sizeof(ptr_t)) {
if (bm & 1) {
ptr_t q;
LOAD_PTR_OR_CONTINUE(q, current_p);
FIXUP_POINTER(q);
if (ADDR_LT(least_ha, q) && ADDR_LT(q, greatest_ha)) {
PUSH_CONTENTS(q, mark_stack_top, mark_stack_limit, current_p);
}
}
}
if (GC_ext_descriptors[env].ed_continued) {
/*
* Push an entry with the rest of the descriptor back onto the stack.
* Thus we never do too much work at once. Note that we also cannot
* overflow the mark stack unless we actually mark something.
*/
mark_stack_top = GC_custom_push_proc(
GC_MAKE_PROC(GC_typed_mark_proc_index, env + 1),
(ptr_t *)addr + CPP_WORDSZ, mark_stack_top, mark_stack_limit);
}
return mark_stack_top;
}
GC_API GC_descr GC_CALL
GC_make_descriptor(const GC_word *bm, size_t len)
{
GC_signed_word last_set_bit = (GC_signed_word)len - 1;
GC_descr d;
#if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_store_release)
if (UNLIKELY(!AO_load_acquire(&GC_explicit_typing_initialized))) {
LOCK();
if (!GC_explicit_typing_initialized) {
GC_init_explicit_typing();
AO_store_release(&GC_explicit_typing_initialized, TRUE);
}
UNLOCK();
}
#else
LOCK();
if (UNLIKELY(!GC_explicit_typing_initialized)) {
GC_init_explicit_typing();
GC_explicit_typing_initialized = TRUE;
}
UNLOCK();
#endif
while (last_set_bit >= 0 && !GC_get_bit(bm, (word)last_set_bit))
last_set_bit--;
if (last_set_bit < 0) {
/* No pointers. */
return 0;
}
#if ALIGNMENT == CPP_PTRSZ / 8
{
GC_signed_word i;
for (i = 0; i < last_set_bit; i++) {
if (!GC_get_bit(bm, (word)i))
break;
}
if (i == last_set_bit) {
/*
* The initial section contains all pointers; use the length
* descriptor.
*/
return PTRS_TO_BYTES((word)last_set_bit + 1) | GC_DS_LENGTH;
}
}
#endif
if (last_set_bit < BITMAP_BITS) {
GC_signed_word i;
/*
* Hopefully the common case. Build the bitmap descriptor (with the
* bits reversed).
*/
d = SIGNB;
for (i = last_set_bit - 1; i >= 0; i--) {
d >>= 1;
if (GC_get_bit(bm, (word)i))
d |= SIGNB;
}
d |= GC_DS_BITMAP;
} else {
GC_signed_word index = GC_add_ext_descriptor(bm, (size_t)last_set_bit + 1);
if (UNLIKELY(index < 0)) {
/* Out of memory: use a conservative approximation. */
return PTRS_TO_BYTES((word)last_set_bit + 1) | GC_DS_LENGTH;
}
#ifdef LINT2
if ((word)index > MAX_ENV)
ABORT("GC_add_ext_descriptor() result cannot exceed MAX_ENV");
#endif
d = GC_MAKE_PROC(GC_typed_mark_proc_index, index);
}
return d;
}
static void
set_obj_descr(ptr_t op, GC_descr d)
{
size_t sz;
if (UNLIKELY(NULL == op))
return;
/*
* It is not safe to use `GC_size_map[]` here as the table might be
* updated asynchronously.
*/
sz = GC_size(op);
GC_ASSERT((sz & (GC_GRANULE_BYTES - 1)) == 0 && sz > sizeof(GC_descr));
#ifdef AO_HAVE_store_release
AO_store_release((volatile AO_t *)&op[sz - sizeof(GC_descr)], d);
#else
*(GC_descr *)&op[sz - sizeof(GC_descr)] = d;
#endif
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_malloc_explicitly_typed(size_t lb, GC_descr d)
{
ptr_t op;
GC_ASSERT(GC_explicit_typing_initialized);
if (UNLIKELY(lb < sizeof(ptr_t) - sizeof(GC_descr) + 1)) {
/* Ensure the descriptor does not occupy the first pointer place. */
lb = sizeof(ptr_t) - sizeof(GC_descr) + 1;
}
op = (ptr_t)GC_malloc_kind(SIZET_SAT_ADD(lb, sizeof(GC_descr) - EXTRA_BYTES),
GC_explicit_kind);
set_obj_descr(op, d);
return op;
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_malloc_explicitly_typed_ignore_off_page(size_t lb, GC_descr d)
{
ptr_t op;
if (lb < HBLKSIZE - sizeof(GC_descr))
return GC_malloc_explicitly_typed(lb, d);
GC_ASSERT(GC_explicit_typing_initialized);
/*
* Note that ignore-off-page objects with the requested size of
* at least `HBLKSIZE` do not have `EXTRA_BYTES` added by
* `GC_generic_malloc_aligned()`.
*/
op = (ptr_t)GC_clear_stack(
GC_generic_malloc_aligned(SIZET_SAT_ADD(lb, sizeof(GC_descr)),
GC_explicit_kind, IGNORE_OFF_PAGE, 0));
set_obj_descr(op, d);
return op;
}
/*
* Array descriptors. `GC_array_mark_proc` understands these.
* We may eventually need to add provisions for headers and trailers.
* Hence we provide for tree structured descriptors, though we do not
* really use them currently.
*/
/* This type describes simple array. */
struct LeafDescriptor {
word ld_tag;
#define LEAF_TAG 1
/* Bytes per element; nonzero, multiple of `ALIGNMENT`. */
size_t ld_size;
/* Number of elements. */
size_t ld_nelements;
/* A simple length, bitmap, or procedure descriptor. */
GC_descr ld_descriptor;
};
struct ComplexArrayDescriptor {
word ad_tag;
#define ARRAY_TAG 2
size_t ad_nelements;
union ComplexDescriptor *ad_element_descr;
};
struct SequenceDescriptor {
word sd_tag;
#define SEQUENCE_TAG 3
union ComplexDescriptor *sd_first;
union ComplexDescriptor *sd_second;
};
typedef union ComplexDescriptor {
struct LeafDescriptor ld;
struct ComplexArrayDescriptor ad;
struct SequenceDescriptor sd;
} complex_descriptor;
STATIC complex_descriptor *
GC_make_leaf_descriptor(size_t size, size_t nelements, GC_descr d)
{
complex_descriptor *result
= (complex_descriptor *)GC_malloc_atomic(sizeof(struct LeafDescriptor));
GC_ASSERT(size != 0);
if (UNLIKELY(NULL == result))
return NULL;
result->ld.ld_tag = LEAF_TAG;
result->ld.ld_size = size;
result->ld.ld_nelements = nelements;
result->ld.ld_descriptor = d;
return result;
}
STATIC complex_descriptor *
GC_make_sequence_descriptor(complex_descriptor *first,
complex_descriptor *second)
{
/*
* Note: for a reason, the sanitizer runtime complains of insufficient
* space for `complex_descriptor` if the pointer type of `result` variable
* is changed to.
*/
struct SequenceDescriptor *result = (struct SequenceDescriptor *)GC_malloc(
sizeof(struct SequenceDescriptor));
if (UNLIKELY(NULL == result))
return NULL;
/*
* Cannot result in overly conservative marking, since tags are very
* small integers. Probably faster than maintaining type information.
*/
result->sd_tag = SEQUENCE_TAG;
result->sd_first = first;
result->sd_second = second;
GC_dirty(result);
REACHABLE_AFTER_DIRTY(first);
REACHABLE_AFTER_DIRTY(second);
return (complex_descriptor *)result;
}
#define NO_MEM (-1)
#define SIMPLE 0
#define LEAF 1
#define COMPLEX 2
/*
* Build a descriptor for an array with `nelements` elements, each of
* which can be described by a simple descriptor `d`.
* We try to optimize some common cases. If the result is `COMPLEX`,
* a `complex_descriptor *` value is returned in `*pcomplex_d`.
* If the result is `LEAF`, then a `LeafDescriptor` value is built in the
* structure pointed to by `pleaf`. The tag in the `*pleaf` structure
* is not set. If the result is `SIMPLE`, then a `GC_descr` value is
* returned in `*psimple_d`. If the result is `NO_MEM`, then we failed
* to allocate the descriptor. The implementation assumes `GC_DS_LENGTH`
* is 0. `*pleaf`, `*pcomplex_d` and `*psimple_d` may be used as
* temporaries during the construction.
*/
STATIC int
GC_make_array_descriptor(size_t nelements, size_t size, GC_descr d,
GC_descr *psimple_d, complex_descriptor **pcomplex_d,
struct LeafDescriptor *pleaf)
{
/*
* For larger arrays, we try to combine descriptors of adjacent
* descriptors to speed up marking, and to reduce the amount of space
* needed on the mark stack.
*/
#define OPT_THRESHOLD 50
GC_ASSERT(size != 0);
if ((d & GC_DS_TAGS) == GC_DS_LENGTH) {
if (d == (GC_descr)size) {
/* Note: no overflow is guaranteed by caller. */
*psimple_d = nelements * d;
return SIMPLE;
} else if (0 == d) {
*psimple_d = 0;
return SIMPLE;
}
}
if (nelements <= OPT_THRESHOLD) {
if (nelements <= 1) {
*psimple_d = nelements == 1 ? d : 0;
return SIMPLE;
}
} else if (size <= BITMAP_BITS / 2 && (d & GC_DS_TAGS) != GC_DS_PROC
&& (size & (sizeof(ptr_t) - 1)) == 0) {
complex_descriptor *one_element, *beginning;
int result = GC_make_array_descriptor(
nelements / 2, 2 * size, GC_double_descr(d, BYTES_TO_PTRS(size)),
psimple_d, pcomplex_d, pleaf);
if ((nelements & 1) == 0 || UNLIKELY(NO_MEM == result))
return result;
one_element = GC_make_leaf_descriptor(size, 1, d);
if (UNLIKELY(NULL == one_element))
return NO_MEM;
if (COMPLEX == result) {
beginning = *pcomplex_d;
} else {
beginning
= SIMPLE == result
? GC_make_leaf_descriptor(size, 1, *psimple_d)
: GC_make_leaf_descriptor(pleaf->ld_size, pleaf->ld_nelements,
pleaf->ld_descriptor);
if (UNLIKELY(NULL == beginning))
return NO_MEM;
}
*pcomplex_d = GC_make_sequence_descriptor(beginning, one_element);
if (UNLIKELY(NULL == *pcomplex_d))
return NO_MEM;
return COMPLEX;
}
pleaf->ld_size = size;
pleaf->ld_nelements = nelements;
pleaf->ld_descriptor = d;
return LEAF;
}
struct GC_calloc_typed_descr_s {
complex_descriptor *complex_d; /*< the first field, the only pointer */
struct LeafDescriptor leaf;
GC_descr simple_d;
word alloc_lb; /*< of `size_t` type actually */
GC_signed_word descr_type; /*< of `int` type actually */
};
GC_API int GC_CALL
GC_calloc_prepare_explicitly_typed(struct GC_calloc_typed_descr_s *pctd,
size_t ctd_sz, size_t n, size_t lb,
GC_descr d)
{
GC_STATIC_ASSERT(sizeof(struct GC_calloc_typed_descr_opaque_s)
== sizeof(struct GC_calloc_typed_descr_s));
GC_ASSERT(GC_explicit_typing_initialized);
GC_ASSERT(sizeof(struct GC_calloc_typed_descr_s) == ctd_sz);
(void)ctd_sz; /*< unused currently */
if (UNLIKELY(0 == lb || 0 == n))
lb = n = 1;
if (UNLIKELY((lb | n) > GC_SQRT_SIZE_MAX) /*< fast initial check */
&& n > GC_SIZE_MAX / lb) {
/* `n * lb` overflows. */
pctd->alloc_lb = GC_SIZE_MAX;
pctd->descr_type = NO_MEM;
/* The rest of the fields are unset. */
return 0; /*< failure */
}
pctd->descr_type = GC_make_array_descriptor(n, lb, d, &pctd->simple_d,
&pctd->complex_d, &pctd->leaf);
switch (pctd->descr_type) {
case NO_MEM:
case SIMPLE:
pctd->alloc_lb = (word)lb * n;
break;
case LEAF:
pctd->alloc_lb = SIZET_SAT_ADD(
lb * n, (BYTES_TO_PTRS_ROUNDUP(sizeof(struct LeafDescriptor)) + 1)
* sizeof(ptr_t)
- EXTRA_BYTES);
break;
case COMPLEX:
pctd->alloc_lb = SIZET_SAT_ADD(lb * n, sizeof(ptr_t) - EXTRA_BYTES);
break;
}
return 1; /*< success */
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_calloc_do_explicitly_typed(const struct GC_calloc_typed_descr_s *pctd,
size_t ctd_sz)
{
void *op;
size_t lpw_m1;
GC_ASSERT(sizeof(struct GC_calloc_typed_descr_s) == ctd_sz);
(void)ctd_sz; /*< unused currently */
switch (pctd->descr_type) {
case NO_MEM:
return (*GC_get_oom_fn())((size_t)pctd->alloc_lb);
case SIMPLE:
return GC_malloc_explicitly_typed((size_t)pctd->alloc_lb, pctd->simple_d);
case LEAF:
case COMPLEX:
break;
default:
ABORT_RET("Bad descriptor type");
return NULL;
}
op = GC_malloc_kind((size_t)pctd->alloc_lb, GC_array_kind);
if (UNLIKELY(NULL == op))
return NULL;
lpw_m1 = BYTES_TO_PTRS(GC_size(op)) - 1;
if (pctd->descr_type == LEAF) {
/* Set up the descriptor inside the object itself. */
struct LeafDescriptor *lp
= (struct LeafDescriptor *)((ptr_t *)op + lpw_m1
- BYTES_TO_PTRS_ROUNDUP(
sizeof(struct LeafDescriptor)));
lp->ld_tag = LEAF_TAG;
lp->ld_size = pctd->leaf.ld_size;
lp->ld_nelements = pctd->leaf.ld_nelements;
lp->ld_descriptor = pctd->leaf.ld_descriptor;
/*
* Hold the allocator lock (in the reader mode which should be enough)
* while writing the descriptor `word` to the object to ensure that
* the descriptor contents are seen by `GC_array_mark_proc` as expected.
*/
/*
* TODO: It should be possible to replace locking with the atomic
* operations (with the release barrier here) but, in this case,
* avoiding the acquire barrier in `GC_array_mark_proc` seems to
* be tricky as `GC_mark_some` might be invoked with the world running.
*/
READER_LOCK();
((struct LeafDescriptor **)op)[lpw_m1] = lp;
READER_UNLOCK_RELEASE();
} else {
#ifndef GC_NO_FINALIZATION
READER_LOCK();
((complex_descriptor **)op)[lpw_m1] = pctd->complex_d;
READER_UNLOCK_RELEASE();
GC_dirty((ptr_t *)op + lpw_m1);
REACHABLE_AFTER_DIRTY(pctd->complex_d);
/*
* Make sure the descriptor is cleared once there is any danger
* it may have been collected.
*/
if (UNLIKELY(
GC_general_register_disappearing_link((void **)op + lpw_m1, op)
== GC_NO_MEMORY))
#endif
{
/* Could not register it due to lack of memory. Punt. */
return (*GC_get_oom_fn())((size_t)pctd->alloc_lb);
}
}
return op;
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_calloc_explicitly_typed(size_t n, size_t lb, GC_descr d)
{
struct GC_calloc_typed_descr_s ctd;
(void)GC_calloc_prepare_explicitly_typed(&ctd, sizeof(ctd), n, lb, d);
return GC_calloc_do_explicitly_typed(&ctd, sizeof(ctd));
}
/*
* Return the size of the object described by `complex_d`.
* It would be faster to store this directly, or to compute it as part
* of `GC_push_complex_descriptor`, but hopefully it does not matter.
*/
STATIC size_t
GC_descr_obj_size(complex_descriptor *complex_d)
{
switch (complex_d->ad.ad_tag) {
case LEAF_TAG:
return complex_d->ld.ld_nelements * complex_d->ld.ld_size;
case ARRAY_TAG:
return complex_d->ad.ad_nelements
* GC_descr_obj_size(complex_d->ad.ad_element_descr);
case SEQUENCE_TAG:
return GC_descr_obj_size(complex_d->sd.sd_first)
+ GC_descr_obj_size(complex_d->sd.sd_second);
default:
ABORT_RET("Bad complex descriptor");
return 0;
}
}
/*
* Push descriptors for the object with the complex descriptor onto
* the mark stack. Return `NULL` if the mark stack overflowed.
*/
STATIC mse *
GC_push_complex_descriptor(ptr_t current, complex_descriptor *complex_d,
mse *msp, mse *msl)
{
size_t i, nelements;
size_t sz;
GC_descr d;
complex_descriptor *element_descr;
switch (complex_d->ad.ad_tag) {
case LEAF_TAG:
d = complex_d->ld.ld_descriptor;
nelements = complex_d->ld.ld_nelements;
sz = complex_d->ld.ld_size;
if (UNLIKELY(msl - msp <= (GC_signed_word)nelements))
return NULL;
GC_ASSERT(sz != 0);
for (i = 0; i < nelements; i++) {
msp++;
msp->mse_start = current;
msp->mse_descr = d;
current += sz;
}
break;
case ARRAY_TAG:
element_descr = complex_d->ad.ad_element_descr;
nelements = complex_d->ad.ad_nelements;
sz = GC_descr_obj_size(element_descr);
GC_ASSERT(sz != 0 || 0 == nelements);
for (i = 0; i < nelements; i++) {
msp = GC_push_complex_descriptor(current, element_descr, msp, msl);
if (UNLIKELY(NULL == msp))
return NULL;
current += sz;
}
break;
case SEQUENCE_TAG:
sz = GC_descr_obj_size(complex_d->sd.sd_first);
msp = GC_push_complex_descriptor(current, complex_d->sd.sd_first, msp,
msl);
if (UNLIKELY(NULL == msp))
return NULL;
GC_ASSERT(sz != 0);
current += sz;
msp = GC_push_complex_descriptor(current, complex_d->sd.sd_second, msp,
msl);
break;
default:
ABORT("Bad complex descriptor");
}
return msp;
}
GC_ATTR_NO_SANITIZE_THREAD
static complex_descriptor *
get_complex_descr(ptr_t *p, size_t lpw)
{
return (complex_descriptor *)p[lpw - 1];
}
/* Used by `GC_calloc_do_explicitly_typed()` via `GC_array_kind`. */
STATIC mse *GC_CALLBACK
GC_array_mark_proc(word *addr, mse *mark_stack_top, mse *mark_stack_limit,
word env)
{
size_t sz = HDR(addr)->hb_sz;
size_t lpw = BYTES_TO_PTRS(sz);
complex_descriptor *complex_d = get_complex_descr((ptr_t *)addr, lpw);
mse *orig_mark_stack_top = mark_stack_top;
mse *new_mark_stack_top;
UNUSED_ARG(env);
if (NULL == complex_d) {
/* Found a reference to a free-list entry. Ignore it. */
return orig_mark_stack_top;
}
/*
* In-use counts were already updated when array descriptor was pushed.
* Here we only replace it by subobject descriptors, so no update is
* necessary.
*/
new_mark_stack_top = GC_push_complex_descriptor(
(ptr_t)addr, complex_d, mark_stack_top, mark_stack_limit - 1);
if (NULL == new_mark_stack_top) {
/* Explicitly instruct Clang Static Analyzer that pointer is non-`NULL`. */
if (NULL == mark_stack_top) {
ABORT("Bad mark_stack_top");
}
/*
* Does not fit. Conservatively push the whole array as a unit and
* request a mark stack expansion. This cannot cause a mark stack
* overflow, since it replaces the original array entry.
*/
#ifdef PARALLEL_MARK
/* We might be using a `local_mark_stack` in the parallel collection. */
if (GC_mark_stack + GC_mark_stack_size == mark_stack_limit)
#endif
{
GC_mark_stack_too_small = TRUE;
}
new_mark_stack_top = orig_mark_stack_top + 1;
new_mark_stack_top->mse_start = (ptr_t)addr;
new_mark_stack_top->mse_descr = sz | GC_DS_LENGTH;
} else {
/* Push descriptor itself. */
new_mark_stack_top++;
new_mark_stack_top->mse_start = (ptr_t)((ptr_t *)addr + lpw - 1);
new_mark_stack_top->mse_descr = sizeof(ptr_t) | GC_DS_LENGTH;
}
return new_mark_stack_top;
}
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1999-2001 by Hewlett-Packard Company. All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#include <limits.h>
#include <stdarg.h>
#if defined(SOLARIS) && defined(THREADS)
# include <sys/syscall.h>
#endif
#if defined(UNIX_LIKE) || defined(CYGWIN32) || defined(SYMBIAN) \
|| (defined(CONSOLE_LOG) && defined(MSWIN32))
# include <fcntl.h>
# include <sys/stat.h>
#endif
#if defined(CONSOLE_LOG) && defined(MSWIN32) && !defined(__GNUC__)
# include <io.h>
#endif
#ifdef NONSTOP
# include <floss.h>
#endif
#ifdef THREADS
# if defined(SN_TARGET_PSP2)
GC_INNER WapiMutex GC_allocate_ml_PSP2 = { 0, NULL };
# elif defined(GC_DEFN_ALLOCATE_ML) && !defined(USE_RWLOCK) \
|| defined(SN_TARGET_PS3)
# include <pthread.h>
GC_INNER pthread_mutex_t GC_allocate_ml;
# else
/*
* For other platforms with threads, the allocator lock and, possibly,
* `GC_lock_holder` are defined in the thread support code.
*/
# endif
#endif /* THREADS */
#ifdef DYNAMIC_LOADING
/*
* We need to register the main data segment. Returns `TRUE` unless
* this is done implicitly as part of dynamic library registration.
*/
# define GC_REGISTER_MAIN_STATIC_DATA() GC_register_main_static_data()
#elif defined(GC_DONT_REGISTER_MAIN_STATIC_DATA)
# define GC_REGISTER_MAIN_STATIC_DATA() FALSE
#else
/*
* Do not unnecessarily call `GC_register_main_static_data()` in case
* `dyn_load.c` file is not linked in.
*/
# define GC_REGISTER_MAIN_STATIC_DATA() TRUE
#endif
#ifdef NEED_CANCEL_DISABLE_COUNT
__thread unsigned char GC_cancel_disable_count = 0;
#endif
struct _GC_arrays GC_arrays /* `= { 0 }` */;
GC_INNER unsigned GC_n_mark_procs = GC_RESERVED_MARK_PROCS;
GC_INNER unsigned GC_n_kinds = GC_N_KINDS_INITIAL_VALUE;
ptr_t GC_stackbottom = 0;
#if defined(E2K) && defined(THREADS) || defined(IA64)
GC_INNER ptr_t GC_register_stackbottom = NULL;
#endif
int GC_dont_gc = FALSE;
int GC_dont_precollect = FALSE;
GC_bool GC_quiet = 0; /*< used also in `msvc_dbg.c` file */
#if !defined(NO_CLOCK) || !defined(SMALL_CONFIG)
GC_INNER int GC_print_stats = 0;
#endif
#ifdef MAKE_BACK_GRAPH
# ifdef GC_PRINT_BACK_HEIGHT
GC_INNER GC_bool GC_print_back_height = TRUE;
# else
GC_INNER GC_bool GC_print_back_height = FALSE;
# endif
#endif
#ifndef NO_DEBUGGING
# ifdef GC_DUMP_REGULARLY
GC_INNER GC_bool GC_dump_regularly = TRUE;
# else
GC_INNER GC_bool GC_dump_regularly = FALSE;
# endif
# ifndef NO_CLOCK
/* The time that the collector was initialized at. */
STATIC CLOCK_TYPE GC_init_time = CLOCK_TYPE_INITIALIZER;
# endif
#endif /* !NO_DEBUGGING */
#ifdef KEEP_BACK_PTRS
GC_INNER long GC_backtraces = 0;
#endif
#ifdef FIND_LEAK
int GC_find_leak = 1;
#else
int GC_find_leak = 0;
#endif
#if !defined(NO_FIND_LEAK) && !defined(SHORT_DBG_HDRS)
# ifdef GC_FINDLEAK_DELAY_FREE
GC_INNER GC_bool GC_findleak_delay_free = TRUE;
# else
GC_INNER GC_bool GC_findleak_delay_free = FALSE;
# endif
#endif /* !NO_FIND_LEAK && !SHORT_DBG_HDRS */
#ifdef ALL_INTERIOR_POINTERS
int GC_all_interior_pointers = 1;
#else
int GC_all_interior_pointers = 0;
#endif
#ifdef FINALIZE_ON_DEMAND
int GC_finalize_on_demand = 1;
#else
int GC_finalize_on_demand = 0;
#endif
#ifdef JAVA_FINALIZATION
int GC_java_finalization = 1;
#else
int GC_java_finalization = 0;
#endif
/* All accesses to it should be synchronized to avoid data race. */
GC_finalizer_notifier_proc GC_finalizer_notifier
= (GC_finalizer_notifier_proc)0;
#ifdef GC_FORCE_UNMAP_ON_GCOLLECT
GC_INNER GC_bool GC_force_unmap_on_gcollect = TRUE;
#else
GC_INNER GC_bool GC_force_unmap_on_gcollect = FALSE;
#endif
#ifndef GC_LARGE_ALLOC_WARN_INTERVAL
# define GC_LARGE_ALLOC_WARN_INTERVAL 5
#endif
#ifndef NO_BLACK_LISTING
GC_INNER long GC_large_alloc_warn_interval = GC_LARGE_ALLOC_WARN_INTERVAL;
#endif
STATIC void *GC_CALLBACK
GC_default_oom_fn(size_t bytes_requested)
{
UNUSED_ARG(bytes_requested);
return NULL;
}
/* All accesses to it should be synchronized to avoid data race. */
GC_oom_func GC_oom_fn = GC_default_oom_fn;
#ifdef CAN_HANDLE_FORK
# ifdef HANDLE_FORK
GC_INNER int GC_handle_fork = 1;
# else
GC_INNER int GC_handle_fork = FALSE;
# endif
#elif !defined(HAVE_NO_FORK)
GC_API void GC_CALL
GC_atfork_prepare(void)
{
# ifdef THREADS
ABORT("fork() handling unsupported");
# endif
}
GC_API void GC_CALL
GC_atfork_parent(void)
{
/* Empty. */
}
GC_API void GC_CALL
GC_atfork_child(void)
{
/* Empty. */
}
#endif /* !CAN_HANDLE_FORK && !HAVE_NO_FORK */
GC_API void GC_CALL
GC_set_handle_fork(int value)
{
#ifdef CAN_HANDLE_FORK
if (!GC_is_initialized) {
/* Map all negative values except for -1 to a positive one. */
GC_handle_fork = value >= -1 ? value : 1;
}
#elif defined(THREADS) || (defined(DARWIN) && defined(MPROTECT_VDB))
if (!GC_is_initialized && value) {
# ifndef SMALL_CONFIG
/* Initialize `GC_manual_vdb` and `GC_stderr`. */
GC_init();
# ifndef THREADS
if (GC_manual_vdb)
return;
# endif
# endif
ABORT("fork() handling unsupported");
}
#else
/* No at-fork handler is needed in the single-threaded mode. */
UNUSED_ARG(value);
#endif
}
/*
* Set things up so that `GC_size_map[i] >= granules(i)`, but not too
* much bigger and so that `GC_size_map` contains relatively few
* distinct entries. This was originally stolen from Russ Atkinson's
* Cedar quantization algorithm (but we precompute it).
*/
STATIC void
GC_init_size_map(void)
{
size_t i = 1;
/* Map size 0 to something bigger; this avoids problems at lower levels. */
GC_size_map[0] = 1;
for (; i <= GRANULES_TO_BYTES(GC_TINY_FREELISTS - 1) - EXTRA_BYTES; i++) {
GC_size_map[i] = ALLOC_REQUEST_GRANS(i);
#ifndef _MSC_VER
/* Seems to tickle bug in VC++ 2008 for x86_64. */
GC_ASSERT(GC_size_map[i] < GC_TINY_FREELISTS);
#endif
}
/* We leave the rest of the array to be filled in on demand. */
}
/*
* The following is a gross hack to deal with a problem that can occur
* on machines that are sloppy about stack frame sizes, notably SPARC.
* Bogus pointers may be written to the stack and not cleared for
* a LONG time, because they always fall into holes in stack frames
* that are not written. We partially address this by clearing
* sections of the stack whenever we get control.
*/
#ifndef SMALL_CLEAR_SIZE
/* Clear this many words of the stack every time. */
# define SMALL_CLEAR_SIZE 256
#endif
#if defined(ALWAYS_SMALL_CLEAR_STACK) || defined(STACK_NOT_SCANNED)
GC_API void *GC_CALL
GC_clear_stack(void *arg)
{
# ifndef STACK_NOT_SCANNED
volatile ptr_t dummy[SMALL_CLEAR_SIZE];
BZERO(CAST_AWAY_VOLATILE_PVOID(dummy), sizeof(dummy));
# endif
return arg;
}
#else
# ifdef THREADS
/* Clear this much sometimes. */
# define BIG_CLEAR_SIZE 2048
# else
# define DEGRADE_RATE 50
STATIC word GC_bytes_allocd_at_reset = 0;
/* `GC_gc_no` value when we last did this. */
STATIC word GC_stack_last_cleared = 0;
# endif
# if defined(__APPLE_CC__) && !GC_CLANG_PREREQ(6, 0)
# define CLEARSTACK_LIMIT_MODIFIER volatile /*< to workaround some bug */
# else
# define CLEARSTACK_LIMIT_MODIFIER /*< empty */
# endif
EXTERN_C_BEGIN
void *GC_clear_stack_inner(void *, CLEARSTACK_LIMIT_MODIFIER ptr_t);
EXTERN_C_END
# ifndef ASM_CLEAR_CODE
/*
* Clear the stack up to about `limit`. Return `arg`. This function is
* not `static` because it could also be erroneously defined in `.S` file,
* so this error would be caught by the linker.
*/
void *
GC_clear_stack_inner(void *arg, CLEARSTACK_LIMIT_MODIFIER ptr_t limit)
{
# define CLEAR_SIZE 213 /*< granularity */
volatile ptr_t dummy[CLEAR_SIZE];
BZERO(CAST_AWAY_VOLATILE_PVOID(dummy), sizeof(dummy));
if (HOTTER_THAN((/* no volatile */ ptr_t)limit, GC_approx_sp())) {
(void)GC_clear_stack_inner(arg, limit);
}
/*
* Make sure the recursive call is not a tail call, and the `bzero` call
* is not recognized as dead code.
*/
# if defined(CPPCHECK)
GC_noop1(ADDR(dummy[0]));
# else
GC_noop1(COVERT_DATAFLOW(ADDR(dummy)));
# endif
return arg;
}
# endif /* !ASM_CLEAR_CODE */
# ifdef THREADS
/* Used to occasionally clear a bigger chunk. */
/* TODO: Should be more random than it is... */
static unsigned
next_random_no(void)
{
# ifdef AO_HAVE_fetch_and_add1
static volatile AO_t random_no;
return (unsigned)AO_fetch_and_add1(&random_no) % 13;
# else
static unsigned random_no = 0;
return (random_no++) % 13;
# endif
}
# endif /* THREADS */
GC_API void *GC_CALL
GC_clear_stack(void *arg)
{
/* Note: this is hotter than the actual stack pointer. */
ptr_t sp = GC_approx_sp();
# ifdef THREADS
volatile ptr_t dummy[SMALL_CLEAR_SIZE];
# endif
/*
* Extra bytes we clear every time. This clears our own activation
* record, and should cause more frequent clearing near the cold end
* of the stack, a good thing.
*/
# define SLOP 400
/*
* We make `GC_high_water` this much hotter than we really saw it,
* to cover for the GC noise above our current frame.
*/
# define GC_SLOP 4000
/*
* We restart the clearing process after this many bytes of allocation.
* Otherwise very heavily recursive programs with sparse stacks may
* result in heaps that grow almost without bounds. As the heap gets
* larger, collection frequency decreases, thus clearing frequency
* would decrease, thus more junk remains accessible, thus the heap
* gets larger...
*/
# define CLEAR_THRESHOLD 100000
# ifdef THREADS
if (next_random_no() == 0) {
ptr_t limit = sp;
MAKE_HOTTER(limit, BIG_CLEAR_SIZE * sizeof(ptr_t));
/*
* Make it sufficiently aligned for assembly implementations
* of `GC_clear_stack_inner`.
*/
limit = PTR_ALIGN_DOWN(limit, 0x10);
return GC_clear_stack_inner(arg, limit);
}
BZERO(CAST_AWAY_VOLATILE_PVOID(dummy), sizeof(dummy));
# else
if (GC_gc_no != GC_stack_last_cleared) {
/* Start things over, so we clear the entire stack again. */
if (UNLIKELY(NULL == GC_high_water))
GC_high_water = (ptr_t)GC_stackbottom;
GC_min_sp = GC_high_water;
GC_stack_last_cleared = GC_gc_no;
GC_bytes_allocd_at_reset = GC_bytes_allocd;
}
/* Adjust `GC_high_water`. */
GC_ASSERT(GC_high_water != NULL);
MAKE_COOLER(GC_high_water, PTRS_TO_BYTES(DEGRADE_RATE) + GC_SLOP);
if (HOTTER_THAN(sp, GC_high_water))
GC_high_water = sp;
MAKE_HOTTER(GC_high_water, GC_SLOP);
{
ptr_t limit = GC_min_sp;
MAKE_HOTTER(limit, SLOP);
if (HOTTER_THAN(limit, sp)) {
limit = PTR_ALIGN_DOWN(limit, 0x10);
GC_min_sp = sp;
return GC_clear_stack_inner(arg, limit);
}
}
if (GC_bytes_allocd - GC_bytes_allocd_at_reset > CLEAR_THRESHOLD) {
/* Restart clearing process, but limit how much clearing we do. */
GC_min_sp = sp;
MAKE_HOTTER(GC_min_sp, CLEAR_THRESHOLD / 4);
if (HOTTER_THAN(GC_min_sp, GC_high_water))
GC_min_sp = GC_high_water;
GC_bytes_allocd_at_reset = GC_bytes_allocd;
}
# endif
return arg;
}
#endif /* !ALWAYS_SMALL_CLEAR_STACK && !STACK_NOT_SCANNED */
GC_API void *GC_CALL
GC_base(void *p)
{
ptr_t r = (ptr_t)p;
struct hblk *h;
bottom_index *bi;
hdr *hhdr;
ptr_t limit;
size_t sz;
if (UNLIKELY(!GC_is_initialized))
return NULL;
h = HBLKPTR(r);
GET_BI(r, bi);
hhdr = HDR_FROM_BI(bi, r);
if (NULL == hhdr)
return NULL;
/*
* If it is a pointer to the middle of a large object, then move it
* to the beginning.
*/
if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
h = GC_find_starting_hblk(h, &hhdr);
r = (ptr_t)h;
}
if (HBLK_IS_FREE(hhdr))
return NULL;
/* Make sure `r` points to the beginning of the object. */
r = PTR_ALIGN_DOWN(r, sizeof(ptr_t));
sz = hhdr->hb_sz;
r -= HBLKDISPL(r) % sz;
limit = r + sz;
if ((ADDR_LT((ptr_t)(h + 1), limit) && sz <= HBLKSIZE)
|| ADDR_GE((ptr_t)p, limit))
return NULL;
return r;
}
GC_API int GC_CALL
GC_is_heap_ptr(const void *p)
{
bottom_index *bi;
GC_ASSERT(GC_is_initialized);
GET_BI(p, bi);
return HDR_FROM_BI(bi, p) != 0;
}
GC_API size_t GC_CALL
GC_size(const void *p)
{
const hdr *hhdr;
/* Accept `NULL` for compatibility with `malloc_usable_size()`. */
if (UNLIKELY(NULL == p))
return 0;
hhdr = HDR(p);
return hhdr->hb_sz;
}
/*
* These getters remain unsynchronized for compatibility (since some clients
* could call some of them from a GC callback holding the allocator lock).
*/
GC_API size_t GC_CALL
GC_get_heap_size(void)
{
/*
* Ignore the memory space returned to OS (i.e. count only the space
* owned by the garbage collector).
*/
return (size_t)(GC_heapsize - GC_unmapped_bytes);
}
GC_API size_t GC_CALL
GC_get_obtained_from_os_bytes(void)
{
return (size_t)GC_our_mem_bytes;
}
GC_API size_t GC_CALL
GC_get_free_bytes(void)
{
/* Ignore the memory space returned to OS. */
return (size_t)(GC_large_free_bytes - GC_unmapped_bytes);
}
GC_API size_t GC_CALL
GC_get_unmapped_bytes(void)
{
return (size_t)GC_unmapped_bytes;
}
GC_API size_t GC_CALL
GC_get_bytes_since_gc(void)
{
return (size_t)GC_bytes_allocd;
}
GC_API size_t GC_CALL
GC_get_total_bytes(void)
{
return (size_t)(GC_bytes_allocd + GC_bytes_allocd_before_gc);
}
#ifndef GC_GET_HEAP_USAGE_NOT_NEEDED
GC_API size_t GC_CALL
GC_get_size_map_at(int i)
{
if ((unsigned)i > MAXOBJBYTES)
return GC_SIZE_MAX;
return GRANULES_TO_BYTES(GC_size_map[i]);
}
GC_API void GC_CALL
GC_get_heap_usage_safe(GC_word *pheap_size, GC_word *pfree_bytes,
GC_word *punmapped_bytes, GC_word *pbytes_since_gc,
GC_word *ptotal_bytes)
{
READER_LOCK();
if (pheap_size != NULL)
*pheap_size = GC_heapsize - GC_unmapped_bytes;
if (pfree_bytes != NULL)
*pfree_bytes = GC_large_free_bytes - GC_unmapped_bytes;
if (punmapped_bytes != NULL)
*punmapped_bytes = GC_unmapped_bytes;
if (pbytes_since_gc != NULL)
*pbytes_since_gc = GC_bytes_allocd;
if (ptotal_bytes != NULL)
*ptotal_bytes = GC_bytes_allocd + GC_bytes_allocd_before_gc;
READER_UNLOCK();
}
/* Fill in GC statistics provided the destination is of enough size. */
static void
fill_prof_stats(struct GC_prof_stats_s *pstats)
{
pstats->heapsize_full = GC_heapsize;
pstats->free_bytes_full = GC_large_free_bytes;
pstats->unmapped_bytes = GC_unmapped_bytes;
pstats->bytes_allocd_since_gc = GC_bytes_allocd;
pstats->allocd_bytes_before_gc = GC_bytes_allocd_before_gc;
pstats->non_gc_bytes = GC_non_gc_bytes;
pstats->gc_no = GC_gc_no; /*< could be -1 */
# ifdef PARALLEL_MARK
pstats->markers_m1 = (word)((GC_signed_word)GC_markers_m1);
# else
/* A single marker. */
pstats->markers_m1 = 0;
# endif
pstats->bytes_reclaimed_since_gc
= GC_bytes_found > 0 ? (word)GC_bytes_found : 0;
pstats->reclaimed_bytes_before_gc = GC_reclaimed_bytes_before_gc;
pstats->expl_freed_bytes_since_gc = GC_bytes_freed; /*< since gc-7.7 */
pstats->obtained_from_os_bytes = GC_our_mem_bytes; /*< since gc-8.2 */
}
# include <string.h> /*< for `memset()` */
GC_API size_t GC_CALL
GC_get_prof_stats(struct GC_prof_stats_s *pstats, size_t stats_sz)
{
struct GC_prof_stats_s stats;
READER_LOCK();
fill_prof_stats(stats_sz >= sizeof(stats) ? pstats : &stats);
READER_UNLOCK();
if (stats_sz == sizeof(stats)) {
return sizeof(stats);
} else if (stats_sz > sizeof(stats)) {
/* Fill in the remaining part with -1. */
memset((char *)pstats + sizeof(stats), 0xff, stats_sz - sizeof(stats));
return sizeof(stats);
} else {
if (LIKELY(stats_sz > 0))
BCOPY(&stats, pstats, stats_sz);
return stats_sz;
}
}
# ifdef THREADS
GC_API size_t GC_CALL
GC_get_prof_stats_unsafe(struct GC_prof_stats_s *pstats, size_t stats_sz)
{
struct GC_prof_stats_s stats;
if (stats_sz >= sizeof(stats)) {
fill_prof_stats(pstats);
if (stats_sz > sizeof(stats))
memset((char *)pstats + sizeof(stats), 0xff, stats_sz - sizeof(stats));
return sizeof(stats);
} else {
if (LIKELY(stats_sz > 0)) {
fill_prof_stats(&stats);
BCOPY(&stats, pstats, stats_sz);
}
return stats_sz;
}
}
# endif /* THREADS */
#endif /* !GC_GET_HEAP_USAGE_NOT_NEEDED */
#if defined(THREADS) && !defined(SIGNAL_BASED_STOP_WORLD)
/* The collector does not use signals to suspend and restart threads. */
GC_API void GC_CALL
GC_set_suspend_signal(int sig)
{
UNUSED_ARG(sig);
}
GC_API void GC_CALL
GC_set_thr_restart_signal(int sig)
{
UNUSED_ARG(sig);
}
GC_API int GC_CALL
GC_get_suspend_signal(void)
{
return -1;
}
GC_API int GC_CALL
GC_get_thr_restart_signal(void)
{
return -1;
}
#endif /* THREADS && !SIGNAL_BASED_STOP_WORLD */
#if !defined(_MAX_PATH) && defined(ANY_MSWIN)
# define _MAX_PATH MAX_PATH
#endif
#ifdef GC_READ_ENV_FILE
/* This works for Win32/WinCE for now. Really useful only for WinCE. */
# ifndef GC_ENVFILE_MAXLEN
# define GC_ENVFILE_MAXLEN 0x4000
# endif
# define GC_ENV_FILE_EXT ".gc.env"
/* The routine initializes `GC_envfile_content` from the `.gc.env` file. */
STATIC void
GC_envfile_init(void)
{
# ifdef ANY_MSWIN
HANDLE hFile;
char *content;
size_t ofs, len;
DWORD nBytesRead;
TCHAR path[_MAX_PATH + 0x10]; /*< buffer for file path with extension */
size_t bytes_to_get;
GC_ASSERT(I_HOLD_LOCK());
len = (size_t)GetModuleFileName(NULL /* `hModule` */, path, _MAX_PATH + 1);
/* If `GetModuleFileName()` failed, then len is 0. */
if (len > 4 && path[len - 4] == (TCHAR)'.') {
/* Strip the executable file extension. */
len -= 4;
}
BCOPY(TEXT(GC_ENV_FILE_EXT), &path[len], sizeof(TEXT(GC_ENV_FILE_EXT)));
hFile = CreateFile(path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE,
NULL /* `lpSecurityAttributes` */, OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL, NULL /* `hTemplateFile` */);
if (hFile == INVALID_HANDLE_VALUE) {
/* The file is absent or the operation failed. */
return;
}
len = (size_t)GetFileSize(hFile, NULL);
if (len <= 1 || len >= GC_ENVFILE_MAXLEN) {
CloseHandle(hFile);
/* Invalid file length - ignoring the file content. */
return;
}
/*
* At this execution point, `GC_setpagesize()` and `GC_init_win32()`
* must already be called (for `GET_MEM()` to work correctly).
*/
GC_ASSERT(GC_page_size != 0);
bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(len + 1);
content = GC_os_get_mem(bytes_to_get);
if (content == NULL) {
CloseHandle(hFile);
/* An allocation failure. */
return;
}
ofs = 0;
nBytesRead = (DWORD)-1L;
/* Last `ReadFile()` call should clear `nBytesRead` on success. */
while (ReadFile(hFile, content + ofs, (DWORD)(len - ofs + 1), &nBytesRead,
NULL /* `lpOverlapped` */)
&& nBytesRead != 0) {
if ((ofs += (size_t)nBytesRead) > len)
break;
}
CloseHandle(hFile);
if (ofs != len || nBytesRead != 0) {
/* TODO: Recycle content. */
/* Read operation has failed - ignoring the file content. */
return;
}
content[ofs] = '\0';
while (ofs-- > 0) {
if (content[ofs] == '\r' || content[ofs] == '\n')
content[ofs] = '\0';
}
GC_ASSERT(NULL == GC_envfile_content);
GC_envfile_length = len + 1;
GC_envfile_content = content;
# endif
}
GC_INNER char *
GC_envfile_getenv(const char *name)
{
char *p;
const char *end_of_content;
size_t namelen;
# ifndef NO_GETENV
/* Try the standard `getenv()` first. */
p = getenv(name);
if (p != NULL)
return *p != '\0' ? p : NULL;
# endif
p = GC_envfile_content;
if (NULL == p) {
/* The `.gc.env` file is absent (or empty). */
return NULL;
}
namelen = strlen(name);
if (0 == namelen) {
/* A sanity check. */
return NULL;
}
for (end_of_content = p + GC_envfile_length;
ADDR_LT((ptr_t)p, (ptr_t)end_of_content); p += strlen(p) + 1) {
if (strncmp(p, name, namelen) == 0 && *(p += namelen) == '=') {
/* The match is found; skip "=". */
p++;
return *p != '\0' ? p : NULL;
}
/* If not matching then skip to the next line. */
}
GC_ASSERT(p == end_of_content);
/* No match is found. */
return NULL;
}
#endif /* GC_READ_ENV_FILE */
GC_API int GC_CALL
GC_is_init_called(void)
{
return (int)GC_is_initialized;
}
#if defined(GC_WIN32_THREADS) \
&& ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE))
GC_INNER CRITICAL_SECTION GC_write_cs;
#endif
#ifndef DONT_USE_ATEXIT
STATIC void
GC_exit_check(void)
{
if (GC_find_leak && !GC_skip_collect_atexit) {
# ifdef THREADS
/*
* Check that the thread executing at-exit functions is the same as
* the one performed the GC initialization, otherwise the latter
* thread might already be dead but still registered and this, as
* a consequence, might cause a signal delivery fail when suspending
* the threads on platforms that do not guarantee `ESRCH` returned
* if the signal is not delivered. It should also prevent
* "Collecting from unknown thread" abort in `GC_push_all_stacks()`.
*/
if (!GC_is_main_thread() || !GC_thread_is_registered())
return;
# endif
GC_gcollect();
}
}
#endif /* !DONT_USE_ATEXIT */
#if defined(UNIX_LIKE) && !defined(NO_DEBUGGING)
static void
looping_handler(int sig)
{
GC_err_printf("Caught signal %d: looping in handler\n", sig);
for (;;) {
/* Empty. */
}
}
static GC_bool installed_looping_handler = FALSE;
static void
maybe_install_looping_handler(void)
{
/*
* Install looping handler before the write fault handler,
* so we handle write faults correctly.
*/
if (!installed_looping_handler && GETENV("GC_LOOP_ON_ABORT") != NULL) {
GC_set_and_save_fault_handler(looping_handler);
installed_looping_handler = TRUE;
}
}
#else /* !UNIX_LIKE */
# define maybe_install_looping_handler()
#endif
#define GC_DEFAULT_STDERR_FD 2
#ifdef KOS
# define GC_DEFAULT_STDOUT_FD GC_DEFAULT_STDERR_FD
#else
# define GC_DEFAULT_STDOUT_FD 1
#endif
#if !defined(OS2) && !defined(GC_ANDROID_LOG) && !defined(NN_PLATFORM_CTR) \
&& !defined(NINTENDO_SWITCH) \
&& (!defined(MSWIN32) || defined(CONSOLE_LOG)) && !defined(MSWINCE)
STATIC int GC_stdout = GC_DEFAULT_STDOUT_FD;
STATIC int GC_stderr = GC_DEFAULT_STDERR_FD;
STATIC int GC_log = GC_DEFAULT_STDERR_FD;
# ifndef MSWIN32
GC_API void GC_CALL
GC_set_log_fd(int fd)
{
GC_log = fd;
}
# endif
#endif
#ifdef MSGBOX_ON_ERROR
STATIC void
GC_win32_MessageBoxA(const char *msg, const char *caption, unsigned flags)
{
# ifndef DONT_USE_USER32_DLL
/* Use static binding to `user32.dll` file. */
(void)MessageBoxA(NULL, msg, caption, flags);
# else
/* This simplifies linking - resolve `MessageBoxA()` at run-time. */
HINSTANCE hU32 = LoadLibrary(TEXT("user32.dll"));
if (hU32) {
FARPROC pfn = GetProcAddress(hU32, "MessageBoxA");
if (pfn)
(void)(*(int(WINAPI *)(HWND, LPCSTR, LPCSTR, UINT))(GC_funcptr_uint)pfn)(
NULL /* `hWnd` */, msg, caption, flags);
(void)FreeLibrary(hU32);
}
# endif
}
#endif /* MSGBOX_ON_ERROR */
#if defined(THREADS) && defined(UNIX_LIKE) && !defined(NO_GETCONTEXT)
static void
callee_saves_pushed_dummy_fn(ptr_t data, void *context)
{
UNUSED_ARG(data);
UNUSED_ARG(context);
}
#endif
#ifdef MANUAL_VDB
static GC_bool manual_vdb_allowed = TRUE;
#else
static GC_bool manual_vdb_allowed = FALSE;
#endif
GC_API void GC_CALL
GC_set_manual_vdb_allowed(int value)
{
manual_vdb_allowed = (GC_bool)value;
}
GC_API int GC_CALL
GC_get_manual_vdb_allowed(void)
{
return (int)manual_vdb_allowed;
}
GC_API unsigned GC_CALL
GC_get_supported_vdbs(void)
{
#ifdef GC_DISABLE_INCREMENTAL
return GC_VDB_NONE;
#else
# if defined(CPPCHECK)
/* Workaround a warning about redundant `| 0`. */
volatile unsigned zero = 0;
# endif
return
# if defined(CPPCHECK)
zero
# else
0
# endif
# ifndef NO_MANUAL_VDB
| GC_VDB_MANUAL
# endif
# ifdef DEFAULT_VDB
| GC_VDB_DEFAULT
# endif
# ifdef MPROTECT_VDB
| GC_VDB_MPROTECT
# endif
# ifdef GWW_VDB
| GC_VDB_GWW
# endif
# ifdef PROC_VDB
| GC_VDB_PROC
# endif
# ifdef SOFT_VDB
| GC_VDB_SOFT
# endif
;
#endif
}
#ifndef GC_DISABLE_INCREMENTAL
static void
set_incremental_mode_on(void)
{
GC_ASSERT(I_HOLD_LOCK());
# ifndef NO_MANUAL_VDB
if (manual_vdb_allowed) {
GC_manual_vdb = TRUE;
GC_incremental = TRUE;
} else
# endif
/* else */ {
/*
* For `GWW_VDB` on Win32, this needs to happen before any heap memory
* is allocated.
*/
GC_incremental = GC_dirty_init();
}
}
#endif /* !GC_DISABLE_INCREMENTAL */
STATIC word
GC_parse_mem_size_arg(const char *str)
{
word result;
char *endptr;
char ch;
if ('\0' == *str)
return GC_WORD_MAX; /*< bad value */
result = (word)STRTOULL(str, &endptr, 10);
ch = *endptr;
if (ch != '\0') {
if (*(endptr + 1) != '\0')
return GC_WORD_MAX;
/* Allow "k", "M" or "G" suffix. */
switch (ch) {
case 'K':
case 'k':
result <<= 10;
break;
#if CPP_WORDSZ >= 32
case 'M':
case 'm':
result <<= 20;
break;
case 'G':
case 'g':
result <<= 30;
break;
#endif
default:
result = GC_WORD_MAX;
}
}
return result;
}
#define GC_LOG_STD_NAME "gc.log"
GC_API void GC_CALL
GC_init(void)
{
word initial_heap_sz;
IF_CANCEL(int cancel_state;)
if (LIKELY(GC_is_initialized))
return;
#ifdef REDIRECT_MALLOC
{
static GC_bool init_started = FALSE;
if (init_started)
ABORT("Redirected malloc() called during GC init");
init_started = TRUE;
}
#endif
#if defined(GC_INITIAL_HEAP_SIZE) && !defined(CPPCHECK)
initial_heap_sz = GC_INITIAL_HEAP_SIZE;
#else
initial_heap_sz = MINHINCR * HBLKSIZE;
#endif
DISABLE_CANCEL(cancel_state);
/*
* Note that although we are nominally called with the allocator lock
* held, now it is only really acquired once a second thread is created.
* And the initialization code needs to run before then. Thus we really
* do not hold any locks, and can safely initialize them here.
*/
#ifdef THREADS
# ifndef GC_ALWAYS_MULTITHREADED
GC_ASSERT(!GC_need_to_lock);
# endif
{
# if !defined(GC_BUILTIN_ATOMIC) && defined(HP_PA) \
&& (defined(USE_SPIN_LOCK) || defined(NEED_FAULT_HANDLER_LOCK))
AO_TS_t ts_init = AO_TS_INITIALIZER;
/* Arrays can only be initialized when declared. */
# ifdef USE_SPIN_LOCK
BCOPY(&ts_init, (/* no volatile */ void *)&GC_allocate_lock,
sizeof(GC_allocate_lock));
# endif
# ifdef NEED_FAULT_HANDLER_LOCK
BCOPY(&ts_init, (/* no volatile */ void *)&GC_fault_handler_lock,
sizeof(GC_fault_handler_lock));
# endif
# else
# ifdef USE_SPIN_LOCK
GC_allocate_lock = AO_TS_INITIALIZER;
# endif
# ifdef NEED_FAULT_HANDLER_LOCK
GC_fault_handler_lock = AO_TS_INITIALIZER;
# endif
# endif
}
# ifdef SN_TARGET_PS3
{
pthread_mutexattr_t mattr;
if (pthread_mutexattr_init(&mattr) != 0)
ABORT("pthread_mutexattr_init failed");
if (pthread_mutex_init(&GC_allocate_ml, &mattr) != 0)
ABORT("pthread_mutex_init failed");
(void)pthread_mutexattr_destroy(&mattr);
}
# endif
#endif /* THREADS */
#if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS)
# ifndef SPIN_COUNT
# define SPIN_COUNT 4000
# endif
# ifdef USE_RWLOCK
/* TODO: Probably use `SRWLOCK_INIT` instead. */
InitializeSRWLock(&GC_allocate_ml);
# elif defined(MSWINRT_FLAVOR)
InitializeCriticalSectionAndSpinCount(&GC_allocate_ml, SPIN_COUNT);
# else
{
# ifndef MSWINCE
FARPROC pfn = 0;
HMODULE hK32 = GetModuleHandle(TEXT("kernel32.dll"));
if (hK32)
pfn = GetProcAddress(hK32, "InitializeCriticalSectionAndSpinCount");
if (pfn) {
(*(BOOL(WINAPI *)(LPCRITICAL_SECTION, DWORD))(GC_funcptr_uint)pfn)(
&GC_allocate_ml, SPIN_COUNT);
} else
# endif /* !MSWINCE */
/* else */ InitializeCriticalSection(&GC_allocate_ml);
}
# endif
#endif /* GC_WIN32_THREADS && !GC_PTHREADS */
#if defined(GC_WIN32_THREADS) \
&& ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE))
InitializeCriticalSection(&GC_write_cs);
#endif
#if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)
/* Just to set `GC_lock_holder`. */
LOCK();
#endif
#ifdef DYNAMIC_POINTER_MASK
if (0 == GC_pointer_mask)
GC_pointer_mask = GC_WORD_MAX;
#endif
GC_setpagesize();
#ifdef MSWIN32
GC_init_win32();
#endif
#ifdef GC_READ_ENV_FILE
GC_envfile_init();
#endif
#if !defined(NO_CLOCK) || !defined(SMALL_CONFIG)
# ifdef GC_PRINT_VERBOSE_STATS
/*
* This is useful for debugging and profiling on platforms with
* missing `getenv()` (like WinCE).
*/
GC_print_stats = VERBOSE;
# else
if (GETENV("GC_PRINT_VERBOSE_STATS") != NULL) {
GC_print_stats = VERBOSE;
} else if (GETENV("GC_PRINT_STATS") != NULL) {
GC_print_stats = 1;
}
# endif
#endif
#if ((defined(UNIX_LIKE) && !defined(GC_ANDROID_LOG)) \
|| (defined(CONSOLE_LOG) && defined(MSWIN32)) || defined(CYGWIN32) \
|| defined(SYMBIAN)) \
&& !defined(SMALL_CONFIG)
{
const char *fname = TRUSTED_STRING(GETENV("GC_LOG_FILE"));
# ifdef GC_LOG_TO_FILE_ALWAYS
if (NULL == fname)
fname = GC_LOG_STD_NAME;
# else
if (fname != NULL)
# endif
{
# if defined(_MSC_VER)
int log_d = _open(fname, O_CREAT | O_WRONLY | O_APPEND);
# else
int log_d = open(fname, O_CREAT | O_WRONLY | O_APPEND, 0644);
# endif
if (log_d < 0) {
GC_err_printf("Failed to open %s as log file\n", fname);
} else {
const char *str;
GC_log = log_d;
str = GETENV("GC_ONLY_LOG_TO_FILE");
# ifdef GC_ONLY_LOG_TO_FILE
/*
* The similar environment variable set to "0"
* overrides the effect of the macro defined.
*/
if (str != NULL && str[0] == '0' && str[1] == '\0')
# else
/*
* Otherwise setting the environment variable to anything other
* than "0" will prevent from redirecting `stdout` and `stderr`
* to the collector log file.
*/
if (str == NULL || (str[0] == '0' && str[1] == '\0'))
# endif
{
GC_stdout = log_d;
GC_stderr = log_d;
}
}
}
}
#endif
#if !defined(NO_DEBUGGING) && !defined(GC_DUMP_REGULARLY)
if (GETENV("GC_DUMP_REGULARLY") != NULL) {
GC_dump_regularly = TRUE;
}
#endif
#ifdef KEEP_BACK_PTRS
{
const char *str = GETENV("GC_BACKTRACES");
if (str != NULL) {
GC_backtraces = atol(str);
if (str[0] == '\0')
GC_backtraces = 1;
}
}
#endif
#ifndef NO_FIND_LEAK
if (GETENV("GC_FIND_LEAK") != NULL) {
GC_find_leak = 1;
}
# ifndef SHORT_DBG_HDRS
if (GETENV("GC_FINDLEAK_DELAY_FREE") != NULL) {
GC_findleak_delay_free = TRUE;
}
# endif
#endif
if (GETENV("GC_ALL_INTERIOR_POINTERS") != NULL) {
GC_all_interior_pointers = 1;
}
if (GETENV("GC_DONT_GC") != NULL) {
#if defined(LINT2) \
&& !(defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED))
GC_disable();
#else
GC_dont_gc = 1;
#endif
}
#if !defined(SMALL_CONFIG) && !defined(GC_PRINT_BACK_HEIGHT)
if (GETENV("GC_PRINT_BACK_HEIGHT") != NULL) {
# ifdef MAKE_BACK_GRAPH
GC_print_back_height = TRUE;
# else
GC_err_printf("Back height is not available!\n");
# endif
}
#endif
{
const char *str = GETENV("GC_TRACE");
if (str != NULL) {
#ifndef ENABLE_TRACE
WARN("Tracing not enabled: Ignoring GC_TRACE value\n", 0);
#else
ptr_t p = MAKE_CPTR(STRTOULL(str, NULL, 16));
if (ADDR(p) < 0x1000)
WARN("Unlikely trace address: %p\n", p);
GC_trace_ptr = p;
#endif
}
}
#ifdef GC_COLLECT_AT_MALLOC
{
const char *str = GETENV("GC_COLLECT_AT_MALLOC");
if (str != NULL) {
size_t min_lb = (size_t)STRTOULL(str, NULL, 10);
if (min_lb > 0)
GC_dbg_collect_at_malloc_min_lb = min_lb;
}
}
#endif
#if !defined(GC_DISABLE_INCREMENTAL) && !defined(NO_CLOCK)
{
const char *str = GETENV("GC_PAUSE_TIME_TARGET");
if (str != NULL) {
long time_limit = atol(str);
if (time_limit > 0) {
GC_time_limit = (unsigned long)time_limit;
}
}
}
#endif
#ifndef SMALL_CONFIG
{
const char *str = GETENV("GC_FULL_FREQUENCY");
if (str != NULL) {
int full_freq = atoi(str);
if (full_freq > 0)
GC_full_freq = full_freq;
}
}
#endif
#ifndef NO_BLACK_LISTING
{
char const *str = GETENV("GC_LARGE_ALLOC_WARN_INTERVAL");
if (str != NULL) {
long interval = atol(str);
if (interval <= 0) {
WARN("GC_LARGE_ALLOC_WARN_INTERVAL environment variable has"
" bad value - ignoring\n",
0);
} else {
GC_large_alloc_warn_interval = interval;
}
}
}
#endif
{
const char *str = GETENV("GC_FREE_SPACE_DIVISOR");
if (str != NULL) {
int space_divisor = atoi(str);
if (space_divisor > 0)
GC_free_space_divisor = (unsigned)space_divisor;
}
}
#ifdef USE_MUNMAP
{
const char *str = GETENV("GC_UNMAP_THRESHOLD");
if (str != NULL) {
if (str[0] == '0' && str[1] == '\0') {
/* "0" is used to disable unmapping. */
GC_unmap_threshold = 0;
} else {
int unmap_threshold = atoi(str);
if (unmap_threshold > 0)
GC_unmap_threshold = (unsigned)unmap_threshold;
}
}
}
{
const char *str = GETENV("GC_FORCE_UNMAP_ON_GCOLLECT");
if (str != NULL) {
if (str[0] == '0' && str[1] == '\0') {
/* "0" is used to turn off the mode. */
GC_force_unmap_on_gcollect = FALSE;
} else {
GC_force_unmap_on_gcollect = TRUE;
}
}
}
{
const char *str = GETENV("GC_USE_ENTIRE_HEAP");
if (str != NULL) {
if (str[0] == '0' && str[1] == '\0') {
/* "0" is used to turn off the mode. */
GC_use_entire_heap = FALSE;
} else {
GC_use_entire_heap = TRUE;
}
}
}
#endif
#if !defined(NO_DEBUGGING) && !defined(NO_CLOCK)
GET_TIME(GC_init_time);
#endif
maybe_install_looping_handler();
#if ALIGNMENT > GC_DS_TAGS
/* Adjust normal object descriptor for extra allocation. */
if (EXTRA_BYTES != 0)
GC_obj_kinds[NORMAL].ok_descriptor
= ((~(word)ALIGNMENT) + 1) | GC_DS_LENGTH;
#endif
GC_exclude_static_roots_inner(beginGC_arrays, endGC_arrays);
GC_exclude_static_roots_inner(beginGC_obj_kinds, endGC_obj_kinds);
#if defined(USE_PROC_FOR_LIBRARIES) && defined(LINUX) && defined(THREADS)
/*
* TODO: `USE_PROC_FOR_LIBRARIES` with LinuxThreads performs poorly!
* If thread stacks are cached, they tend to be scanned in entirety
* as part of the root set. This will grow them to maximum size, and
* is generally not desirable.
*/
#endif
#if !defined(THREADS) || !(defined(SN_TARGET_PS3) || defined(SN_TARGET_PSP2))
if (NULL == GC_stackbottom) {
GC_stackbottom = GC_get_main_stack_base();
# if (defined(LINUX) || defined(HPUX)) && defined(IA64)
GC_register_stackbottom = GC_get_register_stack_base();
# endif
} else {
# if (defined(LINUX) || defined(HPUX)) && defined(IA64)
if (NULL == GC_register_stackbottom) {
WARN("GC_register_stackbottom should be set with GC_stackbottom\n", 0);
/*
* The following may fail, since we may rely on alignment properties
* that may not hold with `GC_stackbottom` value set by client.
*/
GC_register_stackbottom = GC_get_register_stack_base();
}
# endif
}
#endif
#if !defined(CPPCHECK)
GC_STATIC_ASSERT(sizeof(size_t) <= sizeof(ptrdiff_t));
# ifdef AO_HAVE_store
/*
* As of now, `hb_descr`, `mse_descr` and `hb_marks[i]` might be treated
* as variables of `word` type but might be accessed atomically.
*/
GC_STATIC_ASSERT(sizeof(AO_t) == sizeof(word));
# endif
GC_STATIC_ASSERT(sizeof(ptrdiff_t) == sizeof(word));
GC_STATIC_ASSERT(sizeof(GC_signed_word) == sizeof(word));
GC_STATIC_ASSERT(sizeof(word) * 8 == CPP_WORDSZ);
GC_STATIC_ASSERT(sizeof(ptr_t) * 8 == CPP_PTRSZ);
GC_STATIC_ASSERT(sizeof(ptr_t) == sizeof(GC_uintptr_t));
GC_STATIC_ASSERT(sizeof(GC_oom_func) == sizeof(GC_funcptr_uint));
# ifdef FUNCPTR_IS_DATAPTR
GC_STATIC_ASSERT(sizeof(ptr_t) == sizeof(GC_funcptr_uint));
# endif
GC_STATIC_ASSERT((word)(-1) > (word)0); /*< `word` should be unsigned */
/*
* We no longer check for `(void *)-1 > NULL` since all pointers
* are explicitly cast to `word` in every less/greater comparison.
*/
GC_STATIC_ASSERT((GC_signed_word)(-1) < (GC_signed_word)0);
#endif
GC_STATIC_ASSERT(sizeof(struct hblk) == HBLKSIZE);
#ifndef THREADS
GC_ASSERT(!HOTTER_THAN(GC_stackbottom, GC_approx_sp()));
#endif
GC_init_headers();
#ifdef SEARCH_FOR_DATA_START
/*
* For `MPROTECT_VDB`, the temporary fault handler should be installed
* first, before the write fault one in `GC_dirty_init`.
*/
if (GC_REGISTER_MAIN_STATIC_DATA())
GC_init_linux_data_start();
#endif
#ifndef GC_DISABLE_INCREMENTAL
if (GC_incremental || GETENV("GC_ENABLE_INCREMENTAL") != NULL) {
set_incremental_mode_on();
GC_ASSERT(0 == GC_bytes_allocd);
}
#endif
/*
* Add the initial guess of root sets. Do this first, since `sbrk(0)`
* might be used.
*/
if (GC_REGISTER_MAIN_STATIC_DATA())
GC_register_data_segments();
GC_bl_init();
GC_mark_init();
{
const char *str = GETENV("GC_INITIAL_HEAP_SIZE");
if (str != NULL) {
word value = GC_parse_mem_size_arg(str);
if (GC_WORD_MAX == value) {
WARN("Bad initial heap size %s - ignoring\n", str);
} else {
initial_heap_sz = value;
}
}
}
{
const char *str = GETENV("GC_MAXIMUM_HEAP_SIZE");
if (str != NULL) {
word max_heap_sz = GC_parse_mem_size_arg(str);
if (max_heap_sz < initial_heap_sz || GC_WORD_MAX == max_heap_sz) {
WARN("Bad maximum heap size %s - ignoring\n", str);
} else {
if (0 == GC_max_retries)
GC_max_retries = 2;
GC_set_max_heap_size(max_heap_sz);
}
}
}
if (initial_heap_sz != 0) {
if (!GC_expand_hp_inner(divHBLKSZ(initial_heap_sz))) {
GC_err_printf("Can't start up: not enough memory\n");
EXIT();
} else {
GC_requested_heapsize += initial_heap_sz;
}
}
if (GC_all_interior_pointers)
GC_initialize_offsets();
GC_register_displacement_inner(0);
#ifdef REDIR_MALLOC_AND_LINUXTHREADS
if (!GC_all_interior_pointers) {
/* TLS ABI uses "pointer-sized" offsets for `dtv`. */
GC_register_displacement_inner(sizeof(void *));
}
#endif
GC_init_size_map();
GC_is_initialized = TRUE;
#ifdef THREADS
# if defined(LINT2) \
&& !(defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED))
LOCK();
GC_thr_init();
UNLOCK();
# else
GC_thr_init();
# endif
#endif
COND_DUMP;
/* Get black list set up and/or the incremental GC started. */
if (!GC_dont_precollect || GC_incremental) {
#if defined(DYNAMIC_LOADING) && defined(DARWIN)
GC_ASSERT(0 == GC_bytes_allocd);
#endif
GC_gcollect_inner();
}
#if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)
UNLOCK();
#endif
#if defined(THREADS) && defined(UNIX_LIKE) && !defined(NO_GETCONTEXT)
/* Ensure `getcontext_works` is set to avoid potential data race. */
if (GC_dont_gc || GC_dont_precollect)
GC_with_callee_saves_pushed(callee_saves_pushed_dummy_fn, NULL);
#endif
#ifndef DONT_USE_ATEXIT
if (GC_find_leak) {
/*
* This is to give us at least one chance to detect leaks.
* This may report some very benign leaks, but...
*/
atexit(GC_exit_check);
}
#endif
/*
* The rest of this again assumes we do not really hold the allocator
* lock.
*/
#ifdef THREADS
/* Initialize thread-local allocation. */
GC_init_parallel();
#endif
#if defined(DYNAMIC_LOADING) && defined(DARWIN)
/*
* This must be called *without* the allocator lock held and before
* any threads are created.
*/
GC_init_dyld();
#endif
RESTORE_CANCEL(cancel_state);
/*
* It is not safe to allocate any object till completion of `GC_init`
* (in particular by `GC_thr_init`), i.e. before `GC_init_dyld()` call
* and initialization of the incremental mode (if any).
*/
#if defined(GWW_VDB) && !defined(KEEP_BACK_PTRS)
GC_ASSERT(GC_bytes_allocd + GC_bytes_allocd_before_gc == 0);
#endif
}
GC_API void GC_CALL
GC_enable_incremental(void)
{
#if !defined(GC_DISABLE_INCREMENTAL) && !defined(KEEP_BACK_PTRS)
/*
* If we are keeping back pointers, the collector itself dirties all pages
* on which objects have been marked, making the incremental collection
* pointless.
*/
if (!GC_find_leak_inner && NULL == GETENV("GC_DISABLE_INCREMENTAL")) {
LOCK();
if (!GC_incremental) {
GC_setpagesize();
/* TODO: Should we skip enabling incremental if win32s? */
/* Install the looping handler before write fault handler! */
maybe_install_looping_handler();
if (!GC_is_initialized) {
/* Indicate the intention to turn it on. */
GC_incremental = TRUE;
UNLOCK();
GC_init();
LOCK();
} else {
set_incremental_mode_on();
}
/* Cannot easily do it if `GC_dont_gc`. */
if (GC_incremental && !GC_dont_gc) {
IF_CANCEL(int cancel_state;)
DISABLE_CANCEL(cancel_state);
if (GC_bytes_allocd > 0) {
/* There may be unmarked reachable objects. */
GC_gcollect_inner();
} else {
/*
* We are OK in assuming everything is clean since nothing can
* point to an unmarked object.
*/
# ifdef CHECKSUMS
GC_read_dirty(FALSE);
# else
GC_read_dirty(TRUE);
# endif
}
RESTORE_CANCEL(cancel_state);
}
}
UNLOCK();
return;
}
#endif
GC_init();
}
GC_API void GC_CALL
GC_start_mark_threads(void)
{
#ifdef PARALLEL_MARK
IF_CANCEL(int cancel_state;)
DISABLE_CANCEL(cancel_state);
LOCK();
GC_start_mark_threads_inner();
UNLOCK();
RESTORE_CANCEL(cancel_state);
#else
/* No action since parallel markers are disabled (or no POSIX `fork`). */
GC_ASSERT(I_DONT_HOLD_LOCK());
#endif
}
#ifndef GC_NO_DEINIT
GC_API void GC_CALL
GC_deinit(void)
{
/* Prevent duplicate resource close. */
if (!GC_is_initialized)
return;
BZERO(&GC_arrays, sizeof(GC_arrays)); /*< clears GC_is_initialized */
GC_gc_no = 0;
GC_dont_gc = FALSE;
GC_non_gc_bytes = 0;
GC_reset_obj_kinds();
GC_n_mark_procs = GC_RESERVED_MARK_PROCS;
GC_reset_freelist();
# ifdef CHECKSUMS
GC_reset_check_page();
# endif
# ifdef THREADS
GC_reset_threads();
# endif
# ifdef THREAD_LOCAL_ALLOC
GC_reset_thread_local_initialization();
# endif
# if defined(GC_WIN32_THREADS) && (defined(MSWIN32) || defined(MSWINCE))
# if !defined(CONSOLE_LOG) || defined(MSWINCE)
DeleteCriticalSection(&GC_write_cs);
# endif
# if !defined(GC_PTHREADS) && !defined(USE_RWLOCK)
DeleteCriticalSection(&GC_allocate_ml);
# endif
# endif
}
#endif
#if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
STATIC HANDLE GC_log = 0;
# ifdef THREADS
# if defined(PARALLEL_MARK) && !defined(GC_ALWAYS_MULTITHREADED)
# define IF_NEED_TO_LOCK(x) \
if (GC_parallel || GC_need_to_lock) \
x
# else
# define IF_NEED_TO_LOCK(x) \
if (GC_need_to_lock) \
x
# endif
# else
# define IF_NEED_TO_LOCK(x)
# endif /* !THREADS */
# ifdef MSWINRT_FLAVOR
# include <windows.storage.h>
/*
* This API function is defined in platform `roapi.h` file, but we cannot
* include it here since it does not compile in C.
*/
DECLSPEC_IMPORT HRESULT WINAPI
RoGetActivationFactory(HSTRING activatableClassId, REFIID iid, void **factory);
static GC_bool
getWinRTLogPath(wchar_t *buf, size_t bufLen)
{
static const GUID kIID_IApplicationDataStatics
= { 0x5612147B, 0xE843, 0x45E3, 0x94, 0xD8, 0x06,
0x16, 0x9E, 0x3C, 0x8E, 0x17 };
static const GUID kIID_IStorageItem
= { 0x4207A996, 0xCA2F, 0x42F7, 0xBD, 0xE8, 0x8B,
0x10, 0x45, 0x7A, 0x7F, 0x30 };
GC_bool result = FALSE;
HSTRING_HEADER appDataClassNameHeader;
HSTRING appDataClassName;
__x_ABI_CWindows_CStorage_CIApplicationDataStatics *appDataStatics = 0;
GC_ASSERT(bufLen > 0);
if (SUCCEEDED(WindowsCreateStringReference(
RuntimeClass_Windows_Storage_ApplicationData,
(sizeof(RuntimeClass_Windows_Storage_ApplicationData) - 1)
/ sizeof(wchar_t),
&appDataClassNameHeader, &appDataClassName))
&& SUCCEEDED(RoGetActivationFactory(
appDataClassName, &kIID_IApplicationDataStatics, &appDataStatics))) {
__x_ABI_CWindows_CStorage_CIApplicationData *appData = NULL;
__x_ABI_CWindows_CStorage_CIStorageFolder *tempFolder = NULL;
__x_ABI_CWindows_CStorage_CIStorageItem *tempFolderItem = NULL;
HSTRING tempPath = NULL;
if (SUCCEEDED(
appDataStatics->lpVtbl->get_Current(appDataStatics, &appData))
&& SUCCEEDED(
appData->lpVtbl->get_TemporaryFolder(appData, &tempFolder))
&& SUCCEEDED(tempFolder->lpVtbl->QueryInterface(
tempFolder, &kIID_IStorageItem, &tempFolderItem))
&& SUCCEEDED(
tempFolderItem->lpVtbl->get_Path(tempFolderItem, &tempPath))) {
UINT32 tempPathLen;
const wchar_t *tempPathBuf
= WindowsGetStringRawBuffer(tempPath, &tempPathLen);
buf[0] = '\0';
if (wcsncat_s(buf, bufLen, tempPathBuf, tempPathLen) == 0
&& wcscat_s(buf, bufLen, L"\\") == 0
&& wcscat_s(buf, bufLen, TEXT(GC_LOG_STD_NAME)) == 0)
result = TRUE;
WindowsDeleteString(tempPath);
}
if (tempFolderItem != NULL)
tempFolderItem->lpVtbl->Release(tempFolderItem);
if (tempFolder != NULL)
tempFolder->lpVtbl->Release(tempFolder);
if (appData != NULL)
appData->lpVtbl->Release(appData);
appDataStatics->lpVtbl->Release(appDataStatics);
}
return result;
}
# endif /* MSWINRT_FLAVOR */
STATIC HANDLE
GC_CreateLogFile(void)
{
HANDLE hFile;
# ifdef MSWINRT_FLAVOR
TCHAR pathBuf[_MAX_PATH + 0x10]; /*< buffer for file path plus extension */
hFile = INVALID_HANDLE_VALUE;
if (getWinRTLogPath(pathBuf, _MAX_PATH + 1)) {
CREATEFILE2_EXTENDED_PARAMETERS extParams;
BZERO(&extParams, sizeof(extParams));
extParams.dwSize = sizeof(extParams);
extParams.dwFileAttributes = FILE_ATTRIBUTE_NORMAL;
extParams.dwFileFlags
= GC_print_stats == VERBOSE ? 0 : FILE_FLAG_WRITE_THROUGH;
hFile = CreateFile2(pathBuf, GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS,
&extParams);
}
# else
TCHAR *logPath;
# if defined(NO_GETENV_WIN32) && defined(CPPCHECK)
# define appendToFile FALSE
# else
BOOL appendToFile = FALSE;
# endif
# if !defined(NO_GETENV_WIN32) || !defined(OLD_WIN32_LOG_FILE)
TCHAR pathBuf[_MAX_PATH + 0x10]; /*< buffer for file path plus extension */
logPath = pathBuf;
# endif
/* Use `GetEnvironmentVariable` instead of `GETENV` for Unicode support. */
# ifndef NO_GETENV_WIN32
if (GetEnvironmentVariable(TEXT("GC_LOG_FILE"), pathBuf, _MAX_PATH + 1) - 1U
< (DWORD)_MAX_PATH) {
appendToFile = TRUE;
} else
# endif
/* else */ {
/* Environment var not found or its value too long. */
# ifdef OLD_WIN32_LOG_FILE
logPath = TEXT(GC_LOG_STD_NAME);
# else
int len
= (int)GetModuleFileName(NULL /* `hModule` */, pathBuf, _MAX_PATH + 1);
/* If `GetModuleFileName()` has failed, then len is 0. */
if (len > 4 && pathBuf[len - 4] == (TCHAR)'.') {
/* Strip the executable file extension. */
len -= 4;
}
BCOPY(TEXT(".") TEXT(GC_LOG_STD_NAME), &pathBuf[len],
sizeof(TEXT(".") TEXT(GC_LOG_STD_NAME)));
# endif
}
hFile = CreateFile(logPath, GENERIC_WRITE, FILE_SHARE_READ,
NULL /* `lpSecurityAttributes` */,
appendToFile ? OPEN_ALWAYS : CREATE_ALWAYS,
GC_print_stats == VERBOSE
? FILE_ATTRIBUTE_NORMAL
:
/* immediately flush writes unless very verbose */
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_WRITE_THROUGH,
NULL /* `hTemplateFile` */);
# ifndef NO_GETENV_WIN32
if (appendToFile && hFile != INVALID_HANDLE_VALUE) {
LONG posHigh = 0;
/* Seek to the file end (ignoring any error). */
(void)SetFilePointer(hFile, 0, &posHigh, FILE_END);
}
# endif
# undef appendToFile
# endif
return hFile;
}
STATIC int
GC_write(const char *buf, size_t len)
{
BOOL res;
DWORD written;
# if defined(THREADS) && defined(GC_ASSERTIONS)
/* This is to prevent infinite recursion at abort. */
static GC_bool inside_write = FALSE;
if (inside_write)
return -1;
# endif
if (0 == len)
return 0;
IF_NEED_TO_LOCK(EnterCriticalSection(&GC_write_cs));
# if defined(THREADS) && defined(GC_ASSERTIONS)
if (GC_write_disabled) {
inside_write = TRUE;
ABORT("Assertion failure: GC_write called with write_disabled");
}
# endif
if (0 == GC_log) {
GC_log = GC_CreateLogFile();
}
if (GC_log == INVALID_HANDLE_VALUE) {
IF_NEED_TO_LOCK(LeaveCriticalSection(&GC_write_cs));
# ifdef NO_DEBUGGING
/*
* Ignore open log failure (e.g., it might be caused by read-only folder
* of the client application).
*/
return 0;
# else
return -1;
# endif
}
res = WriteFile(GC_log, buf, (DWORD)len, &written, NULL);
# if defined(_MSC_VER) && defined(_DEBUG) && !defined(NO_CRT) \
&& !defined(NO_CRTDBGREPORT)
# ifdef MSWINCE
/* There is no `CrtDbgReport()` in WinCE. */
{
WCHAR wbuf[1024];
/* Always use Unicode variant of `OutputDebugString()`. */
wbuf[MultiByteToWideChar(CP_ACP, 0 /* `dwFlags` */, buf, len, wbuf,
sizeof(wbuf) / sizeof(wbuf[0]) - 1)]
= 0;
OutputDebugStringW(wbuf);
}
# else
_CrtDbgReport(_CRT_WARN, NULL, 0, NULL, "%.*s", len, buf);
# endif
# endif
IF_NEED_TO_LOCK(LeaveCriticalSection(&GC_write_cs));
return res ? (int)written : -1;
}
/* TODO: This is pretty ugly... */
# define WRITE(f, buf, len) GC_write(buf, len)
#elif defined(OS2)
STATIC FILE *GC_stdout = NULL;
STATIC FILE *GC_stderr = NULL;
STATIC FILE *GC_log = NULL;
/* Initialize `GC_log` (and the friends) passed to `GC_write()`. */
STATIC void
GC_set_files(void)
{
if (GC_stdout == NULL) {
GC_stdout = stdout;
}
if (GC_stderr == NULL) {
GC_stderr = stderr;
}
if (GC_log == NULL) {
GC_log = stderr;
}
}
GC_INLINE int
GC_write(FILE *f, const char *buf, size_t len)
{
int res = fwrite(buf, 1, len, f);
fflush(f);
return res;
}
# define WRITE(f, buf, len) (GC_set_files(), GC_write(f, buf, len))
#elif defined(GC_ANDROID_LOG)
# include <android/log.h>
# ifndef GC_ANDROID_LOG_TAG
# define GC_ANDROID_LOG_TAG "BDWGC"
# endif
# define GC_stdout ANDROID_LOG_DEBUG
# define GC_stderr ANDROID_LOG_ERROR
# define GC_log GC_stdout
# define WRITE(level, buf, unused_len) \
__android_log_write(level, GC_ANDROID_LOG_TAG, buf)
#elif defined(NN_PLATFORM_CTR)
int n3ds_log_write(const char *text, int length);
# define WRITE(level, buf, len) n3ds_log_write(buf, len)
#elif defined(NINTENDO_SWITCH)
int switch_log_write(const char *text, int length);
# define WRITE(level, buf, len) switch_log_write(buf, len)
#else
# if !defined(ECOS) && !defined(NOSYS) && !defined(PLATFORM_WRITE) \
&& !defined(SN_TARGET_PSP2)
# include <errno.h>
# endif
STATIC int
GC_write(int fd, const char *buf, size_t len)
{
# if defined(ECOS) || defined(PLATFORM_WRITE) || defined(SN_TARGET_PSP2) \
|| defined(NOSYS)
UNUSED_ARG(fd);
# ifdef ECOS
/* FIXME: This seems to be defined nowhere at present. */
/* `_Jv_diag_write(buf, len);` */
# else
/* No writing. */
# endif
UNUSED_ARG(buf);
return (int)len;
# else
size_t bytes_written = 0;
IF_CANCEL(int cancel_state;)
DISABLE_CANCEL(cancel_state);
while (bytes_written < len) {
int result;
# if defined(SOLARIS) && defined(THREADS)
result = syscall(SYS_write, fd, buf + bytes_written, len - bytes_written);
# elif defined(_MSC_VER)
result = _write(fd, buf + bytes_written, (unsigned)(len - bytes_written));
# else
result = (int)write(fd, buf + bytes_written, len - bytes_written);
# endif
if (result < 0) {
if (EAGAIN == errno) {
/* Resource is temporarily unavailable. */
continue;
}
RESTORE_CANCEL(cancel_state);
return -1;
}
# ifdef LINT2
if ((unsigned)result > len - bytes_written)
ABORT("write() result cannot be bigger than requested length");
# endif
bytes_written += (unsigned)result;
}
RESTORE_CANCEL(cancel_state);
return (int)bytes_written;
# endif
}
# define WRITE(f, buf, len) GC_write(f, buf, len)
#endif /* !MSWINCE && !OS2 && !GC_ANDROID_LOG */
#ifndef GC_DISABLE_SNPRINTF
# define BUFSZ 1024
# if defined(DJGPP) || defined(__STRICT_ANSI__)
/* `vsnprintf` is missing in DJGPP (v2.0.3). */
# define GC_VSNPRINTF(buf, bufsz, format, args) vsprintf(buf, format, args)
# elif defined(_MSC_VER)
# ifdef MSWINCE
/* `_vsnprintf` is deprecated in WinCE. */
# define GC_VSNPRINTF StringCchVPrintfA
# else
# define GC_VSNPRINTF _vsnprintf
# endif
# else
# define GC_VSNPRINTF vsnprintf
# endif
/*
* A variant of `printf` that is unlikely to call `malloc`, and is thus
* safer to call from the collector in case `malloc` has been bound to
* `GC_malloc`. Floating-point arguments and formats should be avoided,
* since the conversion is more likely to allocate memory.
* Assumes that no more than `BUFSZ - 1` characters are written at once.
*/
# define GC_PRINTF_FILLBUF(buf, format) \
do { \
va_list args; \
va_start(args, format); \
(buf)[sizeof(buf) - 1] = 0x15; /*< guard */ \
(void)GC_VSNPRINTF(buf, sizeof(buf) - 1, format, args); \
va_end(args); \
if ((buf)[sizeof(buf) - 1] != 0x15) \
ABORT("GC_printf clobbered stack"); \
} while (0)
# define DECL_BUF_AND_PRINTF_TO(buf, format) \
char buf[BUFSZ + 1]; \
GC_PRINTF_FILLBUF(buf, format)
#else
/*
* At most, when `vsnprintf()` is unavailable, we could only print the
* format string as is, not handling the format specifiers (if any), thus
* skipping the rest of the `printf` arguments.
*/
# define DECL_BUF_AND_PRINTF_TO(buf, format) const char *buf = (format)
#endif /* GC_DISABLE_SNPRINTF */
void
GC_printf(const char *format, ...)
{
if (!GC_quiet) {
DECL_BUF_AND_PRINTF_TO(buf, format);
#ifdef NACL
(void)WRITE(GC_stdout, buf, strlen(buf));
/* Ignore errors silently. */
#else
if (WRITE(GC_stdout, buf, strlen(buf)) < 0
# if defined(CYGWIN32) || (defined(CONSOLE_LOG) && defined(MSWIN32))
&& GC_stdout != GC_DEFAULT_STDOUT_FD
# endif
) {
ABORT("write to stdout failed");
}
#endif
}
}
void
GC_err_printf(const char *format, ...)
{
DECL_BUF_AND_PRINTF_TO(buf, format);
GC_err_puts(buf);
}
void
GC_log_printf(const char *format, ...)
{
DECL_BUF_AND_PRINTF_TO(buf, format);
#ifdef NACL
(void)WRITE(GC_log, buf, strlen(buf));
#else
if (WRITE(GC_log, buf, strlen(buf)) < 0
# if defined(CYGWIN32) || (defined(CONSOLE_LOG) && defined(MSWIN32))
&& GC_log != GC_DEFAULT_STDERR_FD
# endif
) {
ABORT("write to GC log failed");
}
#endif
}
#ifndef GC_ANDROID_LOG
# define GC_warn_printf GC_err_printf
#else
GC_INNER void
GC_info_log_printf(const char *format, ...)
{
DECL_BUF_AND_PRINTF_TO(buf, format);
(void)WRITE(ANDROID_LOG_INFO, buf, 0 /* unused */);
}
GC_INNER void
GC_verbose_log_printf(const char *format, ...)
{
DECL_BUF_AND_PRINTF_TO(buf, format);
/* Note: write errors are ignored. */
(void)WRITE(ANDROID_LOG_VERBOSE, buf, 0);
}
STATIC void
GC_warn_printf(const char *format, ...)
{
DECL_BUF_AND_PRINTF_TO(buf, format);
(void)WRITE(ANDROID_LOG_WARN, buf, 0);
}
#endif /* GC_ANDROID_LOG */
void
GC_err_puts(const char *s)
{
/* Note: write errors are ignored. */
(void)WRITE(GC_stderr, s, strlen(s));
}
STATIC void GC_CALLBACK
GC_default_warn_proc(const char *msg, GC_uintptr_t arg)
{
/* TODO: Add assertion on argument to comply with `msg` (format). */
GC_warn_printf(msg, arg);
}
GC_INNER GC_warn_proc GC_current_warn_proc = GC_default_warn_proc;
GC_API void GC_CALLBACK
GC_ignore_warn_proc(const char *msg, GC_uintptr_t arg)
{
if (GC_print_stats) {
/* Do not ignore warnings if stats printing is on. */
GC_default_warn_proc(msg, arg);
}
}
GC_API void GC_CALL
GC_set_warn_proc(GC_warn_proc p)
{
GC_ASSERT(NONNULL_ARG_NOT_NULL(p));
LOCK();
GC_current_warn_proc = p;
UNLOCK();
}
GC_API GC_warn_proc GC_CALL
GC_get_warn_proc(void)
{
GC_warn_proc result;
READER_LOCK();
result = GC_current_warn_proc;
READER_UNLOCK();
return result;
}
/*
* Print (or display) a message before abnormal exit (including abort).
* Invoked from `ABORT(msg)` macro (where `msg` is non-`NULL`) and from
* `EXIT()` macro (`msg` is `NULL` in that case).
*/
STATIC void GC_CALLBACK
GC_default_on_abort(const char *msg)
{
#if !defined(SMALL_CONFIG)
# ifndef DONT_USE_ATEXIT
/* Disable at-exit garbage collection. */
GC_skip_collect_atexit = TRUE;
# endif
if (msg != NULL) {
# ifdef MSGBOX_ON_ERROR
GC_win32_MessageBoxA(msg, "Fatal error in GC", MB_ICONERROR | MB_OK);
/* Also duplicate `msg` to the collector log file. */
# endif
# ifndef GC_ANDROID_LOG
/*
* Avoid calling `GC_err_printf()` here, as `GC_on_abort()` could
* be called from it. Note 1: this is not an atomic output.
* Note 2: possible write errors are ignored.
*/
# if defined(GC_WIN32_THREADS) && defined(GC_ASSERTIONS) \
&& ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE))
if (!GC_write_disabled)
# endif
{
if (WRITE(GC_stderr, msg, strlen(msg)) >= 0)
(void)WRITE(GC_stderr, "\n", 1);
}
# else
__android_log_assert("*" /* `cond` */, GC_ANDROID_LOG_TAG, "%s\n", msg);
# endif
# if defined(HAIKU) && !defined(DONT_CALL_DEBUGGER)
/*
* This will cause the crash reason to appear in any debug reports
* generated (by the default system application crash dialog).
*/
debugger(msg);
# endif
}
# if !defined(NO_DEBUGGING) && !defined(GC_ANDROID_LOG)
if (GETENV("GC_LOOP_ON_ABORT") != NULL) {
/*
* In many cases it is easier to debug a running process.
* It is arguably nicer to sleep, but that makes it harder to look
* at the thread if the debugger does not know much about threads.
*/
for (;;) {
/* Empty. */
}
}
# endif
#else
UNUSED_ARG(msg);
#endif
}
#ifndef SMALL_CONFIG
GC_abort_func GC_on_abort = GC_default_on_abort;
#endif
GC_API void GC_CALL
GC_set_abort_func(GC_abort_func fn)
{
GC_ASSERT(NONNULL_ARG_NOT_NULL(fn));
LOCK();
#ifndef SMALL_CONFIG
GC_on_abort = fn;
#else
UNUSED_ARG(fn);
#endif
UNLOCK();
}
GC_API GC_abort_func GC_CALL
GC_get_abort_func(void)
{
GC_abort_func fn;
READER_LOCK();
#ifndef SMALL_CONFIG
fn = GC_on_abort;
GC_ASSERT(fn != 0);
#else
fn = GC_default_on_abort;
#endif
READER_UNLOCK();
return fn;
}
#if defined(NEED_SNPRINTF_SLDS) /* && GC_DISABLE_SNPRINTF */
GC_INNER void
GC_snprintf_s_ld_s(char *buf, size_t buf_sz, const char *prefix, long lv,
const char *suffix)
{
size_t len = strlen(prefix);
GC_ASSERT(buf_sz > 0);
/* Copy the prefix. */
if (UNLIKELY(len >= buf_sz))
len = buf_sz - 1;
BCOPY(prefix, buf, len);
buf += len;
buf_sz -= len;
/* Handle sign of the number. */
if (lv >= 0) {
lv = -lv;
} else if (LIKELY(buf_sz > 1)) {
*(buf++) = '-';
buf_sz--;
}
/* Convert the decimal number to string. (A trivial implementation.) */
{
char num_buf[20];
size_t pos = sizeof(num_buf);
do {
long r = lv / 10;
if (UNLIKELY(0 == pos))
break; /*< overflow */
num_buf[--pos] = (char)(r * 10 - lv + '0');
lv = r;
} while (lv < 0);
len = sizeof(num_buf) - pos;
if (UNLIKELY(len >= buf_sz))
len = buf_sz - 1;
BCOPY(&num_buf[pos], buf, len);
}
buf += len;
buf_sz -= len;
/* Copy the suffix (if any). */
len = strlen(suffix);
if (len > 0) {
if (UNLIKELY(len >= buf_sz))
len = buf_sz - 1;
BCOPY(suffix, buf, len);
buf += len;
}
*buf = '\0';
}
#endif /* NEED_SNPRINTF_SLDS */
GC_API void GC_CALL
GC_enable(void)
{
LOCK();
/* Ensure no counter underflow. */
GC_ASSERT(GC_dont_gc != 0);
GC_dont_gc--;
if (!GC_dont_gc && GC_heapsize > GC_heapsize_on_gc_disable)
WARN("Heap grown by %" WARN_PRIuPTR " KiB while GC was disabled\n",
(GC_heapsize - GC_heapsize_on_gc_disable) >> 10);
UNLOCK();
}
GC_API void GC_CALL
GC_disable(void)
{
LOCK();
if (!GC_dont_gc)
GC_heapsize_on_gc_disable = GC_heapsize;
GC_dont_gc++;
UNLOCK();
}
GC_API int GC_CALL
GC_is_disabled(void)
{
return GC_dont_gc != 0;
}
/* Helper procedures for new kind creation. */
GC_API void **GC_CALL
GC_new_free_list_inner(void)
{
void *result;
GC_ASSERT(I_HOLD_LOCK());
result = GC_INTERNAL_MALLOC((MAXOBJGRANULES + 1) * sizeof(ptr_t), PTRFREE);
if (NULL == result)
ABORT("Failed to allocate free list for new kind");
BZERO(result, (MAXOBJGRANULES + 1) * sizeof(ptr_t));
return (void **)result;
}
GC_API void **GC_CALL
GC_new_free_list(void)
{
void **result;
LOCK();
result = GC_new_free_list_inner();
UNLOCK();
return result;
}
GC_API unsigned GC_CALL
GC_new_kind_inner(void **fl, GC_word descr, int adjust, int clear)
{
unsigned result = GC_n_kinds;
GC_ASSERT(NONNULL_ARG_NOT_NULL(fl));
GC_ASSERT(!adjust || 1 == adjust);
/*
* If an object is not needed to be cleared (when moved to the free list),
* then its descriptor should be zero to denote a pointer-free object
* (and, as a consequence, the size of the object should not be added to
* the descriptor template).
*/
GC_ASSERT(1 == clear || (0 == descr && !adjust && !clear));
if (result < MAXOBJKINDS) {
GC_ASSERT(result > 0);
GC_n_kinds++;
GC_obj_kinds[result].ok_freelist = fl;
GC_obj_kinds[result].ok_reclaim_list = 0;
GC_obj_kinds[result].ok_descriptor = descr;
GC_obj_kinds[result].ok_relocate_descr = (GC_bool)adjust;
GC_obj_kinds[result].ok_init = (GC_bool)clear;
#ifdef ENABLE_DISCLAIM
GC_obj_kinds[result].ok_mark_unconditionally = FALSE;
GC_obj_kinds[result].ok_disclaim_proc = 0;
#endif
} else {
ABORT("Too many kinds");
}
return result;
}
GC_API unsigned GC_CALL
GC_new_kind(void **fl, GC_word descr, int adjust, int clear)
{
unsigned result;
LOCK();
result = GC_new_kind_inner(fl, descr, adjust, clear);
UNLOCK();
return result;
}
GC_API unsigned GC_CALL
GC_new_proc_inner(GC_mark_proc proc)
{
unsigned result = GC_n_mark_procs;
if (result < GC_MAX_MARK_PROCS) {
GC_n_mark_procs++;
GC_mark_procs[result] = proc;
} else {
ABORT("Too many mark procedures");
}
return result;
}
GC_API unsigned GC_CALL
GC_new_proc(GC_mark_proc proc)
{
unsigned result;
LOCK();
result = GC_new_proc_inner(proc);
UNLOCK();
return result;
}
GC_API void *GC_CALL
GC_call_with_alloc_lock(GC_fn_type fn, void *client_data)
{
void *result;
LOCK();
result = fn(client_data);
UNLOCK();
return result;
}
#ifdef THREADS
GC_API void GC_CALL
GC_alloc_lock(void)
{
LOCK();
}
GC_API void GC_CALL
GC_alloc_unlock(void)
{
UNLOCK();
}
GC_API void *GC_CALL
GC_call_with_reader_lock(GC_fn_type fn, void *client_data, int release)
{
void *result;
READER_LOCK();
result = fn(client_data);
# ifdef HAS_REAL_READER_LOCK
if (release) {
READER_UNLOCK_RELEASE();
# ifdef LINT2
GC_noop1((unsigned)release);
# endif
return result;
}
# else
UNUSED_ARG(release);
# endif
READER_UNLOCK();
return result;
}
#endif /* THREADS */
GC_ATTR_NOINLINE
GC_API void *GC_CALL
GC_call_with_stack_base(GC_stack_base_func fn, void *arg)
{
struct GC_stack_base base;
void *result;
STORE_APPROX_SP_TO(*(volatile ptr_t *)&base.mem_base);
#ifdef IA64
base.reg_base = GC_save_regs_in_stack();
/*
* TODO: Unnecessarily flushes register stack, but that probably
* does not hurt.
*/
#elif defined(E2K)
{
unsigned long long sz_ull;
GET_PROCEDURE_STACK_SIZE_INNER(&sz_ull);
base.reg_base = NUMERIC_TO_VPTR(sz_ull);
}
#endif
result = (*(GC_stack_base_func volatile *)&fn)(&base, arg);
/*
* Strongly discourage the compiler from treating the above as
* a tail call.
*/
GC_noop1(COVERT_DATAFLOW(ADDR(&base)));
return result;
}
#ifndef THREADS
GC_INNER ptr_t GC_blocked_sp = NULL;
# ifdef IA64
STATIC ptr_t GC_blocked_register_sp = NULL;
# endif
GC_INNER struct GC_traced_stack_sect_s *GC_traced_stack_sect = NULL;
/* This is nearly the same as in `pthread_support.c` file. */
GC_ATTR_NOINLINE
GC_API void *GC_CALL
GC_call_with_gc_active(GC_fn_type fn, void *client_data)
{
struct GC_traced_stack_sect_s stacksect;
GC_ASSERT(GC_is_initialized);
/*
* Adjust our stack bottom pointer (this could happen if
* `GC_get_main_stack_base()` is unimplemented or broken for
* the platform). Note: `stacksect` variable is reused here.
*/
STORE_APPROX_SP_TO(*(volatile ptr_t *)&stacksect.saved_stack_ptr);
if (HOTTER_THAN(GC_stackbottom, stacksect.saved_stack_ptr))
GC_stackbottom = stacksect.saved_stack_ptr;
if (GC_blocked_sp == NULL) {
/* We are not inside `GC_do_blocking()` - do nothing more. */
client_data = (*(GC_fn_type volatile *)&fn)(client_data);
/* Prevent treating the above as a tail call. */
GC_noop1(COVERT_DATAFLOW(ADDR(&stacksect)));
return client_data; /*< result */
}
/* Setup new "stack section". */
stacksect.saved_stack_ptr = GC_blocked_sp;
# ifdef IA64
/* This is the same as in `GC_call_with_stack_base()`. */
stacksect.backing_store_end = GC_save_regs_in_stack();
/* Unnecessarily flushes register stack, but that probably does not hurt. */
stacksect.saved_backing_store_ptr = GC_blocked_register_sp;
# endif
stacksect.prev = GC_traced_stack_sect;
GC_blocked_sp = NULL;
GC_traced_stack_sect = &stacksect;
client_data = (*(GC_fn_type volatile *)&fn)(client_data);
GC_ASSERT(GC_blocked_sp == NULL);
GC_ASSERT(GC_traced_stack_sect == &stacksect);
# if defined(CPPCHECK)
GC_noop1_ptr(GC_traced_stack_sect);
GC_noop1_ptr(GC_blocked_sp);
# endif
/* Restore original "stack section". */
GC_traced_stack_sect = stacksect.prev;
# ifdef IA64
GC_blocked_register_sp = stacksect.saved_backing_store_ptr;
# endif
GC_blocked_sp = stacksect.saved_stack_ptr;
return client_data; /*< result */
}
/* This is nearly the same as in `pthread_support.c` file. */
STATIC void
GC_do_blocking_inner(ptr_t data, void *context)
{
UNUSED_ARG(context);
GC_ASSERT(GC_is_initialized);
GC_ASSERT(GC_blocked_sp == NULL);
# ifdef SPARC
GC_blocked_sp = GC_save_regs_in_stack();
# else
GC_blocked_sp = GC_approx_sp();
# ifdef IA64
GC_blocked_register_sp = GC_save_regs_in_stack();
# endif
# endif
((struct blocking_data *)data)->client_data /*< result */
= ((struct blocking_data *)data)
->fn(((struct blocking_data *)data)->client_data);
GC_ASSERT(GC_blocked_sp != NULL);
# if defined(CPPCHECK)
GC_noop1_ptr(GC_blocked_sp);
# endif
GC_blocked_sp = NULL;
}
GC_API void GC_CALL
GC_set_stackbottom(void *gc_thread_handle, const struct GC_stack_base *sb)
{
GC_ASSERT(sb->mem_base != NULL);
GC_ASSERT(NULL == gc_thread_handle || &GC_stackbottom == gc_thread_handle);
GC_ASSERT(NULL == GC_blocked_sp
&& NULL == GC_traced_stack_sect); /*< for now */
UNUSED_ARG(gc_thread_handle);
GC_stackbottom = (char *)sb->mem_base;
# ifdef IA64
GC_register_stackbottom = (ptr_t)sb->reg_base;
# endif
}
GC_API void *GC_CALL
GC_get_my_stackbottom(struct GC_stack_base *sb)
{
GC_ASSERT(GC_is_initialized);
sb->mem_base = GC_stackbottom;
# ifdef IA64
sb->reg_base = GC_register_stackbottom;
# elif defined(E2K)
sb->reg_base = NULL;
# endif
return &GC_stackbottom; /*< `gc_thread_handle` */
}
#endif /* !THREADS */
GC_API void *GC_CALL
GC_do_blocking(GC_fn_type fn, void *client_data)
{
struct blocking_data my_data;
my_data.fn = fn;
my_data.client_data = client_data;
GC_with_callee_saves_pushed(GC_do_blocking_inner, (ptr_t)(&my_data));
return my_data.client_data; /*< result */
}
#if !defined(NO_DEBUGGING)
GC_API void GC_CALL
GC_dump(void)
{
READER_LOCK();
GC_dump_named(NULL);
READER_UNLOCK();
}
GC_API void GC_CALL
GC_dump_named(const char *name)
{
# ifndef NO_CLOCK
CLOCK_TYPE current_time;
GET_TIME(current_time);
# endif
if (name != NULL) {
GC_printf("\n***GC Dump %s\n", name);
} else {
GC_printf("\n***GC Dump collection #%lu\n", (unsigned long)GC_gc_no);
}
# ifndef NO_CLOCK
/* Note that the time is wrapped in ~49 days if `sizeof(long) == 4`. */
GC_printf("Time since GC init: %lu ms\n",
MS_TIME_DIFF(current_time, GC_init_time));
# endif
GC_printf("\n***Static roots:\n");
GC_print_static_roots();
GC_printf("\n***Heap sections:\n");
GC_print_heap_sects();
GC_printf("\n***Free blocks:\n");
GC_print_hblkfreelist();
GC_printf("\n***Blocks in use:\n");
GC_print_block_list();
# ifndef GC_NO_FINALIZATION
GC_dump_finalization();
# endif
}
#endif /* !NO_DEBUGGING */
GC_API GC_word GC_CALL
GC_get_memory_use(void)
{
word bytes;
READER_LOCK();
GC_ASSERT(GC_heapsize >= GC_large_free_bytes);
bytes = GC_heapsize - GC_large_free_bytes;
READER_UNLOCK();
return bytes;
}
/* Getter functions for the public read-only variables. */
GC_API GC_word GC_CALL
GC_get_gc_no(void)
{
return GC_gc_no;
}
#ifndef PARALLEL_MARK
GC_API void GC_CALL
GC_set_markers_count(unsigned markers)
{
UNUSED_ARG(markers);
}
#endif
GC_API int GC_CALL
GC_get_parallel(void)
{
#ifdef THREADS
return GC_parallel;
#else
return 0;
#endif
}
/*
* Setter and getter functions for the public R/W function variables.
* These functions are synchronized (like `GC_set_warn_proc()` and
* `GC_get_warn_proc()`).
*/
GC_API void GC_CALL
GC_set_oom_fn(GC_oom_func fn)
{
GC_ASSERT(NONNULL_ARG_NOT_NULL(fn));
LOCK();
GC_oom_fn = fn;
UNLOCK();
}
GC_API GC_oom_func GC_CALL
GC_get_oom_fn(void)
{
GC_oom_func fn;
READER_LOCK();
fn = GC_oom_fn;
READER_UNLOCK();
return fn;
}
GC_API void GC_CALL
GC_set_on_heap_resize(GC_on_heap_resize_proc fn)
{
/* `fn` may be 0 (means no event notifier). */
LOCK();
GC_on_heap_resize = fn;
UNLOCK();
}
GC_API GC_on_heap_resize_proc GC_CALL
GC_get_on_heap_resize(void)
{
GC_on_heap_resize_proc fn;
READER_LOCK();
fn = GC_on_heap_resize;
READER_UNLOCK();
return fn;
}
GC_API void GC_CALL
GC_set_finalizer_notifier(GC_finalizer_notifier_proc fn)
{
/* `fn` may be 0 (means no finalizer notifier). */
LOCK();
GC_finalizer_notifier = fn;
UNLOCK();
}
GC_API GC_finalizer_notifier_proc GC_CALL
GC_get_finalizer_notifier(void)
{
GC_finalizer_notifier_proc fn;
READER_LOCK();
fn = GC_finalizer_notifier;
READER_UNLOCK();
return fn;
}
/*
* Setter and getter functions for the public numeric R/W variables.
* It is safe to call these functions even before `GC_INIT()`.
* These functions are unsynchronized and, if called after `GC_INIT()`,
* should be typically invoked inside the context of
* `GC_call_with_alloc_lock()` (or `GC_call_with_reader_lock()` in case
* of the getters) to prevent data race (unless it is guaranteed the
* collector is not multi-threaded at that execution point).
*/
GC_API void GC_CALL
GC_set_find_leak(int value)
{
/* `value` is of boolean type. */
#ifdef NO_FIND_LEAK
if (value)
ABORT("Find-leak mode is unsupported");
#else
GC_find_leak = value;
#endif
}
GC_API int GC_CALL
GC_get_find_leak(void)
{
return GC_find_leak_inner;
}
GC_API void GC_CALL
GC_set_all_interior_pointers(int value)
{
GC_all_interior_pointers = value ? 1 : 0;
if (GC_is_initialized) {
/*
* It is not recommended to change `GC_all_interior_pointers` value
* after the collector is initialized but it seems it could work
* correctly even after switching the mode.
*/
LOCK();
/* Note: this resets manual offsets as well. */
GC_initialize_offsets();
#ifndef NO_BLACK_LISTING
if (!GC_all_interior_pointers)
GC_bl_init_no_interiors();
#endif
UNLOCK();
}
}
GC_API int GC_CALL
GC_get_all_interior_pointers(void)
{
return GC_all_interior_pointers;
}
GC_API void GC_CALL
GC_set_finalize_on_demand(int value)
{
/* Note: -1 was used to retrieve old value in gc-7.2. */
GC_ASSERT(value != -1);
/* `value` is of boolean type. */
GC_finalize_on_demand = value;
}
GC_API int GC_CALL
GC_get_finalize_on_demand(void)
{
return GC_finalize_on_demand;
}
GC_API void GC_CALL
GC_set_java_finalization(int value)
{
/* Note: -1 was used to retrieve old value in gc-7.2. */
GC_ASSERT(value != -1);
/* `value` is of boolean type. */
GC_java_finalization = value;
}
GC_API int GC_CALL
GC_get_java_finalization(void)
{
return GC_java_finalization;
}
GC_API void GC_CALL
GC_set_dont_expand(int value)
{
/* Note: -1 was used to retrieve old value in gc-7.2. */
GC_ASSERT(value != -1);
/* `value` is of boolean type. */
GC_dont_expand = value;
}
GC_API int GC_CALL
GC_get_dont_expand(void)
{
return GC_dont_expand;
}
GC_API void GC_CALL
GC_set_no_dls(int value)
{
/* Note: -1 was used to retrieve old value in gc-7.2. */
GC_ASSERT(value != -1);
/* `value` is of boolean type. */
GC_no_dls = value;
}
GC_API int GC_CALL
GC_get_no_dls(void)
{
return GC_no_dls;
}
GC_API void GC_CALL
GC_set_non_gc_bytes(GC_word value)
{
GC_non_gc_bytes = value;
}
GC_API GC_word GC_CALL
GC_get_non_gc_bytes(void)
{
return GC_non_gc_bytes;
}
GC_API void GC_CALL
GC_set_free_space_divisor(GC_word value)
{
GC_ASSERT(value > 0);
GC_free_space_divisor = value;
}
GC_API GC_word GC_CALL
GC_get_free_space_divisor(void)
{
return GC_free_space_divisor;
}
GC_API void GC_CALL
GC_set_max_retries(GC_word value)
{
/* Note: -1 was used to retrieve old value in gc-7.2. */
GC_ASSERT((GC_signed_word)value != -1);
GC_max_retries = value;
}
GC_API GC_word GC_CALL
GC_get_max_retries(void)
{
return GC_max_retries;
}
GC_API void GC_CALL
GC_set_dont_precollect(int value)
{
/* Note: -1 was used to retrieve old value in gc-7.2. */
GC_ASSERT(value != -1);
/* `value` is of boolean type. */
GC_dont_precollect = value;
}
GC_API int GC_CALL
GC_get_dont_precollect(void)
{
return GC_dont_precollect;
}
GC_API void GC_CALL
GC_set_full_freq(int value)
{
GC_ASSERT(value >= 0);
GC_full_freq = value;
}
GC_API int GC_CALL
GC_get_full_freq(void)
{
return GC_full_freq;
}
GC_API void GC_CALL
GC_set_time_limit(unsigned long value)
{
/* Note: -1 was used to retrieve old value in gc-7.2. */
GC_ASSERT((long)value != -1L);
GC_time_limit = value;
}
GC_API unsigned long GC_CALL
GC_get_time_limit(void)
{
return GC_time_limit;
}
GC_API void GC_CALL
GC_set_force_unmap_on_gcollect(int value)
{
GC_force_unmap_on_gcollect = (GC_bool)value;
}
GC_API int GC_CALL
GC_get_force_unmap_on_gcollect(void)
{
return (int)GC_force_unmap_on_gcollect;
}
GC_API GC_OOM_ABORT_THROW_ATTRIBUTE void GC_CALL
GC_abort_on_oom(void)
{
GC_err_printf("Insufficient memory for the allocation\n");
EXIT();
}
GC_API size_t GC_CALL
GC_get_hblk_size(void)
{
return (size_t)HBLKSIZE;
}
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1995 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
* Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#if (defined(MPROTECT_VDB) && !defined(MSWIN32) && !defined(MSWINCE)) \
|| (defined(SOLARIS) && defined(THREADS)) || defined(OPENBSD)
# include <signal.h>
#endif
#if defined(UNIX_LIKE) || defined(CYGWIN32) || defined(NACL) \
|| defined(SYMBIAN)
# include <fcntl.h>
#endif
#ifdef LINUX
# include <ctype.h>
#endif
/*
* Blatantly OS-dependent routines, except for those that are related
* to dynamic loading.
*/
#ifdef IRIX5
# include <malloc.h> /*< for locking */
# include <sys/uio.h>
#endif
#if defined(MMAP_SUPPORTED) || defined(ADD_HEAP_GUARD_PAGES)
# if defined(USE_MUNMAP) && !defined(USE_MMAP) && !defined(CPPCHECK)
# error Invalid config: USE_MUNMAP requires USE_MMAP
# endif
# include <sys/mman.h>
# include <sys/stat.h>
#endif
#if defined(LINUX) && defined(SPECIFIC_MAIN_STACKBOTTOM) \
|| defined(ADD_HEAP_GUARD_PAGES) || defined(MMAP_SUPPORTED) \
|| defined(NEED_PROC_MAPS)
# include <errno.h>
#endif
#if defined(DARWIN) && !defined(DYNAMIC_LOADING) \
&& !defined(GC_DONT_REGISTER_MAIN_STATIC_DATA)
# include <mach-o/getsect.h> /*< for `get_etext` and friends */
#endif
#ifdef DJGPP
/*
* Apparently necessary for djgpp 2.01. May cause problems with
* other versions.
*/
typedef long unsigned int caddr_t;
#endif
#ifdef NO_EXECUTE_PERMISSION
STATIC GC_bool GC_pages_executable = FALSE;
#else
STATIC GC_bool GC_pages_executable = TRUE;
#endif
/* Note: it is undefined later on `GC_pages_executable` real use. */
#define IGNORE_PAGES_EXECUTABLE 1
#if ((defined(LINUX) && defined(SPECIFIC_MAIN_STACKBOTTOM) \
|| defined(NEED_PROC_MAPS) || defined(PROC_VDB) || defined(SOFT_VDB)) \
&& !defined(PROC_READ)) \
|| defined(CPPCHECK)
/* Note: should probably call the real `read()`, if later is wrapped. */
# define PROC_READ read
#endif
#if defined(LINUX) && defined(SPECIFIC_MAIN_STACKBOTTOM) \
|| defined(NEED_PROC_MAPS)
/*
* Repeatedly perform a `read()` call until the buffer is filled up,
* or we encounter EOF (end of file) or an error.
*/
STATIC ssize_t
GC_repeat_read(int f, char *buf, size_t count)
{
size_t num_read = 0;
ASSERT_CANCEL_DISABLED();
while (num_read < count) {
ssize_t result = PROC_READ(f, buf + num_read, count - num_read);
if (result < 0)
return result;
if (0 == result)
break;
# ifdef LINT2
if ((size_t)result > count - num_read)
ABORT("read() result cannot be bigger than requested length");
# endif
num_read += (size_t)result;
}
return num_read;
}
#endif /* LINUX && SPECIFIC_MAIN_STACKBOTTOM || NEED_PROC_MAPS */
#ifdef NEED_PROC_MAPS
/*
* We need to parse `/proc/self/maps` pseudo-file, either to find
* dynamic libraries, and/or to find the register backing store
* base (the IA-64 case). Do it once here.
*/
# ifndef SINGLE_THREADED_PROCESS
/*
* Determine the length of a file by incrementally reading it into a buffer.
* This would be silly to use it on a file supporting `lseek`, but Linux
* `/proc` files usually do not. As of Linux 4.15.0, `lseek(SEEK_END)` fails
* for `/proc/self/maps` file.
*/
STATIC size_t
GC_get_file_len(int f)
{
size_t total = 0;
# define GET_FILE_LEN_BUF_SZ 500
char buf[GET_FILE_LEN_BUF_SZ];
ASSERT_CANCEL_DISABLED();
for (;;) {
ssize_t result = PROC_READ(f, buf, sizeof(buf));
if (result < 0) {
/* An error has occurred. */
return 0;
}
if (0 == result)
break;
# ifdef LINT2
if ((size_t)result >= GC_SIZE_MAX - total)
ABORT("Too big file is passed to GC_get_file_len");
# endif
total += (size_t)result;
}
return total;
}
STATIC size_t
GC_get_maps_len(void)
{
int f = open("/proc/self/maps", O_RDONLY);
size_t result;
if (f < 0) {
/* Treat missing file as empty. */
return 0;
}
result = GC_get_file_len(f);
close(f);
return result;
}
# endif /* !SINGLE_THREADED_PROCESS */
GC_INNER const char *
GC_get_maps(void)
{
ssize_t result;
static char *maps_buf = NULL;
static size_t maps_buf_sz = 1;
size_t maps_size;
# ifndef SINGLE_THREADED_PROCESS
size_t old_maps_size = 0;
# endif
/* The buffer is essentially `static`, so there must be a single client. */
GC_ASSERT(I_HOLD_LOCK());
/*
* Note that in the presence of threads in the process (even if the
* collector itself is built single-threaded), the `maps` file can
* essentially shrink asynchronously and unexpectedly as threads
* that we already think of as dead release their stacks.
* And there is no easy way to read the entire file atomically.
* This is arguably a misfeature of the `/proc/self/maps` interface.
* Since we expect the file can grow asynchronously in rare cases,
* it should suffice to first determine the size (using `read()`),
* and then to reread the file. If the size is inconsistent, then
* we have to retry. This only matters with threads enabled, and
* if we use this to locate the data roots (not the default).
*/
# ifndef SINGLE_THREADED_PROCESS
/* Determine the initial size of `/proc/self/maps` file. */
maps_size = GC_get_maps_len();
if (0 == maps_size)
ABORT("Cannot determine length of /proc/self/maps");
# else
maps_size = 4000; /*< guess */
# endif
/*
* Read `/proc/self/maps` file, growing `maps_buf` as necessary.
* Note that we may not allocate conventionally, and thus cannot
* use `stdio` functionality.
*/
do {
int f;
while (maps_size >= maps_buf_sz) {
# ifdef LINT2
/* Workaround passing tainted `maps_buf` to a tainted sink. */
GC_noop1_ptr(maps_buf);
# else
GC_scratch_recycle_no_gww(maps_buf, maps_buf_sz);
# endif
/* Grow only by powers of 2, since we leak "too small" buffers. */
while (maps_size >= maps_buf_sz)
maps_buf_sz *= 2;
maps_buf = GC_scratch_alloc(maps_buf_sz);
if (NULL == maps_buf)
ABORT_ARG1("Insufficient space for /proc/self/maps buffer",
", %lu bytes requested", (unsigned long)maps_buf_sz);
# ifndef SINGLE_THREADED_PROCESS
/*
* Recompute initial length, since we allocated.
* This can only happen a few times per program execution.
*/
maps_size = GC_get_maps_len();
if (0 == maps_size)
ABORT("Cannot determine length of /proc/self/maps");
# endif
}
GC_ASSERT(maps_buf_sz >= maps_size + 1);
f = open("/proc/self/maps", O_RDONLY);
if (-1 == f)
ABORT_ARG1("Cannot open /proc/self/maps", ": errno= %d", errno);
# ifndef SINGLE_THREADED_PROCESS
old_maps_size = maps_size;
# endif
maps_size = 0;
do {
result = GC_repeat_read(f, maps_buf, maps_buf_sz - 1);
if (result < 0) {
ABORT_ARG1("Failed to read /proc/self/maps", ": errno= %d", errno);
}
maps_size += (size_t)result;
} while ((size_t)result == maps_buf_sz - 1);
close(f);
if (0 == maps_size)
ABORT("Empty /proc/self/maps");
# ifndef SINGLE_THREADED_PROCESS
if (maps_size > old_maps_size) {
/* This might be caused by e.g. thread creation. */
WARN("Unexpected asynchronous /proc/self/maps growth"
" (to %" WARN_PRIuPTR " bytes)\n",
maps_size);
}
# endif
} while (maps_size >= maps_buf_sz
# ifndef SINGLE_THREADED_PROCESS
|| maps_size < old_maps_size
# endif
);
maps_buf[maps_size] = '\0';
return maps_buf;
}
/*
* `GC_parse_map_entry` parses an entry from `/proc/self/maps` file so we
* can locate all writable data segments that belong to shared libraries.
* The format of one of these entries and the fields we care about
* is as follows:
* ```
* XXXXXXXX-XXXXXXXX r-xp 00000000 30:05 260537 name-of-mapping...\n
* ^^^^^^^^ ^^^^^^^^ ^^^^ ^^
* *p_start *p_end *p_prot *p_maj_dev
* ```
*
* Note that since about August 2003 kernels, the columns no longer have
* fixed offsets on 64-bit kernels. Hence we no longer rely on fixed
* offsets anywhere, which is safer anyway.
*/
# if defined(DYNAMIC_LOADING) && defined(USE_PROC_FOR_LIBRARIES) \
|| defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) \
|| (defined(CHECK_SOFT_VDB) && defined(MPROTECT_VDB)) \
|| defined(REDIR_MALLOC_AND_LINUXTHREADS)
GC_INNER const char *
GC_parse_map_entry(const char *maps_ptr, ptr_t *p_start, ptr_t *p_end,
const char **p_prot, unsigned *p_maj_dev,
const char **p_mapping_name)
{
const unsigned char *start_start, *end_start, *maj_dev_start;
const unsigned char *p; /*< unsigned for `isspace`, `isxdigit` */
if (maps_ptr == NULL || *maps_ptr == '\0') {
return NULL;
}
p = (const unsigned char *)maps_ptr;
while (isspace(*p))
++p;
start_start = p;
GC_ASSERT(isxdigit(*start_start));
*p_start = (ptr_t)strtoul((const char *)start_start, (char **)&p, 16);
GC_ASSERT(*p == '-');
++p;
end_start = p;
GC_ASSERT(isxdigit(*end_start));
*p_end = (ptr_t)strtoul((const char *)end_start, (char **)&p, 16);
GC_ASSERT(isspace(*p));
while (isspace(*p))
++p;
GC_ASSERT(*p == 'r' || *p == '-');
*p_prot = (const char *)p;
/* Skip past protection field to offset field. */
while (!isspace(*p))
++p;
while (isspace(*p))
p++;
GC_ASSERT(isxdigit(*p));
/* Skip past offset field, which we ignore. */
while (!isspace(*p))
++p;
while (isspace(*p))
p++;
maj_dev_start = p;
GC_ASSERT(isxdigit(*maj_dev_start));
*p_maj_dev = strtoul((const char *)maj_dev_start, NULL, 16);
if (p_mapping_name != NULL) {
while (*p && *p != '\n' && *p != '/' && *p != '[')
p++;
*p_mapping_name = (const char *)p;
}
while (*p && *p++ != '\n') {
/* Empty. */
}
return (const char *)p;
}
# endif /* REDIRECT_MALLOC || DYNAMIC_LOADING || IA64 || ... */
# if defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) \
|| (defined(CHECK_SOFT_VDB) && defined(MPROTECT_VDB))
GC_INNER GC_bool
GC_enclosing_writable_mapping(ptr_t addr, ptr_t *startp, ptr_t *endp)
{
const char *prot;
ptr_t my_start, my_end;
const char *maps_ptr;
unsigned maj_dev;
GC_ASSERT(I_HOLD_LOCK());
maps_ptr = GC_get_maps();
for (;;) {
maps_ptr = GC_parse_map_entry(maps_ptr, &my_start, &my_end, &prot,
&maj_dev, NULL);
if (NULL == maps_ptr)
break;
if (ADDR_INSIDE(addr, my_start, my_end)) {
if (prot[1] != 'w' || maj_dev != 0)
break;
*startp = my_start;
*endp = my_end;
return TRUE;
}
}
return FALSE;
}
# endif /* IA64 || INCLUDE_LINUX_THREAD_DESCR */
# ifdef REDIR_MALLOC_AND_LINUXTHREADS
GC_INNER GC_bool
GC_text_mapping(const char *nm, ptr_t *startp, ptr_t *endp)
{
size_t nm_len;
const char *prot, *map_path;
ptr_t my_start, my_end;
unsigned int maj_dev;
const char *maps_ptr;
GC_ASSERT(I_HOLD_LOCK());
maps_ptr = GC_get_maps();
nm_len = strlen(nm);
for (;;) {
maps_ptr = GC_parse_map_entry(maps_ptr, &my_start, &my_end, &prot,
&maj_dev, &map_path);
if (NULL == maps_ptr)
break;
if (prot[0] == 'r' && prot[1] == '-' && prot[2] == 'x') {
const char *p = map_path;
/* Set `p` to point just past last slash, if any. */
while (*p != '\0' && *p != '\n' && *p != ' ' && *p != '\t') {
++p;
}
while (ADDR_GE((ptr_t)p, (ptr_t)map_path) && *p != '/') {
--p;
}
++p;
if (strncmp(nm, p, nm_len) == 0) {
*startp = my_start;
*endp = my_end;
return TRUE;
}
}
}
return FALSE;
}
# endif /* REDIR_MALLOC_AND_LINUXTHREADS */
# ifdef IA64
static ptr_t
backing_store_base_from_proc(void)
{
ptr_t my_start, my_end;
GC_ASSERT(I_HOLD_LOCK());
if (!GC_enclosing_writable_mapping(GC_save_regs_in_stack(), &my_start,
&my_end)) {
GC_COND_LOG_PRINTF("Failed to find backing store base from /proc\n");
return 0;
}
return my_start;
}
# endif
#endif /* NEED_PROC_MAPS */
#if defined(SEARCH_FOR_DATA_START)
/*
* The i686 case can be handled without a search. The Alpha case used to
* be handled differently as well, but the rules changed for recent Linux
* versions. This seems to be the easiest way to cover all versions.
*/
# if defined(LINUX) || defined(HURD)
/*
* Some Linux distributions arrange to define `__data_start`.
* Some define `data_start` as a weak symbol. The latter is technically
* broken, since the user program may define `data_start`, in which
* case we lose. Nonetheless, we try both, preferring `__data_start`.
* We assume gcc-compatible pragmas.
*/
EXTERN_C_BEGIN
# pragma weak __data_start
# pragma weak data_start
extern int __data_start[], data_start[];
EXTERN_C_END
# elif defined(NETBSD)
EXTERN_C_BEGIN
extern char **environ;
EXTERN_C_END
# endif
ptr_t GC_data_start = NULL;
GC_INNER void
GC_init_linux_data_start(void)
{
ptr_t data_end = DATAEND;
# if (defined(LINUX) || defined(HURD)) && defined(USE_PROG_DATA_START)
/*
* Try the easy approaches first. However, this may lead to wrong
* data start value if the collector code is put into a shared library
* (directly or indirectly) which is linked with `-Bsymbolic-functions`
* option. Thus, the following is not used by default.
*/
if (COVERT_DATAFLOW(ADDR(__data_start)) != 0) {
GC_data_start = (ptr_t)(__data_start);
} else {
GC_data_start = (ptr_t)(data_start);
}
if (COVERT_DATAFLOW(ADDR(GC_data_start)) != 0) {
if (ADDR_LT(data_end, GC_data_start))
ABORT_ARG2("Wrong __data_start/_end pair", ": %p .. %p",
(void *)GC_data_start, (void *)data_end);
return;
}
# ifdef DEBUG_ADD_DEL_ROOTS
GC_log_printf("__data_start not provided\n");
# endif
# endif /* LINUX */
if (GC_no_dls) {
/*
* Not needed, avoids the `SIGSEGV` caused by `GC_find_limit` which
* complicates debugging.
*/
GC_data_start = data_end; /*< set data root size to 0 */
return;
}
# ifdef NETBSD
/*
* This may need to be `environ`, without the underscore, for
* some versions.
*/
GC_data_start = (ptr_t)GC_find_limit(&environ, FALSE);
# else
GC_data_start = (ptr_t)GC_find_limit(data_end, FALSE);
# endif
}
#endif /* SEARCH_FOR_DATA_START */
#ifdef ECOS
static void *
tiny_sbrk(ptrdiff_t increment)
{
/*
* TODO: This is a simple way of allocating memory which is compatible with
* ECOS early releases. Later releases use a more sophisticated means of
* allocating memory than this simple static allocator, but this method is
* at least bound to work.
*/
void *p = &GC_ecos_memory[GC_ecos_brk_idx];
GC_ASSERT(GC_ecos_brk_idx <= sizeof(GC_ecos_memory));
if ((size_t)increment > sizeof(GC_ecos_memory) - GC_ecos_brk_idx)
return NULL;
GC_ecos_brk_idx += (size_t)increment;
return p;
}
# define sbrk tiny_sbrk
#endif /* ECOS */
#if defined(ADDRESS_SANITIZER) \
&& (defined(UNIX_LIKE) || defined(NEED_FIND_LIMIT) \
|| defined(MPROTECT_VDB)) \
&& !defined(CUSTOM_ASAN_DEF_OPTIONS)
EXTERN_C_BEGIN
GC_API const char *__asan_default_options(void);
EXTERN_C_END
/*
* To tell ASan to allow the collector to use its own `SIGBUS` and `SIGSEGV`
* handlers. The function is exported just to be visible to ASan library.
*/
GC_API const char *
__asan_default_options(void)
{
return "allow_user_segv_handler=1";
}
#endif
#ifdef OPENBSD
static struct sigaction old_segv_act;
STATIC JMP_BUF GC_jmp_buf_openbsd;
STATIC void
GC_fault_handler_openbsd(int sig)
{
UNUSED_ARG(sig);
LONGJMP(GC_jmp_buf_openbsd, 1);
}
static volatile int firstpass;
/*
* Return first addressable location that is greater than `p` or return
* `bound`.
*/
STATIC ptr_t
GC_skip_hole_openbsd(ptr_t p, ptr_t bound)
{
static volatile ptr_t result;
struct sigaction act;
size_t pgsz;
GC_ASSERT(I_HOLD_LOCK());
pgsz = (size_t)sysconf(_SC_PAGESIZE);
GC_ASSERT(ADDR(bound) >= (word)pgsz);
act.sa_handler = GC_fault_handler_openbsd;
sigemptyset(&act.sa_mask);
act.sa_flags = SA_NODEFER | SA_RESTART;
/* `act.sa_restorer` is deprecated and should not be initialized. */
sigaction(SIGSEGV, &act, &old_segv_act);
firstpass = 1;
result = PTR_ALIGN_DOWN(p, pgsz);
if (SETJMP(GC_jmp_buf_openbsd) != 0 || firstpass) {
firstpass = 0;
if (ADDR_GE(result, bound - pgsz)) {
result = bound;
} else {
/*
* Notes: no overflow is expected; do not use compound assignment
* with `volatile`-qualified left operand.
*/
result = result + pgsz;
GC_noop1((word)(unsigned char)(*result));
}
}
sigaction(SIGSEGV, &old_segv_act, 0);
return result;
}
#endif /* OPENBSD */
#ifdef OS2
# include <stddef.h>
# if !defined(__IBMC__) && !defined(__WATCOMC__) /*< e.g. EMX */
struct exe_hdr {
unsigned short magic_number;
unsigned short padding[29];
long new_exe_offset;
};
# define E_MAGIC(x) (x).magic_number
# define EMAGIC 0x5A4D
# define E_LFANEW(x) (x).new_exe_offset
struct e32_exe {
unsigned char magic_number[2];
unsigned char byte_order;
unsigned char word_order;
unsigned long exe_format_level;
unsigned short cpu;
unsigned short os;
unsigned long padding1[13];
unsigned long object_table_offset;
unsigned long object_count;
unsigned long padding2[31];
};
# define E32_MAGIC1(x) (x).magic_number[0]
# define E32MAGIC1 'L'
# define E32_MAGIC2(x) (x).magic_number[1]
# define E32MAGIC2 'X'
# define E32_BORDER(x) (x).byte_order
# define E32LEBO 0
# define E32_WORDER(x) (x).word_order
# define E32LEWO 0
# define E32_CPU(x) (x).cpu
# define E32CPU286 1
# define E32_OBJTAB(x) (x).object_table_offset
# define E32_OBJCNT(x) (x).object_count
struct o32_obj {
unsigned long size;
unsigned long base;
unsigned long flags;
unsigned long pagemap;
unsigned long mapsize;
unsigned long reserved;
};
# define O32_FLAGS(x) (x).flags
# define OBJREAD 0x0001L
# define OBJWRITE 0x0002L
# define OBJINVALID 0x0080L
# define O32_SIZE(x) (x).size
# define O32_BASE(x) (x).base
# else /* IBM's compiler */
/* A kludge to get around what appears to be a header file bug. */
# ifndef WORD
# define WORD unsigned short
# endif
# ifndef DWORD
# define DWORD unsigned long
# endif
# define EXE386 1
# include <exe386.h>
# include <newexe.h>
# endif /* __IBMC__ */
# define INCL_DOSERRORS
# define INCL_DOSEXCEPTIONS
# define INCL_DOSFILEMGR
# define INCL_DOSMEMMGR
# define INCL_DOSMISC
# define INCL_DOSMODULEMGR
# define INCL_DOSPROCESS
# include <os2.h>
#endif /* OS2 */
GC_INNER size_t GC_page_size = 0;
#ifdef REAL_PAGESIZE_NEEDED
GC_INNER size_t GC_real_page_size = 0;
#endif
#ifdef SOFT_VDB
STATIC unsigned GC_log_pagesize = 0;
#endif
#ifdef ANY_MSWIN
# ifndef VER_PLATFORM_WIN32_CE
# define VER_PLATFORM_WIN32_CE 3
# endif
# if defined(MSWINCE) && defined(THREADS)
GC_INNER GC_bool GC_dont_query_stack_min = FALSE;
# endif
GC_INNER SYSTEM_INFO GC_sysinfo;
# ifndef CYGWIN32
# define is_writable(prot) \
((prot) == PAGE_READWRITE || (prot) == PAGE_WRITECOPY \
|| (prot) == PAGE_EXECUTE_READWRITE \
|| (prot) == PAGE_EXECUTE_WRITECOPY)
/*
* Return the number of bytes that are writable starting at `p`.
* The pointer `p` is assumed to be page-aligned. If `base` is not `NULL`,
* then `*base` becomes the beginning of the allocation region containing `p`.
*/
STATIC word
GC_get_writable_length(ptr_t p, ptr_t *base)
{
MEMORY_BASIC_INFORMATION buf;
word result;
word protect;
result = VirtualQuery(p, &buf, sizeof(buf));
if (result != sizeof(buf))
ABORT("Weird VirtualQuery result");
if (base != 0)
*base = (ptr_t)(buf.AllocationBase);
protect = buf.Protect & ~(word)(PAGE_GUARD | PAGE_NOCACHE);
if (!is_writable(protect) || buf.State != MEM_COMMIT)
return 0;
return buf.RegionSize;
}
GC_API int GC_CALL
GC_get_stack_base(struct GC_stack_base *sb)
{
/*
* Note: this function should not acquire the allocator lock as it is
* used by `GC_DllMain`.
*/
ptr_t trunc_sp;
word size;
/*
* Set page size if it is not ready (so client can use this function even
* before the collector is initialized).
*/
if (!GC_page_size)
GC_setpagesize();
trunc_sp = PTR_ALIGN_DOWN(GC_approx_sp(), GC_page_size);
/*
* FIXME: This will not work if called from a deeply recursive
* client code (and the committed stack space has grown).
*/
size = GC_get_writable_length(trunc_sp, 0);
GC_ASSERT(size != 0);
sb->mem_base = trunc_sp + size;
return GC_SUCCESS;
}
# else /* CYGWIN32 */
GC_API int GC_CALL
GC_get_stack_base(struct GC_stack_base *sb)
{
/*
* An alternate variant for Cygwin (adapted from Dave Korn's gcc version
* of boehm-gc).
*/
# ifdef X86_64
sb->mem_base = ((NT_TIB *)NtCurrentTeb())->StackBase;
# else
void *_tlsbase;
__asm__("movl %%fs:4, %0" : "=r"(_tlsbase));
sb->mem_base = _tlsbase;
# endif
return GC_SUCCESS;
}
# endif /* CYGWIN32 */
# define HAVE_GET_STACK_BASE
#elif defined(OS2)
static int
os2_getpagesize(void)
{
ULONG result[1];
if (DosQuerySysInfo(QSV_PAGE_SIZE, QSV_PAGE_SIZE, (void *)result,
sizeof(ULONG))
!= NO_ERROR) {
WARN("DosQuerySysInfo failed\n", 0);
result[0] = 4096;
}
return (int)result[0];
}
#endif /* !ANY_MSWIN && OS2 */
GC_INNER void
GC_setpagesize(void)
{
#ifdef ANY_MSWIN
GetSystemInfo(&GC_sysinfo);
# ifdef ALT_PAGESIZE_USED
/*
* Allocations made with `mmap()` are aligned to the allocation
* granularity, which (at least on Win64) is not the same as the
* page size. Probably we could distinguish the allocation
* granularity from the actual page size, but in practice there
* is no good reason to make allocations smaller than
* `dwAllocationGranularity`, so we just use it instead of the
* actual page size here (as Cygwin itself does in many cases).
*/
GC_page_size = (size_t)GC_sysinfo.dwAllocationGranularity;
# ifdef REAL_PAGESIZE_NEEDED
GC_real_page_size = (size_t)GC_sysinfo.dwPageSize;
GC_ASSERT(GC_page_size >= GC_real_page_size);
# endif
# else
GC_page_size = (size_t)GC_sysinfo.dwPageSize;
# endif
# if defined(MSWINCE) && !defined(_WIN32_WCE_EMULATION)
{
OSVERSIONINFO verInfo;
/* Check the current WinCE version. */
verInfo.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
if (!GetVersionEx(&verInfo))
ABORT("GetVersionEx failed");
if (verInfo.dwPlatformId == VER_PLATFORM_WIN32_CE
&& verInfo.dwMajorVersion < 6) {
/*
* Only the first 32 MB of address space belongs to the
* current process (unless WinCE 6.0+ or emulation).
*/
GC_sysinfo.lpMaximumApplicationAddress = (LPVOID)((word)32 << 20);
# ifdef THREADS
/*
* On some old WinCE versions, it is observed that
* `VirtualQuery()` calls do not work properly when used to
* get thread current stack committed minimum.
*/
if (verInfo.dwMajorVersion < 5)
GC_dont_query_stack_min = TRUE;
# endif
}
}
# endif
#else
# ifdef ALT_PAGESIZE_USED
# ifdef REAL_PAGESIZE_NEEDED
GC_real_page_size = (size_t)GETPAGESIZE();
# endif
/* It is acceptable to fake it. */
GC_page_size = HBLKSIZE;
# else
GC_page_size = (size_t)GETPAGESIZE();
# if !defined(CPPCHECK)
if (0 == GC_page_size)
ABORT("getpagesize failed");
# endif
# endif
#endif /* !ANY_MSWIN */
#ifdef SOFT_VDB
{
size_t pgsize;
unsigned log_pgsize = 0;
# if !defined(CPPCHECK)
if (((GC_page_size - 1) & GC_page_size) != 0) {
/* Not a power of two. */
ABORT("Invalid page size");
}
# endif
for (pgsize = GC_page_size; pgsize > 1; pgsize >>= 1)
log_pgsize++;
GC_log_pagesize = log_pgsize;
}
#endif
}
#ifdef EMBOX
# include <kernel/thread/thread_stack.h>
# include <pthread.h>
GC_API int GC_CALL
GC_get_stack_base(struct GC_stack_base *sb)
{
pthread_t self = pthread_self();
void *stack_addr = thread_stack_get(self);
/* TODO: Use `pthread_getattr_np`, `pthread_attr_getstack` alternatively. */
# ifdef STACK_GROWS_UP
sb->mem_base = stack_addr;
# else
sb->mem_base = (ptr_t)stack_addr + thread_stack_get_size(self);
# endif
return GC_SUCCESS;
}
# define HAVE_GET_STACK_BASE
#endif /* EMBOX */
#ifdef OS2
GC_API int GC_CALL
GC_get_stack_base(struct GC_stack_base *sb)
{
PTIB ptib; /*< thread information block */
PPIB ppib;
if (DosGetInfoBlocks(&ptib, &ppib) != NO_ERROR) {
WARN("DosGetInfoBlocks failed\n", 0);
return GC_UNIMPLEMENTED;
}
sb->mem_base = ptib->tib_pstacklimit;
return GC_SUCCESS;
}
# define HAVE_GET_STACK_BASE
#endif /* OS2 */
#ifdef SERENITY
# include <serenity.h>
GC_API int GC_CALL
GC_get_stack_base(struct GC_stack_base *sb)
{
uintptr_t base;
size_t size;
if (get_stack_bounds(&base, &size) < 0) {
WARN("get_stack_bounds failed\n", 0);
return GC_UNIMPLEMENTED;
}
sb->mem_base = base + size;
return GC_SUCCESS;
}
# define HAVE_GET_STACK_BASE
#endif /* SERENITY */
#if defined(NEED_FIND_LIMIT) \
|| (defined(UNIX_LIKE) && !defined(NO_DEBUGGING)) \
|| (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) \
|| (defined(WRAP_MARK_SOME) && defined(NO_SEH_AVAILABLE))
# include <signal.h>
# ifdef USE_SEGV_SIGACT
# ifndef OPENBSD
static struct sigaction old_segv_act;
# endif
# ifdef USE_BUS_SIGACT
static struct sigaction old_bus_act;
# endif
# else
static GC_fault_handler_t old_segv_hand;
# ifdef HAVE_SIGBUS
static GC_fault_handler_t old_bus_hand;
# endif
# endif /* !USE_SEGV_SIGACT */
GC_INNER void
GC_set_and_save_fault_handler(GC_fault_handler_t h)
{
# ifdef USE_SEGV_SIGACT
struct sigaction act;
act.sa_handler = h;
# ifdef SIGACTION_FLAGS_NODEFER_HACK
/* Was necessary for Solaris 2.3 and very temporary NetBSD bugs. */
act.sa_flags = SA_RESTART | SA_NODEFER;
# else
act.sa_flags = SA_RESTART;
# endif
(void)sigemptyset(&act.sa_mask);
/* `act.sa_restorer` is deprecated and should not be initialized. */
# if defined(IRIX5) && defined(THREADS)
/*
* Older versions have a bug related to retrieving and setting
* a handler at the same time.
*/
(void)sigaction(SIGSEGV, 0, &old_segv_act);
(void)sigaction(SIGSEGV, &act, 0);
# else
(void)sigaction(SIGSEGV, &act, &old_segv_act);
# ifdef USE_BUS_SIGACT
/*
* `pthreads` library does not exist under Irix 5.x, so we do not have
* to worry of the multi-threaded case.
*/
(void)sigaction(SIGBUS, &act, &old_bus_act);
# endif
# endif /* !IRIX5 || !THREADS */
# else
old_segv_hand = signal(SIGSEGV, h);
# ifdef HAVE_SIGBUS
old_bus_hand = signal(SIGBUS, h);
# endif
# endif /* !USE_SEGV_SIGACT */
# if defined(CPPCHECK) && defined(ADDRESS_SANITIZER)
GC_noop1((word)(GC_funcptr_uint)(&__asan_default_options));
# endif
}
#endif /* NEED_FIND_LIMIT || UNIX_LIKE || WRAP_MARK_SOME */
#if defined(NEED_FIND_LIMIT) \
|| (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) \
|| (defined(WRAP_MARK_SOME) && defined(NO_SEH_AVAILABLE))
GC_INNER JMP_BUF GC_jmp_buf;
STATIC void
GC_fault_handler(int sig)
{
UNUSED_ARG(sig);
LONGJMP(GC_jmp_buf, 1);
}
GC_INNER void
GC_setup_temporary_fault_handler(void)
{
/*
* Handler is process-wide, so this should only happen in one thread
* at a time.
*/
GC_ASSERT(I_HOLD_LOCK());
GC_set_and_save_fault_handler(GC_fault_handler);
}
GC_INNER void
GC_reset_fault_handler(void)
{
# ifdef USE_SEGV_SIGACT
(void)sigaction(SIGSEGV, &old_segv_act, 0);
# ifdef USE_BUS_SIGACT
(void)sigaction(SIGBUS, &old_bus_act, 0);
# endif
# else
(void)signal(SIGSEGV, old_segv_hand);
# ifdef HAVE_SIGBUS
(void)signal(SIGBUS, old_bus_hand);
# endif
# endif
}
#endif /* NEED_FIND_LIMIT || USE_PROC_FOR_LIBRARIES || WRAP_MARK_SOME */
#if defined(NEED_FIND_LIMIT) \
|| (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS))
# define MIN_PAGE_SIZE 256 /*< smallest conceivable page size, in bytes */
/*
* Return the first non-addressable location greater than `p` (if `up`)
* or the smallest location `q` such that [`q`,`p`) is addressable (if
* not `up`). We assume that `p` (if `up`) or `p - 1` (if not `up`) is
* addressable.
*/
GC_ATTR_NO_SANITIZE_ADDR
STATIC ptr_t
GC_find_limit_with_bound(ptr_t p, GC_bool up, ptr_t bound)
{
/*
* This is safer if `static`, since otherwise it may not be preserved
* across the `longjmp`. Can safely be `static` since it is only called
* with the allocator lock held.
*/
static volatile ptr_t result;
GC_ASSERT(up ? ADDR(bound) >= MIN_PAGE_SIZE
: ADDR(bound) <= ~(word)MIN_PAGE_SIZE);
GC_ASSERT(I_HOLD_LOCK());
result = PTR_ALIGN_DOWN(p, MIN_PAGE_SIZE);
GC_setup_temporary_fault_handler();
if (SETJMP(GC_jmp_buf) == 0) {
for (;;) {
if (up) {
if (ADDR_GE(result, bound - MIN_PAGE_SIZE)) {
result = bound;
break;
}
/*
* Notes: no overflow is expected; do not use compound assignment
* with `volatile`-qualified left operand.
*/
result = result + MIN_PAGE_SIZE;
} else {
if (ADDR_GE(bound + MIN_PAGE_SIZE, result)) {
/*
* This is to compensate further result increment (we do not
* modify `up` variable since it might be clobbered by `setjmp()`
* otherwise).
*/
result = bound - MIN_PAGE_SIZE;
break;
}
/* See the notes for the case when `up` is `TRUE`. */
result = result - MIN_PAGE_SIZE;
}
GC_noop1((word)(unsigned char)(*result));
}
}
GC_reset_fault_handler();
return up ? result : result + MIN_PAGE_SIZE;
}
void *
GC_find_limit(void *p, int up)
{
ptr_t bound;
# ifdef CHERI_PURECAP
bound = (ptr_t)cheri_address_set(p, cheri_base_get(p)
+ (up ? cheri_length_get(p) : 0));
# else
bound = up ? MAKE_CPTR(GC_WORD_MAX) : NULL;
# endif
return GC_find_limit_with_bound((ptr_t)p, (GC_bool)up, bound);
}
#endif /* NEED_FIND_LIMIT || USE_PROC_FOR_LIBRARIES */
#if defined(HPUX) && defined(IA64)
# include <sys/param.h>
# include <sys/pstat.h>
GC_INNER ptr_t
GC_get_register_stack_base(void)
{
struct pst_vm_status vm_status;
int i = 0;
while (pstat_getprocvm(&vm_status, sizeof(vm_status), 0, i++) == 1) {
if (vm_status.pst_type == PS_RSESTACK) {
return (ptr_t)vm_status.pst_vaddr;
}
}
/* Old way to get the register stack bottom. */
GC_ASSERT(GC_stackbottom != NULL);
return PTR_ALIGN_DOWN(GC_stackbottom - BACKING_STORE_DISPLACEMENT - 1,
BACKING_STORE_ALIGNMENT);
}
#endif /* HPUX && IA64 */
#if defined(LINUX) && defined(IA64)
# ifdef USE_LIBC_PRIVATES
EXTERN_C_BEGIN
# pragma weak __libc_ia64_register_backing_store_base
extern ptr_t __libc_ia64_register_backing_store_base;
EXTERN_C_END
# endif
GC_INNER ptr_t
GC_get_register_stack_base(void)
{
ptr_t result;
GC_ASSERT(I_HOLD_LOCK());
# ifdef USE_LIBC_PRIVATES
{
ptr_t *p_libc_ia64_register_backing_store_base
= &__libc_ia64_register_backing_store_base;
# ifdef CPPCHECK
/*
* Workaround a warning that the address of the global symbol
* (which is a weak one) cannot be null.
*/
GC_noop1_ptr(&p_libc_ia64_register_backing_store_base);
# endif
if (p_libc_ia64_register_backing_store_base != NULL
&& __libc_ia64_register_backing_store_base != NULL) {
/*
* `glibc` 2.2.4 has a bug such that for dynamically linked
* executables `__libc_ia64_register_backing_store_base` is
* defined but uninitialized during constructor calls.
* Hence we check for both nonzero address and value.
*/
return __libc_ia64_register_backing_store_base;
}
}
# endif
result = backing_store_base_from_proc();
if (0 == result) {
/* This works better than a constant displacement heuristic. */
result = (ptr_t)GC_find_limit(GC_save_regs_in_stack(), FALSE);
}
return result;
}
#endif /* LINUX && IA64 */
#ifdef SPECIFIC_MAIN_STACKBOTTOM
# ifdef HPUX
# include <sys/param.h>
# include <sys/pstat.h>
static ptr_t
os_main_stackbottom(void)
{
struct pst_vm_status vm_status;
int i = 0;
while (pstat_getprocvm(&vm_status, sizeof(vm_status), 0, i++) == 1) {
if (vm_status.pst_type == PS_STACK)
return (ptr_t)vm_status.pst_vaddr;
}
/* Old way to get the stack bottom. */
# ifdef STACK_GROWS_UP
return (ptr_t)GC_find_limit(GC_approx_sp(), FALSE);
# else
return (ptr_t)GC_find_limit(GC_approx_sp(), TRUE /* `up` */);
# endif
}
# elif defined(LINUX)
# include <sys/stat.h>
/* Number of fields preceding `startstack` one in `/proc/self/stat` file. */
# define STAT_SKIP 27
# ifdef USE_LIBC_PRIVATES
EXTERN_C_BEGIN
# pragma weak __libc_stack_end
extern ptr_t __libc_stack_end;
EXTERN_C_END
# endif
static ptr_t
os_main_stackbottom(void)
{
/*
* We read the stack bottom value from `/proc/self/stat` file.
* We do this using direct I/O system calls in order to avoid
* calling `malloc` in case `REDIRECT_MALLOC` is defined.
*/
# define STAT_BUF_SIZE 4096
unsigned char stat_buf[STAT_BUF_SIZE];
int f;
word addr;
ssize_t i, buf_offset = 0, len;
/*
* First try the easy way. This should work for `glibc` 2.2.
* This fails in a prelinked (`prelink` command) executable
* since the correct value of `__libc_stack_end` never becomes
* visible to us. The second test is a workaround for this.
*/
# ifdef USE_LIBC_PRIVATES
ptr_t *p_libc_stack_end = &__libc_stack_end;
# ifdef CPPCHECK
GC_noop1_ptr(&p_libc_stack_end);
# endif
if (p_libc_stack_end != NULL && __libc_stack_end != NULL) {
# ifdef IA64
/*
* Some versions of `glibc` set the address 16 bytes too low
* while the initialization code is running.
*/
if ((ADDR(__libc_stack_end) & 0xfff) + 0x10 < 0x1000) {
return __libc_stack_end + 0x10;
} else {
/* It is not safe to add 16 bytes. Thus, fall back to using `/proc`. */
}
# elif defined(SPARC)
/*
* Older versions of `glibc` for 64-bit SPARC do not set this
* variable correctly, it gets set to either zero or one.
*/
if (ADDR(__libc_stack_end) != 1)
return __libc_stack_end;
# else
return __libc_stack_end;
# endif
}
# endif
f = open("/proc/self/stat", O_RDONLY);
if (-1 == f)
ABORT_ARG1("Could not open /proc/self/stat", ": errno= %d", errno);
len = GC_repeat_read(f, (char *)stat_buf, sizeof(stat_buf));
if (len < 0)
ABORT_ARG1("Failed to read /proc/self/stat", ": errno= %d", errno);
close(f);
/*
* Skip the required number of fields. This number is hopefully constant
* across all Linux implementations.
*/
for (i = 0; i < STAT_SKIP; ++i) {
while (buf_offset < len && isspace(stat_buf[buf_offset++])) {
/* Empty. */
}
while (buf_offset < len && !isspace(stat_buf[buf_offset++])) {
/* Empty. */
}
}
/* Skip spaces. */
while (buf_offset < len && isspace(stat_buf[buf_offset])) {
buf_offset++;
}
/* Find the end of the number and cut the buffer there. */
for (i = 0; buf_offset + i < len; i++) {
if (!isdigit(stat_buf[buf_offset + i]))
break;
}
if (buf_offset + i >= len)
ABORT("Could not parse /proc/self/stat");
stat_buf[buf_offset + i] = '\0';
addr = (word)STRTOULL((char *)stat_buf + buf_offset, NULL, 10);
if (addr < 0x100000 || addr % ALIGNMENT != 0)
ABORT_ARG1("Absurd stack bottom value", ": 0x%lx", (unsigned long)addr);
return MAKE_CPTR(addr);
}
# elif defined(QNX)
static ptr_t
os_main_stackbottom(void)
{
/*
* TODO: This approach is not very exact but it works for the tests,
* at least, unlike other available heuristics.
*/
return (ptr_t)__builtin_frame_address(0);
}
# elif defined(FREEBSD)
# include <sys/sysctl.h>
/*
* This uses an undocumented `sysctl` call, but at least one expert
* believes it will stay.
*/
static ptr_t
os_main_stackbottom(void)
{
int nm[2] = { CTL_KERN, KERN_USRSTACK };
ptr_t base;
size_t len = sizeof(ptr_t);
int r = sysctl(nm, 2, &base, &len, NULL, 0);
if (r != 0)
ABORT("Error getting main stack base");
return base;
}
# endif
#endif /* SPECIFIC_MAIN_STACKBOTTOM */
#if defined(ECOS) || defined(NOSYS)
GC_INNER ptr_t
GC_get_main_stack_base(void)
{
return STACKBOTTOM;
}
# define GET_MAIN_STACKBASE_SPECIAL
#elif defined(SYMBIAN)
EXTERN_C_BEGIN
extern int GC_get_main_symbian_stack_base(void);
EXTERN_C_END
GC_INNER ptr_t
GC_get_main_stack_base(void)
{
return (ptr_t)GC_get_main_symbian_stack_base();
}
# define GET_MAIN_STACKBASE_SPECIAL
#elif defined(EMSCRIPTEN)
# include <emscripten/stack.h>
GC_INNER ptr_t
GC_get_main_stack_base(void)
{
return (ptr_t)emscripten_stack_get_base();
}
# define GET_MAIN_STACKBASE_SPECIAL
#elif !defined(ANY_MSWIN) && !defined(EMBOX) && !defined(OS2) \
&& !(defined(OPENBSD) && defined(THREADS)) && !defined(SERENITY) \
&& (!(defined(SOLARIS) && defined(THREADS)) || defined(_STRICT_STDC))
# if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
&& (defined(THREADS) || defined(USE_GET_STACKBASE_FOR_MAIN))
# include <pthread.h>
# ifdef HAVE_PTHREAD_NP_H
# include <pthread_np.h> /*< for `pthread_attr_get_np()` */
# endif
# elif defined(DARWIN) && !defined(NO_PTHREAD_GET_STACKADDR_NP)
/*
* We could use `pthread_get_stackaddr_np` even in case of a single-threaded
* collector build (there is no `-lpthread` option on Darwin).
*/
# include <pthread.h>
# undef STACKBOTTOM
# define STACKBOTTOM (ptr_t) pthread_get_stackaddr_np(pthread_self())
# endif
GC_INNER ptr_t
GC_get_main_stack_base(void)
{
ptr_t result;
# if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
&& (defined(USE_GET_STACKBASE_FOR_MAIN) \
|| (defined(THREADS) && !defined(REDIRECT_MALLOC)))
pthread_attr_t attr;
void *stackaddr;
size_t size;
# ifdef HAVE_PTHREAD_ATTR_GET_NP
if (pthread_attr_init(&attr) == 0
&& (pthread_attr_get_np(pthread_self(), &attr) == 0
? TRUE
: (pthread_attr_destroy(&attr), FALSE)))
# else /* HAVE_PTHREAD_GETATTR_NP */
if (pthread_getattr_np(pthread_self(), &attr) == 0)
# endif
{
if (pthread_attr_getstack(&attr, &stackaddr, &size) == 0
&& stackaddr != NULL) {
(void)pthread_attr_destroy(&attr);
# ifndef STACK_GROWS_UP
stackaddr = (char *)stackaddr + size;
# endif
return (ptr_t)stackaddr;
}
(void)pthread_attr_destroy(&attr);
}
WARN("pthread_getattr_np or pthread_attr_getstack failed"
" for main thread\n",
0);
# endif
# ifdef STACKBOTTOM
result = STACKBOTTOM;
# else
# ifdef HEURISTIC1
# define STACKBOTTOM_ALIGNMENT_M1 ((word)STACK_GRAN - 1)
# ifdef STACK_GROWS_UP
result = PTR_ALIGN_DOWN(GC_approx_sp(), STACKBOTTOM_ALIGNMENT_M1 + 1);
# else
result = PTR_ALIGN_UP(GC_approx_sp(), STACKBOTTOM_ALIGNMENT_M1 + 1);
# endif
# elif defined(SPECIFIC_MAIN_STACKBOTTOM)
result = os_main_stackbottom();
# elif defined(HEURISTIC2)
{
ptr_t sp = GC_approx_sp();
# ifdef STACK_GROWS_UP
result = (ptr_t)GC_find_limit(sp, FALSE);
# else
result = (ptr_t)GC_find_limit(sp, TRUE /* `up` */);
# endif
# if defined(HEURISTIC2_LIMIT) && !defined(CPPCHECK)
if (HOTTER_THAN(HEURISTIC2_LIMIT, result)
&& HOTTER_THAN(sp, HEURISTIC2_LIMIT))
result = HEURISTIC2_LIMIT;
# endif
}
# elif defined(STACK_NOT_SCANNED) || defined(CPPCHECK)
result = NULL;
# else
# error None of HEURISTIC* and *STACKBOTTOM defined!
# endif
# if !defined(STACK_GROWS_UP) && !defined(CPPCHECK)
if (NULL == result)
result = MAKE_CPTR((GC_signed_word)(-sizeof(ptr_t)));
# endif
# endif
# if !defined(CPPCHECK)
GC_ASSERT(HOTTER_THAN(GC_approx_sp(), result));
# endif
return result;
}
# define GET_MAIN_STACKBASE_SPECIAL
#endif /* !ANY_MSWIN && !EMBOX && !OS2 && !SERENITY */
#if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
&& defined(THREADS) && !defined(HAVE_GET_STACK_BASE)
# include <pthread.h>
# ifdef HAVE_PTHREAD_NP_H
# include <pthread_np.h>
# endif
GC_API int GC_CALL
GC_get_stack_base(struct GC_stack_base *b)
{
pthread_attr_t attr;
size_t size;
# ifdef HAVE_PTHREAD_ATTR_GET_NP
if (pthread_attr_init(&attr) != 0)
ABORT("pthread_attr_init failed");
if (pthread_attr_get_np(pthread_self(), &attr) != 0) {
WARN("pthread_attr_get_np failed\n", 0);
(void)pthread_attr_destroy(&attr);
return GC_UNIMPLEMENTED;
}
# else /* HAVE_PTHREAD_GETATTR_NP */
if (pthread_getattr_np(pthread_self(), &attr) != 0) {
WARN("pthread_getattr_np failed\n", 0);
return GC_UNIMPLEMENTED;
}
# endif
if (pthread_attr_getstack(&attr, &b->mem_base, &size) != 0) {
ABORT("pthread_attr_getstack failed");
}
(void)pthread_attr_destroy(&attr);
# ifndef STACK_GROWS_UP
b->mem_base = (char *)b->mem_base + size;
# endif
# ifdef IA64
/*
* We could try `backing_store_base_from_proc`, but that is safe only
* if no mappings are being asynchronously created. Subtracting the size
* from the stack base does not work for at least the main thread.
*/
LOCK();
{
IF_CANCEL(int cancel_state;)
ptr_t bsp;
ptr_t next_stack;
DISABLE_CANCEL(cancel_state);
bsp = GC_save_regs_in_stack();
next_stack = GC_greatest_stack_base_below(bsp);
if (NULL == next_stack) {
b->reg_base = GC_find_limit(bsp, FALSE);
} else {
/*
* Avoid walking backwards into preceding memory stack and
* growing it.
*/
b->reg_base = GC_find_limit_with_bound(bsp, FALSE, next_stack);
}
RESTORE_CANCEL(cancel_state);
}
UNLOCK();
# elif defined(E2K)
b->reg_base = NULL;
# endif
return GC_SUCCESS;
}
# define HAVE_GET_STACK_BASE
#endif /* THREADS && (HAVE_PTHREAD_ATTR_GET_NP || HAVE_PTHREAD_GETATTR_NP) */
#if defined(DARWIN) && defined(THREADS) \
&& !defined(NO_PTHREAD_GET_STACKADDR_NP)
# include <pthread.h>
GC_API int GC_CALL
GC_get_stack_base(struct GC_stack_base *b)
{
/*
* `pthread_get_stackaddr_np()` should return stack bottom (highest
* stack address plus 1).
*/
b->mem_base = pthread_get_stackaddr_np(pthread_self());
GC_ASSERT(HOTTER_THAN(GC_approx_sp(), (ptr_t)b->mem_base));
return GC_SUCCESS;
}
# define HAVE_GET_STACK_BASE
#endif /* DARWIN && THREADS && !NO_PTHREAD_GET_STACKADDR_NP */
#if defined(OPENBSD) && defined(THREADS)
# include <pthread.h>
# include <pthread_np.h>
# include <sys/signal.h>
GC_API int GC_CALL
GC_get_stack_base(struct GC_stack_base *sb)
{
stack_t stack;
/* Find the stack using `pthread_stackseg_np()`. */
if (pthread_stackseg_np(pthread_self(), &stack))
ABORT("pthread_stackseg_np(self) failed");
sb->mem_base = stack.ss_sp;
return GC_SUCCESS;
}
# define HAVE_GET_STACK_BASE
#endif /* OPENBSD && THREADS */
#if defined(SOLARIS) && defined(THREADS) && !defined(_STRICT_STDC)
# include <pthread.h>
# include <thread.h>
/*
* These variables are used to cache `ss_sp` value for the primordial
* thread (it is better not to call `thr_stksegment()` twice for this
* thread - see JDK bug #4352906).
* Note: `stackbase_main_self` set to zero means `stackbase_main_ss_sp`
* value is unset.
*/
static pthread_t stackbase_main_self = 0;
static void *stackbase_main_ss_sp = NULL;
# ifdef CAN_HANDLE_FORK
GC_INNER void
GC_stackbase_info_update_after_fork(void)
{
if (stackbase_main_self == GC_parent_pthread_self) {
/* The primordial thread has forked the process. */
stackbase_main_self = pthread_self();
} else {
stackbase_main_self = 0;
}
}
# endif
GC_API int GC_CALL
GC_get_stack_base(struct GC_stack_base *b)
{
stack_t s;
pthread_t self = pthread_self();
if (self == stackbase_main_self) {
/*
* If the client calls `GC_get_stack_base()` from the main thread,
* then just return the cached value.
*/
b->mem_base = stackbase_main_ss_sp;
GC_ASSERT(b->mem_base != NULL);
return GC_SUCCESS;
}
if (thr_stksegment(&s)) {
/*
* According to the manual, the only failure error code returned is
* `EAGAIN` meaning "the information is not available due to the thread
* is not yet completely initialized or it is an internal thread" - this
* should not happen here.
*/
ABORT("thr_stksegment failed");
}
/* `s.ss_sp` holds the pointer to the stack bottom. */
GC_ASSERT(HOTTER_THAN(GC_approx_sp(), (ptr_t)s.ss_sp));
if (!stackbase_main_self && thr_main() != 0) {
/*
* Cache the stack bottom pointer for the primordial thread
* (this is done during `GC_init`, so there is no race).
*/
stackbase_main_ss_sp = s.ss_sp;
stackbase_main_self = self;
}
b->mem_base = s.ss_sp;
return GC_SUCCESS;
}
# define HAVE_GET_STACK_BASE
#endif /* SOLARIS && THREADS */
#if defined(RTEMS) && defined(THREADS)
GC_API int GC_CALL
GC_get_stack_base(struct GC_stack_base *sb)
{
sb->mem_base = rtems_get_stack_bottom();
return GC_SUCCESS;
}
# define HAVE_GET_STACK_BASE
#endif /* RTEMS && THREADS */
#ifndef HAVE_GET_STACK_BASE
# ifdef NEED_FIND_LIMIT
GC_API int GC_CALL
GC_get_stack_base(struct GC_stack_base *b)
{
IF_CANCEL(int cancel_state;)
/*
* Note: using the `GC_find_limit` variant is risky; in the IA-64 case,
* e.g., there is no guard page between the stack of one thread and the
* register backing store of the next; thus this is likely to identify way
* too large a "stack" and thus at least result in disastrous performance.
*/
/* TODO: Implement better strategies here. */
LOCK();
/* TODO: `DISABLE_CANCEL` may be unnecessary? */
DISABLE_CANCEL(cancel_state);
# ifdef STACK_GROWS_UP
b->mem_base = GC_find_limit(GC_approx_sp(), FALSE);
# else
b->mem_base = GC_find_limit(GC_approx_sp(), TRUE /* `up` */);
# endif
# ifdef IA64
b->reg_base = GC_find_limit(GC_save_regs_in_stack(), FALSE);
# elif defined(E2K)
b->reg_base = NULL;
# endif
RESTORE_CANCEL(cancel_state);
UNLOCK();
return GC_SUCCESS;
}
# else /* !NEED_FIND_LIMIT */
GC_API int GC_CALL
GC_get_stack_base(struct GC_stack_base *b)
{
# if defined(GET_MAIN_STACKBASE_SPECIAL) && !defined(THREADS) \
&& !defined(IA64)
b->mem_base = GC_get_main_stack_base();
return GC_SUCCESS;
# else
UNUSED_ARG(b);
return GC_UNIMPLEMENTED;
# endif
}
# endif
#endif /* !HAVE_GET_STACK_BASE */
#ifndef GET_MAIN_STACKBASE_SPECIAL
GC_INNER ptr_t
GC_get_main_stack_base(void)
{
/* Default implementation. */
struct GC_stack_base sb;
if (GC_get_stack_base(&sb) != GC_SUCCESS)
ABORT("GC_get_stack_base failed");
GC_ASSERT(HOTTER_THAN(GC_approx_sp(), (ptr_t)sb.mem_base));
return (ptr_t)sb.mem_base;
}
#endif /* !GET_MAIN_STACKBASE_SPECIAL */
/*
* Register static data segment(s) as roots. If more data segments are
* added later, then they need to be registered at that point (as we do
* with SunOS dynamic loading), or `GC_mark_roots` needs to check for them.
*/
#ifdef ANY_MSWIN
# if defined(GWW_VDB)
# ifndef MEM_WRITE_WATCH
# define MEM_WRITE_WATCH 0x200000
# endif
# ifndef WRITE_WATCH_FLAG_RESET
# define WRITE_WATCH_FLAG_RESET 1
# endif
/*
* Since we cannot easily check whether `ULONG_PTR` and `SIZE_T` are
* defined in Win32 `basetsd.h` file, we define own `ULONG_PTR`.
*/
# define GC_ULONG_PTR word
typedef UINT(WINAPI *GetWriteWatch_type)(DWORD, PVOID,
GC_ULONG_PTR /* `SIZE_T` */, PVOID *,
GC_ULONG_PTR *, PULONG);
static FARPROC GetWriteWatch_func;
static DWORD GetWriteWatch_alloc_flag;
# define GC_GWW_AVAILABLE() (GetWriteWatch_func != 0)
static void
detect_GetWriteWatch(void)
{
static GC_bool done;
HMODULE hK32;
if (done)
return;
# if defined(MPROTECT_VDB)
{
char *str = GETENV("GC_USE_GETWRITEWATCH");
# if defined(GC_PREFER_MPROTECT_VDB)
if (NULL == str || (*str == '0' && *(str + 1) == '\0')) {
/*
* `GC_USE_GETWRITEWATCH` environment variable is unset or set to "0".
* Falling back to `MPROTECT_VDB` strategy.
*/
done = TRUE;
/* This should work as if `GWW_VDB` macro is not defined. */
return;
}
# else
if (str != NULL && *str == '0' && *(str + 1) == '\0') {
/*
* `GC_USE_GETWRITEWATCH` environment variable is set "0".
* Falling back to `MPROTECT_VDB` strategy.
*/
done = TRUE;
return;
}
# endif
}
# endif
# if defined(MSWINRT_FLAVOR) && defined(FUNCPTR_IS_DATAPTR)
{
MEMORY_BASIC_INFORMATION memInfo;
SIZE_T result = VirtualQuery(CAST_THRU_UINTPTR(void *, GetProcAddress),
&memInfo, sizeof(memInfo));
if (result != sizeof(memInfo))
ABORT("Weird VirtualQuery result");
hK32 = (HMODULE)memInfo.AllocationBase;
}
# else
hK32 = GetModuleHandle(TEXT("kernel32.dll"));
# endif
if (hK32 != (HMODULE)0
&& (GetWriteWatch_func = GetProcAddress(hK32, "GetWriteWatch")) != 0) {
void *page;
GC_ASSERT(GC_page_size != 0);
/*
* Also check whether `VirtualAlloc()` accepts `MEM_WRITE_WATCH`,
* as some versions of `kernel32.dll` library have one but not the other,
* making the feature completely broken.
*/
page = VirtualAlloc(NULL, GC_page_size, MEM_WRITE_WATCH | MEM_RESERVE,
PAGE_READWRITE);
if (page != NULL) {
PVOID pages[16];
GC_ULONG_PTR count = sizeof(pages) / sizeof(PVOID);
DWORD page_size;
/*
* Check that it actually works. In spite of some documentation
* it actually seems to exist on Win2K.
* This test may be unnecessary, but...
*/
if ((*(GetWriteWatch_type)(GC_funcptr_uint)GetWriteWatch_func)(
WRITE_WATCH_FLAG_RESET, page, GC_page_size, pages, &count,
&page_size)
!= 0) {
/* `GetWriteWatch()` always fails. */
GetWriteWatch_func = 0;
} else {
GetWriteWatch_alloc_flag = MEM_WRITE_WATCH;
}
VirtualFree(page, 0 /* `dwSize` */, MEM_RELEASE);
} else {
/* `GetWriteWatch` will be useless. */
GetWriteWatch_func = 0;
}
}
done = TRUE;
}
# else
# define GetWriteWatch_alloc_flag 0
# endif /* !GWW_VDB */
# ifdef MSWIN32
/*
* Unfortunately, we have to handle win32s very differently from Windows NT,
* since `VirtualQuery()` has very different semantics. In particular,
* under win32s a `VirtualQuery()` call on an unmapped page returns
* an invalid result. Under Windows NT, `GC_register_data_segments()` is
* a no-op and all real work is done by `GC_register_dynamic_libraries()`.
* Under win32s, we cannot find the data segments associated with DLL files.
* We register the main data segment here.
*/
GC_INNER GC_bool GC_no_win32_dlls = FALSE;
GC_INNER GC_bool GC_wnt = FALSE;
GC_INNER void
GC_init_win32(void)
{
# if defined(_WIN64) || (defined(_MSC_VER) && _MSC_VER >= 1800)
/*
* MS Visual Studio 2013 deprecates `GetVersion`, but on the other hand
* it cannot be used to target pre-Win2K.
*/
GC_wnt = TRUE;
# else
/*
* Set `GC_wnt`. If we are running under win32s, assume that no DLL file
* will be loaded. I doubt anyone still runs win32s, but...
*/
DWORD v = GetVersion();
GC_wnt = !(v & (DWORD)0x80000000UL);
GC_no_win32_dlls |= ((!GC_wnt) && (v & 0xff) <= 3);
# endif
# ifdef USE_MUNMAP
if (GC_no_win32_dlls) {
/*
* Turn off unmapping for safety (since may not work well with
* `GlobalAlloc()`).
*/
GC_unmap_threshold = 0;
}
# endif
}
/*
* Return the smallest address `p` such that `VirtualQuery()` returns
* correct results for all addresses between `p` and `start`.
* Assumes `VirtualQuery()` returns correct information for `start`.
*/
STATIC ptr_t
GC_least_described_address(ptr_t start)
{
ptr_t limit = (ptr_t)GC_sysinfo.lpMinimumApplicationAddress;
ptr_t p = PTR_ALIGN_DOWN(start, GC_page_size);
GC_ASSERT(GC_page_size != 0);
for (;;) {
MEMORY_BASIC_INFORMATION buf;
size_t result;
ptr_t q;
if (UNLIKELY(ADDR(p) <= (word)GC_page_size))
break; /*< avoid underflow */
q = p - GC_page_size;
if (ADDR_LT(q, limit))
break;
result = VirtualQuery((LPVOID)q, &buf, sizeof(buf));
if (result != sizeof(buf) || 0 == buf.AllocationBase)
break;
p = (ptr_t)buf.AllocationBase;
}
return p;
}
STATIC void
GC_register_root_section(ptr_t static_root)
{
ptr_t p, base, limit;
GC_ASSERT(I_HOLD_LOCK());
if (!GC_no_win32_dlls)
return;
p = GC_least_described_address(static_root);
base = limit = p;
while (ADDR_LT(p, (ptr_t)GC_sysinfo.lpMaximumApplicationAddress)) {
MEMORY_BASIC_INFORMATION buf;
size_t result = VirtualQuery((LPVOID)p, &buf, sizeof(buf));
if (result != sizeof(buf) || 0 == buf.AllocationBase
|| GC_is_heap_base(buf.AllocationBase))
break;
if (ADDR(p) > GC_WORD_MAX - buf.RegionSize) {
/* Avoid overflow. */
break;
}
if (buf.State == MEM_COMMIT && is_writable(buf.Protect)) {
if (p != limit) {
if (base != limit)
GC_add_roots_inner(base, limit, FALSE);
base = p;
}
limit = p + buf.RegionSize;
}
p += buf.RegionSize;
}
if (base != limit)
GC_add_roots_inner(base, limit, FALSE);
}
# endif /* MSWIN32 */
# if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
/*
* We maintain a linked list of `AllocationBase` values (that we know)
* correspond to `malloc` heap sections. Currently this is only called
* during a collection. But there is some hope that for long-running
* programs we will eventually see most heap sections.
*
* In the long run, it would be more reliable to occasionally walk
* the `malloc` heap with `HeapWalk()` on the default heap.
* But that apparently works only for NT-based Windows.
*/
/* Note: initialized to approximate largest root size. */
# define GC_INITIAL_MAX_ROOT_SIZE 100000
STATIC size_t GC_max_root_size = GC_INITIAL_MAX_ROOT_SIZE;
/* In the long run, a better data structure would also be nice... */
STATIC struct GC_malloc_heap_list {
void *allocation_base;
struct GC_malloc_heap_list *next;
} *GC_malloc_heap_l = NULL;
/*
* Is `p` the base of one of the `malloc` heap sections we already
* know about?
*/
STATIC GC_bool
GC_is_malloc_heap_base(const void *p)
{
struct GC_malloc_heap_list *q;
for (q = GC_malloc_heap_l; q != NULL; q = q->next) {
if (q->allocation_base == p)
return TRUE;
}
return FALSE;
}
STATIC void *
GC_get_allocation_base(void *p)
{
MEMORY_BASIC_INFORMATION buf;
size_t result = VirtualQuery(p, &buf, sizeof(buf));
if (result != sizeof(buf)) {
ABORT("Weird VirtualQuery result");
}
return buf.AllocationBase;
}
GC_INNER void
GC_add_current_malloc_heap(void)
{
struct GC_malloc_heap_list *new_l = (struct GC_malloc_heap_list *)malloc(
sizeof(struct GC_malloc_heap_list));
void *candidate;
if (NULL == new_l)
return;
/* Explicitly set to suppress "maybe-uninitialized" gcc warning. */
new_l->allocation_base = NULL;
candidate = GC_get_allocation_base(new_l);
if (GC_is_malloc_heap_base(candidate)) {
/* Try a little harder to find `malloc` heap. */
size_t req_size = 10000;
do {
void *p = malloc(req_size);
if (NULL == p) {
free(new_l);
return;
}
candidate = GC_get_allocation_base(p);
free(p);
req_size *= 2;
} while (GC_is_malloc_heap_base(candidate)
&& req_size < GC_max_root_size / 10 && req_size < 500000);
if (GC_is_malloc_heap_base(candidate)) {
free(new_l);
return;
}
}
GC_COND_LOG_PRINTF("Found new system malloc AllocationBase at %p\n",
candidate);
new_l->allocation_base = candidate;
new_l->next = GC_malloc_heap_l;
GC_malloc_heap_l = new_l;
}
# ifndef GC_NO_DEINIT
/*
* Free all the linked list nodes. Could be invoked at process exit
* to avoid memory leak complains of a dynamic code analysis tool.
*/
STATIC void
GC_free_malloc_heap_list(void)
{
struct GC_malloc_heap_list *q = GC_malloc_heap_l;
GC_malloc_heap_l = NULL;
GC_max_root_size = GC_INITIAL_MAX_ROOT_SIZE;
while (q != NULL) {
struct GC_malloc_heap_list *next = q->next;
free(q);
q = next;
}
}
# endif
# endif /* USE_WINALLOC && !REDIRECT_MALLOC */
GC_INNER GC_bool
GC_is_heap_base(const void *p)
{
size_t i;
# if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
if (GC_root_size > GC_max_root_size)
GC_max_root_size = GC_root_size;
if (GC_is_malloc_heap_base(p))
return TRUE;
# endif
for (i = 0; i < GC_n_heap_bases; i++) {
if (GC_heap_bases[i] == p)
return TRUE;
}
return FALSE;
}
GC_INNER void
GC_register_data_segments(void)
{
# ifdef MSWIN32
/* Note: any other GC global variable would fit too. */
GC_register_root_section((ptr_t)&GC_pages_executable);
# endif
}
#endif /* ANY_MSWIN */
#ifdef DATASTART_USES_XGETDATASTART
# ifdef CHERI_PURECAP
# include <link.h>
/*
* The CheriBSD LLVM compiler declares `etext`, `edata` and `end` as
* typeless variables. If the collector library is statically linked
* with the executable, these capabilities are compiled with the
* read-only permissions and bounds that span the `.data` and `.bss`
* sections. If the collector is compiled as a shared library, these
* symbols are compiled with zero bounds and cannot be dereferenced;
* instead, the read-only capability returned by the loader is used.
*/
struct scan_bounds_s {
word start_addr;
word end_addr;
ptr_t ld_cap;
};
static int
ld_cap_search(struct dl_phdr_info *info, size_t size, void *cd)
{
struct scan_bounds_s *region = (struct scan_bounds_s *)cd;
ptr_t load_ptr = (ptr_t)info->dlpi_addr;
UNUSED_ARG(size);
if (!SPANNING_CAPABILITY(load_ptr, region->start_addr, region->end_addr))
return 0;
region->ld_cap = (ptr_t)cheri_bounds_set(
cheri_address_set(load_ptr, region->start_addr),
region->end_addr - region->start_addr);
return 1; /*< stop */
}
static ptr_t
derive_cap_from_ldr(ptr_t range_start, ptr_t range_end)
{
word scan_start = ADDR(range_start);
word scan_end = ADDR(range_end);
struct scan_bounds_s region;
/* If symbols already span the required range, return one of them. */
if (SPANNING_CAPABILITY(range_start, scan_start, scan_end))
return range_start;
if (SPANNING_CAPABILITY(range_end, scan_start, scan_end))
return range_end;
/*
* Fall-back option: derive `.data` plus `.bss` end pointer from the
* read-only capability provided by loader.
*/
region.start_addr = scan_start;
region.end_addr = scan_end;
region.ld_cap = NULL; /*< prevent a compiler warning */
if (!dl_iterate_phdr(ld_cap_search, &region))
ABORT("Cannot find static roots for capability system");
GC_ASSERT(region.ld_cap != NULL);
return region.ld_cap;
}
# endif /* CHERI_PURECAP */
GC_INNER ptr_t
GC_SysVGetDataStart(size_t max_page_size, ptr_t etext_ptr)
{
volatile ptr_t result;
GC_ASSERT(max_page_size % ALIGNMENT == 0);
result = PTR_ALIGN_UP(etext_ptr, ALIGNMENT);
# ifdef CHERI_PURECAP
result = derive_cap_from_ldr(result, DATAEND);
# endif
GC_setup_temporary_fault_handler();
if (SETJMP(GC_jmp_buf) == 0) {
/*
* Note that this is not equivalent to just adding `max_page_size` to
* `etext_ptr` because the latter is not guaranteed to be multiple of
* the page size.
*/
ptr_t next_page = PTR_ALIGN_UP(result, max_page_size);
# ifdef FREEBSD
/*
* It is unclear whether this should be identical to the below, or
* whether it should apply to non-x86 architectures. For now we
* do not assume that there is always an empty page after `etext`.
* But in some cases there actually seems to be slightly more.
* It also deals with holes between read-only and writable data.
*
* Try reading at the address. This should happen before there is
* another thread.
*/
for (; ADDR_LT(next_page, DATAEND); next_page += max_page_size) {
GC_noop1((word)(*(volatile unsigned char *)next_page));
}
# else
result = next_page + (ADDR(result) & ((word)max_page_size - 1));
/* Try writing to the address. */
{
# ifdef AO_HAVE_fetch_and_add
volatile AO_t zero = 0;
(void)AO_fetch_and_add((volatile AO_t *)result, zero);
# else
/* Fall back to non-atomic fetch-and-store. */
char v = *result;
# ifdef CPPCHECK
GC_noop1_ptr(&v);
# endif
*result = v;
# endif
}
# endif
GC_reset_fault_handler();
} else {
GC_reset_fault_handler();
/*
* We got here via a `longjmp`. The address is not readable.
* This is known to happen under Solaris 2.4 with gcc, which places
* string constants in the `text` segment, but after `etext`.
* Use plan B. Note that we now know there is a gap between `text`
* and `data` segments, so plan A brought us something.
*/
# ifdef CHERI_PURECAP
result = (ptr_t)GC_find_limit(cheri_address_set(result, ADDR(DATAEND)),
FALSE);
# else
result = (ptr_t)GC_find_limit(DATAEND, FALSE);
# endif
}
return (ptr_t)CAST_AWAY_VOLATILE_PVOID(result);
}
#endif /* DATASTART_USES_XGETDATASTART */
#if defined(OS2)
GC_INNER void
GC_register_data_segments(void)
{
PTIB ptib;
PPIB ppib;
HMODULE module_handle;
# define PBUFSIZ 512
UCHAR path[PBUFSIZ];
FILE *myexefile;
struct exe_hdr hdrdos; /*< MSDOS header */
struct e32_exe hdr386; /*< real header for my executable */
struct o32_obj seg; /*< current segment */
int nsegs;
# if defined(CPPCHECK)
hdrdos.padding[0] = 0; /*< to prevent "field unused" warnings */
hdr386.exe_format_level = 0;
hdr386.os = 0;
hdr386.padding1[0] = 0;
hdr386.padding2[0] = 0;
seg.pagemap = 0;
seg.mapsize = 0;
seg.reserved = 0;
# endif
if (DosGetInfoBlocks(&ptib, &ppib) != NO_ERROR) {
ABORT("DosGetInfoBlocks failed");
}
module_handle = ppib->pib_hmte;
if (DosQueryModuleName(module_handle, PBUFSIZ, path) != NO_ERROR) {
ABORT("DosQueryModuleName failed");
}
myexefile = fopen(path, "rb");
if (myexefile == 0) {
ABORT_ARG1("Failed to open executable", ": %s", path);
}
if (fread((char *)&hdrdos, 1, sizeof(hdrdos), myexefile) < sizeof(hdrdos)) {
ABORT_ARG1("Could not read MSDOS header", " from: %s", path);
}
if (E_MAGIC(hdrdos) != EMAGIC) {
ABORT_ARG1("Bad DOS magic number", " in file: %s", path);
}
if (fseek(myexefile, E_LFANEW(hdrdos), SEEK_SET) != 0) {
ABORT_ARG1("Bad DOS magic number", " in file: %s", path);
}
if (fread((char *)&hdr386, 1, sizeof(hdr386), myexefile) < sizeof(hdr386)) {
ABORT_ARG1("Could not read OS/2 header", " from: %s", path);
}
if (E32_MAGIC1(hdr386) != E32MAGIC1 || E32_MAGIC2(hdr386) != E32MAGIC2) {
ABORT_ARG1("Bad OS/2 magic number", " in file: %s", path);
}
if (E32_BORDER(hdr386) != E32LEBO || E32_WORDER(hdr386) != E32LEWO) {
ABORT_ARG1("Bad byte order in executable", " file: %s", path);
}
if (E32_CPU(hdr386) == E32CPU286) {
ABORT_ARG1("GC cannot handle 80286 executables", ": %s", path);
}
if (fseek(myexefile, E_LFANEW(hdrdos) + E32_OBJTAB(hdr386), SEEK_SET) != 0) {
ABORT_ARG1("Seek to object table failed", " in file: %s", path);
}
for (nsegs = E32_OBJCNT(hdr386); nsegs > 0; nsegs--) {
int flags;
if (fread((char *)&seg, 1, sizeof(seg), myexefile) < sizeof(seg)) {
ABORT_ARG1("Could not read obj table entry", " from file: %s", path);
}
flags = O32_FLAGS(seg);
if (!(flags & OBJWRITE))
continue;
if (!(flags & OBJREAD))
continue;
if (flags & OBJINVALID) {
GC_err_printf("Object with invalid pages?\n");
continue;
}
GC_add_roots_inner((ptr_t)O32_BASE(seg),
(ptr_t)(O32_BASE(seg) + O32_SIZE(seg)), FALSE);
}
(void)fclose(myexefile);
}
#elif defined(OPENBSD)
GC_INNER void
GC_register_data_segments(void)
{
/*
* Depending on arch alignment, there can be multiple holes between
* `DATASTART` and `DATAEND`. Scan in `DATASTART` .. `DATAEND` and
* register each region.
*/
ptr_t region_start = DATASTART;
GC_ASSERT(I_HOLD_LOCK());
if (ADDR(region_start) - 1U >= ADDR(DATAEND))
ABORT_ARG2("Wrong DATASTART/END pair", ": %p .. %p", (void *)region_start,
(void *)DATAEND);
for (;;) {
ptr_t region_end = GC_find_limit_with_bound(region_start, TRUE, DATAEND);
GC_add_roots_inner(region_start, region_end, FALSE);
if (ADDR_GE(region_end, DATAEND))
break;
region_start = GC_skip_hole_openbsd(region_end, DATAEND);
}
}
#elif !defined(ANY_MSWIN)
GC_INNER void
GC_register_data_segments(void)
{
GC_ASSERT(I_HOLD_LOCK());
# if !defined(DYNAMIC_LOADING) && defined(GC_DONT_REGISTER_MAIN_STATIC_DATA)
/*
* Avoid even referencing `DATASTART` and `DATAEND` as they are
* unnecessary and cause linker errors when bitcode is enabled.
* `GC_register_data_segments` is not called anyway.
*/
# elif defined(DYNAMIC_LOADING) && (defined(DARWIN) || defined(HAIKU))
/* No-op. `GC_register_main_static_data()` always returns `FALSE`. */
# elif defined(REDIRECT_MALLOC) && defined(SOLARIS) && defined(THREADS)
/*
* As of Solaris 2.3, the Solaris threads implementation allocates
* the data structure for the initial thread with `sbrk` at the
* process startup. It needs to be scanned, so that we do not lose
* some `malloc`-allocated data structures hanging from it.
* We are on thin ice here...
*/
GC_ASSERT(DATASTART);
{
ptr_t p = (ptr_t)sbrk(0);
if (ADDR_LT(DATASTART, p))
GC_add_roots_inner(DATASTART, p, FALSE);
}
# else
/*
* Note: subtract one is to also check for `NULL` without a compiler
* warning.
*/
if (ADDR(DATASTART) - 1U >= ADDR(DATAEND)) {
ABORT_ARG2("Wrong DATASTART/END pair", ": %p .. %p", (void *)DATASTART,
(void *)DATAEND);
}
GC_add_roots_inner(DATASTART, DATAEND, FALSE);
# ifdef GC_HAVE_DATAREGION2
if (ADDR(DATASTART2) - 1U >= ADDR(DATAEND2))
ABORT_ARG2("Wrong DATASTART/END2 pair", ": %p .. %p", (void *)DATASTART2,
(void *)DATAEND2);
GC_add_roots_inner(DATASTART2, DATAEND2, FALSE);
# endif
# endif
/*
* Dynamic libraries are added at every collection, since they
* may change.
*/
}
#endif /* !ANY_MSWIN && !OPENBSD && !OS2 */
/* Auxiliary routines for obtaining memory from OS. */
#ifdef NEED_UNIX_GET_MEM
# define SBRK_ARG_T ptrdiff_t
# if defined(MMAP_SUPPORTED)
# ifdef USE_MMAP_FIXED
/*
* Seems to yield better performance on Solaris 2, but can be unreliable
* if something is already mapped at the address.
*/
# define GC_MMAP_FLAGS MAP_FIXED | MAP_PRIVATE
# else
# define GC_MMAP_FLAGS MAP_PRIVATE
# endif
# ifdef USE_MMAP_ANON
# define zero_fd -1
# if defined(MAP_ANONYMOUS) && !defined(CPPCHECK)
# define OPT_MAP_ANON MAP_ANONYMOUS
# else
# define OPT_MAP_ANON MAP_ANON
# endif
# else
static int zero_fd = -1;
# define OPT_MAP_ANON 0
# endif
# ifndef MSWIN_XBOX1
# if defined(SYMBIAN) && !defined(USE_MMAP_ANON)
EXTERN_C_BEGIN
extern char *GC_get_private_path_and_zero_file(void);
EXTERN_C_END
# endif
STATIC void *
GC_unix_mmap_get_mem(size_t bytes)
{
void *result;
static word last_addr = HEAP_START;
# ifndef USE_MMAP_ANON
static GC_bool initialized = FALSE;
if (UNLIKELY(!initialized)) {
# ifdef SYMBIAN
char *path = GC_get_private_path_and_zero_file();
if (path != NULL) {
zero_fd = open(path, O_RDWR | O_CREAT, 0644);
free(path);
}
# else
zero_fd = open("/dev/zero", O_RDONLY);
# endif
if (zero_fd == -1)
ABORT("Could not open /dev/zero");
if (fcntl(zero_fd, F_SETFD, FD_CLOEXEC) == -1)
WARN("Could not set FD_CLOEXEC for /dev/zero\n", 0);
initialized = TRUE;
}
# endif
GC_ASSERT(GC_page_size != 0);
if (bytes & (GC_page_size - 1))
ABORT("Bad GET_MEM arg");
/*
* Note: it is essential for CHERI to have only address part in
* `last_addr` without metadata (thus the variable is of `word` type
* intentionally), otherwise `mmap()` fails setting `errno` to `EPROT`.
*/
result
= mmap(MAKE_CPTR(last_addr), bytes,
(PROT_READ | PROT_WRITE) | (GC_pages_executable ? PROT_EXEC : 0),
GC_MMAP_FLAGS | OPT_MAP_ANON, zero_fd, 0 /* `offset` */);
# undef IGNORE_PAGES_EXECUTABLE
if (UNLIKELY(MAP_FAILED == result)) {
if (HEAP_START == last_addr && GC_pages_executable
&& (EACCES == errno || EPERM == errno))
ABORT("Cannot allocate executable pages");
return NULL;
}
# ifdef LINUX
GC_ASSERT(ADDR(result) <= ~(word)(GC_page_size - 1) - bytes);
/* The following `PTR_ALIGN_UP()` cannot overflow. */
# else
if (UNLIKELY(ADDR(result) > ~(word)(GC_page_size - 1) - bytes)) {
/*
* Oops. We got the end of the address space. This is not usable
* by arbitrary C code, since one-past-end pointers do not work,
* so we discard it and try again. Leave the last page mapped,
* so we cannot repeat.
*/
(void)munmap(result, ~(GC_page_size - 1) - (size_t)ADDR(result));
return GC_unix_mmap_get_mem(bytes);
}
# endif
if ((ADDR(result) % HBLKSIZE) != 0)
ABORT("Memory returned by mmap is not aligned to HBLKSIZE");
last_addr = ADDR(result) + bytes;
GC_ASSERT((last_addr & (GC_page_size - 1)) == 0);
return result;
}
# endif /* !MSWIN_XBOX1 */
# endif /* MMAP_SUPPORTED */
# if defined(USE_MMAP)
GC_INNER void *
GC_unix_get_mem(size_t bytes)
{
return GC_unix_mmap_get_mem(bytes);
}
# else /* !USE_MMAP */
STATIC void *
GC_unix_sbrk_get_mem(size_t bytes)
{
void *result;
# ifdef IRIX5
/*
* Bare `sbrk()` is not thread-safe. Play by `malloc` rules.
* The equivalent may be needed on other systems as well.
*/
__LOCK_MALLOC();
# endif
{
ptr_t cur_brk = (ptr_t)sbrk(0);
SBRK_ARG_T lsbs = ADDR(cur_brk) & (GC_page_size - 1);
GC_ASSERT(GC_page_size != 0);
if (UNLIKELY((SBRK_ARG_T)bytes < 0)) {
/* Value of `bytes` is too big. */
result = NULL;
goto out;
}
if (lsbs != 0) {
if ((ptr_t)sbrk((SBRK_ARG_T)GC_page_size - lsbs) == (ptr_t)(-1)) {
result = NULL;
goto out;
}
}
# ifdef ADD_HEAP_GUARD_PAGES
/*
* This is useful for catching severe memory overwrite problems
* that span heap sections. It should not otherwise be turned on.
*/
{
ptr_t guard = (ptr_t)sbrk((SBRK_ARG_T)GC_page_size);
if (mprotect(guard, GC_page_size, PROT_NONE) != 0)
ABORT("ADD_HEAP_GUARD_PAGES: mprotect failed");
}
# endif
result = sbrk((SBRK_ARG_T)bytes);
if (UNLIKELY(ADDR(result) == GC_WORD_MAX))
result = NULL;
}
out:
# ifdef IRIX5
__UNLOCK_MALLOC();
# endif
return result;
}
GC_INNER void *
GC_unix_get_mem(size_t bytes)
{
# if defined(MMAP_SUPPORTED)
/* By default, we try both `sbrk` and `mmap`, in that order. */
static GC_bool sbrk_failed = FALSE;
void *result = NULL;
if (GC_pages_executable) {
/*
* If the allocated memory should have the execute permission,
* then `sbrk()` cannot be used.
*/
return GC_unix_mmap_get_mem(bytes);
}
if (!sbrk_failed)
result = GC_unix_sbrk_get_mem(bytes);
if (NULL == result) {
sbrk_failed = TRUE;
result = GC_unix_mmap_get_mem(bytes);
if (NULL == result) {
/* Try `sbrk()` again, in case `sbrk` memory became available. */
result = GC_unix_sbrk_get_mem(bytes);
}
}
return result;
# else /* !MMAP_SUPPORTED */
return GC_unix_sbrk_get_mem(bytes);
# endif
}
# endif /* !USE_MMAP */
#endif /* NEED_UNIX_GET_MEM */
#if defined(OS2)
GC_INNER void *
GC_get_mem(size_t bytes)
{
void *result = NULL;
int retry;
GC_ASSERT(GC_page_size != 0);
bytes = SIZET_SAT_ADD(bytes, GC_page_size);
for (retry = 0;; retry++) {
if (DosAllocMem(&result, bytes,
(PAG_READ | PAG_WRITE | PAG_COMMIT)
| (GC_pages_executable ? PAG_EXECUTE : 0))
== NO_ERROR
&& LIKELY(result != NULL))
break;
/*
* TODO: Unclear the purpose of the retry. (Probably, if `DosAllocMem`
* returns memory at address zero, then just retry once.)
*/
if (retry >= 1)
return NULL;
}
return HBLKPTR((ptr_t)result + GC_page_size - 1);
}
#elif defined(MSWIN_XBOX1)
GC_INNER void *
GC_get_mem(size_t bytes)
{
if (UNLIKELY(0 == bytes))
return NULL;
return VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_TOP_DOWN, PAGE_READWRITE);
}
#elif defined(MSWINCE)
GC_INNER void *
GC_get_mem(size_t bytes)
{
void *result = NULL; /*< initialized to prevent a compiler warning */
size_t i;
GC_ASSERT(GC_page_size != 0);
bytes = ROUNDUP_PAGESIZE(bytes);
/* Try to find reserved, uncommitted pages. */
for (i = 0; i < GC_n_heap_bases; i++) {
if (((word)(-(GC_signed_word)GC_heap_lengths[i])
& (GC_sysinfo.dwAllocationGranularity - 1))
>= bytes) {
result = GC_heap_bases[i] + GC_heap_lengths[i];
break;
}
}
if (i == GC_n_heap_bases) {
/* Reserve more pages. */
size_t res_bytes
= SIZET_SAT_ADD(bytes, (size_t)GC_sysinfo.dwAllocationGranularity - 1)
& ~((size_t)GC_sysinfo.dwAllocationGranularity - 1);
/*
* If we ever support `MPROTECT_VDB` here, we will probably need
* to ensure that `res_bytes` is greater (strictly) than `bytes`,
* so that `VirtualProtect()` never spans regions. It seems to be
* fine for a `VirtualFree()` argument to span regions, so we
* should be OK for now.
*/
result = VirtualAlloc(NULL, res_bytes, MEM_RESERVE | MEM_TOP_DOWN,
GC_pages_executable ? PAGE_EXECUTE_READWRITE
: PAGE_READWRITE);
if (HBLKDISPL(result) != 0) {
/*
* If I read the documentation correctly, this can only happen
* if `HBLKSIZE` is greater than 64 KB or not a power of 2.
*/
ABORT("Bad VirtualAlloc result");
}
if (GC_n_heap_bases >= MAX_HEAP_SECTS)
ABORT("Too many heap sections");
if (UNLIKELY(NULL == result))
return NULL;
GC_heap_bases[GC_n_heap_bases] = (ptr_t)result;
GC_heap_lengths[GC_n_heap_bases] = 0;
GC_n_heap_bases++;
}
/* Commit pages. */
result = VirtualAlloc(result, bytes, MEM_COMMIT,
GC_pages_executable ? PAGE_EXECUTE_READWRITE
: PAGE_READWRITE);
# undef IGNORE_PAGES_EXECUTABLE
if (HBLKDISPL(result) != 0)
ABORT("Bad VirtualAlloc result");
if (LIKELY(result != NULL))
GC_heap_lengths[i] += bytes;
return result;
}
#elif defined(CYGWIN32) || defined(MSWIN32)
# ifdef USE_GLOBAL_ALLOC
# define GLOBAL_ALLOC_TEST 1
# else
# define GLOBAL_ALLOC_TEST GC_no_win32_dlls
# endif
# if (defined(GC_USE_MEM_TOP_DOWN) && defined(USE_WINALLOC)) \
|| defined(CPPCHECK)
/*
* Use `GC_USE_MEM_TOP_DOWN` for better 64-bit testing.
* Otherwise all addresses tend to end up in the first 4 GB, hiding bugs.
*/
DWORD GC_mem_top_down = MEM_TOP_DOWN;
# else
# define GC_mem_top_down 0
# endif /* !GC_USE_MEM_TOP_DOWN */
GC_INNER void *
GC_get_mem(size_t bytes)
{
void *result;
# ifndef USE_WINALLOC
result = GC_unix_get_mem(bytes);
# else
# if defined(MSWIN32) && !defined(MSWINRT_FLAVOR)
if (GLOBAL_ALLOC_TEST) {
/*
* `VirtualAlloc()` does not like `PAGE_EXECUTE_READWRITE`.
* There are also unconfirmed rumors of other problems, so we
* dodge the issue.
*/
result = GlobalAlloc(0, SIZET_SAT_ADD(bytes, HBLKSIZE));
/* Align it at `HBLKSIZE` boundary (`NULL` value remains unchanged). */
result = PTR_ALIGN_UP((ptr_t)result, HBLKSIZE);
} else
# endif
/* else */ {
/*
* `VirtualProtect()` only works on regions returned by a single
* `VirtualAlloc()` call. Thus we allocate one extra page, which will
* prevent merging of blocks in separate regions, and eliminate any
* temptation to call `VirtualProtect()` on a range spanning regions.
* This wastes a small amount of memory, and risks increased
* fragmentation. But better alternatives would require effort.
*/
# ifdef MPROTECT_VDB
/*
* We cannot check for `GC_incremental` here (because
* `GC_enable_incremental()` might be called some time later after
* the collector initialization).
*/
# ifdef GWW_VDB
# define VIRTUAL_ALLOC_PAD (GC_GWW_AVAILABLE() ? 0 : 1)
# else
# define VIRTUAL_ALLOC_PAD 1
# endif
# else
# define VIRTUAL_ALLOC_PAD 0
# endif
/*
* Pass `MEM_WRITE_WATCH` only if `GetWriteWatch`-based VDB is
* enabled and `GetWriteWatch()` is available. Otherwise we waste
* resources or possibly cause `VirtualAlloc()` to fail (observed
* in Windows 2000 SP2).
*/
result = VirtualAlloc(
NULL, SIZET_SAT_ADD(bytes, VIRTUAL_ALLOC_PAD),
MEM_COMMIT | MEM_RESERVE | GetWriteWatch_alloc_flag | GC_mem_top_down,
GC_pages_executable ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE);
# undef IGNORE_PAGES_EXECUTABLE
}
# endif
if (HBLKDISPL(result) != 0)
ABORT("Bad VirtualAlloc result");
if (GC_n_heap_bases >= MAX_HEAP_SECTS)
ABORT("Too many heap sections");
if (LIKELY(result != NULL))
GC_heap_bases[GC_n_heap_bases++] = (ptr_t)result;
return result;
}
#endif /* CYGWIN32 || MSWIN32 */
#if (defined(ANY_MSWIN) || defined(MSWIN_XBOX1)) && !defined(GC_NO_DEINIT)
GC_API void GC_CALL
GC_win32_free_heap(void)
{
# if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
GC_free_malloc_heap_list();
# endif
# if defined(CYGWIN32) || defined(MSWIN32)
# ifndef MSWINRT_FLAVOR
# ifdef MSWIN32
if (GLOBAL_ALLOC_TEST)
# endif
{
while (GC_n_heap_bases > 0) {
GC_n_heap_bases--;
# ifdef CYGWIN32
/* FIXME: Is it OK to use non-GC `free()` here? */
# else
GlobalFree(GC_heap_bases[GC_n_heap_bases]);
# endif
GC_heap_bases[GC_n_heap_bases] = 0;
}
return;
}
# endif /* !MSWINRT_FLAVOR */
# ifndef CYGWIN32
/* Avoiding `VirtualAlloc` leak. */
while (GC_n_heap_bases > 0) {
VirtualFree(GC_heap_bases[--GC_n_heap_bases], 0, MEM_RELEASE);
GC_heap_bases[GC_n_heap_bases] = 0;
}
# endif
# endif
}
#endif
#if (defined(USE_MUNMAP) || defined(MPROTECT_VDB)) && !defined(USE_WINALLOC)
# define ABORT_ON_REMAP_FAIL(C_msg_prefix, start_addr, len) \
ABORT_ARG3(C_msg_prefix " failed", " at %p (length %lu), errno= %d", \
(void *)(start_addr), (unsigned long)(len), errno)
#endif
#ifdef USE_MUNMAP
# if !defined(NN_PLATFORM_CTR) && !defined(MSWIN32) && !defined(MSWINCE) \
&& !defined(MSWIN_XBOX1)
# ifdef SN_TARGET_PS3
# include <sys/memory.h>
# else
# include <sys/mman.h>
# endif
# include <sys/stat.h>
# endif
/*
* Compute a page-aligned starting address for the memory unmap
* operation on a block of size `bytes` starting at `start`.
* Return `NULL` if the block is too small to make this feasible.
*/
STATIC ptr_t
GC_unmap_start(ptr_t start, size_t bytes)
{
ptr_t result;
GC_ASSERT(GC_page_size != 0);
result = PTR_ALIGN_UP(start, GC_page_size);
if (ADDR_LT(start + bytes, result + GC_page_size))
return NULL;
return result;
}
/*
* We assume that `GC_remap` is called on exactly the same range as
* the previous call to `GC_unmap`. It is safe to consistently round
* the endpoints in both places.
*/
static void
block_unmap_inner(ptr_t start_addr, size_t len)
{
if (0 == start_addr)
return;
# ifdef USE_WINALLOC
/*
* Under Win32/WinCE we commit (map) and decommit (unmap) memory
* using `VirtualAlloc()` and `VirtualFree()`. These functions
* work on individual allocations of virtual memory, made
* previously using `VirtualAlloc()` with the `MEM_RESERVE` flag.
* The ranges we need to (de)commit may span several of these
* allocations; therefore we use `VirtualQuery()` to check
* allocation lengths, and split up the range as necessary.
*/
while (len != 0) {
MEMORY_BASIC_INFORMATION mem_info;
word free_len;
if (VirtualQuery(start_addr, &mem_info, sizeof(mem_info))
!= sizeof(mem_info))
ABORT("Weird VirtualQuery result");
free_len = (len < mem_info.RegionSize) ? len : mem_info.RegionSize;
if (!VirtualFree(start_addr, free_len, MEM_DECOMMIT))
ABORT("VirtualFree failed");
GC_unmapped_bytes += free_len;
start_addr += free_len;
len -= free_len;
}
# else
if (len != 0) {
# ifdef SN_TARGET_PS3
ps3_free_mem(start_addr, len);
# elif defined(AIX) || defined(COSMO) || defined(CYGWIN32) \
|| defined(HPUX) \
|| (defined(LINUX) && !defined(PREFER_MMAP_PROT_NONE))
/*
* On AIX, `mmap(PROT_NONE)` fails with `ENOMEM` unless the
* environment variable `XPG_SUS_ENV` is set to `ON`.
* On Cygwin, calling `mmap()` with the new protection flags on
* an existing memory map with `MAP_FIXED` is broken.
* However, calling `mprotect()` on the given address range
* with `PROT_NONE` seems to work fine. On Linux, low `RLIMIT_AS`
* value may lead to `mmap()` failure.
*/
# if (defined(COSMO) || defined(LINUX)) \
&& !defined(FORCE_MPROTECT_BEFORE_MADVISE)
/* On Linux, at least, `madvise()` should be sufficient. */
# else
if (mprotect(start_addr, len, PROT_NONE))
ABORT_ON_REMAP_FAIL("unmap: mprotect", start_addr, len);
# endif
# if !defined(CYGWIN32)
/*
* On Linux (and some other platforms probably), `mprotect(PROT_NONE)`
* is just disabling access to the pages but not returning them to OS.
*/
if (madvise(start_addr, len, MADV_DONTNEED) == -1)
ABORT_ON_REMAP_FAIL("unmap: madvise", start_addr, len);
# endif
# else
/*
* We immediately remap it to prevent an intervening `mmap()` from
* accidentally grabbing the same address space.
*/
void *result = mmap(start_addr, len, PROT_NONE,
MAP_PRIVATE | MAP_FIXED | OPT_MAP_ANON, zero_fd,
0 /* `offset` */);
if (UNLIKELY(MAP_FAILED == result))
ABORT_ON_REMAP_FAIL("unmap: mmap", start_addr, len);
if (result != start_addr)
ABORT("unmap: mmap() result differs from start_addr");
# if defined(CPPCHECK) || defined(LINT2)
/* Explicitly store the resource handle to a global variable. */
GC_noop1_ptr(result);
# endif
# endif
GC_unmapped_bytes += len;
}
# endif
}
/* Compute end address for an unmap operation on the indicated block. */
GC_INLINE ptr_t
GC_unmap_end(ptr_t start, size_t bytes)
{
return (ptr_t)HBLK_PAGE_ALIGNED(start + bytes);
}
GC_INNER void
GC_unmap(ptr_t start, size_t bytes)
{
ptr_t start_addr = GC_unmap_start(start, bytes);
ptr_t end_addr = GC_unmap_end(start, bytes);
block_unmap_inner(start_addr, (size_t)(end_addr - start_addr));
}
GC_INNER void
GC_remap(ptr_t start, size_t bytes)
{
ptr_t start_addr = GC_unmap_start(start, bytes);
ptr_t end_addr = GC_unmap_end(start, bytes);
word len = (word)(end_addr - start_addr);
if (0 == start_addr) {
return;
}
/* FIXME: Handle out-of-memory correctly (at least for Win32). */
# ifdef USE_WINALLOC
while (len != 0) {
MEMORY_BASIC_INFORMATION mem_info;
word alloc_len;
ptr_t result;
if (VirtualQuery(start_addr, &mem_info, sizeof(mem_info))
!= sizeof(mem_info))
ABORT("Weird VirtualQuery result");
alloc_len = (len < mem_info.RegionSize) ? len : mem_info.RegionSize;
result = (ptr_t)VirtualAlloc(start_addr, alloc_len, MEM_COMMIT,
GC_pages_executable ? PAGE_EXECUTE_READWRITE
: PAGE_READWRITE);
if (result != start_addr) {
if (GetLastError() == ERROR_NOT_ENOUGH_MEMORY
|| GetLastError() == ERROR_OUTOFMEMORY) {
ABORT("Not enough memory to process remapping");
} else {
ABORT("VirtualAlloc remapping failed");
}
}
# ifdef LINT2
GC_noop1_ptr(result);
# endif
GC_ASSERT(GC_unmapped_bytes >= alloc_len);
GC_unmapped_bytes -= alloc_len;
start_addr += alloc_len;
len -= alloc_len;
}
# undef IGNORE_PAGES_EXECUTABLE
# else
/* It was already remapped with `PROT_NONE`. */
{
# if !defined(SN_TARGET_PS3) && !defined(FORCE_MPROTECT_BEFORE_MADVISE) \
&& (defined(LINUX) && !defined(PREFER_MMAP_PROT_NONE) \
|| defined(COSMO))
/* Nothing to unprotect as `madvise()` is just a hint. */
# elif defined(COSMO) || defined(NACL) || defined(NETBSD)
/*
* NaCl does not expose `mprotect`, but `mmap` should work fine.
* In case of NetBSD, `mprotect` fails (unlike `mmap`) even without
* `PROT_EXEC` if PaX `MPROTECT` feature is enabled.
*/
void *result = mmap(
start_addr, len,
(PROT_READ | PROT_WRITE) | (GC_pages_executable ? PROT_EXEC : 0),
MAP_PRIVATE | MAP_FIXED | OPT_MAP_ANON, zero_fd, 0 /* `offset` */);
if (UNLIKELY(MAP_FAILED == result))
ABORT_ON_REMAP_FAIL("remap: mmap", start_addr, len);
if (result != start_addr)
ABORT("remap: mmap() result differs from start_addr");
# if defined(CPPCHECK) || defined(LINT2)
GC_noop1_ptr(result);
# endif
# undef IGNORE_PAGES_EXECUTABLE
# else
if (mprotect(start_addr, len,
(PROT_READ | PROT_WRITE)
| (GC_pages_executable ? PROT_EXEC : 0)))
ABORT_ON_REMAP_FAIL("remap: mprotect", start_addr, len);
# undef IGNORE_PAGES_EXECUTABLE
# endif /* !NACL */
}
GC_ASSERT(GC_unmapped_bytes >= len);
GC_unmapped_bytes -= len;
# endif
}
GC_INNER void
GC_unmap_gap(ptr_t start1, size_t bytes1, ptr_t start2, size_t bytes2)
{
ptr_t start1_addr = GC_unmap_start(start1, bytes1);
ptr_t end1_addr = GC_unmap_end(start1, bytes1);
ptr_t start2_addr = GC_unmap_start(start2, bytes2);
ptr_t start_addr = end1_addr;
ptr_t end_addr = start2_addr;
GC_ASSERT(start1 + bytes1 == start2);
if (0 == start1_addr)
start_addr = GC_unmap_start(start1, bytes1 + bytes2);
if (0 == start2_addr)
end_addr = GC_unmap_end(start1, bytes1 + bytes2);
block_unmap_inner(start_addr, (size_t)(end_addr - start_addr));
}
#endif /* USE_MUNMAP */
/*
* Routine for pushing any additional roots. In the multi-threaded
* environment, this is also responsible for marking from thread stacks.
*/
#ifndef THREADS
# if defined(EMSCRIPTEN) && defined(EMSCRIPTEN_ASYNCIFY)
# include <emscripten.h>
static void
scan_regs_cb(void *begin, void *finish)
{
GC_push_all_stack((ptr_t)begin, (ptr_t)finish);
}
STATIC void GC_CALLBACK
GC_default_push_other_roots(void)
{
/* Note: this needs `-sASYNCIFY` linker flag. */
emscripten_scan_registers(scan_regs_cb);
}
# else
# define GC_default_push_other_roots 0
# endif
#else /* THREADS */
# if defined(SN_TARGET_PS3)
STATIC void GC_CALLBACK
GC_default_push_other_roots(void)
{
ABORT("GC_default_push_other_roots is not implemented");
}
GC_INNER void
GC_push_thread_structures(void)
{
ABORT("GC_push_thread_structures is not implemented");
}
# else /* GC_PTHREADS, etc. */
STATIC void GC_CALLBACK
GC_default_push_other_roots(void)
{
GC_push_all_stacks();
}
# endif
#endif /* THREADS */
GC_push_other_roots_proc GC_push_other_roots = GC_default_push_other_roots;
GC_API void GC_CALL
GC_set_push_other_roots(GC_push_other_roots_proc fn)
{
GC_push_other_roots = fn;
}
GC_API GC_push_other_roots_proc GC_CALL
GC_get_push_other_roots(void)
{
return GC_push_other_roots;
}
#if defined(SOFT_VDB) && !defined(NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK) \
|| (defined(GLIBC_2_19_TSX_BUG) && defined(GC_PTHREADS_PARAMARK))
GC_INNER int
GC_parse_version(int *pminor, const char *pverstr)
{
char *endp;
unsigned long value = strtoul(pverstr, &endp, 10);
int major = (int)value;
if (major < 0 || (char *)pverstr == endp || (unsigned)major != value) {
/* Parse error. */
return -1;
}
if (*endp != '.') {
/* No minor part. */
*pminor = -1;
} else {
value = strtoul(endp + 1, &endp, 10);
*pminor = (int)value;
if (*pminor < 0 || (unsigned)(*pminor) != value) {
return -1;
}
}
return major;
}
#endif
/*
* Routines for accessing dirty bits on virtual pages. There are 6 ways to
* maintain this information, as of now:
*
* - `DEFAULT_VDB`: A simple dummy implementation that treats every page
* as possibly dirty. This makes incremental collection useless, but
* the implementation is still correct.
*
* - `MANUAL_VDB`: Stacks and static data are always considered dirty.
* Heap pages are considered dirty if `GC_dirty(p)` has been called on
* some `p` pointing to somewhere inside an object on that page.
* A `GC_dirty()` call on a large object directly dirties only a single
* page, but for the manual VDB we are careful to treat an object with
* a dirty page as completely dirty. In order to avoid races, an object
* must be marked dirty after it is written, and a reference to the
* object must be kept on a stack or in a register in the interim.
* With threads enabled, an object directly reachable from the stack at
* the time of a collection is treated as dirty. In the single-threaded
* mode, it suffices to ensure that no collection can take place between
* the pointer assignment and the `GC_dirty()` call.
*
* - `PROC_VDB`: Use the `/proc` facility for reading dirty bits.
* Only works under some SVR4 variants. Even then, it may be too slow
* to be entirely satisfactory. Requires reading dirty bits for entire
* address space. Implementations tend to assume that the client is
* a (slow) debugger.
*
* - `SOFT_VDB`: Use the `/proc` facility for reading soft-dirty PTEs
* (page table entries). Works on Linux 3.18+ if the kernel is
* properly configured. The proposed implementation iterates over
* `GC_heap_sects` and `GC_static_roots` examining the soft-dirty bit
* of the `word` elements in `/proc/self/pagemap` file corresponding to
* the pages of the sections; finally all soft-dirty bits of the process
* are cleared (by writing some special value to `/proc/self/clear_refs`
* file). In case the soft-dirty bit is not supported by the kernel,
* `MPROTECT_VDB` may be defined as a fall back strategy.
*
* - `MPROTECT_VDB`: Protect pages and then catch the faults to keep
* track of dirtied pages. The implementation (and implementability)
* is highly system-dependent. This usually fails when system calls
* write to a protected page. We prevent the `read` system call from
* doing so. It is the clients responsibility to make sure that other
* system calls are similarly protected or write only to the stack.
*
* - `GWW_VDB`: Use the Win32 `GetWriteWatch` function, if available, to
* read dirty bits. In case it is not available (because we are
* running on Windows 95, Windows 2000 or earlier), `MPROTECT_VDB` may
* be defined as a fall back strategy.
*/
#if (defined(CHECKSUMS) && defined(GWW_VDB)) || defined(PROC_VDB)
/* Add all pages in `pht2` to `pht1`. */
STATIC void
GC_or_pages(page_hash_table pht1, const word *pht2)
{
size_t i;
for (i = 0; i < PHT_SIZE; i++)
pht1[i] |= pht2[i];
}
#endif /* CHECKSUMS && GWW_VDB || PROC_VDB */
#ifdef GWW_VDB
/*
* Note: this is still susceptible to overflow, if there are very large
* allocations, and everything is dirty.
*/
# define GC_GWW_BUF_LEN (MAXHINCR * HBLKSIZE / 4096 /* x86 page size */)
static PVOID gww_buf[GC_GWW_BUF_LEN];
# ifndef MPROTECT_VDB
# define GC_gww_dirty_init GC_dirty_init
# endif
GC_INNER GC_bool
GC_gww_dirty_init(void)
{
/* No assumption about the allocator lock. */
detect_GetWriteWatch();
return GC_GWW_AVAILABLE();
}
GC_INLINE void
GC_gww_read_dirty(GC_bool output_unneeded)
{
size_t i;
GC_ASSERT(I_HOLD_LOCK());
if (!output_unneeded)
BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
for (i = 0; i < GC_n_heap_sects; ++i) {
GC_ULONG_PTR count;
do {
PVOID *pages = gww_buf;
DWORD page_size;
count = GC_GWW_BUF_LEN;
/*
* `GetWriteWatch()` is documented as returning nonzero when
* it fails, but the documentation does not explicitly say why
* it would fail or what its behavior will be if it fails.
* It does appear to fail, at least on recent Win2K instances,
* if the underlying memory was not allocated with the appropriate
* flag. This is common if `GC_enable_incremental` is called
* shortly after the collector initialization. To avoid modifying
* the interface, we silently work around such a failure, it only
* affects the initial (small) heap allocation. If there are
* more dirty pages than will fit in the buffer, this is not
* treated as a failure; we must check the page count in the
* loop condition. Since each partial call will reset the
* status of some pages, this should eventually terminate even
* in the overflow case.
*/
if ((*(GetWriteWatch_type)(GC_funcptr_uint)GetWriteWatch_func)(
WRITE_WATCH_FLAG_RESET, GC_heap_sects[i].hs_start,
GC_heap_sects[i].hs_bytes, pages, &count, &page_size)
!= 0) {
static int warn_count = 0;
static const struct hblk *last_warned = NULL;
struct hblk *start = (struct hblk *)GC_heap_sects[i].hs_start;
size_t nblocks = divHBLKSZ(GC_heap_sects[i].hs_bytes);
if (i != 0 && last_warned != start && warn_count++ < 5) {
last_warned = start;
WARN("GC_gww_read_dirty unexpectedly failed at %p:"
" Falling back to marking all pages dirty\n",
start);
}
if (!output_unneeded) {
size_t j;
for (j = 0; j < nblocks; ++j) {
size_t index = PHT_HASH(start + j);
set_pht_entry_from_index(GC_grungy_pages, index);
}
}
/* Done with this section. */
count = 1;
} else if (!output_unneeded) { /*< succeeded */
const PVOID *pages_end = pages + count;
while (pages != pages_end) {
struct hblk *h = (struct hblk *)(*pages++);
ptr_t h_end = (ptr_t)h + page_size;
do {
set_pht_entry_from_index(GC_grungy_pages, PHT_HASH(h));
h++;
} while (ADDR_LT((ptr_t)h, h_end));
}
}
} while (count == GC_GWW_BUF_LEN);
/*
* FIXME: It is unclear from Microsoft's documentation if this loop
* is useful. We suspect the call just fails if the buffer fills up.
* But that should still be handled correctly.
*/
}
# ifdef CHECKSUMS
GC_ASSERT(!output_unneeded);
GC_or_pages(GC_written_pages, GC_grungy_pages);
# endif
}
#elif defined(SOFT_VDB)
static int clear_refs_fd = -1;
# define GC_GWW_AVAILABLE() (clear_refs_fd != -1)
#else
# define GC_GWW_AVAILABLE() FALSE
#endif /* !GWW_VDB && !SOFT_VDB */
#ifdef DEFAULT_VDB
/*
* The client asserts that unallocated pages in the heap are never
* written.
*/
GC_INNER GC_bool
GC_dirty_init(void)
{
GC_VERBOSE_LOG_PRINTF("Initializing DEFAULT_VDB...\n");
/* `GC_dirty_pages` and `GC_grungy_pages` are already cleared. */
return TRUE;
}
#endif /* DEFAULT_VDB */
#if !defined(NO_MANUAL_VDB) || defined(MPROTECT_VDB)
# if !defined(THREADS) || defined(HAVE_LOCKFREE_AO_OR)
# ifdef MPROTECT_VDB
# define async_set_pht_entry_from_index(db, index) \
set_pht_entry_from_index_concurrent_volatile(db, index)
# else
# define async_set_pht_entry_from_index(db, index) \
set_pht_entry_from_index_concurrent(db, index)
# endif
# elif defined(NEED_FAULT_HANDLER_LOCK)
/*
* We need to lock around the bitmap update (in the write fault
* handler or `GC_dirty`) in order to avoid the risk of losing a bit.
* We do this with a test-and-set spin lock if possible.
*/
static void
async_set_pht_entry_from_index(volatile page_hash_table db, size_t index)
{
GC_acquire_dirty_lock();
set_pht_entry_from_index(db, index);
GC_release_dirty_lock();
}
# else /* THREADS && !NEED_FAULT_HANDLER_LOCK */
# error No test_and_set operation: Introduces a race.
# endif
#endif /* !NO_MANUAL_VDB || MPROTECT_VDB */
#ifdef MPROTECT_VDB
/*
* This implementation maintains dirty bits itself by catching write
* faults and keeping track of them. We assume nobody else catches
* `SIGBUS` or `SIGSEGV`. We assume no write faults occur in system
* calls. This means that clients must ensure that system calls do
* not write to the write-protected heap. Probably the best way to
* do this is to ensure that system calls write at most to
* pointer-free objects in the heap, and do even that only if we are
* on a platform on which those are not protected (or the collector
* is built with `DONT_PROTECT_PTRFREE` defined). We assume the page
* size is a multiple of `HBLKSIZE`.
*/
# ifdef DARWIN
/* `#define BROKEN_EXCEPTION_HANDLING` */
/*
* Using `vm_protect` (a `mach` `syscall`) over `mprotect` (a BSD `syscall`)
* seems to decrease the likelihood of some of the problems described below.
*/
# include <mach/vm_map.h>
STATIC mach_port_t GC_task_self = 0;
# define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
if (vm_protect(GC_task_self, (vm_address_t)(addr), (vm_size_t)(len), \
FALSE, \
VM_PROT_READ | ((allow_write) ? VM_PROT_WRITE : 0) \
| (GC_pages_executable ? VM_PROT_EXECUTE : 0)) \
== KERN_SUCCESS) { \
} else \
ABORT(C_msg_prefix "vm_protect() failed")
# elif !defined(USE_WINALLOC)
# include <sys/mman.h>
# if !defined(AIX) && !defined(CYGWIN32) && !defined(HAIKU)
# include <sys/syscall.h>
# endif
# define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
if (mprotect((caddr_t)(addr), (size_t)(len), \
PROT_READ | ((allow_write) ? PROT_WRITE : 0) \
| (GC_pages_executable ? PROT_EXEC : 0)) \
>= 0) { \
} else if (GC_pages_executable) { \
ABORT_ON_REMAP_FAIL(C_msg_prefix "mprotect vdb executable pages", \
addr, len); \
} else \
ABORT_ON_REMAP_FAIL(C_msg_prefix "mprotect vdb", addr, len)
# undef IGNORE_PAGES_EXECUTABLE
# else /* USE_WINALLOC */
static DWORD protect_junk;
# define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
if (VirtualProtect(addr, len, \
GC_pages_executable \
? ((allow_write) ? PAGE_EXECUTE_READWRITE \
: PAGE_EXECUTE_READ) \
: (allow_write) ? PAGE_READWRITE \
: PAGE_READONLY, \
&protect_junk)) { \
} else \
ABORT_ARG1(C_msg_prefix "VirtualProtect failed", ": errcode= 0x%X", \
(unsigned)GetLastError())
# endif /* USE_WINALLOC */
# define PROTECT(addr, len) PROTECT_INNER(addr, len, FALSE, "")
# define UNPROTECT(addr, len) PROTECT_INNER(addr, len, TRUE, "un-")
# if defined(MSWIN32)
typedef LPTOP_LEVEL_EXCEPTION_FILTER SIG_HNDLR_PTR;
# undef SIG_DFL
# define SIG_DFL ((LPTOP_LEVEL_EXCEPTION_FILTER)(~(GC_funcptr_uint)0))
# elif defined(MSWINCE)
typedef LONG(WINAPI *SIG_HNDLR_PTR)(struct _EXCEPTION_POINTERS *);
# undef SIG_DFL
# define SIG_DFL ((SIG_HNDLR_PTR)(~(GC_funcptr_uint)0))
# elif defined(DARWIN)
# ifdef BROKEN_EXCEPTION_HANDLING
typedef void (*SIG_HNDLR_PTR)();
# endif
# else
typedef void (*SIG_HNDLR_PTR)(int, siginfo_t *, void *);
typedef void (*PLAIN_HNDLR_PTR)(int);
# endif /* !DARWIN && !MSWIN32 && !MSWINCE */
# ifndef DARWIN
/* Also old `MSWIN32` `ACCESS_VIOLATION` filter. */
STATIC SIG_HNDLR_PTR GC_old_segv_handler = 0;
# ifdef USE_BUS_SIGACT
STATIC SIG_HNDLR_PTR GC_old_bus_handler = 0;
STATIC GC_bool GC_old_bus_handler_used_si = FALSE;
# endif
# if !defined(MSWIN32) && !defined(MSWINCE)
STATIC GC_bool GC_old_segv_handler_used_si = FALSE;
# endif
# endif /* !DARWIN */
# ifdef THREADS
/*
* This function is used only by the fault handler. Potential data
* race between this function and `GC_install_header`, `GC_remove_header`
* should not be harmful because the added or removed header should be
* already unprotected.
*/
GC_ATTR_NO_SANITIZE_THREAD
static GC_bool
is_header_found_async(const void *p)
{
# ifdef HASH_TL
hdr *result;
GET_HDR(p, result);
return result != NULL;
# else
return HDR_INNER(p) != NULL;
# endif
}
# else
# define is_header_found_async(p) (HDR(p) != NULL)
# endif /* !THREADS */
# ifndef DARWIN
# if !defined(MSWIN32) && !defined(MSWINCE)
# include <errno.h>
# ifdef USE_BUS_SIGACT
# define SIG_OK (sig == SIGBUS || sig == SIGSEGV)
# else
/* Catch `SIGSEGV` but ignore `SIGBUS`. */
# define SIG_OK (sig == SIGSEGV)
# endif
# if defined(FREEBSD) || defined(OPENBSD)
# ifndef SEGV_ACCERR
# define SEGV_ACCERR 2
# endif
# if defined(AARCH64) || defined(ARM32) || defined(MIPS) \
|| (__FreeBSD__ >= 7 || defined(OPENBSD))
# define CODE_OK (si->si_code == SEGV_ACCERR)
# elif defined(POWERPC)
/* Pretend that we are AIM. */
# define AIM
# include <machine/trap.h>
# define CODE_OK \
(si->si_code == EXC_DSI || si->si_code == SEGV_ACCERR)
# else
# define CODE_OK \
(si->si_code == BUS_PAGE_FAULT || si->si_code == SEGV_ACCERR)
# endif
# elif defined(OSF1)
# define CODE_OK (si->si_code == 2) /*< experimentally determined */
# elif defined(IRIX5)
# define CODE_OK (si->si_code == EACCES)
# elif defined(AIX) || defined(COSMO) || defined(CYGWIN32) \
|| defined(HAIKU) || defined(HURD) || defined(LINUX) \
|| defined(NETBSD)
/*
* Linux/i686: Empirically `c.trapno == 14`, but is that useful?
* Should probably consider alignment issues on other architectures.
*/
# define CODE_OK TRUE
# elif defined(HPUX)
# define CODE_OK \
(si->si_code == SEGV_ACCERR || si->si_code == BUS_ADRERR \
|| si->si_code == BUS_UNKNOWN || si->si_code == SEGV_UNKNOWN \
|| si->si_code == BUS_OBJERR)
# elif defined(SUNOS5SIGS)
# define CODE_OK (si->si_code == SEGV_ACCERR)
# endif
# ifndef NO_GETCONTEXT
# include <ucontext.h>
# endif
STATIC void
GC_write_fault_handler(int sig, siginfo_t *si, void *raw_sc)
# else /* MSWIN32 || MSWINCE */
# define SIG_OK \
(exc_info->ExceptionRecord->ExceptionCode == STATUS_ACCESS_VIOLATION)
# define CODE_OK \
(exc_info->ExceptionRecord->ExceptionInformation[0] \
== 1) /*< write fault */
STATIC LONG WINAPI
GC_write_fault_handler(struct _EXCEPTION_POINTERS *exc_info)
# endif
{
# if !defined(MSWIN32) && !defined(MSWINCE)
char *addr = (char *)si->si_addr;
# else
char *addr = (char *)exc_info->ExceptionRecord->ExceptionInformation[1];
# endif
if (SIG_OK && CODE_OK) {
struct hblk *h = HBLK_PAGE_ALIGNED(addr);
GC_bool in_allocd_block;
size_t i;
GC_ASSERT(GC_page_size != 0);
# ifdef CHECKSUMS
GC_record_fault(h);
# endif
# ifdef SUNOS5SIGS
/* Address is only within the correct physical page. */
in_allocd_block = FALSE;
for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
if (is_header_found_async(&h[i])) {
in_allocd_block = TRUE;
break;
}
}
# else
in_allocd_block = is_header_found_async(addr);
# endif
if (!in_allocd_block) {
/*
* FIXME: We should make sure that we invoke the old handler with the
* appropriate calling sequence, which often depends on `SA_SIGINFO`.
*/
/* Heap blocks now begin and end on page boundaries. */
SIG_HNDLR_PTR old_handler;
# if defined(MSWIN32) || defined(MSWINCE)
old_handler = GC_old_segv_handler;
# else
GC_bool used_si;
# ifdef USE_BUS_SIGACT
if (sig == SIGBUS) {
old_handler = GC_old_bus_handler;
used_si = GC_old_bus_handler_used_si;
} else
# endif
/* else */ {
old_handler = GC_old_segv_handler;
used_si = GC_old_segv_handler_used_si;
}
# endif
if ((GC_funcptr_uint)old_handler == (GC_funcptr_uint)SIG_DFL) {
# if !defined(MSWIN32) && !defined(MSWINCE)
ABORT_ARG1("Unexpected segmentation fault outside heap", " at %p",
(void *)addr);
# else
return EXCEPTION_CONTINUE_SEARCH;
# endif
} else {
/*
* FIXME: This code should probably check if the old signal handler
* used the traditional style and if so, call it using that style.
*/
# if defined(MSWIN32) || defined(MSWINCE)
return (*old_handler)(exc_info);
# else
if (used_si)
((SIG_HNDLR_PTR)old_handler)(sig, si, raw_sc);
else
/* FIXME: Should pass nonstandard arguments as well. */
((PLAIN_HNDLR_PTR)(GC_funcptr_uint)old_handler)(sig);
return;
# endif
}
}
UNPROTECT(h, GC_page_size);
/*
* We need to make sure that no collection occurs between the
* `UNPROTECT()` call and the setting of the dirty bit.
* Otherwise a write by a third thread might go unnoticed.
* Reversing the order is just as bad, since we would end up
* unprotecting a page in a collection cycle during which it is not
* marked. Currently we do this by disabling the thread stopping
* signals while this handler is running. An alternative might be
* to record the fact that we are about to unprotect, or have just
* unprotected a page in the collector's thread structure, and then
* to have the thread stopping code set the dirty flag, if necessary.
*/
for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
size_t index = PHT_HASH(h + i);
async_set_pht_entry_from_index(GC_dirty_pages, index);
}
/*
* The `write()` may not take place before dirty bits are read.
* But then we will fault again...
*/
# if defined(MSWIN32) || defined(MSWINCE)
return EXCEPTION_CONTINUE_EXECUTION;
# else
return;
# endif
}
# if defined(MSWIN32) || defined(MSWINCE)
return EXCEPTION_CONTINUE_SEARCH;
# else
ABORT_ARG1("Unexpected bus error or segmentation fault", " at %p",
(void *)addr);
# endif
}
# if defined(GC_WIN32_THREADS) && !defined(CYGWIN32)
GC_INNER void
GC_set_write_fault_handler(void)
{
SetUnhandledExceptionFilter(GC_write_fault_handler);
}
# endif
# ifdef SOFT_VDB
static GC_bool soft_dirty_init(void);
# endif
GC_INNER GC_bool
GC_dirty_init(void)
{
# if !defined(MSWIN32) && !defined(MSWINCE)
struct sigaction act, oldact;
# endif
GC_ASSERT(I_HOLD_LOCK());
# ifdef COUNT_PROTECTED_REGIONS
GC_ASSERT(GC_page_size != 0);
if ((GC_signed_word)(GC_heapsize / (word)GC_page_size)
>= ((GC_signed_word)GC_UNMAPPED_REGIONS_SOFT_LIMIT
- GC_num_unmapped_regions)
* 2) {
GC_COND_LOG_PRINTF("Cannot turn on GC incremental mode"
" as heap contains too many pages\n");
return FALSE;
}
# endif
# if !defined(MSWIN32) && !defined(MSWINCE)
act.sa_flags = SA_RESTART | SA_SIGINFO;
act.sa_sigaction = GC_write_fault_handler;
(void)sigemptyset(&act.sa_mask);
# ifdef SIGNAL_BASED_STOP_WORLD
/*
* Arrange to postpone the signal while we are in a write fault handler.
* This effectively makes the handler atomic w.r.t. stopping the world
* for the collection.
*/
(void)sigaddset(&act.sa_mask, GC_get_suspend_signal());
# endif
# endif /* !MSWIN32 */
GC_VERBOSE_LOG_PRINTF(
"Initializing mprotect virtual dirty bit implementation\n");
if (GC_page_size % HBLKSIZE != 0) {
ABORT("Page size not multiple of HBLKSIZE");
}
# ifdef GWW_VDB
if (GC_gww_dirty_init()) {
GC_COND_LOG_PRINTF("Using GetWriteWatch()\n");
return TRUE;
}
# elif defined(SOFT_VDB)
# ifdef CHECK_SOFT_VDB
if (!soft_dirty_init())
ABORT("Soft-dirty bit support is missing");
# else
if (soft_dirty_init()) {
GC_COND_LOG_PRINTF("Using soft-dirty bit feature\n");
return TRUE;
}
# endif
# endif
# ifdef MSWIN32
GC_old_segv_handler = SetUnhandledExceptionFilter(GC_write_fault_handler);
if (GC_old_segv_handler != NULL) {
GC_COND_LOG_PRINTF("Replaced other UnhandledExceptionFilter\n");
} else {
GC_old_segv_handler = SIG_DFL;
}
# elif defined(MSWINCE)
{
/* `MPROTECT_VDB` is unsupported for WinCE at present. */
/* FIXME: Implement (if possible). */
}
# else
/* `act.sa_restorer` is deprecated and should not be initialized. */
# if defined(IRIX5) && defined(THREADS)
sigaction(SIGSEGV, 0, &oldact);
sigaction(SIGSEGV, &act, 0);
# else
{
int res = sigaction(SIGSEGV, &act, &oldact);
if (res != 0)
ABORT("Sigaction failed");
}
# endif
if (oldact.sa_flags & SA_SIGINFO) {
GC_old_segv_handler = oldact.sa_sigaction;
GC_old_segv_handler_used_si = TRUE;
} else {
GC_old_segv_handler = (SIG_HNDLR_PTR)(GC_funcptr_uint)oldact.sa_handler;
GC_old_segv_handler_used_si = FALSE;
}
if ((GC_funcptr_uint)GC_old_segv_handler == (GC_funcptr_uint)SIG_IGN) {
WARN("Previously ignored segmentation violation!?\n", 0);
GC_old_segv_handler = (SIG_HNDLR_PTR)(GC_funcptr_uint)SIG_DFL;
}
if ((GC_funcptr_uint)GC_old_segv_handler != (GC_funcptr_uint)SIG_DFL) {
GC_VERBOSE_LOG_PRINTF("Replaced other SIGSEGV handler\n");
}
# ifdef USE_BUS_SIGACT
sigaction(SIGBUS, &act, &oldact);
if ((oldact.sa_flags & SA_SIGINFO) != 0) {
GC_old_bus_handler = oldact.sa_sigaction;
GC_old_bus_handler_used_si = TRUE;
} else {
GC_old_bus_handler = (SIG_HNDLR_PTR)(GC_funcptr_uint)oldact.sa_handler;
}
if ((GC_funcptr_uint)GC_old_bus_handler == (GC_funcptr_uint)SIG_IGN) {
WARN("Previously ignored bus error!?\n", 0);
GC_old_bus_handler = (SIG_HNDLR_PTR)(GC_funcptr_uint)SIG_DFL;
} else if ((GC_funcptr_uint)GC_old_bus_handler != (GC_funcptr_uint)SIG_DFL) {
GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n");
}
# endif
# endif /* !MSWIN32 && !MSWINCE */
# if defined(CPPCHECK) && defined(ADDRESS_SANITIZER)
GC_noop1((word)(GC_funcptr_uint)(&__asan_default_options));
# endif
return TRUE;
}
# endif /* !DARWIN */
STATIC void
GC_protect_heap(void)
{
size_t i;
GC_ASSERT(GC_page_size != 0);
for (i = 0; i < GC_n_heap_sects; i++) {
ptr_t start = GC_heap_sects[i].hs_start;
size_t len = GC_heap_sects[i].hs_bytes;
struct hblk *current;
struct hblk *current_start; /*< start of block to be protected */
ptr_t limit;
GC_ASSERT((ADDR(start) & (GC_page_size - 1)) == 0);
GC_ASSERT((len & (GC_page_size - 1)) == 0);
# ifndef DONT_PROTECT_PTRFREE
/*
* We avoid protecting pointer-free objects unless the page size
* differs from `HBLKSIZE`.
*/
if (GC_page_size != HBLKSIZE) {
PROTECT(start, len);
continue;
}
# endif
current_start = (struct hblk *)start;
limit = start + len;
for (current = current_start;;) {
size_t nblocks = 0;
GC_bool is_ptrfree = TRUE;
if (ADDR_LT((ptr_t)current, limit)) {
hdr *hhdr;
GET_HDR(current, hhdr);
if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
/*
* This can happen only if we are at the beginning of a heap
* segment, and a block spans heap segments. We will handle
* that block as part of the preceding segment.
*/
GC_ASSERT(current_start == current);
current_start = ++current;
continue;
}
if (HBLK_IS_FREE(hhdr)) {
GC_ASSERT(modHBLKSZ(hhdr->hb_sz) == 0);
nblocks = divHBLKSZ(hhdr->hb_sz);
} else {
nblocks = OBJ_SZ_TO_BLOCKS(hhdr->hb_sz);
is_ptrfree = IS_PTRFREE(hhdr);
}
}
if (is_ptrfree) {
if (ADDR_LT((ptr_t)current_start, (ptr_t)current)) {
# ifdef DONT_PROTECT_PTRFREE
ptr_t cur_aligned = PTR_ALIGN_UP((ptr_t)current, GC_page_size);
current_start = HBLK_PAGE_ALIGNED(current_start);
/*
* Adjacent free blocks might be protected too because
* of the alignment by the page size.
*/
PROTECT(current_start, cur_aligned - (ptr_t)current_start);
# else
PROTECT(current_start, (ptr_t)current - (ptr_t)current_start);
# endif
}
if (ADDR_GE((ptr_t)current, limit))
break;
}
current += nblocks;
if (is_ptrfree)
current_start = current;
}
}
}
# if defined(CAN_HANDLE_FORK) && defined(DARWIN) && defined(THREADS) \
|| defined(COUNT_PROTECTED_REGIONS)
/* Remove protection for the entire heap not updating `GC_dirty_pages`. */
STATIC void
GC_unprotect_all_heap(void)
{
size_t i;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_auto_incremental);
for (i = 0; i < GC_n_heap_sects; i++) {
UNPROTECT(GC_heap_sects[i].hs_start, GC_heap_sects[i].hs_bytes);
}
}
# endif
# ifdef COUNT_PROTECTED_REGIONS
GC_INNER void
GC_handle_protected_regions_limit(void)
{
GC_ASSERT(GC_page_size != 0);
/*
* To prevent exceeding the limit of `vm.max_map_count`, the most
* trivial (though highly restrictive) way is to turn off the
* incremental collection mode (based on `mprotect`) once the number
* of pages in the heap reaches that limit.
*/
if (GC_auto_incremental && !GC_GWW_AVAILABLE()
&& (GC_signed_word)(GC_heapsize / (word)GC_page_size)
>= ((GC_signed_word)GC_UNMAPPED_REGIONS_SOFT_LIMIT
- GC_num_unmapped_regions)
* 2) {
GC_unprotect_all_heap();
# ifdef DARWIN
GC_task_self = 0;
# endif
GC_incremental = FALSE;
WARN("GC incremental mode is turned off"
" to prevent hitting VM maps limit\n",
0);
}
}
# endif /* COUNT_PROTECTED_REGIONS */
#endif /* MPROTECT_VDB */
#if !defined(THREADS) && (defined(PROC_VDB) || defined(SOFT_VDB))
static pid_t saved_proc_pid; /*< `pid` used to compose `/proc` file names */
#endif
#ifdef PROC_VDB
/*
* This implementation assumes the Solaris new structured `/proc`
* pseudo-file-system from which we can read page modified bits.
* This facility is far from optimal (e.g. we would like to get the
* info for only some of the address space), but it avoids intercepting
* system calls.
*/
# include <errno.h>
# include <sys/signal.h>
# include <sys/stat.h>
# include <sys/syscall.h>
# ifdef GC_NO_SYS_FAULT_H
/* This exists only to check `PROC_VDB` code compilation (on Linux). */
# define PG_MODIFIED 1
struct prpageheader {
long dummy[2]; /*< `pr_tstamp` */
long pr_nmap;
long pr_npage;
};
struct prasmap {
GC_uintptr_t pr_vaddr;
size_t pr_npage;
char dummy1[64 + 8]; /*< `pr_mapname`, `pr_offset` */
int pr_mflags;
int pr_pagesize;
int dummy2[2]; /*< `pr_shmid`, `pr_filler` */
};
# else
/* Use the new structured `/proc` definitions. */
# include <procfs.h>
# endif
# define INITIAL_BUF_SZ 8192
STATIC size_t GC_proc_buf_size = INITIAL_BUF_SZ;
STATIC char *GC_proc_buf = NULL;
STATIC int GC_proc_fd = -1;
static GC_bool
proc_dirty_open_files(void)
{
char buf[6 + 20 + 9 + 1];
pid_t pid = getpid();
GC_snprintf_s_ld_s(buf, sizeof(buf), "/proc/", (long)pid, "/pagedata");
GC_proc_fd = open(buf, O_RDONLY);
if (-1 == GC_proc_fd) {
WARN("/proc open failed; cannot enable GC incremental mode\n", 0);
return FALSE;
}
if (syscall(SYS_fcntl, GC_proc_fd, F_SETFD, FD_CLOEXEC) == -1)
WARN("Could not set FD_CLOEXEC for /proc\n", 0);
# ifndef THREADS
/* Updated on success only. */
saved_proc_pid = pid;
# endif
return TRUE;
}
# ifdef CAN_HANDLE_FORK
GC_INNER void
GC_dirty_update_child(void)
{
GC_ASSERT(I_HOLD_LOCK());
if (-1 == GC_proc_fd) {
/* The GC incremental mode is off. */
return;
}
close(GC_proc_fd);
if (!proc_dirty_open_files()) {
/* Should be safe to turn it off. */
GC_incremental = FALSE;
}
}
# endif /* CAN_HANDLE_FORK */
GC_INNER GC_bool
GC_dirty_init(void)
{
GC_ASSERT(I_HOLD_LOCK());
if (GC_bytes_allocd != 0 || GC_bytes_allocd_before_gc != 0) {
memset(GC_written_pages, 0xff, sizeof(page_hash_table));
GC_VERBOSE_LOG_PRINTF(
"Allocated %lu bytes: all pages may have been written\n",
(unsigned long)(GC_bytes_allocd + GC_bytes_allocd_before_gc));
}
if (!proc_dirty_open_files())
return FALSE;
GC_proc_buf = GC_scratch_alloc(GC_proc_buf_size);
if (GC_proc_buf == NULL)
ABORT("Insufficient space for /proc read");
return TRUE;
}
GC_INLINE void
GC_proc_read_dirty(GC_bool output_unneeded)
{
size_t i, nmaps;
ssize_t pagedata_len;
char *bufp = GC_proc_buf;
GC_ASSERT(I_HOLD_LOCK());
# ifndef THREADS
/*
* If the current `pid` differs from the saved one, then we are in
* the forked (child) process, the current `/proc` file should be
* closed, the new one should be opened with the updated path.
* Note, this is not needed for the multi-threaded case because
* `fork_child_proc()` reopens the file right after `fork()` call.
*/
if (getpid() != saved_proc_pid
&& (-1 == GC_proc_fd /*< no need to retry */
|| (close(GC_proc_fd), !proc_dirty_open_files()))) {
/* Failed to reopen the file. Punt! */
if (!output_unneeded)
memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
memset(GC_written_pages, 0xff, sizeof(page_hash_table));
return;
}
# endif
for (;;) {
char *new_buf;
size_t new_size;
pagedata_len = PROC_READ(GC_proc_fd, bufp, GC_proc_buf_size);
if (LIKELY(pagedata_len != -1))
break;
if (errno != E2BIG) {
WARN("read /proc failed, errno= %" WARN_PRIdPTR "\n",
(GC_signed_word)errno);
/* Punt. */
if (!output_unneeded)
memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
memset(GC_written_pages, 0xff, sizeof(page_hash_table));
return;
}
/* Retry with larger buffer. */
new_size = 2 * GC_proc_buf_size;
/*
* Alternatively, we could use `fstat()` to determine the required
* buffer size.
*/
# ifdef DEBUG_DIRTY_BITS
GC_log_printf("Growing proc buf to %lu bytes at collection #%lu\n",
(unsigned long)new_size, (unsigned long)GC_gc_no + 1);
# endif
new_buf = GC_scratch_alloc(new_size);
if (new_buf != NULL) {
GC_scratch_recycle_no_gww(bufp, GC_proc_buf_size);
GC_proc_buf = bufp = new_buf;
GC_proc_buf_size = new_size;
}
}
GC_ASSERT((size_t)pagedata_len <= GC_proc_buf_size);
/* Copy dirty bits into `GC_grungy_pages`. */
BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
nmaps = (size_t)(((struct prpageheader *)bufp)->pr_nmap);
# ifdef DEBUG_DIRTY_BITS
GC_log_printf("Proc VDB read: pr_nmap= %u, pr_npage= %ld\n", (unsigned)nmaps,
((struct prpageheader *)bufp)->pr_npage);
# endif
# if defined(GC_NO_SYS_FAULT_H) && defined(CPPCHECK)
GC_noop1(((struct prpageheader *)bufp)->dummy[0]);
# endif
bufp += sizeof(struct prpageheader);
for (i = 0; i < nmaps; i++) {
struct prasmap *map = (struct prasmap *)bufp;
ptr_t vaddr, limit;
unsigned long npages = 0;
unsigned pagesize;
bufp += sizeof(struct prasmap);
/* Ensure no buffer overrun. */
if (bufp - GC_proc_buf < pagedata_len)
npages = (unsigned long)map->pr_npage;
if (bufp - GC_proc_buf > pagedata_len - (ssize_t)npages)
ABORT("Wrong pr_nmap or pr_npage read from /proc");
vaddr = (ptr_t)map->pr_vaddr;
pagesize = (unsigned)map->pr_pagesize;
# if defined(GC_NO_SYS_FAULT_H) && defined(CPPCHECK)
GC_noop1(map->dummy1[0] + map->dummy2[0]);
# endif
# ifdef DEBUG_DIRTY_BITS
GC_log_printf("pr_vaddr= %p, npage= %lu, mflags= 0x%x, pagesize= 0x%x\n",
(void *)vaddr, npages, map->pr_mflags, pagesize);
# endif
if (0 == pagesize || ((pagesize - 1) & pagesize) != 0)
ABORT("Wrong pagesize read from /proc");
limit = vaddr + pagesize * npages;
for (; ADDR_LT(vaddr, limit); vaddr += pagesize) {
if ((*bufp++) & PG_MODIFIED) {
struct hblk *h;
ptr_t next_vaddr = vaddr + pagesize;
# ifdef DEBUG_DIRTY_BITS
GC_log_printf("dirty page at: %p\n", (void *)vaddr);
# endif
for (h = (struct hblk *)vaddr; ADDR_LT((ptr_t)h, next_vaddr); h++) {
size_t index = PHT_HASH(h);
set_pht_entry_from_index(GC_grungy_pages, index);
}
}
}
/*
* According to the new structured `pagedata` file format, an 8-byte
* alignment is enforced (preceding the next `struct prasmap`)
* regardless of the pointer size.
*/
bufp = PTR_ALIGN_UP(bufp, 8);
}
# ifdef DEBUG_DIRTY_BITS
GC_log_printf("Proc VDB read done\n");
# endif
/* Update `GC_written_pages` (even if `output_unneeded`). */
GC_or_pages(GC_written_pages, GC_grungy_pages);
}
#endif /* PROC_VDB */
#ifdef SOFT_VDB
# ifndef VDB_BUF_SZ
# define VDB_BUF_SZ 16384
# endif
static int
open_proc_fd(pid_t pid, const char *slash_filename, int mode)
{
int f;
char buf[6 + 20 + 11 + 1];
GC_snprintf_s_ld_s(buf, sizeof(buf), "/proc/", (long)pid, slash_filename);
f = open(buf, mode);
if (-1 == f) {
WARN("/proc/self%s open failed; cannot enable GC incremental mode\n",
slash_filename);
} else if (fcntl(f, F_SETFD, FD_CLOEXEC) == -1) {
WARN("Could not set FD_CLOEXEC for /proc\n", 0);
}
return f;
}
# include <stdint.h> /*< for `uint64_t` */
typedef uint64_t pagemap_elem_t;
static pagemap_elem_t *soft_vdb_buf;
static int pagemap_fd;
static GC_bool
soft_dirty_open_files(void)
{
pid_t pid = getpid();
clear_refs_fd = open_proc_fd(pid, "/clear_refs", O_WRONLY);
if (-1 == clear_refs_fd)
return FALSE;
pagemap_fd = open_proc_fd(pid, "/pagemap", O_RDONLY);
if (-1 == pagemap_fd) {
close(clear_refs_fd);
clear_refs_fd = -1;
return FALSE;
}
# ifndef THREADS
/* Updated on success only. */
saved_proc_pid = pid;
# endif
return TRUE;
}
# ifdef CAN_HANDLE_FORK
GC_INNER void
GC_dirty_update_child(void)
{
GC_ASSERT(I_HOLD_LOCK());
if (-1 == clear_refs_fd) {
/* The GC incremental mode is off. */
return;
}
close(clear_refs_fd);
close(pagemap_fd);
if (!soft_dirty_open_files())
GC_incremental = FALSE;
}
# endif /* CAN_HANDLE_FORK */
/* Clear soft-dirty bits from the task's PTEs. */
static void
clear_soft_dirty_bits(void)
{
ssize_t res = write(clear_refs_fd, "4\n", 2);
if (res != 2)
ABORT_ARG1("Failed to write to /proc/self/clear_refs", ": errno= %d",
res < 0 ? errno : 0);
}
/* The bit 55 of the 64-bit `qword` of `pagemap` file is the soft-dirty one. */
# define PM_SOFTDIRTY_MASK ((pagemap_elem_t)1 << 55)
static GC_bool
detect_soft_dirty_supported(ptr_t vaddr)
{
off_t fpos;
pagemap_elem_t buf[1];
GC_ASSERT(GC_log_pagesize != 0);
/* Make it dirty. */
*vaddr = 1;
fpos = (off_t)((ADDR(vaddr) >> GC_log_pagesize) * sizeof(pagemap_elem_t));
for (;;) {
/* Read the relevant PTE from the `pagemap` file. */
if (lseek(pagemap_fd, fpos, SEEK_SET) == (off_t)(-1))
return FALSE;
if (PROC_READ(pagemap_fd, buf, sizeof(buf)) != (int)sizeof(buf))
return FALSE;
/* Is the soft-dirty bit unset? */
if ((buf[0] & PM_SOFTDIRTY_MASK) == 0)
return FALSE;
if (0 == *vaddr)
break;
/*
* Retry to check that writing to `clear_refs` works as expected.
* This malfunction of the soft-dirty bits implementation is
* observed on some Linux kernels on Power9 (e.g. in Fedora 36).
*/
clear_soft_dirty_bits();
*vaddr = 0;
}
return TRUE; /*< success */
}
# ifndef NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK
# include <string.h> /*< for strcmp() */
# include <sys/utsname.h>
/* Ensure the linux (kernel) major/minor version is as given or higher. */
static GC_bool
ensure_min_linux_ver(int major, int minor)
{
struct utsname info;
int actual_major;
int actual_minor = -1;
if (uname(&info) == -1) {
/* `uname()` has failed, should not happen actually. */
return FALSE;
}
if (strcmp(info.sysname, "Linux")) {
WARN("Cannot ensure Linux version as running on other OS: %s\n",
info.sysname);
return FALSE;
}
actual_major = GC_parse_version(&actual_minor, info.release);
return actual_major > major
|| (actual_major == major && actual_minor >= minor);
}
# endif
# ifdef MPROTECT_VDB
static GC_bool
soft_dirty_init(void)
# else
GC_INNER GC_bool
GC_dirty_init(void)
# endif
{
# if defined(MPROTECT_VDB) && !defined(CHECK_SOFT_VDB)
char *str = GETENV("GC_USE_GETWRITEWATCH");
# ifdef GC_PREFER_MPROTECT_VDB
if (NULL == str || (*str == '0' && *(str + 1) == '\0')) {
/* The environment variable is unset or set to "0". */
return FALSE;
}
# else
if (str != NULL && *str == '0' && *(str + 1) == '\0') {
/* The environment variable is set "0". */
return FALSE;
}
# endif
# endif
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(NULL == soft_vdb_buf);
# ifndef NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK
if (!ensure_min_linux_ver(3, 18)) {
GC_COND_LOG_PRINTF(
"Running on old kernel lacking correct soft-dirty bit support\n");
return FALSE;
}
# endif
if (!soft_dirty_open_files())
return FALSE;
soft_vdb_buf = (pagemap_elem_t *)GC_scratch_alloc(VDB_BUF_SZ);
if (NULL == soft_vdb_buf)
ABORT("Insufficient space for /proc pagemap buffer");
if (!detect_soft_dirty_supported((ptr_t)soft_vdb_buf)) {
GC_COND_LOG_PRINTF("Soft-dirty bit is not supported by kernel\n");
/* Release the resources. */
GC_scratch_recycle_no_gww(soft_vdb_buf, VDB_BUF_SZ);
soft_vdb_buf = NULL;
close(clear_refs_fd);
clear_refs_fd = -1;
close(pagemap_fd);
return FALSE;
}
return TRUE;
}
static off_t pagemap_buf_fpos; /*< valid only if `pagemap_buf_len > 0` */
static size_t pagemap_buf_len;
/*
* Read bytes from `/proc/self/pagemap` file at given file position.
* `len` is the maximum number of bytes to read; `*pres` is the amount
* of bytes actually read (always bigger than 0 but never exceeds `len`);
* `next_fpos_hint` is the file position of the next bytes block to read
* ahead if possible (0 means no information provided).
*/
static const pagemap_elem_t *
pagemap_buffered_read(size_t *pres, off_t fpos, size_t len,
off_t next_fpos_hint)
{
ssize_t res;
size_t ofs;
GC_ASSERT(GC_page_size != 0);
GC_ASSERT(len > 0);
if (pagemap_buf_fpos <= fpos
&& fpos < pagemap_buf_fpos + (off_t)pagemap_buf_len) {
/* The requested data is already in the buffer. */
ofs = (size_t)(fpos - pagemap_buf_fpos);
res = (ssize_t)(pagemap_buf_fpos + pagemap_buf_len - fpos);
} else {
off_t aligned_pos = fpos
& ~(off_t)(GC_page_size < VDB_BUF_SZ ? GC_page_size - 1
: VDB_BUF_SZ - 1);
for (;;) {
size_t count;
if ((0 == pagemap_buf_len
|| pagemap_buf_fpos + (off_t)pagemap_buf_len != aligned_pos)
&& lseek(pagemap_fd, aligned_pos, SEEK_SET) == (off_t)(-1))
ABORT_ARG2("Failed to lseek /proc/self/pagemap",
": offset= %lu, errno= %d", (unsigned long)fpos, errno);
/* How much to read at once? */
ofs = (size_t)(fpos - aligned_pos);
GC_ASSERT(ofs < VDB_BUF_SZ);
if (next_fpos_hint > aligned_pos
&& next_fpos_hint - aligned_pos < VDB_BUF_SZ) {
count = VDB_BUF_SZ;
} else {
count = len + ofs;
if (count > VDB_BUF_SZ)
count = VDB_BUF_SZ;
}
GC_ASSERT(count % sizeof(pagemap_elem_t) == 0);
res = PROC_READ(pagemap_fd, soft_vdb_buf, count);
if (res > (ssize_t)ofs)
break;
if (res <= 0)
ABORT_ARG1("Failed to read /proc/self/pagemap", ": errno= %d",
res < 0 ? errno : 0);
/* Retry (once) w/o page-alignment. */
aligned_pos = fpos;
}
/* Save the buffer (file window) position and size. */
pagemap_buf_fpos = aligned_pos;
pagemap_buf_len = (size_t)res;
res -= (ssize_t)ofs;
}
GC_ASSERT(ofs % sizeof(pagemap_elem_t) == 0);
*pres = (size_t)res < len ? (size_t)res : len;
return &soft_vdb_buf[ofs / sizeof(pagemap_elem_t)];
}
static void
soft_set_grungy_pages(ptr_t start, ptr_t limit, ptr_t next_start_hint,
GC_bool is_static_root)
{
ptr_t vaddr = (ptr_t)HBLK_PAGE_ALIGNED(start);
off_t next_fpos_hint = (off_t)((ADDR(next_start_hint) >> GC_log_pagesize)
* sizeof(pagemap_elem_t));
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(modHBLKSZ(ADDR(start)) == 0);
GC_ASSERT(GC_log_pagesize != 0);
while (ADDR_LT(vaddr, limit)) {
size_t res;
ptr_t limit_buf;
word vlen_p = ADDR(limit) - ADDR(vaddr) + GC_page_size - 1;
const pagemap_elem_t *bufp = pagemap_buffered_read(
&res,
(off_t)((ADDR(vaddr) >> GC_log_pagesize) * sizeof(pagemap_elem_t)),
(size_t)((vlen_p >> GC_log_pagesize) * sizeof(pagemap_elem_t)),
next_fpos_hint);
if (res % sizeof(pagemap_elem_t) != 0) {
/* Punt. */
memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
WARN("Incomplete read of pagemap, not multiple of entry size\n", 0);
break;
}
limit_buf = vaddr + ((res / sizeof(pagemap_elem_t)) << GC_log_pagesize);
for (; ADDR_LT(vaddr, limit_buf); vaddr += GC_page_size, bufp++) {
if ((*bufp & PM_SOFTDIRTY_MASK) != 0) {
struct hblk *h;
ptr_t next_vaddr = vaddr + GC_page_size;
if (UNLIKELY(ADDR_LT(limit, next_vaddr))) {
next_vaddr = limit;
}
/*
* If the bit is set, the respective PTE was written to
* since clearing the soft-dirty bits.
*/
# ifdef DEBUG_DIRTY_BITS
if (is_static_root)
GC_log_printf("static root dirty page at: %p\n", (void *)vaddr);
# endif
h = (struct hblk *)vaddr;
if (UNLIKELY(ADDR_LT(vaddr, start))) {
h = (struct hblk *)start;
}
for (; ADDR_LT((ptr_t)h, next_vaddr); h++) {
size_t index = PHT_HASH(h);
/*
* Filter out the blocks without pointers. It might worth for
* the case when the heap is large enough for the hash collisions
* to occur frequently. Thus, off by default.
*/
# if defined(FILTER_PTRFREE_HBLKS_IN_SOFT_VDB) || defined(CHECKSUMS) \
|| defined(DEBUG_DIRTY_BITS)
if (!is_static_root) {
hdr *hhdr;
# ifdef CHECKSUMS
set_pht_entry_from_index(GC_written_pages, index);
# endif
GET_HDR(h, hhdr);
if (NULL == hhdr)
continue;
(void)GC_find_starting_hblk(h, &hhdr);
if (HBLK_IS_FREE(hhdr) || IS_PTRFREE(hhdr))
continue;
# ifdef DEBUG_DIRTY_BITS
GC_log_printf("dirty page (hblk) at: %p\n", (void *)h);
# endif
}
# else
UNUSED_ARG(is_static_root);
# endif
set_pht_entry_from_index(GC_grungy_pages, index);
}
} else {
# if defined(CHECK_SOFT_VDB) /* `&& defined(MPROTECT_VDB)` */
/*
* Ensure that each clean page according to the soft-dirty VDB is
* also identified such by the `mprotect`-based one.
*/
if (!is_static_root
&& get_pht_entry_from_index(GC_dirty_pages, PHT_HASH(vaddr))) {
ptr_t my_start, my_end; /*< the values are not used */
/*
* There could be a hash collision, thus we need to verify the
* page is clean using slow `GC_get_maps()`.
*/
if (GC_enclosing_writable_mapping(vaddr, &my_start, &my_end)) {
ABORT("Inconsistent soft-dirty against mprotect dirty bits");
}
}
# endif
}
}
/* Read the next portion of `pagemap` file if incomplete. */
}
}
GC_INLINE void
GC_soft_read_dirty(GC_bool output_unneeded)
{
GC_ASSERT(I_HOLD_LOCK());
# ifndef THREADS
/* Similar as for `GC_proc_read_dirty`. */
if (getpid() != saved_proc_pid
&& (-1 == clear_refs_fd /*< no need to retry */
|| (close(clear_refs_fd), close(pagemap_fd),
!soft_dirty_open_files()))) {
/* Failed to reopen the files. */
if (!output_unneeded) {
/* Punt. */
memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
# ifdef CHECKSUMS
memset(GC_written_pages, 0xff, sizeof(page_hash_table));
# endif
}
return;
}
# endif
if (!output_unneeded) {
size_t i;
BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
pagemap_buf_len = 0; /*< invalidate `soft_vdb_buf` */
for (i = 0; i < GC_n_heap_sects; ++i) {
ptr_t start = GC_heap_sects[i].hs_start;
soft_set_grungy_pages(
start, start + GC_heap_sects[i].hs_bytes,
i + 1 < GC_n_heap_sects ? GC_heap_sects[i + 1].hs_start : NULL,
FALSE);
}
# ifndef NO_VDB_FOR_STATIC_ROOTS
for (i = 0; i < n_root_sets; ++i) {
soft_set_grungy_pages(
(ptr_t)HBLKPTR(GC_static_roots[i].r_start), GC_static_roots[i].r_end,
i + 1 < n_root_sets ? GC_static_roots[i + 1].r_start : NULL, TRUE);
}
# endif
}
clear_soft_dirty_bits();
}
#endif /* SOFT_VDB */
#ifndef NO_MANUAL_VDB
GC_INNER GC_bool GC_manual_vdb = FALSE;
void
GC_dirty_inner(const void *p)
{
size_t index = PHT_HASH(p);
# if defined(MPROTECT_VDB)
/*
* Do not update `GC_dirty_pages` if it should be followed by the
* page unprotection.
*/
GC_ASSERT(GC_manual_vdb);
# endif
async_set_pht_entry_from_index(GC_dirty_pages, index);
}
#endif /* !NO_MANUAL_VDB */
#ifndef GC_DISABLE_INCREMENTAL
GC_INNER void
GC_read_dirty(GC_bool output_unneeded)
{
GC_ASSERT(I_HOLD_LOCK());
# ifdef DEBUG_DIRTY_BITS
GC_log_printf("read dirty begin\n");
# endif
if (GC_manual_vdb
# if defined(MPROTECT_VDB)
|| !GC_GWW_AVAILABLE()
# endif
) {
if (!output_unneeded)
BCOPY(CAST_AWAY_VOLATILE_PVOID(GC_dirty_pages), GC_grungy_pages,
sizeof(GC_dirty_pages));
BZERO(CAST_AWAY_VOLATILE_PVOID(GC_dirty_pages), sizeof(GC_dirty_pages));
# ifdef MPROTECT_VDB
if (!GC_manual_vdb)
GC_protect_heap();
# endif
return;
}
# ifdef GWW_VDB
GC_gww_read_dirty(output_unneeded);
# elif defined(PROC_VDB)
GC_proc_read_dirty(output_unneeded);
# elif defined(SOFT_VDB)
GC_soft_read_dirty(output_unneeded);
# endif
# if defined(CHECK_SOFT_VDB) /* `&& defined(MPROTECT_VDB)` */
BZERO(CAST_AWAY_VOLATILE_PVOID(GC_dirty_pages), sizeof(GC_dirty_pages));
GC_protect_heap();
# endif
}
# if !defined(NO_VDB_FOR_STATIC_ROOTS) && !defined(PROC_VDB)
GC_INNER GC_bool
GC_is_vdb_for_static_roots(void)
{
if (GC_manual_vdb)
return FALSE;
# if defined(MPROTECT_VDB)
/* Currently used only in conjunction with `SOFT_VDB`. */
return GC_GWW_AVAILABLE();
# else
# ifndef LINT2
GC_ASSERT(GC_incremental);
# endif
return TRUE;
# endif
}
# endif
GC_INNER GC_bool
GC_page_was_dirty(struct hblk *h)
{
size_t index;
# ifdef DEFAULT_VDB
if (!GC_manual_vdb)
return TRUE;
# elif defined(PROC_VDB)
/* Unless manual VDB is on, the bitmap covers all process memory. */
if (GC_manual_vdb)
# endif
{
if (NULL == HDR(h))
return TRUE;
}
index = PHT_HASH(h);
return get_pht_entry_from_index(GC_grungy_pages, index);
}
# if defined(CHECKSUMS) || defined(PROC_VDB)
GC_INNER GC_bool
GC_page_was_ever_dirty(struct hblk *h)
{
# if defined(GWW_VDB) || defined(PROC_VDB) || defined(SOFT_VDB)
size_t index;
# ifdef MPROTECT_VDB
if (!GC_GWW_AVAILABLE())
return TRUE;
# endif
# if defined(PROC_VDB)
if (GC_manual_vdb)
# endif
{
if (NULL == HDR(h))
return TRUE;
}
index = PHT_HASH(h);
return get_pht_entry_from_index(GC_written_pages, index);
# else
/* TODO: Implement for `MANUAL_VDB` case. */
UNUSED_ARG(h);
return TRUE;
# endif
}
# endif /* CHECKSUMS || PROC_VDB */
GC_INNER void
GC_remove_protection(struct hblk *h, size_t nblocks, GC_bool is_ptrfree)
{
# ifdef MPROTECT_VDB
struct hblk *current;
struct hblk *h_trunc; /*< truncated to page boundary */
ptr_t h_end; /*< page boundary following the block end */
# endif
# ifndef PARALLEL_MARK
GC_ASSERT(I_HOLD_LOCK());
# endif
# ifdef MPROTECT_VDB
/*
* Note it is not allowed to call `GC_printf` (and the friends)
* in this function, see Win32 `GC_stop_world` for the details.
*/
# ifdef DONT_PROTECT_PTRFREE
if (is_ptrfree)
return;
# endif
if (!GC_auto_incremental || GC_GWW_AVAILABLE())
return;
GC_ASSERT(GC_page_size != 0);
h_trunc = HBLK_PAGE_ALIGNED(h);
h_end = PTR_ALIGN_UP((ptr_t)(h + nblocks), GC_page_size);
/*
* Note that we cannot examine `GC_dirty_pages` to check whether the
* page at `h_trunc` has already been marked dirty as there could be
* a hash collision.
*/
for (current = h_trunc; ADDR_LT((ptr_t)current, h_end); ++current) {
size_t index = PHT_HASH(current);
# ifndef DONT_PROTECT_PTRFREE
if (!is_ptrfree
|| !ADDR_INSIDE((ptr_t)current, (ptr_t)h, (ptr_t)(h + nblocks)))
# endif
{
async_set_pht_entry_from_index(GC_dirty_pages, index);
}
}
UNPROTECT(h_trunc, h_end - (ptr_t)h_trunc);
# else
/* Ignore write hints. They do not help us here. */
UNUSED_ARG(h);
UNUSED_ARG(nblocks);
UNUSED_ARG(is_ptrfree);
# endif
}
#endif /* !GC_DISABLE_INCREMENTAL */
#if defined(MPROTECT_VDB) && defined(DARWIN)
/*
* The following sources were used as a "reference" for this exception
* handling code:
* - Apple's mach/xnu documentation;
* - Timothy J. Wood's "Mach Exception Handlers 101" post to the omnigroup's
* macosx-dev list;
* - macosx-nat.c from Apple's GDB source code.
*/
/*
* The bug that caused all this trouble should now be fixed.
* This should eventually be removed if all goes well.
*/
# include <mach/exception.h>
# include <mach/mach.h>
# include <mach/mach_error.h>
# include <mach/task.h>
EXTERN_C_BEGIN
/*
* Some of the following prototypes are missing in any header, although
* they are documented. Some are in platform `mach/exc.h` file.
*/
extern boolean_t exc_server(mach_msg_header_t *, mach_msg_header_t *);
extern kern_return_t exception_raise(mach_port_t, mach_port_t, mach_port_t,
exception_type_t, exception_data_t,
mach_msg_type_number_t);
extern kern_return_t exception_raise_state(
mach_port_t, mach_port_t, mach_port_t, exception_type_t, exception_data_t,
mach_msg_type_number_t, thread_state_flavor_t *, thread_state_t,
mach_msg_type_number_t, thread_state_t, mach_msg_type_number_t *);
extern kern_return_t exception_raise_state_identity(
mach_port_t, mach_port_t, mach_port_t, exception_type_t, exception_data_t,
mach_msg_type_number_t, thread_state_flavor_t *, thread_state_t,
mach_msg_type_number_t, thread_state_t, mach_msg_type_number_t *);
GC_API_OSCALL kern_return_t catch_exception_raise(
mach_port_t exception_port, mach_port_t thread, mach_port_t task,
exception_type_t exception, exception_data_t code,
mach_msg_type_number_t code_count);
GC_API_OSCALL kern_return_t catch_exception_raise_state(
mach_port_name_t exception_port, int exception, exception_data_t code,
mach_msg_type_number_t codeCnt, int flavor, thread_state_t old_state,
int old_stateCnt, thread_state_t new_state, int new_stateCnt);
GC_API_OSCALL kern_return_t catch_exception_raise_state_identity(
mach_port_name_t exception_port, mach_port_t thread, mach_port_t task,
int exception, exception_data_t code, mach_msg_type_number_t codeCnt,
int flavor, thread_state_t old_state, int old_stateCnt,
thread_state_t new_state, int new_stateCnt);
EXTERN_C_END
/* These should never be called, but just in case... */
GC_API_OSCALL kern_return_t
catch_exception_raise_state(mach_port_name_t exception_port, int exception,
exception_data_t code,
mach_msg_type_number_t codeCnt, int flavor,
thread_state_t old_state, int old_stateCnt,
thread_state_t new_state, int new_stateCnt)
{
UNUSED_ARG(exception_port);
UNUSED_ARG(exception);
UNUSED_ARG(code);
UNUSED_ARG(codeCnt);
UNUSED_ARG(flavor);
UNUSED_ARG(old_state);
UNUSED_ARG(old_stateCnt);
UNUSED_ARG(new_state);
UNUSED_ARG(new_stateCnt);
ABORT_RET("Unexpected catch_exception_raise_state invocation");
return KERN_INVALID_ARGUMENT;
}
GC_API_OSCALL kern_return_t
catch_exception_raise_state_identity(
mach_port_name_t exception_port, mach_port_t thread, mach_port_t task,
int exception, exception_data_t code, mach_msg_type_number_t codeCnt,
int flavor, thread_state_t old_state, int old_stateCnt,
thread_state_t new_state, int new_stateCnt)
{
UNUSED_ARG(exception_port);
UNUSED_ARG(thread);
UNUSED_ARG(task);
UNUSED_ARG(exception);
UNUSED_ARG(code);
UNUSED_ARG(codeCnt);
UNUSED_ARG(flavor);
UNUSED_ARG(old_state);
UNUSED_ARG(old_stateCnt);
UNUSED_ARG(new_state);
UNUSED_ARG(new_stateCnt);
ABORT_RET("Unexpected catch_exception_raise_state_identity invocation");
return KERN_INVALID_ARGUMENT;
}
# define MAX_EXCEPTION_PORTS 16
static struct {
mach_msg_type_number_t count;
exception_mask_t masks[MAX_EXCEPTION_PORTS];
exception_handler_t ports[MAX_EXCEPTION_PORTS];
exception_behavior_t behaviors[MAX_EXCEPTION_PORTS];
thread_state_flavor_t flavors[MAX_EXCEPTION_PORTS];
} GC_old_exc_ports;
STATIC struct ports_s {
void (*volatile os_callback[3])(void);
mach_port_t exception;
# if defined(THREADS)
mach_port_t reply;
# endif
} GC_ports = { { /*< this is to prevent stripping these routines as dead */
(void (*)(void))catch_exception_raise,
(void (*)(void))catch_exception_raise_state,
(void (*)(void))catch_exception_raise_state_identity },
# ifdef THREADS
0 /* `exception` */,
# endif
0 };
typedef struct {
mach_msg_header_t head;
} GC_msg_t;
typedef enum {
GC_MP_NORMAL,
GC_MP_DISCARDING,
GC_MP_STOPPED
} GC_mprotect_state_t;
# ifdef THREADS
/*
* FIXME: 1 and 2 seem to be safe to use in the `msgh_id` field, but it
* is not documented. Use the source and see if they should be OK.
*/
# define ID_STOP 1
# define ID_RESUME 2
/* This value is only used on the reply port. */
# define ID_ACK 3
STATIC GC_mprotect_state_t GC_mprotect_state = GC_MP_NORMAL;
/* The following should *only* be called when the world is stopped. */
STATIC void
GC_mprotect_thread_notify(mach_msg_id_t id)
{
struct buf_s {
GC_msg_t msg;
mach_msg_trailer_t trailer;
} buf;
mach_msg_return_t r;
/* remote, local */
buf.msg.head.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND, 0);
buf.msg.head.msgh_size = sizeof(buf.msg);
buf.msg.head.msgh_remote_port = GC_ports.exception;
buf.msg.head.msgh_local_port = MACH_PORT_NULL;
buf.msg.head.msgh_id = id;
r = mach_msg(&buf.msg.head, MACH_SEND_MSG | MACH_RCV_MSG | MACH_RCV_LARGE,
sizeof(buf.msg), sizeof(buf), GC_ports.reply,
MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
if (r != MACH_MSG_SUCCESS)
ABORT("mach_msg failed in GC_mprotect_thread_notify");
if (buf.msg.head.msgh_id != ID_ACK)
ABORT("Invalid ack in GC_mprotect_thread_notify");
}
/* Should only be called by the `mprotect` thread. */
STATIC void
GC_mprotect_thread_reply(void)
{
GC_msg_t msg;
mach_msg_return_t r;
/* remote, local */
msg.head.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND, 0);
msg.head.msgh_size = sizeof(msg);
msg.head.msgh_remote_port = GC_ports.reply;
msg.head.msgh_local_port = MACH_PORT_NULL;
msg.head.msgh_id = ID_ACK;
r = mach_msg(&msg.head, MACH_SEND_MSG, sizeof(msg), 0, MACH_PORT_NULL,
MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
if (r != MACH_MSG_SUCCESS)
ABORT("mach_msg failed in GC_mprotect_thread_reply");
}
GC_INNER void
GC_mprotect_stop(void)
{
GC_mprotect_thread_notify(ID_STOP);
}
GC_INNER void
GC_mprotect_resume(void)
{
GC_mprotect_thread_notify(ID_RESUME);
}
# ifdef CAN_HANDLE_FORK
GC_INNER void
GC_dirty_update_child(void)
{
GC_ASSERT(I_HOLD_LOCK());
if (0 == GC_task_self) {
/* The GC incremental mode is off. */
return;
}
GC_ASSERT(GC_mprotect_state == GC_MP_NORMAL);
GC_task_self = mach_task_self(); /*< needed by `UNPROTECT()` */
GC_unprotect_all_heap();
/* Restore the old task exception ports. */
/* TODO: Should we do it in `fork_prepare_proc`/`fork_parent_proc`? */
if (GC_old_exc_ports.count > 0) {
/* TODO: Should we check `GC_old_exc_ports.count <= 1`? */
if (task_set_exception_ports(
GC_task_self, GC_old_exc_ports.masks[0], GC_old_exc_ports.ports[0],
GC_old_exc_ports.behaviors[0], GC_old_exc_ports.flavors[0])
!= KERN_SUCCESS)
ABORT("task_set_exception_ports failed (in child)");
}
/* TODO: Re-enable incremental mode in child. */
GC_task_self = 0;
GC_incremental = FALSE;
}
# endif /* CAN_HANDLE_FORK */
# else
/* The compiler should optimize away any `GC_mprotect_state` computations. */
# define GC_mprotect_state GC_MP_NORMAL
# endif /* !THREADS */
struct mp_reply_s {
mach_msg_header_t head;
char data[256];
};
struct mp_msg_s {
mach_msg_header_t head;
mach_msg_body_t msgh_body;
char data[1024];
};
STATIC void *
GC_mprotect_thread(void *arg)
{
mach_msg_return_t r;
/*
* These two structures contain some private kernel data. We do not need
* to access any of it so we do not bother defining a proper structure.
* The correct definitions are in the `xnu` source code.
*/
struct mp_reply_s reply;
struct mp_msg_s msg;
mach_msg_id_t id;
if (ADDR(arg) == GC_WORD_MAX)
return 0; /*< to prevent a compiler warning */
# if defined(CPPCHECK)
reply.data[0] = 0; /*< to prevent "field unused" warnings */
msg.data[0] = 0;
# endif
# if defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID)
(void)pthread_setname_np("GC-mprotect");
# endif
# if defined(THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
GC_darwin_register_self_mach_handler();
# endif
for (;;) {
r = mach_msg(
&msg.head,
MACH_RCV_MSG | MACH_RCV_LARGE
| (GC_mprotect_state == GC_MP_DISCARDING ? MACH_RCV_TIMEOUT : 0),
0, sizeof(msg), GC_ports.exception,
GC_mprotect_state == GC_MP_DISCARDING ? 0 : MACH_MSG_TIMEOUT_NONE,
MACH_PORT_NULL);
id = r == MACH_MSG_SUCCESS ? msg.head.msgh_id : -1;
# if defined(THREADS)
if (GC_mprotect_state == GC_MP_DISCARDING) {
if (r == MACH_RCV_TIMED_OUT) {
GC_mprotect_state = GC_MP_STOPPED;
GC_mprotect_thread_reply();
continue;
}
if (r == MACH_MSG_SUCCESS && (id == ID_STOP || id == ID_RESUME))
ABORT("Out of order mprotect thread request");
}
# endif /* THREADS */
if (r != MACH_MSG_SUCCESS) {
ABORT_ARG2("mach_msg failed", ": errcode= %d (%s)", (int)r,
mach_error_string(r));
}
switch (id) {
# if defined(THREADS)
case ID_STOP:
if (GC_mprotect_state != GC_MP_NORMAL)
ABORT("Called mprotect_stop when state wasn't normal");
GC_mprotect_state = GC_MP_DISCARDING;
break;
case ID_RESUME:
if (GC_mprotect_state != GC_MP_STOPPED)
ABORT("Called mprotect_resume when state wasn't stopped");
GC_mprotect_state = GC_MP_NORMAL;
GC_mprotect_thread_reply();
break;
# endif /* THREADS */
default:
/* Handle the message (it calls `catch_exception_raise`). */
if (!exc_server(&msg.head, &reply.head))
ABORT("exc_server failed");
/* Send the reply. */
r = mach_msg(&reply.head, MACH_SEND_MSG, reply.head.msgh_size, 0,
MACH_PORT_NULL, MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
if (r != MACH_MSG_SUCCESS) {
/*
* This will fail if the thread dies, but the thread should
* not die...
*/
# ifdef BROKEN_EXCEPTION_HANDLING
GC_err_printf("mach_msg failed with %d %s while sending "
"exc reply\n",
(int)r, mach_error_string(r));
# else
ABORT("mach_msg failed while sending exception reply");
# endif
}
}
}
}
/*
* All this `SIGBUS` code should not be necessary. All protection
* faults should be going through the `mach` exception handler.
* However, it seems a `SIGBUS` is occasionally sent for some unknown
* reason; even more odd, it seems to be meaningless and safe to ignore.
*/
# ifdef BROKEN_EXCEPTION_HANDLING
/*
* Updates to this are not atomic, but the `SIGBUS` signals seem pretty rare.
* Even if this does not get updated property, it is not really a problem.
*/
STATIC int GC_sigbus_count = 0;
STATIC void
GC_darwin_sigbus(int num, siginfo_t *sip, void *context)
{
if (num != SIGBUS)
ABORT("Got a non-sigbus signal in the sigbus handler");
/*
* Ugh... some seem safe to ignore, but too many in a row probably means
* trouble. `GC_sigbus_count` is reset for each `mach` exception that
* is handled.
*/
if (GC_sigbus_count >= 8)
ABORT("Got many SIGBUS signals in a row!");
GC_sigbus_count++;
WARN("Ignoring SIGBUS\n", 0);
}
# endif /* BROKEN_EXCEPTION_HANDLING */
GC_INNER GC_bool
GC_dirty_init(void)
{
kern_return_t r;
mach_port_t me;
pthread_t thread;
pthread_attr_t attr;
exception_mask_t mask;
GC_ASSERT(I_HOLD_LOCK());
# if defined(CAN_HANDLE_FORK) && !defined(THREADS)
if (GC_handle_fork) {
/*
* To both support GC incremental mode and GC functions usage in
* the forked child process, `pthread_atfork` should be used to
* install handlers that switch off `GC_incremental` in the child
* gracefully (unprotecting all pages and clearing
* `GC_mach_handler_thread`). For now, we just disable incremental
* mode if `fork()` handling is requested by the client.
*/
WARN("Can't turn on GC incremental mode as fork()"
" handling requested\n",
0);
return FALSE;
}
# endif
GC_VERBOSE_LOG_PRINTF("Initializing mach/darwin mprotect"
" virtual dirty bit implementation\n");
# ifdef BROKEN_EXCEPTION_HANDLING
WARN("Enabling workarounds for various darwin exception handling bugs\n", 0);
# endif
if (GC_page_size % HBLKSIZE != 0) {
ABORT("Page size not multiple of HBLKSIZE");
}
GC_task_self = me = mach_task_self();
GC_ASSERT(me != 0);
r = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, &GC_ports.exception);
/* TODO: Call `WARN()` and return `FALSE` in case of a failure. */
if (r != KERN_SUCCESS)
ABORT("mach_port_allocate failed (exception port)");
r = mach_port_insert_right(me, GC_ports.exception, GC_ports.exception,
MACH_MSG_TYPE_MAKE_SEND);
if (r != KERN_SUCCESS)
ABORT("mach_port_insert_right failed (exception port)");
# if defined(THREADS)
r = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, &GC_ports.reply);
if (r != KERN_SUCCESS)
ABORT("mach_port_allocate failed (reply port)");
# endif
/* The exceptions we want to catch. */
mask = EXC_MASK_BAD_ACCESS;
r = task_get_exception_ports(me, mask, GC_old_exc_ports.masks,
&GC_old_exc_ports.count, GC_old_exc_ports.ports,
GC_old_exc_ports.behaviors,
GC_old_exc_ports.flavors);
if (r != KERN_SUCCESS)
ABORT("task_get_exception_ports failed");
r = task_set_exception_ports(me, mask, GC_ports.exception, EXCEPTION_DEFAULT,
GC_MACH_THREAD_STATE);
if (r != KERN_SUCCESS)
ABORT("task_set_exception_ports failed");
if (pthread_attr_init(&attr) != 0)
ABORT("pthread_attr_init failed");
if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0)
ABORT("pthread_attr_setdetachedstate failed");
/* This will call the real `pthreads` routine, not our wrapper. */
if (GC_inner_pthread_create(&thread, &attr, GC_mprotect_thread, NULL) != 0)
ABORT("pthread_create failed");
(void)pthread_attr_destroy(&attr);
/* Setup the handler for ignoring the meaningless `SIGBUS` signals. */
# ifdef BROKEN_EXCEPTION_HANDLING
{
struct sigaction sa, oldsa;
sa.sa_handler = (SIG_HNDLR_PTR)GC_darwin_sigbus;
sigemptyset(&sa.sa_mask);
sa.sa_flags = SA_RESTART | SA_SIGINFO;
/* `sa.sa_restorer` is deprecated and should not be initialized. */
if (sigaction(SIGBUS, &sa, &oldsa) < 0)
ABORT("sigaction failed");
if ((GC_funcptr_uint)oldsa.sa_handler != (GC_funcptr_uint)SIG_DFL) {
GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n");
}
}
# endif /* BROKEN_EXCEPTION_HANDLING */
# if defined(CPPCHECK)
GC_noop1((word)(GC_funcptr_uint)GC_ports.os_callback[0]);
# endif
return TRUE;
}
/*
* The source code for Apple's GDB was used as a reference for the
* exception forwarding code. This code is similar to be GDB code only
* because there is only one way to do it.
*/
STATIC kern_return_t
GC_forward_exception(mach_port_t thread, mach_port_t task,
exception_type_t exception, exception_data_t data,
mach_msg_type_number_t data_count)
{
size_t i;
kern_return_t r;
mach_port_t port;
exception_behavior_t behavior;
thread_state_flavor_t flavor;
thread_state_data_t thread_state;
mach_msg_type_number_t thread_state_count = THREAD_STATE_MAX;
for (i = 0; i < (size_t)GC_old_exc_ports.count; i++) {
if ((GC_old_exc_ports.masks[i] & ((exception_mask_t)1 << exception)) != 0)
break;
}
if (i == (size_t)GC_old_exc_ports.count)
ABORT("No handler for exception!");
port = GC_old_exc_ports.ports[i];
behavior = GC_old_exc_ports.behaviors[i];
flavor = GC_old_exc_ports.flavors[i];
if (behavior == EXCEPTION_STATE || behavior == EXCEPTION_STATE_IDENTITY) {
r = thread_get_state(thread, flavor, thread_state, &thread_state_count);
if (r != KERN_SUCCESS)
ABORT("thread_get_state failed in forward_exception");
}
switch (behavior) {
case EXCEPTION_STATE:
r = exception_raise_state(port, thread, task, exception, data, data_count,
&flavor, thread_state, thread_state_count,
thread_state, &thread_state_count);
break;
case EXCEPTION_STATE_IDENTITY:
r = exception_raise_state_identity(
port, thread, task, exception, data, data_count, &flavor, thread_state,
thread_state_count, thread_state, &thread_state_count);
break;
/* `case EXCEPTION_DEFAULT:` - the default signal handlers. */
default:
/* The user-supplied signal handlers. */
r = exception_raise(port, thread, task, exception, data, data_count);
}
if (behavior == EXCEPTION_STATE || behavior == EXCEPTION_STATE_IDENTITY) {
r = thread_set_state(thread, flavor, thread_state, thread_state_count);
if (r != KERN_SUCCESS)
ABORT("thread_set_state failed in forward_exception");
}
return r;
}
# define FWD() GC_forward_exception(thread, task, exception, code, code_count)
# ifdef ARM32
# define DARWIN_EXC_STATE ARM_EXCEPTION_STATE
# define DARWIN_EXC_STATE_COUNT ARM_EXCEPTION_STATE_COUNT
# define DARWIN_EXC_STATE_T arm_exception_state_t
# define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(far)
# elif defined(AARCH64)
# define DARWIN_EXC_STATE ARM_EXCEPTION_STATE64
# define DARWIN_EXC_STATE_COUNT ARM_EXCEPTION_STATE64_COUNT
# define DARWIN_EXC_STATE_T arm_exception_state64_t
# define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(far)
# elif defined(POWERPC)
# if CPP_WORDSZ == 32
# define DARWIN_EXC_STATE PPC_EXCEPTION_STATE
# define DARWIN_EXC_STATE_COUNT PPC_EXCEPTION_STATE_COUNT
# define DARWIN_EXC_STATE_T ppc_exception_state_t
# else
# define DARWIN_EXC_STATE PPC_EXCEPTION_STATE64
# define DARWIN_EXC_STATE_COUNT PPC_EXCEPTION_STATE64_COUNT
# define DARWIN_EXC_STATE_T ppc_exception_state64_t
# endif
# define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(dar)
# elif defined(I386) || defined(X86_64)
# if CPP_WORDSZ == 32
# if defined(i386_EXCEPTION_STATE_COUNT) \
&& !defined(x86_EXCEPTION_STATE32_COUNT)
/* Use old naming convention for i686. */
# define DARWIN_EXC_STATE i386_EXCEPTION_STATE
# define DARWIN_EXC_STATE_COUNT i386_EXCEPTION_STATE_COUNT
# define DARWIN_EXC_STATE_T i386_exception_state_t
# else
# define DARWIN_EXC_STATE x86_EXCEPTION_STATE32
# define DARWIN_EXC_STATE_COUNT x86_EXCEPTION_STATE32_COUNT
# define DARWIN_EXC_STATE_T x86_exception_state32_t
# endif
# else
# define DARWIN_EXC_STATE x86_EXCEPTION_STATE64
# define DARWIN_EXC_STATE_COUNT x86_EXCEPTION_STATE64_COUNT
# define DARWIN_EXC_STATE_T x86_exception_state64_t
# endif
# define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(faultvaddr)
# elif !defined(CPPCHECK)
# error FIXME for non-arm/ppc/x86 darwin
# endif
/*
* This violates the namespace rules but there is not anything that can
* be done about it. The exception handling stuff is hard-coded to
* call this. `catch_exception_raise`, `catch_exception_raise_state`
* and `catch_exception_raise_state_identity` are called from OS.
*/
GC_API_OSCALL kern_return_t
catch_exception_raise(mach_port_t exception_port, mach_port_t thread,
mach_port_t task, exception_type_t exception,
exception_data_t code, mach_msg_type_number_t code_count)
{
kern_return_t r;
char *addr;
thread_state_flavor_t flavor = DARWIN_EXC_STATE;
mach_msg_type_number_t exc_state_count = DARWIN_EXC_STATE_COUNT;
DARWIN_EXC_STATE_T exc_state;
UNUSED_ARG(exception_port);
UNUSED_ARG(task);
if (exception != EXC_BAD_ACCESS || code[0] != KERN_PROTECTION_FAILURE) {
# ifdef DEBUG_EXCEPTION_HANDLING
/* We are not interested in, pass it on to the old handler. */
GC_log_printf("Exception: 0x%x Code: 0x%x 0x%x in catch...\n", exception,
code_count > 0 ? code[0] : -1,
code_count > 1 ? code[1] : -1);
# else
UNUSED_ARG(code_count);
# endif
return FWD();
}
r = thread_get_state(thread, flavor, (natural_t *)&exc_state,
&exc_state_count);
if (r != KERN_SUCCESS) {
/*
* The thread is supposed to be suspended while the exception
* handler is called. This should not fail.
*/
# ifdef BROKEN_EXCEPTION_HANDLING
GC_err_printf("thread_get_state failed in catch_exception_raise\n");
return KERN_SUCCESS;
# else
ABORT("thread_get_state failed in catch_exception_raise");
# endif
}
/* This is the address that caused the fault. */
addr = (char *)exc_state.DARWIN_EXC_STATE_DAR;
if (!is_header_found_async(addr)) {
/*
* Ugh... just like the `SIGBUS` problem above, it seems we get
* a bogus `KERN_PROTECTION_FAILURE` every once and a while.
* We wait till we get a bunch in a row before doing anything
* about it. If a "real" fault ever occurs, it will just keep
* faulting over and over, and we will hit the limit pretty quickly.
*/
# ifdef BROKEN_EXCEPTION_HANDLING
static const char *last_fault;
static int last_fault_count;
if (addr != last_fault) {
last_fault = addr;
last_fault_count = 0;
}
if (++last_fault_count < 32) {
if (last_fault_count == 1)
WARN("Ignoring KERN_PROTECTION_FAILURE at %p\n", addr);
return KERN_SUCCESS;
}
GC_err_printf("Unexpected KERN_PROTECTION_FAILURE at %p; aborting...\n",
(void *)addr);
/*
* Cannot pass it along to the signal handler because that is ignoring
* `SIGBUS` signals. We also should not call `ABORT()` here as signals
* do not always work too well from the exception handler.
*/
EXIT();
# else
/*
* Pass it along to the next exception handler (which should call
* `SIGBUS`/`SIGSEGV`).
*/
return FWD();
# endif /* !BROKEN_EXCEPTION_HANDLING */
}
# ifdef BROKEN_EXCEPTION_HANDLING
/* Reset the number of consecutive `SIGBUS` signals. */
GC_sigbus_count = 0;
# endif
GC_ASSERT(GC_page_size != 0);
if (GC_mprotect_state == GC_MP_NORMAL) {
/* The common case. */
struct hblk *h = HBLK_PAGE_ALIGNED(addr);
size_t i;
# ifdef CHECKSUMS
GC_record_fault(h);
# endif
UNPROTECT(h, GC_page_size);
for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
size_t index = PHT_HASH(h + i);
async_set_pht_entry_from_index(GC_dirty_pages, index);
}
} else if (GC_mprotect_state == GC_MP_DISCARDING) {
/*
* Lie to the thread for now. No sense `UNPROTECT`'ing the memory
* when we are just going to `PROTECT()` it again later.
* The thread will just fault again once it resumes.
*/
} else {
/* Should not happen, I do not think. */
GC_err_printf("KERN_PROTECTION_FAILURE while world is stopped\n");
return FWD();
}
return KERN_SUCCESS;
}
# undef FWD
# ifndef NO_DESC_CATCH_EXCEPTION_RAISE
/*
* These symbols should have `REFERENCED_DYNAMICALLY` (0x10) bit set to
* let strip know they are not to be stripped.
*/
__asm__(".desc _catch_exception_raise, 0x10");
__asm__(".desc _catch_exception_raise_state, 0x10");
__asm__(".desc _catch_exception_raise_state_identity, 0x10");
# endif
#endif /* DARWIN && MPROTECT_VDB */
GC_API int GC_CALL
GC_incremental_protection_needs(void)
{
GC_ASSERT(GC_is_initialized);
#ifdef MPROTECT_VDB
# if defined(GWW_VDB) || (defined(SOFT_VDB) && !defined(CHECK_SOFT_VDB))
/* Only if the incremental mode is already switched on. */
if (GC_GWW_AVAILABLE())
return GC_PROTECTS_NONE;
# endif
# ifndef DONT_PROTECT_PTRFREE
if (GC_page_size != HBLKSIZE)
return GC_PROTECTS_POINTER_HEAP | GC_PROTECTS_PTRFREE_HEAP;
# endif
return GC_PROTECTS_POINTER_HEAP;
#else
return GC_PROTECTS_NONE;
#endif
}
GC_API unsigned GC_CALL
GC_get_actual_vdb(void)
{
#ifndef GC_DISABLE_INCREMENTAL
if (GC_incremental) {
# ifndef NO_MANUAL_VDB
if (GC_manual_vdb)
return GC_VDB_MANUAL;
# endif
# ifdef MPROTECT_VDB
# ifdef GWW_VDB
if (GC_GWW_AVAILABLE())
return GC_VDB_GWW;
# endif
# ifdef SOFT_VDB
if (GC_GWW_AVAILABLE())
return GC_VDB_SOFT;
# endif
return GC_VDB_MPROTECT;
# elif defined(GWW_VDB)
return GC_VDB_GWW;
# elif defined(SOFT_VDB)
return GC_VDB_SOFT;
# elif defined(PROC_VDB)
return GC_VDB_PROC;
# else /* DEFAULT_VDB */
return GC_VDB_DEFAULT;
# endif
}
#endif
return GC_VDB_NONE;
}
#ifdef ECOS
/* Undo `sbrk()` redirection. */
# undef sbrk
#endif
GC_API void GC_CALL
GC_set_pages_executable(int value)
{
GC_ASSERT(!GC_is_initialized);
/*
* Even if `IGNORE_PAGES_EXECUTABLE` macro is defined,
* `GC_pages_executable` is touched here to prevent a compiler warning.
*/
GC_pages_executable = (GC_bool)(value != 0);
}
GC_API int GC_CALL
GC_get_pages_executable(void)
{
/*
* `GC_get_pages_executable` is defined after all the places
* where `GC_get_pages_executable` is undefined.
*/
#ifdef IGNORE_PAGES_EXECUTABLE
/* Always allocate executable memory. */
return 1;
#else
return (int)GC_pages_executable;
#endif
}
/*
* Call stack save code for debugging. Should probably be in
* `mach_dep.c` file, but that requires reorganization.
*/
#ifdef NEED_CALLINFO
/*
* I suspect the following works for most Un*x i686 variants, so long as
* the frame pointer is explicitly stored. In the case of gcc, the client
* code should not be compiled with `-fomit-frame-pointer` option.
*/
# if defined(I386) && defined(LINUX) && defined(SAVE_CALL_CHAIN)
struct frame {
struct frame *fr_savfp;
long fr_savpc;
# if NARGS > 0
/* All the arguments go here. */
long fr_arg[NARGS];
# endif
};
# endif
# if defined(SPARC)
# if defined(LINUX)
# if defined(SAVE_CALL_CHAIN)
struct frame {
long fr_local[8];
long fr_arg[6];
struct frame *fr_savfp;
long fr_savpc;
# ifndef __arch64__
char *fr_stret;
# endif
long fr_argd[6];
long fr_argx[0];
};
# endif
# elif defined(DRSNX)
# include <sys/sparc/frame.h>
# elif defined(OPENBSD)
# include <frame.h>
# elif defined(FREEBSD) || defined(NETBSD)
# include <machine/frame.h>
# else
# include <sys/frame.h>
# endif
# if NARGS > 6
# error We only know how to get the first 6 arguments
# endif
# endif /* SPARC */
/*
* Fill in the `pc` and `argument` information for up to `NFRAMES` of
* my callers. Ignore my frame and my callers frame.
*/
# if defined(GC_HAVE_BUILTIN_BACKTRACE)
# ifdef _MSC_VER
EXTERN_C_BEGIN
int backtrace(void *addresses[], int count);
char **backtrace_symbols(void *const addresses[], int count);
EXTERN_C_END
# else
# include <execinfo.h>
# endif
# endif /* GC_HAVE_BUILTIN_BACKTRACE */
# ifdef SAVE_CALL_CHAIN
# if NARGS == 0 && NFRAMES % 2 == 0 /*< no padding */ \
&& defined(GC_HAVE_BUILTIN_BACKTRACE)
# ifdef REDIRECT_MALLOC
/*
* Deal with possible `malloc()` calls in `backtrace()` by omitting
* the infinitely recursing backtrace.
*/
STATIC GC_bool GC_in_save_callers = FALSE;
# if defined(THREADS) && defined(DBG_HDRS_ALL)
GC_INNER void
GC_save_callers_no_unlock(struct callinfo info[NFRAMES])
{
GC_ASSERT(I_HOLD_LOCK());
info[0].ci_pc
= CAST_THRU_UINTPTR(GC_return_addr_t, GC_save_callers_no_unlock);
BZERO(&info[1], sizeof(void *) * (NFRAMES - 1));
}
# endif
# endif /* REDIRECT_MALLOC */
GC_INNER void
GC_save_callers(struct callinfo info[NFRAMES])
{
void *tmp_info[NFRAMES + 1];
int npcs, i;
/*
* `backtrace()` may call `dl_iterate_phdr` which is also used by
* `GC_register_dynamic_libraries()`, and `dl_iterate_phdr` is not
* guaranteed to be reentrant.
*/
GC_ASSERT(I_HOLD_LOCK());
GC_STATIC_ASSERT(sizeof(struct callinfo) == sizeof(void *));
# ifdef REDIRECT_MALLOC
if (GC_in_save_callers) {
info[0].ci_pc = CAST_THRU_UINTPTR(GC_return_addr_t, GC_save_callers);
BZERO(&info[1], sizeof(void *) * (NFRAMES - 1));
return;
}
GC_in_save_callers = TRUE;
/* `backtrace()` might call a redirected `malloc`. */
UNLOCK();
npcs = backtrace((void **)tmp_info, NFRAMES + 1);
LOCK();
# else
npcs = backtrace((void **)tmp_info, NFRAMES + 1);
# endif
/*
* We retrieve `NFRAMES + 1` `pc` values, but discard the first one,
* since it points to our own frame.
*/
i = 0;
if (npcs > 1) {
i = npcs - 1;
BCOPY(&tmp_info[1], info, (unsigned)i * sizeof(void *));
}
BZERO(&info[i], sizeof(void *) * (unsigned)(NFRAMES - i));
# ifdef REDIRECT_MALLOC
GC_in_save_callers = FALSE;
# endif
}
# elif defined(I386) || defined(SPARC)
# if defined(ANY_BSD) && defined(SPARC)
# define FR_SAVFP fr_fp
# define FR_SAVPC fr_pc
# else
# define FR_SAVFP fr_savfp
# define FR_SAVPC fr_savpc
# endif
# if defined(SPARC) && (defined(__arch64__) || defined(__sparcv9))
# define BIAS 2047
# else
# define BIAS 0
# endif
GC_INNER void
GC_save_callers(struct callinfo info[NFRAMES])
{
struct frame *frame;
struct frame *fp;
int nframes = 0;
# ifdef I386
/* We assume this is turned on only with gcc as the compiler. */
asm("movl %%ebp,%0" : "=r"(frame));
fp = frame;
# else /* SPARC */
frame = (struct frame *)GC_save_regs_in_stack();
fp = (struct frame *)((ptr_t)frame->FR_SAVFP + BIAS);
# endif
for (; !HOTTER_THAN((ptr_t)fp, (ptr_t)frame)
# ifndef THREADS
&& !HOTTER_THAN(GC_stackbottom, (ptr_t)fp)
# elif defined(STACK_GROWS_UP)
&& fp != NULL
# endif
&& nframes < NFRAMES;
fp = (struct frame *)((ptr_t)fp->FR_SAVFP + BIAS), nframes++) {
# if NARGS > 0
int i;
# endif
info[nframes].ci_pc = (GC_return_addr_t)fp->FR_SAVPC;
# if NARGS > 0
for (i = 0; i < NARGS; i++) {
info[nframes].ci_arg[i] = GC_HIDE_NZ_POINTER(MAKE_CPTR(fp->fr_arg[i]));
}
# endif
}
if (nframes < NFRAMES)
info[nframes].ci_pc = 0;
}
# endif /* !GC_HAVE_BUILTIN_BACKTRACE */
# endif /* SAVE_CALL_CHAIN */
GC_INNER void
GC_print_callers(struct callinfo info[NFRAMES])
{
int i, reent_cnt;
# if defined(AO_HAVE_fetch_and_add1) && defined(AO_HAVE_fetch_and_sub1)
static volatile AO_t reentry_count = 0;
/*
* Note: alternatively, if available, we may use a thread-local storage,
* thus, enabling concurrent usage of `GC_print_callers()`;
* but practically this has little sense because printing is done into
* a single output stream.
*/
GC_ASSERT(I_DONT_HOLD_LOCK());
reent_cnt = (int)(GC_signed_word)AO_fetch_and_add1(&reentry_count);
# else
static int reentry_count = 0;
/* Note: this could use a different lock. */
LOCK();
reent_cnt = reentry_count++;
UNLOCK();
# endif
# if NFRAMES == 1
GC_err_printf("\tCaller at allocation:\n");
# else
GC_err_printf("\tCall chain at allocation:\n");
# endif
for (i = 0; i < NFRAMES; i++) {
# if defined(LINUX) && !defined(SMALL_CONFIG)
GC_bool stop = FALSE;
# endif
if (0 == info[i].ci_pc)
break;
# if NARGS > 0
{
int j;
GC_err_printf("\t\targs: ");
for (j = 0; j < NARGS; j++) {
void *p = GC_REVEAL_NZ_POINTER(info[i].ci_arg[j]);
if (j != 0)
GC_err_printf(", ");
GC_err_printf("%ld (%p)", (long)(GC_signed_word)ADDR(p), p);
}
GC_err_printf("\n");
}
# endif
if (reent_cnt > 0) {
/*
* We were called either concurrently or during an allocation
* by `backtrace_symbols()` called from `GC_print_callers`; punt.
*/
GC_err_printf("\t\t##PC##= 0x%lx\n", (unsigned long)ADDR(info[i].ci_pc));
continue;
}
{
char buf[40];
char *name;
# if defined(GC_HAVE_BUILTIN_BACKTRACE) \
&& !defined(GC_BACKTRACE_SYMBOLS_BROKEN) && defined(FUNCPTR_IS_DATAPTR)
char **sym_name = backtrace_symbols((void **)&info[i].ci_pc, 1);
if (sym_name != NULL) {
name = sym_name[0];
} else
# endif
/* else */ {
(void)snprintf(buf, sizeof(buf), "##PC##= 0x%lx",
(unsigned long)ADDR(info[i].ci_pc));
buf[sizeof(buf) - 1] = '\0';
name = buf;
}
# if defined(LINUX) && !defined(SMALL_CONFIG)
/* Try for a line number. */
do {
FILE *pipe;
# define EXE_SZ 100
static char exe_name[EXE_SZ];
# define CMD_SZ 200
char cmd_buf[CMD_SZ];
# define RESULT_SZ 200
static char result_buf[RESULT_SZ];
size_t result_len;
const char *old_preload;
# define PRELOAD_SZ 200
char preload_buf[PRELOAD_SZ];
static GC_bool found_exe_name = FALSE;
static GC_bool will_fail = FALSE;
/*
* Try to get it via a hairy and expensive scheme.
* First we get the name of the executable.
*/
if (will_fail)
break;
if (!found_exe_name) {
int ret_code = readlink("/proc/self/exe", exe_name, EXE_SZ);
if (ret_code < 0 || ret_code >= EXE_SZ || exe_name[0] != '/') {
/* Do not try again. */
will_fail = TRUE;
break;
}
exe_name[ret_code] = '\0';
found_exe_name = TRUE;
}
/*
* Then we use `popen()` to start `addr2line -e <exe> <addr>`.
* There are faster ways to do this, but hopefully this is
* not time critical.
*/
(void)snprintf(cmd_buf, sizeof(cmd_buf),
"/usr/bin/addr2line -f -e %s 0x%lx", exe_name,
(unsigned long)ADDR(info[i].ci_pc));
cmd_buf[sizeof(cmd_buf) - 1] = '\0';
old_preload = GETENV("LD_PRELOAD");
if (old_preload != NULL) {
size_t old_len = strlen(old_preload);
if (old_len >= PRELOAD_SZ) {
will_fail = TRUE;
break;
}
BCOPY(old_preload, preload_buf, old_len + 1);
unsetenv("LD_PRELOAD");
}
pipe = popen(cmd_buf, "r");
if (old_preload != NULL
&& setenv("LD_PRELOAD", preload_buf, 0 /* `overwrite` */) == -1) {
WARN("Failed to reset LD_PRELOAD\n", 0);
}
if (NULL == pipe) {
will_fail = TRUE;
break;
}
result_len = fread(result_buf, 1, RESULT_SZ - 1, pipe);
(void)pclose(pipe);
if (0 == result_len) {
will_fail = TRUE;
break;
}
if (result_buf[result_len - 1] == '\n')
--result_len;
result_buf[result_len] = 0;
if (result_buf[0] == '?'
|| (result_buf[result_len - 2] == ':'
&& result_buf[result_len - 1] == '0'))
break;
/* Get rid of embedded newline, if any. Test for "main". */
{
char *nl = strchr(result_buf, '\n');
if (nl != NULL && ADDR_LT(nl, result_buf + result_len)) {
*nl = ':';
}
if (strncmp(result_buf, "main",
nl != NULL
? (size_t)(ADDR(nl) /*< CPPCHECK */
- COVERT_DATAFLOW(ADDR(result_buf)))
: result_len)
== 0) {
stop = TRUE;
}
}
if (result_len < RESULT_SZ - 25) {
/* Add address in the hex format. */
(void)snprintf(&result_buf[result_len],
sizeof(result_buf) - result_len, " [0x%lx]",
(unsigned long)ADDR(info[i].ci_pc));
result_buf[sizeof(result_buf) - 1] = '\0';
}
# if defined(CPPCHECK)
GC_noop1((unsigned char)name[0]);
/* The value of name computed previously is discarded. */
# endif
name = result_buf;
} while (0);
# endif /* LINUX */
GC_err_printf("\t\t%s\n", name);
# if defined(GC_HAVE_BUILTIN_BACKTRACE) \
&& !defined(GC_BACKTRACE_SYMBOLS_BROKEN) && defined(FUNCPTR_IS_DATAPTR)
if (sym_name != NULL) {
/* May call `GC_free()`, `GC_debug_free()`; that is OK. */
free(sym_name);
}
# endif
}
# if defined(LINUX) && !defined(SMALL_CONFIG)
if (stop)
break;
# endif
}
# if defined(AO_HAVE_fetch_and_add1) && defined(AO_HAVE_fetch_and_sub1)
(void)AO_fetch_and_sub1(&reentry_count);
# else
LOCK();
--reentry_count;
UNLOCK();
# endif
}
#endif /* NEED_CALLINFO */
#if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG)
/*
* Dump `/proc/self/maps` file to `GC_stderr`, to enable looking up names
* for addresses in `FIND_LEAK` output.
*/
void
GC_print_address_map(void)
{
const char *maps_ptr;
GC_ASSERT(I_HOLD_LOCK());
maps_ptr = GC_get_maps();
GC_err_printf("---------- Begin address map ----------\n");
GC_err_puts(maps_ptr);
GC_err_printf("---------- End address map ----------\n");
}
#endif /* LINUX && ELF */
/*
* Copyright (c) 2000-2005 by Hewlett-Packard Company. All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#if defined(THREAD_LOCAL_ALLOC)
# if !defined(THREADS) && !defined(CPPCHECK)
# error Invalid config - THREAD_LOCAL_ALLOC requires GC_THREADS
# endif
/*
* Copyright (c) 2000-2005 by Hewlett-Packard Company. All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* Included indirectly from a thread-library-specific file.
* This is the interface for thread-local allocation, whose implementation
* is mostly thread-library-independent. Here we describe only the interface
* that needs to be known and invoked from the thread support layer;
* the actual implementation also exports `GC_malloc` and friends, which are
* declared in `gc.h` file.
*/
#ifndef GC_THREAD_LOCAL_ALLOC_H
#define GC_THREAD_LOCAL_ALLOC_H
#ifdef THREAD_LOCAL_ALLOC
# if defined(USE_HPUX_TLS)
# error USE_HPUX_TLS macro was replaced by USE_COMPILER_TLS
# endif
# include <stdlib.h>
EXTERN_C_BEGIN
# if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC) \
&& !defined(USE_WIN32_COMPILER_TLS) && !defined(USE_COMPILER_TLS) \
&& !defined(USE_CUSTOM_SPECIFIC)
# if defined(GC_WIN32_THREADS)
# if defined(CYGWIN32) && GC_GNUC_PREREQ(4, 0)
# if defined(__clang__)
/* As of Cygwin clang-3.5.2, thread-local storage is unsupported. */
# define USE_PTHREAD_SPECIFIC
# else
# define USE_COMPILER_TLS
# endif
# elif defined(__GNUC__) || defined(MSWINCE)
# define USE_WIN32_SPECIFIC
# else
# define USE_WIN32_COMPILER_TLS
# endif /* !GNU */
# elif defined(HOST_ANDROID)
# if defined(ARM32) \
&& (GC_GNUC_PREREQ(4, 6) || GC_CLANG_PREREQ_FULL(3, 8, 256229))
# define USE_COMPILER_TLS
# elif !defined(__clang__) && !defined(ARM32)
/* TODO: Support clang/arm64. */
# define USE_COMPILER_TLS
# else
# define USE_PTHREAD_SPECIFIC
# endif
# elif defined(LINUX) && GC_GNUC_PREREQ(3, 3) /* `&& !HOST_ANDROID` */
# if defined(ARM32) || defined(AVR32)
/* TODO: Change to USE_COMPILER_TLS on Linux/arm. */
# define USE_PTHREAD_SPECIFIC
# elif defined(AARCH64) && defined(__clang__) && !GC_CLANG_PREREQ(8, 0)
/* To avoid "R_AARCH64_ABS64 used with TLS symbol" linker warnings. */
# define USE_PTHREAD_SPECIFIC
# else
# define USE_COMPILER_TLS
# endif
# elif (defined(COSMO) || defined(FREEBSD) \
|| (defined(NETBSD) && __NetBSD_Version__ >= 600000000 /* 6.0 */)) \
&& (GC_GNUC_PREREQ(4, 4) || GC_CLANG_PREREQ(3, 9))
# define USE_COMPILER_TLS
# elif defined(HPUX)
# ifdef __GNUC__
/* Empirically, as of gcc-3.3, USE_COMPILER_TLS does not work. */
# define USE_PTHREAD_SPECIFIC
# else
# define USE_COMPILER_TLS
# endif
# elif defined(IRIX5) || defined(OPENBSD) || defined(SOLARIS) \
|| defined(NINTENDO_SWITCH) || defined(NN_PLATFORM_CTR)
/* Use our own. */
# define USE_CUSTOM_SPECIFIC
# else
# define USE_PTHREAD_SPECIFIC
# endif
# endif /* !USE_x_SPECIFIC */
# ifndef THREAD_FREELISTS_KINDS
# ifdef ENABLE_DISCLAIM
# define THREAD_FREELISTS_KINDS (NORMAL + 2)
# else
# define THREAD_FREELISTS_KINDS (NORMAL + 1)
# endif
# endif /* !THREAD_FREELISTS_KINDS */
/*
* The first `GC_TINY_FREELISTS` free lists correspond to the first
* `GC_TINY_FREELISTS` multiples of `GC_GRANULE_BYTES`, i.e. we keep
* separate free lists for each multiple of `GC_GRANULE_BYTES` up to
* `(GC_TINY_FREELISTS-1) * GC_GRANULE_BYTES`. After that they may be
* spread out further.
*/
/*
* This should be used for the `tlfs` field in the structure pointed to
* by a `GC_thread` entity. Free lists contain either a pointer or
* a small count reflecting the number of granules allocated at that size:
* - zero means thread-local allocation in use, free list is empty;
* - greater than zero but not greater than `DIRECT_GRANULES` means
* using global allocation, too few objects of this size have been
* allocated by this thread;
* - greater than `DIRECT_GRANULES` but less than `HBLKSIZE` means
* transition to local allocation, equivalent to zero;
* - not less than `HBLKSIZE` means pointer to nonempty free list.
*/
struct thread_local_freelists {
/* Note: preserve `*_freelists` names for some clients. */
void *_freelists[THREAD_FREELISTS_KINDS][GC_TINY_FREELISTS];
# define ptrfree_freelists _freelists[PTRFREE]
# define normal_freelists _freelists[NORMAL]
# ifdef GC_GCJ_SUPPORT
void *gcj_freelists[GC_TINY_FREELISTS];
/* A value used for `gcj_freelists[-1]`; allocation is erroneous. */
# define ERROR_FL GC_WORD_MAX
# endif
/* Do not use local free lists for up to this much allocation. */
# define DIRECT_GRANULES (HBLKSIZE / GC_GRANULE_BYTES)
};
typedef struct thread_local_freelists *GC_tlfs;
# if defined(USE_PTHREAD_SPECIFIC)
# define GC_getspecific pthread_getspecific
# define GC_setspecific pthread_setspecific
# define GC_key_create pthread_key_create
/*
* Explicitly delete the value to stop the TLS (thread-local storage)
* destructor from being called repeatedly.
*/
# define GC_remove_specific(key) (void)pthread_setspecific(key, NULL)
# define GC_remove_specific_after_fork(key, t) \
(void)0 /*< no action needed */
typedef pthread_key_t GC_key_t;
# elif defined(USE_COMPILER_TLS) || defined(USE_WIN32_COMPILER_TLS)
# define GC_getspecific(x) (x)
# define GC_setspecific(key, v) ((key) = (v), 0)
# define GC_key_create(key, d) 0
/* Just to clear the pointer to `tlfs`. */
# define GC_remove_specific(key) (void)GC_setspecific(key, NULL)
# define GC_remove_specific_after_fork(key, t) (void)0
typedef void *GC_key_t;
# elif defined(USE_WIN32_SPECIFIC)
# define GC_getspecific TlsGetValue
/* Note: we assume that zero means success, Win32 API does the opposite. */
# define GC_setspecific(key, v) !TlsSetValue(key, v)
# ifndef TLS_OUT_OF_INDEXES
/* This is currently missing in WinCE. */
# define TLS_OUT_OF_INDEXES (DWORD)0xFFFFFFFF
# endif
# define GC_key_create(key, d) \
((d) != 0 || (*(key) = TlsAlloc()) == TLS_OUT_OF_INDEXES ? -1 : 0)
/* TODO: Is `TlsFree` needed on process exit/detach? */
# define GC_remove_specific(key) (void)GC_setspecific(key, NULL)
# define GC_remove_specific_after_fork(key, t) (void)0
typedef DWORD GC_key_t;
# elif defined(USE_CUSTOM_SPECIFIC)
EXTERN_C_END
/*
* Copyright (c) 2000 by Hewlett-Packard Company. All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* This is a reimplementation of a subset of the
* `pthread_getspecific`/`pthread_setspecific` interface. This appears
* to outperform the standard LinuxThreads one by a significant margin.
* The major restriction is that each thread may only make a single
* `pthread_setspecific()` call on a single key. (The current data
* structure does not really require that. This restriction should be
* easily removable.) We do not currently support the destruction
* functions, though that could be done. We also currently assume that
* only one `pthread_setspecific()` call can be executed at a time,
* though that assumption would be easy to remove by adding a lock.
*/
#ifndef GC_SPECIFIC_H
#define GC_SPECIFIC_H
#if !defined(GC_THREAD_LOCAL_ALLOC_H)
# error specific.h should be included from thread_local_alloc.h
#endif
#include <errno.h>
EXTERN_C_BEGIN
/*
* Called during key creation or by `GC_setspecific()`. For the GC we
* already hold the allocator lock. Currently allocated objects leak
* on thread exit. It is hard to avoid, but OK if we allocate garbage
* collected memory.
*/
#define MALLOC_CLEAR(n) GC_INTERNAL_MALLOC(n, NORMAL)
#define TS_CACHE_SIZE 1024
#define TS_CACHE_HASH(n) ((((n) >> 8) ^ (n)) & (TS_CACHE_SIZE - 1))
#define TS_HASH_SIZE 1024
#define TS_HASH(p) ((unsigned)((ADDR(p) >> 8) ^ ADDR(p)) & (TS_HASH_SIZE - 1))
#ifdef GC_ASSERTIONS
/*
* Thread-local storage is not guaranteed to be scanned by the collector.
* We hide values stored in "specific" entries for a test purpose.
*/
typedef GC_hidden_pointer ts_entry_value_t;
# define TS_HIDE_VALUE(p) GC_HIDE_NZ_POINTER(p)
# define TS_REVEAL_PTR(p) GC_REVEAL_NZ_POINTER(p)
#else
typedef void *ts_entry_value_t;
# define TS_HIDE_VALUE(p) (p)
# define TS_REVEAL_PTR(p) (p)
#endif
/*
* An entry describing a thread-specific value for a given thread.
* All such accessible structures preserve the invariant that if
* either thread is a valid `pthreads` id or `qtid` is a valid
* "quick thread identifier" for a thread, then value holds the
* corresponding thread specific value. This invariant must be
* preserved at all times, since asynchronous reads are allowed.
*/
typedef struct thread_specific_entry {
volatile AO_t qtid; /*< a "quick thread identifier", only for cache */
ts_entry_value_t value;
struct thread_specific_entry *next;
pthread_t thread;
} tse;
/*
* We represent each thread-specific datum as two tables. The first is
* a cache, indexed by a "quick thread identifier". The latter one is
* an easy-to-compute value, which is guaranteed to determine the thread,
* though a thread may correspond to more than one value. We typically
* use the address of a page in the stack. The second is a hash table,
* indexed by `pthread_self()`. It is used only as a backup.
*/
/*
* Return the "quick thread identifier". The default variant.
* Assumes the page size, or at least thread stack separation, is at
* least 4 KB. Must be defined so that it never returns 0. (Page zero
* cannot really be part of any stack, since that would make 0 a valid
* stack pointer.)
*/
#define ts_quick_thread_id() ((size_t)(ADDR(GC_approx_sp()) >> 12))
#define INVALID_QTID ((size_t)0)
#define INVALID_THREADID ((pthread_t)0)
typedef struct thread_specific_data {
tse *volatile cache[TS_CACHE_SIZE]; /*< a faster index to the hash table */
tse *hash[TS_HASH_SIZE];
pthread_mutex_t lock;
} tsd;
typedef tsd *GC_key_t;
#define GC_key_create(key, d) GC_key_create_inner(key)
GC_INNER int GC_key_create_inner(tsd **key_ptr);
/*
* Set the thread-local value associated with `key`.
* Should not be used to overwrite a previously set value.
*/
GC_INNER int GC_setspecific(tsd *key, void *value);
/* This function is called on thread exit. */
#define GC_remove_specific(key) \
GC_remove_specific_after_fork(key, pthread_self())
/*
* Remove thread-specific data for a given thread. This function is called
* at `fork` from the child process for all threads except for the survived
* one.
*/
GC_INNER void GC_remove_specific_after_fork(tsd *key, pthread_t t);
#ifdef CAN_HANDLE_FORK
/*
* Update thread-specific data for the survived thread of the child process.
* Should be called once after removing thread-specific data for other
* threads.
*/
GC_INNER void GC_update_specific_after_fork(tsd *key);
#endif
/*
* An internal variant of `GC_getspecific` that assumes a cache miss.
* Note that even the slow path does not lock.
*/
GC_INNER void *GC_slow_getspecific(tsd *key, size_t qtid,
tse *volatile *entry_ptr);
GC_INLINE void *
GC_getspecific(tsd *key)
{
size_t qtid = ts_quick_thread_id();
tse *volatile *entry_ptr = &key->cache[TS_CACHE_HASH(qtid)];
const tse *entry = *entry_ptr; /*< must be loaded only once */
GC_ASSERT(qtid != INVALID_QTID);
if (LIKELY(entry->qtid == qtid)) {
GC_ASSERT(entry->thread == pthread_self());
return TS_REVEAL_PTR(entry->value);
}
return GC_slow_getspecific(key, qtid, entry_ptr);
}
EXTERN_C_END
#endif /* GC_SPECIFIC_H */
EXTERN_C_BEGIN
# else
# error implement me
# endif
/*
* Each thread structure must be initialized. This call must be made from
* the new thread. Caller should hold the allocator lock.
*/
GC_INNER void GC_init_thread_local(GC_tlfs p);
/*
* Called when a thread is unregistered, or exits. Caller should hold
* the allocator lock.
*/
GC_INNER void GC_destroy_thread_local(GC_tlfs p);
/*
* The thread support layer must arrange to mark thread-local free lists
* explicitly, since the link field is often invisible to the marker.
* It knows how to find all threads; we take care of an individual thread
* free-list structure.
*/
GC_INNER void GC_mark_thread_local_fls_for(GC_tlfs p);
# ifdef GC_ASSERTIONS
GC_bool GC_is_thread_tsd_valid(void *tsd);
void GC_check_tls_for(GC_tlfs p);
# if defined(USE_CUSTOM_SPECIFIC)
void GC_check_tsd_marks(tsd *key);
# endif
# endif /* GC_ASSERTIONS */
# ifndef GC_ATTR_TLS_FAST
# define GC_ATTR_TLS_FAST /*< empty */
# endif
/*
* This is set up by `GC_init_thread_local()`. No need for cleanup
* on thread exit. But the thread support layer makes sure that
* `GC_thread_key` is traced, if necessary.
*/
extern
# if defined(USE_COMPILER_TLS)
__thread GC_ATTR_TLS_FAST
# elif defined(USE_WIN32_COMPILER_TLS)
__declspec(thread) GC_ATTR_TLS_FAST
# endif
GC_key_t GC_thread_key;
EXTERN_C_END
#endif /* THREAD_LOCAL_ALLOC */
#endif /* GC_THREAD_LOCAL_ALLOC_H */
# if defined(USE_COMPILER_TLS)
__thread GC_ATTR_TLS_FAST
# elif defined(USE_WIN32_COMPILER_TLS)
__declspec(thread) GC_ATTR_TLS_FAST
# endif
GC_key_t GC_thread_key;
static GC_bool keys_initialized;
# ifndef GC_NO_DEINIT
GC_INNER void
GC_reset_thread_local_initialization(void)
{
keys_initialized = FALSE;
/* TODO: Dispose resources associated with `GC_thread_key`. */
}
# endif
/*
* Return a single nonempty free list `fl` to the global one pointed to
* by `gfl`.
*/
static void
return_single_freelist(void *fl, void **gfl)
{
if (NULL == *gfl) {
*gfl = fl;
} else {
void *q = fl;
void **qptr;
GC_ASSERT(GC_size(fl) == GC_size(*gfl));
/* Concatenate. */
do {
qptr = &obj_link(q);
q = *qptr;
} while (ADDR(q) >= HBLKSIZE);
GC_ASSERT(NULL == q);
*qptr = *gfl;
*gfl = fl;
}
}
/*
* Recover the contents of the free-list array `fl` into the global one
* `gfl`.
*/
static void
return_freelists(void **fl, void **gfl)
{
int i;
for (i = 1; i < GC_TINY_FREELISTS; ++i) {
if (ADDR(fl[i]) >= HBLKSIZE) {
return_single_freelist(fl[i], &gfl[i]);
}
/*
* Clear `fl[i]`, since the thread structure may hang around.
* Do it in a way that is likely to trap if we access it.
*/
fl[i] = (ptr_t)NUMERIC_TO_VPTR(HBLKSIZE);
}
/* The 0 granule free list really contains 1 granule objects. */
if (ADDR(fl[0]) >= HBLKSIZE
# ifdef GC_GCJ_SUPPORT
&& ADDR(fl[0]) != ERROR_FL
# endif
) {
return_single_freelist(fl[0], &gfl[1]);
}
}
# ifdef USE_PTHREAD_SPECIFIC
/*
* Re-set the TLS value on thread cleanup to allow thread-local allocations
* to happen in the TLS destructors. `GC_unregister_my_thread()` (and
* similar routines) will finally set the `GC_thread_key` to `NULL`
* preventing this destructor from being called repeatedly.
*/
static void
reset_thread_key(void *v)
{
pthread_setspecific(GC_thread_key, v);
}
# else
# define reset_thread_key 0
# endif
GC_INNER void
GC_init_thread_local(GC_tlfs p)
{
int kind, j, res;
GC_ASSERT(I_HOLD_LOCK());
if (UNLIKELY(!keys_initialized)) {
# ifdef USE_CUSTOM_SPECIFIC
/* Ensure proper alignment of a "pushed" GC symbol. */
GC_ASSERT(ADDR(&GC_thread_key) % ALIGNMENT == 0);
# endif
res = GC_key_create(&GC_thread_key, reset_thread_key);
if (COVERT_DATAFLOW(res) != 0) {
ABORT("Failed to create key for local allocator");
}
keys_initialized = TRUE;
}
res = GC_setspecific(GC_thread_key, p);
if (COVERT_DATAFLOW(res) != 0) {
ABORT("Failed to set thread specific allocation pointers");
}
for (j = 0; j < GC_TINY_FREELISTS; ++j) {
for (kind = 0; kind < THREAD_FREELISTS_KINDS; ++kind) {
p->_freelists[kind][j] = NUMERIC_TO_VPTR(1);
}
# ifdef GC_GCJ_SUPPORT
p->gcj_freelists[j] = NUMERIC_TO_VPTR(1);
# endif
}
/*
* The zero-sized free list is handled like the regular free list, to
* ensure that the explicit deallocation works. However, an allocation
* of a `gcj` object with the zero size is always an error.
*/
# ifdef GC_GCJ_SUPPORT
p->gcj_freelists[0] = MAKE_CPTR(ERROR_FL);
# endif
}
GC_INNER void
GC_destroy_thread_local(GC_tlfs p)
{
int kind;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_getspecific(GC_thread_key) == p);
/* We currently only do this from the thread itself. */
GC_STATIC_ASSERT(THREAD_FREELISTS_KINDS <= MAXOBJKINDS);
for (kind = 0; kind < THREAD_FREELISTS_KINDS; ++kind) {
if (kind == (int)GC_n_kinds) {
/* The kind is not created. */
break;
}
return_freelists(p->_freelists[kind], GC_obj_kinds[kind].ok_freelist);
}
# ifdef GC_GCJ_SUPPORT
return_freelists(p->gcj_freelists, (void **)GC_gcjobjfreelist);
# endif
}
STATIC void *
GC_get_tlfs(void)
{
# if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
GC_key_t k = GC_thread_key;
if (UNLIKELY(0 == k)) {
/*
* We have not yet run `GC_init_parallel()`. That means we also
* are not locking, so `GC_malloc_kind_global()` is fairly cheap.
*/
return NULL;
}
return GC_getspecific(k);
# else
if (UNLIKELY(!keys_initialized))
return NULL;
return GC_getspecific(GC_thread_key);
# endif
}
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_malloc_kind(size_t lb, int kind)
{
size_t lg;
void *tsd;
void *result;
# if MAXOBJKINDS > THREAD_FREELISTS_KINDS
if (UNLIKELY(kind >= THREAD_FREELISTS_KINDS))
return GC_malloc_kind_global(lb, kind);
# endif
tsd = GC_get_tlfs();
if (UNLIKELY(NULL == tsd))
return GC_malloc_kind_global(lb, kind);
GC_ASSERT(GC_is_initialized);
GC_ASSERT(GC_is_thread_tsd_valid(tsd));
lg = ALLOC_REQUEST_GRANS(lb);
# if defined(CPPCHECK)
# define MALLOC_KIND_PTRFREE_INIT (void *)1
# else
# define MALLOC_KIND_PTRFREE_INIT NULL
# endif
GC_FAST_MALLOC_GRANS(result, lg, ((GC_tlfs)tsd)->_freelists[kind],
DIRECT_GRANULES, kind, GC_malloc_kind_global(lb, kind),
(void)(kind == PTRFREE ? MALLOC_KIND_PTRFREE_INIT
: (obj_link(result) = 0)));
# ifdef LOG_ALLOCS
GC_log_printf("GC_malloc_kind(%lu, %d) returned %p, recent GC #%lu\n",
(unsigned long)lb, kind, result, (unsigned long)GC_gc_no);
# endif
return result;
}
# ifdef GC_GCJ_SUPPORT
GC_API GC_ATTR_MALLOC void *GC_CALL
GC_gcj_malloc(size_t lb, const void *vtable_ptr)
{
void *result;
void **tiny_fl;
size_t lg;
/*
* Unlike the other thread-local allocation calls, we assume that the
* collector has been explicitly initialized.
*/
GC_ASSERT(GC_gcjobjfreelist != NULL);
# if defined(USE_PTHREAD_SPECIFIC) || defined(USE_WIN32_SPECIFIC)
GC_ASSERT(keys_initialized);
# else
GC_ASSERT(GC_thread_key != 0);
# endif
/*
* `gcj`-style allocation without locks is extremely tricky.
* The fundamental issue is that we may end up marking a free list,
* which has free-list links instead of "vtable" pointers.
* That is usually OK, since the next object on the free list will be
* cleared, and will thus be interpreted as containing a zero descriptor.
* That is fine if the object has not yet been initialized. But there
* are interesting potential races. In the case of incremental
* collection, this seems hopeless, since the marker may run
* asynchronously, and may pick up the pointer to the next free-list
* entry (which it thinks is a "vtable" pointer), get suspended for
* a while, and then see an allocated object instead of the "vtable".
* This may be avoidable with either a handshake with the collector or,
* probably more easily, by moving the free list links to the second
* "pointer-sized" word of each object. The latter is not a universal
* win, since on architecture like Itanium, nonzero offsets are not
* necessarily free. And there may be cache fill order issues.
* For now, we punt with the incremental collection. This probably means
* that the incremental collection should be enabled before we create
* a second thread.
*/
if (UNLIKELY(GC_incremental))
return GC_core_gcj_malloc(lb, vtable_ptr, 0 /* `flags` */);
tiny_fl = ((GC_tlfs)GC_getspecific(GC_thread_key))->gcj_freelists;
lg = ALLOC_REQUEST_GRANS(lb);
/*
* The provided `default_expr` below forces the initialization of the
* "vtable" pointer. This is necessary to ensure some very subtle
* properties required if a garbage collection is run in the middle of
* such an allocation. Here we implicitly also assume atomicity for the
* free list and method pointer assignments. We must update the free list
* before we store the pointer. Otherwise a collection at this point
* would see a corrupted free list. A real memory barrier is not needed,
* since the action of stopping this thread will cause prior writes
* to complete. We assert that any concurrent marker will stop us.
* Thus it is impossible for a mark procedure to see the allocation of the
* next object, but to see this object still containing a free-list pointer.
* Otherwise the marker, by misinterpreting the free-list link as a "vtable"
* pointer, might find a random "mark descriptor" in the next object.
*/
GC_FAST_MALLOC_GRANS(
result, lg, tiny_fl, DIRECT_GRANULES, GC_gcj_kind,
GC_core_gcj_malloc(lb, vtable_ptr, 0 /* `flags` */), do {
AO_compiler_barrier();
*(const void **)result = vtable_ptr;
} while (0));
return result;
}
# endif /* GC_GCJ_SUPPORT */
GC_INNER void
GC_mark_thread_local_fls_for(GC_tlfs p)
{
ptr_t q;
int kind, j;
for (j = 0; j < GC_TINY_FREELISTS; ++j) {
for (kind = 0; kind < THREAD_FREELISTS_KINDS; ++kind) {
/*
* Load the pointer atomically as it might be updated concurrently
* by `GC_FAST_MALLOC_GRANS()`.
*/
q = GC_cptr_load((volatile ptr_t *)&p->_freelists[kind][j]);
if (ADDR(q) > HBLKSIZE)
GC_set_fl_marks(q);
}
# ifdef GC_GCJ_SUPPORT
if (LIKELY(j > 0)) {
q = GC_cptr_load((volatile ptr_t *)&p->gcj_freelists[j]);
if (ADDR(q) > HBLKSIZE)
GC_set_fl_marks(q);
}
# endif
}
}
# if defined(GC_ASSERTIONS)
/* Check that all thread-local free-lists in `p` are completely marked. */
void
GC_check_tls_for(GC_tlfs p)
{
int kind, j;
for (j = 1; j < GC_TINY_FREELISTS; ++j) {
for (kind = 0; kind < THREAD_FREELISTS_KINDS; ++kind) {
GC_check_fl_marks(&p->_freelists[kind][j]);
}
# ifdef GC_GCJ_SUPPORT
GC_check_fl_marks(&p->gcj_freelists[j]);
# endif
}
}
# endif
#endif /* THREAD_LOCAL_ALLOC */
/* Most platform-specific files go here... */
/*
* Copyright (c) 1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996 by Silicon Graphics. All rights reserved.
* Copyright (c) 1998 by Fergus Henderson. All rights reserved.
* Copyright (c) 2000-2010 by Hewlett-Packard Development Company.
* All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* Copyright (c) 1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996 by Silicon Graphics. All rights reserved.
* Copyright (c) 1998 by Fergus Henderson. All rights reserved.
* Copyright (c) 2000-2009 by Hewlett-Packard Development Company.
* All rights reserved.
* Copyright (c) 2009-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/* Private declarations for thread support. */
#ifndef GC_PTHREAD_SUPPORT_H
#define GC_PTHREAD_SUPPORT_H
#ifdef THREADS
# if defined(GC_PTHREADS) || defined(GC_PTHREADS_PARAMARK)
# include <pthread.h>
# endif
# ifdef DARWIN
# include <mach/mach.h>
# include <mach/thread_act.h>
# endif
# ifdef THREAD_LOCAL_ALLOC
# endif
# ifdef THREAD_SANITIZER
# endif
EXTERN_C_BEGIN
# ifdef WOW64_THREAD_CONTEXT_WORKAROUND
# if defined(__WATCOMC__) && defined(GC_WIN32_THREADS)
/*
* Open Watcom (as of v2 2024-11-02 Build) does not define `NT_TIB`
* in the platform `winnt.h` file.
*/
struct GC_NT_TIB_s {
/* `EXCEPTION_REGISTRATION_RECORD *` */ void *ExceptionList; /*< unused */
PVOID StackBase;
PVOID StackLimit;
/* The remaining fields are unused. */
};
typedef struct GC_NT_TIB_s GC_NT_TIB;
# else
# define GC_NT_TIB NT_TIB
# endif
# endif
struct GC_StackContext_Rep {
# if defined(THREAD_SANITIZER) && defined(SIGNAL_BASED_STOP_WORLD)
/*
* A dummy field to avoid TSan false positive about the race
* between `GC_has_other_debug_info` and `GC_suspend_handler_inner`
* (the latter one sets `stack_ptr`).
*/
char dummy[sizeof(oh)];
# endif
/*
* Cold end of the stack (except for main thread on non-Windows).
* On Windows: 0 means entry invalid; not `in_use` implies `stack_end`
* is zero.
*/
# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
volatile
# endif
ptr_t stack_end;
/* Valid only in some platform-specific states. */
ptr_t stack_ptr;
# ifdef GC_WIN32_THREADS
# define ADDR_LIMIT ((ptr_t)GC_WORD_MAX)
/*
* Last known minimum (hottest) address in stack or `ADDR_LIMIT`
* if unset.
*/
ptr_t last_stack_min;
# ifdef I386
/*
* The cold end of the stack saved by `GC_record_stack_base` (never
* modified by `GC_set_stackbottom()`); used for the old way of the
* coroutines support.
*/
ptr_t initial_stack_base;
# endif
# elif defined(DARWIN) && !defined(DARWIN_DONT_PARSE_STACK)
/*
* Result of `GC_FindTopOfStack(0)`; valid only if the thread is blocked;
* non-`NULL` value means already set.
*/
ptr_t topOfStack;
# endif
# if defined(E2K) || defined(IA64)
ptr_t backing_store_end;
ptr_t backing_store_ptr;
# endif
# ifdef GC_WIN32_THREADS
/* For now, alt-stack is not implemented for Win32. */
# else
/* The start of the alt-stack if there is one, `NULL` otherwise. */
ptr_t altstack;
/* Same for the "normal" stack (set by `GC_register_altstack`). */
ptr_t normstack;
/* The size of the alt-stack if exists. */
size_t altstack_size;
size_t normstack_size;
# endif
# ifdef E2K
/* The current offset in the procedure stack. */
size_t ps_ofs;
# endif
# ifndef GC_NO_FINALIZATION
unsigned char finalizer_nested;
/*
* Used by `GC_check_finalizer_nested()` to minimize the level of recursion
* when a client finalizer allocates memory. Initially it is zero and
* `finalizer_nested` is zero.
*/
unsigned short finalizer_skipped;
# endif
/*
* Points to the "frame" data held in stack by the innermost
* `GC_call_with_gc_active()` of this stack (thread); may be `NULL`.
*/
struct GC_traced_stack_sect_s *traced_stack_sect;
};
typedef struct GC_StackContext_Rep *GC_stack_context_t;
# ifdef GC_WIN32_THREADS
typedef DWORD thread_id_t;
# define thread_id_self() GetCurrentThreadId()
# define THREAD_ID_EQUAL(id1, id2) ((id1) == (id2))
/*
* Convert a platform-specific thread `id` (of `thread_id_t` type) to some
* pointer identifying the thread. This is used mostly for a debugging
* purpose when printing thread id.
*/
# define THREAD_ID_TO_VPTR(id) CAST_THRU_UINTPTR(void *, id)
# else
typedef pthread_t thread_id_t;
# define thread_id_self() pthread_self()
# define THREAD_ID_EQUAL(id1, id2) THREAD_EQUAL(id1, id2)
# define THREAD_ID_TO_VPTR(id) PTHREAD_TO_VPTR(id)
# endif
struct GC_Thread_Rep {
union {
# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
/*
* Updated without a lock. We assert that each unused entry has
* invalid `id` of zero and zero `stack_end`.
* Used only if `GC_win32_dll_threads`.
*/
volatile AO_t in_use;
/*
* The same but of the type that matches the first argument of
* `InterlockedExchange()`; `volatile` is omitted because the
* ancient version of the function prototype lacked the qualifier.
*/
LONG long_in_use;
# endif
/*
* Hash table link without `GC_win32_dll_threads`. More recently
* allocated threads with a given `pthreads` id come first.
* (All but the first are guaranteed to be dead, but we may not yet
* have registered the join.)
*/
struct GC_Thread_Rep *next;
} tm; /*< table_management */
GC_stack_context_t crtn;
thread_id_t id; /*< hash table key */
# ifdef DARWIN
mach_port_t mach_thread;
# elif defined(GC_WIN32_THREADS) && defined(GC_PTHREADS)
pthread_t pthread_id;
# elif defined(USE_TKILL_ON_ANDROID)
pid_t kernel_id;
# endif
# ifdef MSWINCE
/*
* According to MSDN docs for WinCE targets:
* - `DuplicateHandle()` is not applicable to thread handles;
* - the value returned by `GetCurrentThreadId()` could be used as
* a "real" thread handle (for `SuspendThread()`, `ResumeThread()`
* and `GetThreadContext()`).
*/
# define THREAD_HANDLE(p) ((HANDLE)(word)(p)->id)
# elif defined(GC_WIN32_THREADS)
HANDLE handle;
# define THREAD_HANDLE(p) ((p)->handle)
# endif /* GC_WIN32_THREADS && !MSWINCE */
/* Note: this is protected by the allocator lock. */
unsigned char flags;
/* Thread has exited (`pthreads` only). */
# define FINISHED 0x1
# ifndef GC_PTHREADS
# define KNOWN_FINISHED(p) FALSE
# else
# define KNOWN_FINISHED(p) (((p)->flags & FINISHED) != 0)
/*
* Thread is treated as detached. Thread may really be detached,
* or it may have been explicitly registered, in which case we can
* deallocate its `GC_Thread_Rep` entity once it unregisters itself,
* since it may not return a pointer to the collector heap.
*/
# define DETACHED 0x2
# endif
# if (defined(GC_HAVE_PTHREAD_EXIT) || !defined(GC_NO_PTHREAD_CANCEL)) \
&& defined(GC_PTHREADS)
/* Collections are disabled while the thread is exiting. */
# define DISABLED_GC 0x10
# endif
/*
* Thread is in the do-blocking state. If the flag is set, the thread
* has stored its stack pointer value and will acquire the allocator lock
* before any pointer manipulation. Thus, it does not need a signal sent
* to stop it.
*/
# define DO_BLOCKING 0x20
# ifdef GC_WIN32_THREADS
/* Thread is suspended by `SuspendThread()`. */
# define IS_SUSPENDED 0x40
# endif
# ifdef SIGNAL_BASED_STOP_WORLD
/*
* The value of `GC_stop_count` when the thread last successfully
* handled a suspend signal.
*/
volatile AO_t last_stop_count;
# ifdef GC_ENABLE_SUSPEND_THREAD
/*
* Note: an odd value means thread was suspended externally;
* incremented on every call of `GC_suspend_thread()` and
* `GC_resume_thread()`; updated with the allocator lock held, but
* could be read from a signal handler.
*/
volatile AO_t ext_suspend_cnt;
# endif
# endif
# ifdef GC_PTHREADS
/*
* The value returned from the thread. Used only to avoid premature
* reclamation of any data it might reference. This is unfortunately
* also the reason we need to intercept join and detach.
*/
void *status;
# endif
# ifdef THREAD_LOCAL_ALLOC
struct thread_local_freelists tlfs GC_ATTR_PTRT_ALIGNED;
# endif
# ifdef NACL
/*
* Grab `NACL_GC_REG_STORAGE_SIZE` pointers off the stack when going
* into a `syscall()`. 20 is more than we need, but it is
* an overestimate in case the instrumented function uses any
* callee-saved registers, they may be pushed to the stack much
* earlier. Also, on x86_64 `push` puts 8 bytes on the stack even
* though our pointers have 4 bytes in length.
*/
# ifdef ARM32
/* Space for `r4` ... `r8`, `r10`, ... `r12`, `r14`. */
# define NACL_GC_REG_STORAGE_SIZE 9
# else
# define NACL_GC_REG_STORAGE_SIZE 20
# endif
ptr_t reg_storage[NACL_GC_REG_STORAGE_SIZE];
# elif defined(PLATFORM_HAVE_GC_REG_STORAGE_SIZE)
word registers[PLATFORM_GC_REG_STORAGE_SIZE]; /*< used externally */
# endif
# if defined(WOW64_THREAD_CONTEXT_WORKAROUND) && defined(MSWINRT_FLAVOR)
GC_NT_TIB *tib;
# endif
# ifdef RETRY_GET_THREAD_CONTEXT /* `&& GC_WIN32_THREADS` */
ptr_t context_sp;
/*
* Populated as part of `GC_suspend()` as resume/suspend loop may be
* needed for `GetThreadContext()` to succeed.
*/
word context_regs[PUSHED_REGS_COUNT];
# endif
};
typedef struct GC_Thread_Rep *GC_thread;
# if defined(GC_PTHREADS) || defined(GC_PTHREADS_PARAMARK)
/* Convert an opaque `pthread_t` value to a pointer identifying the thread. */
# if defined(GC_WIN32_THREADS) && !defined(CYGWIN32) \
&& (defined(GC_WIN32_PTHREADS) || defined(GC_PTHREADS_PARAMARK)) \
&& !defined(__WINPTHREADS_VERSION_MAJOR)
/*
* Using documented internal details of pthreads-win32 library.
* `pthread_t` is a structure.
*/
# define PTHREAD_TO_VPTR(id) ((void *)(id).p)
# else
# define PTHREAD_TO_VPTR(id) CAST_THRU_UINTPTR(void *, id)
# endif
# endif
# ifndef THREAD_TABLE_SZ
/* Note: this is a power of 2 (for speed). */
# define THREAD_TABLE_SZ 256
# endif
# ifdef GC_WIN32_THREADS
# define THREAD_TABLE_INDEX(id) /*< `id` is of `DWORD` type */ \
(int)((((id) >> 8) ^ (id)) % THREAD_TABLE_SZ)
# elif CPP_WORDSZ > 32
# define THREAD_TABLE_INDEX(id) \
(int)(((((NUMERIC_THREAD_ID(id) >> 8) ^ NUMERIC_THREAD_ID(id)) >> 16) \
^ ((NUMERIC_THREAD_ID(id) >> 8) ^ NUMERIC_THREAD_ID(id))) \
% THREAD_TABLE_SZ)
# else
# define THREAD_TABLE_INDEX(id) \
(int)(((NUMERIC_THREAD_ID(id) >> 16) ^ (NUMERIC_THREAD_ID(id) >> 8) \
^ NUMERIC_THREAD_ID(id)) \
% THREAD_TABLE_SZ)
# endif
/*
* The set (hash table) of all known threads. We intercept thread
* creation and join/detach. Protected by the allocator lock.
* Not used if `GC_win32_dll_threads`.
*/
GC_EXTERN GC_thread GC_threads[THREAD_TABLE_SZ];
# ifndef MAX_MARKERS
# define MAX_MARKERS 16
# endif
# ifdef STACKPTR_CORRECTOR_AVAILABLE
GC_EXTERN GC_sp_corrector_proc GC_sp_corrector;
# endif
GC_EXTERN GC_on_thread_event_proc GC_on_thread_event;
# ifdef GC_WIN32_THREADS
# ifdef GC_NO_THREADS_DISCOVERY
# define GC_win32_dll_threads FALSE
# elif defined(GC_DISCOVER_TASK_THREADS)
# define GC_win32_dll_threads TRUE
# else
/*
* `GC_win32_dll_threads` must be set (if needed) at the application
* initialization time, i.e. before any collector or thread calls.
* We make it a "dynamic" option only to avoid multiple library versions.
*/
GC_EXTERN GC_bool GC_win32_dll_threads;
# endif
# ifdef PARALLEL_MARK
GC_EXTERN int GC_available_markers_m1;
/*
* The desired amount of marker threads (including the initiating one).
* Note: the default value (0) means the number of markers should be
* selected automatically.
*/
GC_EXTERN unsigned GC_required_markers_cnt;
/* The cold end of the stack for markers. */
GC_EXTERN ptr_t GC_marker_sp[MAX_MARKERS - 1];
/*
* Last known minimum (hottest) address in stack (or `ADDR_LIMIT` if
* unset) for markers.
*/
GC_EXTERN ptr_t GC_marker_last_stack_min[MAX_MARKERS - 1];
# ifndef GC_PTHREADS_PARAMARK
/*
* A table mapping the helper thread id to the mark helper index
* (linear search is used since the mapping contains only a few entries).
*/
GC_EXTERN thread_id_t GC_marker_Id[MAX_MARKERS - 1];
# endif
# if !defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) && !defined(MSWINCE)
GC_INNER void GC_init_win32_thread_naming(HMODULE hK32);
# endif
# ifdef GC_PTHREADS_PARAMARK
GC_INNER void *GC_mark_thread(void *);
# elif defined(MSWINCE)
GC_INNER DWORD WINAPI GC_mark_thread(LPVOID);
# else
GC_INNER unsigned __stdcall GC_mark_thread(void *);
# endif
# endif /* PARALLEL_MARK */
/*
* Add a thread to `GC_threads`. We assume it was not already there.
* Note: the `id` field should be set by the caller.
*/
GC_INNER GC_thread GC_new_thread(thread_id_t);
GC_INNER void GC_record_stack_base(GC_stack_context_t crtn,
const struct GC_stack_base *sb);
/*
* This may be called from `DllMain`, and hence operates under unusual
* constraints. In particular, it must be lock-free if
* `GC_win32_dll_threads`. Always called from the thread being added.
* If not `GC_win32_dll_threads`, then we already hold the allocator
* lock except possibly during single-threaded startup code.
* Does not initialize thread-local free lists.
*/
GC_INNER GC_thread GC_register_my_thread_inner(const struct GC_stack_base *sb,
thread_id_t self_id);
# ifdef GC_PTHREADS
GC_INNER void GC_win32_cache_self_pthread(thread_id_t);
# else
/*
* Delete a thread from `GC_threads`. We assume it is there.
* (The code intentionally traps if it was not.) It is also safe to delete
* the main thread. If `GC_win32_dll_threads`, it should be called only
* from the thread being deleted (except for `DLL_PROCESS_DETACH` case).
* If a thread has been joined, but we have not yet been notified, then
* there may be more than one thread in the table with the same thread
* id - this is OK because we delete a specific one.
*/
GC_INNER void GC_delete_thread(GC_thread);
# endif
# ifdef CAN_HANDLE_FORK
/* Prepare for a process fork if requested. */
GC_INNER void GC_setup_atfork(void);
# endif
# if !defined(DONT_USE_ATEXIT) || !defined(GC_NO_THREADS_DISCOVERY)
GC_EXTERN thread_id_t GC_main_thread_id;
# endif
# ifndef GC_NO_THREADS_DISCOVERY
/*
* Search in `dll_thread_table` and return the `GC_thread[]` entity
* corresponding to the given thread `id`.
* May be called without a lock, but should be called in contexts in
* those the requested thread cannot be asynchronously deleted, e.g.
* from the thread itself.
*/
GC_INNER GC_thread GC_win32_dll_lookup_thread(thread_id_t id);
# endif
# ifdef MPROTECT_VDB
/*
* Make sure given thread descriptor is not protected by the VDB
* implementation. Used to prevent write faults when the world is
* (partially) stopped, since it may have been stopped with a system
* lock held, and that lock may be required for fault handling.
*/
GC_INNER void GC_win32_unprotect_thread(GC_thread);
# else
# define GC_win32_unprotect_thread(t) (void)(t)
# endif /* !MPROTECT_VDB */
# else
# define GC_win32_dll_threads FALSE
# endif /* !GC_WIN32_THREADS */
# ifdef GC_PTHREADS
# ifdef GC_WIN32_THREADS
/*
* Return a `GC_thread` corresponding to a given `pthread_t`, or
* `NULL` if it is not there. We assume that this is only called for
* `pthreads` ids that have not yet terminated or are still joinable,
* and cannot be terminated concurrently.
*/
GC_INNER GC_thread GC_lookup_by_pthread(pthread_t);
# else
# define GC_lookup_by_pthread(t) GC_lookup_thread(t)
# endif
# endif /* GC_PTHREADS */
/*
* Return a `GC_thread` corresponding to a given thread `id`, or
* `NULL` if it is not there. Caller holds the allocator lock in the
* reader mode, at least, or otherwise inhibits updates. If there is more
* than one thread with the given `id`, we return the most recent one.
*/
GC_INNER GC_thread GC_lookup_thread(thread_id_t id);
# define GC_self_thread_inner() GC_lookup_thread(thread_id_self())
/*
* Wait until an in-progress collection has finished.
* We hold the allocator lock and repeatedly release it in order to wait.
* If `wait_for_all`, then we exit with the allocator lock held and
* no collection is in progress; otherwise we just wait for the current
* collection to finish.
*/
GC_INNER void GC_wait_for_gc_completion(GC_bool wait_for_all);
# ifdef NACL
GC_INNER void GC_nacl_initialize_gc_thread(GC_thread);
GC_INNER void GC_nacl_shutdown_gc_thread(void);
# endif
# if defined(PTHREAD_STOP_WORLD_IMPL) \
&& !defined(NO_SIGNALS_UNBLOCK_IN_MAIN) \
|| defined(GC_EXPLICIT_SIGNALS_UNBLOCK)
/*
* Some targets (e.g., Solaris) might require this to be called when
* doing thread registering from the thread destructor.
*/
GC_INNER void GC_unblock_gc_signals(void);
# endif
# if defined(GC_ENABLE_SUSPEND_THREAD) && defined(SIGNAL_BASED_STOP_WORLD)
GC_INNER void GC_suspend_self_inner(GC_thread me, size_t suspend_cnt);
/*
* A wrapper over `GC_suspend_self_inner()`. Similar to
* `GC_do_blocking_inner()` but assuming the allocator lock is held and
* `fn` is `GC_suspend_self_inner`.
*/
GC_INNER void GC_suspend_self_blocked(ptr_t thread_me, void *context);
# endif
# if defined(GC_PTHREADS) && !defined(PLATFORM_THREADS) \
&& !defined(SN_TARGET_PSP2)
# ifdef GC_PTHREAD_START_STANDALONE
# define GC_INNER_PTHRSTART /*< empty */
# else
# define GC_INNER_PTHRSTART GC_INNER
# endif
GC_INNER_PTHRSTART void *GC_CALLBACK
GC_pthread_start_inner(struct GC_stack_base *sb, void *arg);
/*
* Called from `GC_pthread_start_inner()`. Defined in `pthread_support.c`
* file to minimize the number of files included from `pthread_start.c` file
* (because `sem_t` and `sem_post()` are not used in that file directly).
*/
GC_INNER_PTHRSTART GC_thread
GC_start_rtn_prepare_thread(void *(**pstart)(void *), void **pstart_arg,
struct GC_stack_base *sb, void *arg);
/*
* Called at thread exit. Never called for main thread. That is OK,
* since it results in at most a tiny one-time leak. And LinuxThreads
* implementation does not reclaim the primordial (main) thread resources
* or id anyway.
*/
GC_INNER_PTHRSTART void GC_thread_exit_proc(void *);
# endif /* GC_PTHREADS */
# ifdef DARWIN
# ifndef DARWIN_DONT_PARSE_STACK
GC_INNER ptr_t GC_FindTopOfStack(unsigned long);
# endif
# if defined(PARALLEL_MARK) && !defined(GC_NO_THREADS_DISCOVERY)
/* Note: this is used only by `GC_suspend_thread_list()`. */
GC_INNER GC_bool GC_is_mach_marker(thread_act_t);
# endif
# endif /* DARWIN */
# ifdef PTHREAD_STOP_WORLD_IMPL
GC_INNER void GC_stop_init(void);
# endif
EXTERN_C_END
#endif /* THREADS */
#endif /* GC_PTHREAD_SUPPORT_H */
/* This probably needs more porting work to ppc64. */
#if defined(DARWIN) && defined(THREADS)
# include <mach/machine.h>
# include <sys/sysctl.h>
# if defined(ARM32) && defined(ARM_THREAD_STATE32)
# include <CoreFoundation/CoreFoundation.h>
# endif
/*
* From "Inside Mac OS X - Mach-O Runtime Architecture" published by Apple:
* - Page 49: The space beneath the stack pointer, where a new stack frame
* would normally be allocated, is called the red zone. This area as
* shown in Figure 3-2 may be used for any purpose as long as a new stack
* frame does not need to be added to the stack.
* - Page 50: If a leaf procedure's red zone usage would exceed 224 bytes,
* then it must set up a stack frame just like routines that call other
* routines.
*/
# ifdef POWERPC
# if CPP_WORDSZ == 32
# define PPC_RED_ZONE_SIZE 224
# elif CPP_WORDSZ == 64
# define PPC_RED_ZONE_SIZE 320
# endif
# endif
# ifndef DARWIN_DONT_PARSE_STACK
typedef struct StackFrame {
unsigned long savedSP;
unsigned long savedCR;
unsigned long savedLR;
/* `unsigned long reserved[2];` */
/* `unsigned long savedRTOC;` */
} StackFrame;
GC_INNER ptr_t
GC_FindTopOfStack(unsigned long stack_start)
{
StackFrame *frame = (StackFrame *)MAKE_CPTR(stack_start);
if (NULL == frame) {
# ifdef POWERPC
# if CPP_WORDSZ == 32
__asm__ __volatile__("lwz %0,0(r1)" : "=r"(frame));
# else
__asm__ __volatile__("ld %0,0(r1)" : "=r"(frame));
# endif
# elif defined(ARM32)
volatile ptr_t sp_reg;
__asm__ __volatile__("mov %0, r7\n" : "=r"(sp_reg));
frame = (/* no volatile */ StackFrame *)sp_reg;
# elif defined(AARCH64)
volatile ptr_t sp_reg;
__asm__ __volatile__("mov %0, x29\n" : "=r"(sp_reg));
frame = (/* no volatile */ StackFrame *)sp_reg;
# else
# if defined(CPPCHECK)
GC_noop1_ptr(&frame);
# endif
ABORT("GC_FindTopOfStack(0) is not implemented");
# endif
}
# ifdef DEBUG_THREADS_EXTRA
GC_log_printf("FindTopOfStack start at sp= %p\n", (void *)frame);
# endif
while (frame->savedSP != 0) { /*< stop if no more stack frames */
unsigned long maskedLR;
# ifdef CPPCHECK
GC_noop1(frame->savedCR);
# endif
frame = (StackFrame *)MAKE_CPTR(frame->savedSP);
/*
* We do these next two checks after going to the next frame because
* the `savedLR` for the first stack frame in the loop is not set up
* on purpose, so we should not check it.
*/
maskedLR = frame->savedLR & ~0x3UL;
if (0 == maskedLR || ~0x3UL == maskedLR) {
/* The next `savedLR` is bogus, stop. */
break;
}
}
# ifdef DEBUG_THREADS_EXTRA
GC_log_printf("FindTopOfStack finish at sp= %p\n", (void *)frame);
# endif
return (ptr_t)frame;
}
# endif /* !DARWIN_DONT_PARSE_STACK */
/*
* `GC_query_task_threads` controls whether to obtain the list of the
* threads from the kernel or to use `GC_threads` table.
*/
# ifdef GC_NO_THREADS_DISCOVERY
# define GC_query_task_threads FALSE
# elif defined(GC_DISCOVER_TASK_THREADS)
# define GC_query_task_threads TRUE
# else
STATIC GC_bool GC_query_task_threads = FALSE;
# endif /* !GC_NO_THREADS_DISCOVERY */
GC_API void GC_CALL
GC_use_threads_discovery(void)
{
# ifdef GC_NO_THREADS_DISCOVERY
ABORT("Darwin task-threads-based stop and push unsupported");
# else
# ifndef GC_ALWAYS_MULTITHREADED
GC_ASSERT(!GC_need_to_lock);
# endif
# ifndef GC_DISCOVER_TASK_THREADS
GC_query_task_threads = TRUE;
# endif
GC_init();
# endif
}
# ifndef kCFCoreFoundationVersionNumber_iOS_8_0
# define kCFCoreFoundationVersionNumber_iOS_8_0 1140.1
# endif
# define THREAD_ACT_TO_VPTR(t) THREAD_ID_TO_VPTR(t)
# define MACH_PORT_TO_VPTR(t) THREAD_ID_TO_VPTR(t)
/*
* Evaluates the stack range for a given thread. Returns the lower bound
* and sets `*phi` to the upper one. Sets `*pfound_me` to `TRUE` if this
* is the current thread, otherwise the value is not changed.
*/
STATIC ptr_t
GC_stack_range_for(ptr_t *phi, thread_act_t thread, GC_thread p,
mach_port_t my_thread, ptr_t *paltstack_lo,
ptr_t *paltstack_hi, GC_bool *pfound_me)
{
# ifdef DARWIN_DONT_PARSE_STACK
GC_stack_context_t crtn;
# endif
ptr_t lo;
GC_ASSERT(I_HOLD_LOCK());
if (thread == my_thread) {
GC_ASSERT(NULL == p || (p->flags & DO_BLOCKING) == 0);
lo = GC_approx_sp();
# ifndef DARWIN_DONT_PARSE_STACK
*phi = GC_FindTopOfStack(0);
# endif
*pfound_me = TRUE;
} else if (p != NULL && (p->flags & DO_BLOCKING) != 0) {
lo = p->crtn->stack_ptr;
# ifndef DARWIN_DONT_PARSE_STACK
*phi = p->crtn->topOfStack;
# endif
} else {
/*
* `MACHINE_THREAD_STATE_COUNT` does not seem to be defined everywhere.
* Hence we use our own variant. Alternatively, we could use
* `THREAD_STATE_MAX` (but seems to be not optimal).
*/
kern_return_t kern_result;
GC_THREAD_STATE_T state;
# if defined(ARM32) && defined(ARM_THREAD_STATE32)
/*
* Use `ARM_UNIFIED_THREAD_STATE` on iOS8+ 32-bit targets and on
* 64-bit H/W (iOS7+ 32-bit mode).
*/
size_t size;
static cpu_type_t cputype = 0;
if (cputype == 0) {
sysctlbyname("hw.cputype", &cputype, &size, NULL, 0);
}
if (cputype == CPU_TYPE_ARM64
|| kCFCoreFoundationVersionNumber
>= kCFCoreFoundationVersionNumber_iOS_8_0) {
arm_unified_thread_state_t unified_state;
mach_msg_type_number_t unified_thread_state_count
= ARM_UNIFIED_THREAD_STATE_COUNT;
# if defined(CPPCHECK)
# define GC_ARM_UNIFIED_THREAD_STATE 1
# else
# define GC_ARM_UNIFIED_THREAD_STATE ARM_UNIFIED_THREAD_STATE
# endif
kern_result = thread_get_state(thread, GC_ARM_UNIFIED_THREAD_STATE,
(natural_t *)&unified_state,
&unified_thread_state_count);
# if !defined(CPPCHECK)
if (unified_state.ash.flavor != ARM_THREAD_STATE32) {
ABORT("unified_state flavor should be ARM_THREAD_STATE32");
}
# endif
state = unified_state;
} else
# endif
/* else */ {
mach_msg_type_number_t thread_state_count = GC_MACH_THREAD_STATE_COUNT;
/* Get the thread state (registers, etc.). */
do {
kern_result
= thread_get_state(thread, GC_MACH_THREAD_STATE,
(natural_t *)&state, &thread_state_count);
} while (kern_result == KERN_ABORTED);
}
# ifdef DEBUG_THREADS
GC_log_printf("thread_get_state returns %d\n", kern_result);
# endif
if (kern_result != KERN_SUCCESS)
ABORT("thread_get_state failed");
# if defined(I386)
lo = (ptr_t)state.THREAD_FLD(esp);
# ifndef DARWIN_DONT_PARSE_STACK
*phi = GC_FindTopOfStack(state.THREAD_FLD(esp));
# endif
GC_push_one(state.THREAD_FLD(eax));
GC_push_one(state.THREAD_FLD(ebx));
GC_push_one(state.THREAD_FLD(ecx));
GC_push_one(state.THREAD_FLD(edx));
GC_push_one(state.THREAD_FLD(edi));
GC_push_one(state.THREAD_FLD(esi));
GC_push_one(state.THREAD_FLD(ebp));
# elif defined(X86_64)
lo = (ptr_t)state.THREAD_FLD(rsp);
# ifndef DARWIN_DONT_PARSE_STACK
*phi = GC_FindTopOfStack(state.THREAD_FLD(rsp));
# endif
GC_push_one(state.THREAD_FLD(rax));
GC_push_one(state.THREAD_FLD(rbx));
GC_push_one(state.THREAD_FLD(rcx));
GC_push_one(state.THREAD_FLD(rdx));
GC_push_one(state.THREAD_FLD(rdi));
GC_push_one(state.THREAD_FLD(rsi));
GC_push_one(state.THREAD_FLD(rbp));
/* `rsp` is skipped. */
GC_push_one(state.THREAD_FLD(r8));
GC_push_one(state.THREAD_FLD(r9));
GC_push_one(state.THREAD_FLD(r10));
GC_push_one(state.THREAD_FLD(r11));
GC_push_one(state.THREAD_FLD(r12));
GC_push_one(state.THREAD_FLD(r13));
GC_push_one(state.THREAD_FLD(r14));
GC_push_one(state.THREAD_FLD(r15));
# elif defined(POWERPC)
lo = (ptr_t)(state.THREAD_FLD(r1) - PPC_RED_ZONE_SIZE);
# ifndef DARWIN_DONT_PARSE_STACK
*phi = GC_FindTopOfStack(state.THREAD_FLD(r1));
# endif
GC_push_one(state.THREAD_FLD(r0));
/* `r1` is skipped. */
GC_push_one(state.THREAD_FLD(r2));
GC_push_one(state.THREAD_FLD(r3));
GC_push_one(state.THREAD_FLD(r4));
GC_push_one(state.THREAD_FLD(r5));
GC_push_one(state.THREAD_FLD(r6));
GC_push_one(state.THREAD_FLD(r7));
GC_push_one(state.THREAD_FLD(r8));
GC_push_one(state.THREAD_FLD(r9));
GC_push_one(state.THREAD_FLD(r10));
GC_push_one(state.THREAD_FLD(r11));
GC_push_one(state.THREAD_FLD(r12));
GC_push_one(state.THREAD_FLD(r13));
GC_push_one(state.THREAD_FLD(r14));
GC_push_one(state.THREAD_FLD(r15));
GC_push_one(state.THREAD_FLD(r16));
GC_push_one(state.THREAD_FLD(r17));
GC_push_one(state.THREAD_FLD(r18));
GC_push_one(state.THREAD_FLD(r19));
GC_push_one(state.THREAD_FLD(r20));
GC_push_one(state.THREAD_FLD(r21));
GC_push_one(state.THREAD_FLD(r22));
GC_push_one(state.THREAD_FLD(r23));
GC_push_one(state.THREAD_FLD(r24));
GC_push_one(state.THREAD_FLD(r25));
GC_push_one(state.THREAD_FLD(r26));
GC_push_one(state.THREAD_FLD(r27));
GC_push_one(state.THREAD_FLD(r28));
GC_push_one(state.THREAD_FLD(r29));
GC_push_one(state.THREAD_FLD(r30));
GC_push_one(state.THREAD_FLD(r31));
# elif defined(ARM32)
lo = (ptr_t)state.THREAD_FLD(sp);
# ifndef DARWIN_DONT_PARSE_STACK
*phi = GC_FindTopOfStack(state.THREAD_FLD(r[7])); /*< `fp` */
# endif
{
int j;
for (j = 0; j < 7; j++)
GC_push_one(state.THREAD_FLD(r[j]));
/* Skip `r7` (because iOS uses it as a frame pointer). */
j++;
for (; j <= 12; j++)
GC_push_one(state.THREAD_FLD(r[j]));
}
/* `cpsr`, `pc` and `sp` are skipped. */
GC_push_one(state.THREAD_FLD(lr));
# elif defined(AARCH64)
lo = (ptr_t)state.THREAD_FLD(sp);
# ifndef DARWIN_DONT_PARSE_STACK
*phi = GC_FindTopOfStack(state.THREAD_FLD(fp));
# endif
{
int j;
for (j = 0; j <= 28; j++) {
GC_push_one(state.THREAD_FLD(x[j]));
}
}
/* `cpsr`, `fp`, `pc` and `sp` are skipped. */
GC_push_one(state.THREAD_FLD(lr));
# elif defined(CPPCHECK)
lo = NULL;
# else
# error FIXME for non-arm/ppc/x86 architectures
# endif
}
# ifndef DARWIN_DONT_PARSE_STACK
/* TODO: Determine `p` and handle `altstack` if `!DARWIN_DONT_PARSE_STACK` */
UNUSED_ARG(paltstack_hi);
# else
/*
* `p` is guaranteed to be non-`NULL` regardless of
* `GC_query_task_threads`.
*/
# ifdef CPPCHECK
if (NULL == p)
ABORT("Bad GC_stack_range_for call");
# endif
crtn = p->crtn;
*phi = crtn->stack_end;
if (crtn->altstack != NULL && ADDR_GE(lo, crtn->altstack)
&& ADDR_GE(crtn->altstack + crtn->altstack_size, lo)) {
*paltstack_lo = lo;
*paltstack_hi = crtn->altstack + crtn->altstack_size;
lo = crtn->normstack;
*phi = lo + crtn->normstack_size;
} else
# endif
/* else */ {
*paltstack_lo = NULL;
}
# if defined(STACKPTR_CORRECTOR_AVAILABLE) && defined(DARWIN_DONT_PARSE_STACK)
if (GC_sp_corrector != 0)
GC_sp_corrector((void **)&lo, THREAD_ID_TO_VPTR(p->id));
# endif
# ifdef DEBUG_THREADS
GC_log_printf("Darwin: Stack for thread %p is [%p,%p)\n",
THREAD_ACT_TO_VPTR(thread), (void *)lo, (void *)*phi);
# endif
return lo;
}
GC_INNER void
GC_push_all_stacks(void)
{
ptr_t hi, altstack_lo, altstack_hi;
task_t my_task = current_task();
mach_port_t my_thread = mach_thread_self();
GC_bool found_me = FALSE;
int nthreads = 0;
word total_size = 0;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_thr_initialized);
# ifndef DARWIN_DONT_PARSE_STACK
if (GC_query_task_threads) {
int i;
kern_return_t kern_result;
thread_act_array_t act_list;
mach_msg_type_number_t listcount;
/* Obtain the list of the threads from the kernel. */
kern_result = task_threads(my_task, &act_list, &listcount);
if (kern_result != KERN_SUCCESS)
ABORT("task_threads failed");
for (i = 0; i < (int)listcount; i++) {
thread_act_t thread = act_list[i];
ptr_t lo = GC_stack_range_for(&hi, thread, NULL, my_thread, &altstack_lo,
&altstack_hi, &found_me);
if (lo) {
GC_ASSERT(ADDR_GE(hi, lo));
total_size += hi - lo;
GC_push_all_stack(lo, hi);
}
/* TODO: Handle `altstack`. */
nthreads++;
mach_port_deallocate(my_task, thread);
}
vm_deallocate(my_task, (vm_address_t)act_list,
sizeof(thread_t) * listcount);
} else
# endif /* !DARWIN_DONT_PARSE_STACK */
/* else */ {
int i;
for (i = 0; i < THREAD_TABLE_SZ; i++) {
GC_thread p;
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
GC_ASSERT(THREAD_TABLE_INDEX(p->id) == i);
if (!KNOWN_FINISHED(p)) {
thread_act_t thread = (thread_act_t)(p->mach_thread);
ptr_t lo = GC_stack_range_for(&hi, thread, p, my_thread,
&altstack_lo, &altstack_hi, &found_me);
if (lo) {
GC_ASSERT(ADDR_GE(hi, lo));
total_size += hi - lo;
GC_push_all_stack_sections(lo, hi, p->crtn->traced_stack_sect);
}
if (altstack_lo) {
total_size += altstack_hi - altstack_lo;
GC_push_all_stack(altstack_lo, altstack_hi);
}
nthreads++;
}
}
}
}
mach_port_deallocate(my_task, my_thread);
GC_VERBOSE_LOG_PRINTF("Pushed %d thread stacks\n", nthreads);
if (!found_me && !GC_in_thread_creation)
ABORT("Collecting from unknown thread");
GC_total_stacksize = total_size;
}
# ifndef GC_NO_THREADS_DISCOVERY
# ifdef MPROTECT_VDB
STATIC mach_port_t GC_mach_handler_thread = 0;
STATIC GC_bool GC_use_mach_handler_thread = FALSE;
GC_INNER void
GC_darwin_register_self_mach_handler(void)
{
GC_mach_handler_thread = mach_thread_self();
GC_use_mach_handler_thread = TRUE;
}
# endif /* MPROTECT_VDB */
# ifndef GC_MAX_MACH_THREADS
# define GC_MAX_MACH_THREADS THREAD_TABLE_SZ
# endif
struct GC_mach_thread {
thread_act_t thread;
GC_bool suspended;
};
STATIC struct GC_mach_thread GC_mach_threads[GC_MAX_MACH_THREADS]
= { { 0, FALSE } };
STATIC int GC_mach_threads_count = 0;
/* FIXME: It is better to implement `GC_mach_threads` as a hash set. */
/*
* Return `TRUE` if there is a thread in `act_list` that was not
* in `old_list`.
*/
STATIC GC_bool
GC_suspend_thread_list(thread_act_array_t act_list, int count,
thread_act_array_t old_list, int old_count,
task_t my_task, mach_port_t my_thread)
{
int i;
int j = -1;
GC_bool changed = FALSE;
GC_ASSERT(I_HOLD_LOCK());
for (i = 0; i < count; i++) {
thread_act_t thread = act_list[i];
GC_bool found;
kern_return_t kern_result;
if (thread == my_thread
# ifdef MPROTECT_VDB
|| (GC_mach_handler_thread == thread && GC_use_mach_handler_thread)
# endif
# ifdef PARALLEL_MARK
|| GC_is_mach_marker(thread) /*< ignore the parallel markers */
# endif
) {
/*
* Do not add our one, parallel marker and the handler threads;
* consider it as found (e.g., it was processed earlier).
*/
mach_port_deallocate(my_task, thread);
continue;
}
/* Find the current thread in the old list. */
found = FALSE;
{
/* The previous found thread index. */
int last_found = j;
/* Search for the thread starting from the last found one first. */
while (++j < old_count)
if (old_list[j] == thread) {
found = TRUE;
break;
}
if (!found) {
/* If not found, search in the rest (beginning) of the list. */
for (j = 0; j < last_found; j++)
if (old_list[j] == thread) {
found = TRUE;
break;
}
}
}
if (found) {
/* It is already in the list, skip processing, release `mach` port. */
mach_port_deallocate(my_task, thread);
continue;
}
/* Add it to the `GC_mach_threads` list. */
if (GC_mach_threads_count == GC_MAX_MACH_THREADS)
ABORT("Too many threads");
GC_mach_threads[GC_mach_threads_count].thread = thread;
/* The default is "not suspended". */
GC_mach_threads[GC_mach_threads_count].suspended = FALSE;
changed = TRUE;
# ifdef DEBUG_THREADS
GC_log_printf("Suspending %p\n", THREAD_ACT_TO_VPTR(thread));
# endif
/*
* Unconditionally suspend the thread. It will do no harm if it is
* already suspended by the client logic.
*/
GC_acquire_dirty_lock();
do {
kern_result = thread_suspend(thread);
} while (kern_result == KERN_ABORTED);
GC_release_dirty_lock();
if (kern_result != KERN_SUCCESS) {
/*
* The thread may have quit since the `thread_threads()` call we
* mark as already suspended so it is not dealt with anymore later.
*/
GC_mach_threads[GC_mach_threads_count].suspended = FALSE;
} else {
/* Mark the thread as suspended and require resume. */
GC_mach_threads[GC_mach_threads_count].suspended = TRUE;
if (GC_on_thread_event)
GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED,
THREAD_ACT_TO_VPTR(thread));
}
GC_mach_threads_count++;
}
return changed;
}
# endif /* !GC_NO_THREADS_DISCOVERY */
GC_INNER void
GC_stop_world(void)
{
task_t my_task = current_task();
mach_port_t my_thread = mach_thread_self();
kern_return_t kern_result;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_thr_initialized);
# ifdef DEBUG_THREADS
GC_log_printf("Stopping the world from thread %p\n",
MACH_PORT_TO_VPTR(my_thread));
# endif
# ifdef PARALLEL_MARK
if (GC_parallel) {
GC_acquire_mark_lock();
/* We should have previously waited for it to become zero. */
GC_ASSERT(GC_fl_builder_count == 0);
}
# endif /* PARALLEL_MARK */
if (GC_query_task_threads) {
# ifndef GC_NO_THREADS_DISCOVERY
GC_bool changed;
thread_act_array_t act_list, prev_list;
mach_msg_type_number_t listcount, prevcount;
/*
* Clear out the `mach` threads list table. We do not need to really
* clear `GC_mach_threads[]` as it is used only in the range from 0 to
* `GC_mach_threads_count - 1`, inclusive.
*/
GC_mach_threads_count = 0;
/*
* Loop stopping threads until you have gone over the whole list twice
* without a new one appearing. `thread_create()` will not return (and
* thus the thread stop) until the new thread exists, so there is
* no window whereby you could stop a thread, recognize it is stopped,
* but then have a new thread it created before stopping shows up later.
*/
changed = TRUE;
prev_list = NULL;
prevcount = 0;
do {
kern_result = task_threads(my_task, &act_list, &listcount);
if (kern_result == KERN_SUCCESS) {
changed = GC_suspend_thread_list(act_list, listcount, prev_list,
prevcount, my_task, my_thread);
if (prev_list != NULL) {
/*
* Thread ports are not deallocated by list, unused ports
* deallocated in `GC_suspend_thread_list`, used ones - kept
* in `GC_mach_threads` till `GC_start_world` as otherwise thread
* object change can occur and `GC_start_world` will not find the
* thread to resume which will cause application to hang.
*/
vm_deallocate(my_task, (vm_address_t)prev_list,
sizeof(thread_t) * prevcount);
}
/* Repeat while having changes. */
prev_list = act_list;
prevcount = listcount;
}
} while (changed);
GC_ASSERT(prev_list != 0);
/* The thread ports are not deallocated by list, see above. */
vm_deallocate(my_task, (vm_address_t)act_list,
sizeof(thread_t) * listcount);
# endif /* !GC_NO_THREADS_DISCOVERY */
} else {
unsigned i;
for (i = 0; i < THREAD_TABLE_SZ; i++) {
GC_thread p;
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
if ((p->flags & (FINISHED | DO_BLOCKING)) == 0
&& p->mach_thread != my_thread) {
GC_acquire_dirty_lock();
do {
kern_result = thread_suspend(p->mach_thread);
} while (kern_result == KERN_ABORTED);
GC_release_dirty_lock();
if (kern_result != KERN_SUCCESS)
ABORT("thread_suspend failed");
if (GC_on_thread_event)
GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED,
MACH_PORT_TO_VPTR(p->mach_thread));
}
}
}
}
# ifdef MPROTECT_VDB
if (GC_auto_incremental) {
GC_mprotect_stop();
}
# endif
# ifdef PARALLEL_MARK
if (GC_parallel)
GC_release_mark_lock();
# endif
# ifdef DEBUG_THREADS
GC_log_printf("World stopped from %p\n", MACH_PORT_TO_VPTR(my_thread));
# endif
mach_port_deallocate(my_task, my_thread);
}
GC_INLINE void
GC_thread_resume(thread_act_t thread)
{
kern_return_t kern_result;
# if defined(DEBUG_THREADS) || defined(GC_ASSERTIONS)
struct thread_basic_info info;
mach_msg_type_number_t outCount = THREAD_BASIC_INFO_COUNT;
# ifdef CPPCHECK
info.run_state = 0;
# endif
kern_result = thread_info(thread, THREAD_BASIC_INFO, (thread_info_t)&info,
&outCount);
if (kern_result != KERN_SUCCESS)
ABORT("thread_info failed");
# endif
GC_ASSERT(I_HOLD_LOCK());
# ifdef DEBUG_THREADS
GC_log_printf("Resuming thread %p with state %d\n",
THREAD_ACT_TO_VPTR(thread), info.run_state);
# endif
/* Resume the thread. */
kern_result = thread_resume(thread);
if (kern_result != KERN_SUCCESS) {
WARN("thread_resume(%p) failed: mach port invalid\n", thread);
} else if (GC_on_thread_event) {
GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED,
THREAD_ACT_TO_VPTR(thread));
}
}
GC_INNER void
GC_start_world(void)
{
task_t my_task = current_task();
/* The allocator lock is held continuously since the world stopped. */
GC_ASSERT(I_HOLD_LOCK());
# ifdef DEBUG_THREADS
GC_log_printf("World starting\n");
# endif
# ifdef MPROTECT_VDB
if (GC_auto_incremental) {
GC_mprotect_resume();
}
# endif
if (GC_query_task_threads) {
# ifndef GC_NO_THREADS_DISCOVERY
int i, j;
kern_return_t kern_result;
thread_act_array_t act_list;
mach_msg_type_number_t listcount;
kern_result = task_threads(my_task, &act_list, &listcount);
if (kern_result != KERN_SUCCESS)
ABORT("task_threads failed");
j = (int)listcount;
for (i = 0; i < GC_mach_threads_count; i++) {
thread_act_t thread = GC_mach_threads[i].thread;
if (GC_mach_threads[i].suspended) {
/*
* The thread index found during the previous iteration
* (reaching `listcount` value means no thread found yet).
*/
int last_found = j;
/* Search for the thread starting from the last found one first. */
while (++j < (int)listcount) {
if (act_list[j] == thread)
break;
}
if (j >= (int)listcount) {
/* If not found, search in the rest (beginning) of the list. */
for (j = 0; j < last_found; j++) {
if (act_list[j] == thread)
break;
}
}
if (j != last_found) {
/* The thread is alive, resume it. */
GC_thread_resume(thread);
}
} else {
/*
* This thread has failed to be suspended by `GC_stop_world`,
* no action is needed.
*/
# ifdef DEBUG_THREADS
GC_log_printf("Not resuming thread %p as it is not suspended\n",
THREAD_ACT_TO_VPTR(thread));
# endif
}
mach_port_deallocate(my_task, thread);
}
for (i = 0; i < (int)listcount; i++)
mach_port_deallocate(my_task, act_list[i]);
vm_deallocate(my_task, (vm_address_t)act_list,
sizeof(thread_t) * listcount);
# endif /* !GC_NO_THREADS_DISCOVERY */
} else {
int i;
mach_port_t my_thread = mach_thread_self();
for (i = 0; i < THREAD_TABLE_SZ; i++) {
GC_thread p;
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
if ((p->flags & (FINISHED | DO_BLOCKING)) == 0
&& p->mach_thread != my_thread)
GC_thread_resume(p->mach_thread);
}
}
mach_port_deallocate(my_task, my_thread);
}
# ifdef DEBUG_THREADS
GC_log_printf("World started\n");
# endif
}
#endif /* DARWIN && THREADS */
/*
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1997 by Silicon Graphics. All rights reserved.
* Copyright (c) 2009-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* This is incredibly OS specific code for tracking down data sections in
* dynamic libraries. There appears to be no way of doing this quickly
* without groveling through undocumented data structures. We would argue
* that this is a bug in the design of the `dlopen` interface. This code
* may break in future OS releases. If this matters to you, do not hesitate
* to let your vendor know...
*
* None of this is safe with `dlclose` and incremental collection.
* But, then not much of anything is safe in the presence of `dlclose`.
*/
/*
* BTL: avoid circular redefinition of `dlopen` if `SOLARIS` and `THREADS`
* are both defined.
*/
#undef GC_MUST_RESTORE_REDEFINED_DLOPEN
#if defined(GC_PTHREADS) && !defined(GC_NO_DLOPEN) \
&& !defined(GC_NO_THREAD_REDIRECTS) && !defined(GC_USE_LD_WRAP)
/*
* To support threads in Solaris, `gc.h` file interposes on `dlopen` by
* defining `dlopen` to be `GC_dlopen`. `GC_FirstDLOpenedLinkMap()` needs
* to use the real system `dlopen()`, thus we first undo `dlopen` definition
* by `gc.h` file and restore it at the end of the current file.
*/
# undef dlopen
# define GC_MUST_RESTORE_REDEFINED_DLOPEN
#endif /* !GC_NO_DLOPEN */
#if defined(SOLARISDL) && defined(THREADS) && !defined(SOLARIS) \
&& !defined(CPPCHECK)
# error Fix mutual exclusion with dlopen
#endif
/*
* A user-supplied routine (custom filter) that might be called to
* determine whether a DSO really needs to be scanned by the collector.
* Zero means no filter installed. May be unused on some platforms.
*/
/* FIXME: Add filter support for more platforms. */
STATIC GC_has_static_roots_func GC_has_static_roots = 0;
#ifdef ANY_MSWIN
/*
* We traverse the entire address space and register all segments that
* could possibly have been written to.
*/
STATIC void
GC_cond_add_roots(ptr_t base, ptr_t limit)
{
# ifdef THREADS
ptr_t curr_base = base;
ptr_t next_stack_lo, next_stack_hi;
# else
ptr_t stack_top;
# endif
GC_ASSERT(I_HOLD_LOCK());
if (base == limit)
return;
# ifdef THREADS
for (;;) {
GC_get_next_stack(curr_base, limit, &next_stack_lo, &next_stack_hi);
if (ADDR_GE(next_stack_lo, limit))
break;
if (ADDR_LT(curr_base, next_stack_lo))
GC_add_roots_inner(curr_base, next_stack_lo, TRUE);
curr_base = next_stack_hi;
}
if (ADDR_LT(curr_base, limit))
GC_add_roots_inner(curr_base, limit, TRUE);
# else
stack_top
= PTR_ALIGN_DOWN(GC_approx_sp(), GC_sysinfo.dwAllocationGranularity);
if (ADDR_LT(stack_top, limit) && ADDR_LT(base, GC_stackbottom)) {
/* Part of the stack; ignore it. */
return;
}
GC_add_roots_inner(base, limit, TRUE);
# endif
}
# ifdef DYNAMIC_LOADING
GC_INNER GC_bool
GC_register_main_static_data(void)
{
# if defined(MSWINCE) || defined(CYGWIN32)
return FALSE;
# else
return GC_no_win32_dlls;
# endif
}
# define HAVE_REGISTER_MAIN_STATIC_DATA
# endif /* DYNAMIC_LOADING */
# ifdef DEBUG_VIRTUALQUERY
void
GC_dump_meminfo(MEMORY_BASIC_INFORMATION *buf)
{
GC_printf("BaseAddress= 0x%lx, AllocationBase= 0x%lx,"
" RegionSize= 0x%lx(%lu)\n",
buf->BaseAddress, buf->AllocationBase, buf->RegionSize,
buf->RegionSize);
GC_printf("\tAllocationProtect= 0x%lx, State= 0x%lx, Protect= 0x%lx, "
"Type= 0x%lx\n",
buf->AllocationProtect, buf->State, buf->Protect, buf->Type);
}
# endif /* DEBUG_VIRTUALQUERY */
# if defined(MSWINCE) || defined(CYGWIN32)
/* FIXME: Should we really need to scan `MEM_PRIVATE` sections? */
/*
* For now, we do not add `MEM_PRIVATE` sections to the data roots for
* WinCE because otherwise `SIGSEGV` fault sometimes happens to occur
* in `GC_mark_from()` (and, even if we use `WRAP_MARK_SOME`, WinCE
* prints a "Data Abort" message to the debugging console).
* To workaround that, use `-D GC_REGISTER_MEM_PRIVATE`.
*/
# define GC_wnt TRUE
# endif
GC_INNER void
GC_register_dynamic_libraries(void)
{
ptr_t p, base, limit;
GC_ASSERT(I_HOLD_LOCK());
# ifdef MSWIN32
if (GC_no_win32_dlls)
return;
# endif
p = (ptr_t)GC_sysinfo.lpMinimumApplicationAddress;
base = limit = p;
while (ADDR_LT(p, (ptr_t)GC_sysinfo.lpMaximumApplicationAddress)) {
MEMORY_BASIC_INFORMATION buf;
size_t result = VirtualQuery((LPVOID)p, &buf, sizeof(buf));
# ifdef MSWINCE
if (0 == result) {
if (ADDR(p) > GC_WORD_MAX - GC_sysinfo.dwAllocationGranularity) {
/* An overflow. */
break;
}
/* Page is free; advance to the next possible allocation base. */
p = PTR_ALIGN_UP(p + 1, GC_sysinfo.dwAllocationGranularity);
} else
# endif
/* else */ {
DWORD protect;
if (result != sizeof(buf))
ABORT("Weird VirtualQuery result");
if (ADDR(p) > GC_WORD_MAX - buf.RegionSize) {
/* An overflow. */
break;
}
protect = buf.Protect;
if (buf.State == MEM_COMMIT
&& (protect == PAGE_EXECUTE_READWRITE
|| protect == PAGE_EXECUTE_WRITECOPY || protect == PAGE_READWRITE
|| protect == PAGE_WRITECOPY)
&& (buf.Type == MEM_IMAGE
# ifdef GC_REGISTER_MEM_PRIVATE
|| (protect == PAGE_READWRITE && buf.Type == MEM_PRIVATE)
# else
/*
* There is some evidence that we cannot always ignore
* `MEM_PRIVATE` sections under Windows ME and predecessors.
* Hence we now also check for that case.
*/
|| (!GC_wnt && buf.Type == MEM_PRIVATE)
# endif
)
&& !GC_is_heap_base(buf.AllocationBase)) {
# ifdef DEBUG_VIRTUALQUERY
GC_dump_meminfo(&buf);
# endif
if (p != limit) {
GC_cond_add_roots(base, limit);
base = p;
}
limit = p + buf.RegionSize;
}
p += buf.RegionSize;
}
}
GC_cond_add_roots(base, limit);
}
#elif defined(DYNAMIC_LOADING) /* `&& !ANY_MSWIN` */
# if !(defined(CPPCHECK) || defined(AIX) || defined(DARWIN) || defined(DGUX) \
|| defined(IRIX5) || defined(HAIKU) || defined(HPUX) || defined(HURD) \
|| defined(NACL) || defined(OSF1) || defined(SCO_ELF) \
|| defined(SERENITY) || defined(SOLARISDL) \
|| ((defined(ANY_BSD) || defined(LINUX)) && defined(__ELF__)) \
|| (defined(OPENBSD) && defined(M68K)))
# error Finding data segments of dynamic libraries is unsupported on target
# endif
# if defined(DARWIN) && !defined(USE_DYLD_TO_BIND) \
&& !defined(NO_DYLD_BIND_FULLY_IMAGE)
# include <dlfcn.h>
# endif
# ifdef SOLARISDL
# include <dlfcn.h>
# include <link.h>
# include <sys/elf.h>
# endif
# if defined(NETBSD)
# include <dlfcn.h>
# include <machine/elf_machdep.h>
# include <sys/param.h>
# define ELFSIZE ARCH_ELFSIZE
# endif
# if defined(OPENBSD)
# include <sys/param.h>
# if (OpenBSD >= 200519) && !defined(HAVE_DL_ITERATE_PHDR)
# define HAVE_DL_ITERATE_PHDR
# endif
# endif /* OPENBSD */
# if defined(DGUX) || defined(HURD) || defined(NACL) || defined(SCO_ELF) \
|| defined(SERENITY) \
|| ((defined(ANY_BSD) || defined(LINUX)) && defined(__ELF__))
# include <stddef.h>
# if !defined(OPENBSD) && !defined(HOST_ANDROID)
/*
* OpenBSD does not have platform `elf.h` file; include `link.h` file below
* is sufficient. Exclude this on Android because platform `linker.h` file
* below includes its own variant.
*/
# include <elf.h>
# endif
# ifdef HOST_ANDROID
/*
* If you do not need the "dynamic loading" feature, you may build the
* collector with `-D IGNORE_DYNAMIC_LOADING`.
*/
# ifdef BIONIC_ELFDATA_REDEF_BUG
/*
* Workaround a problem in Bionic (as of Android 4.2) which has mismatching
* `ELF_DATA` definitions in platform `sys/exec_elf.h` and `asm/elf.h` files
* included from `linker.h` file (similar to `EM_ALPHA`).
*/
# include <asm/elf.h>
# include <linux/elf-em.h>
# undef ELF_DATA
# undef EM_ALPHA
# endif
# include <link.h>
# endif /* HOST_ANDROID */
# if ((defined(HOST_ANDROID) && !defined(GC_DONT_DEFINE_LINK_MAP) \
&& !(__ANDROID_API__ >= 21)) \
|| defined(SERENITY)) \
&& !defined(USE_PROC_FOR_LIBRARIES)
/*
* `link_map` and `r_debug` are defined in `link.h` file of NDK r10+.
* `bionic/linker/linker.h` file defines them too but the header itself
* is a C++ one starting from Android 4.3.
*/
struct link_map {
uintptr_t l_addr;
char *l_name;
uintptr_t l_ld;
struct link_map *l_next;
struct link_map *l_prev;
};
struct r_debug {
int32_t r_version;
struct link_map *r_map;
/* `void (*r_brk)(void);` */
/* `int32_t r_state;` */
/* `uintptr_t r_ldbase;` */
};
# define LINK_MAP_R_DEBUG_DEFINED
# endif /* __ANDROID_API__ >= 21 || SERENITY */
# ifndef HOST_ANDROID
/*
* Workaround missing `extern "C"` around `_DYNAMIC` symbol in platform
* `link.h` file of some Linux hosts.
*/
EXTERN_C_BEGIN
# include <link.h>
EXTERN_C_END
# endif /* !HOST_ANDROID */
# endif
/*
* Newer versions of GNU/Linux define this macro. We define it similarly
* for all ELF systems that do not.
*/
# ifndef ElfW
# if defined(FREEBSD)
# if __ELF_WORD_SIZE == 32
# define ElfW(type) Elf32_##type
# else
# define ElfW(type) Elf64_##type
# endif
# elif defined(NETBSD) || defined(OPENBSD)
# if ELFSIZE == 32
# define ElfW(type) Elf32_##type
# elif ELFSIZE == 64
# define ElfW(type) Elf64_##type
# else
# error Missing ELFSIZE define
# endif
# else
# if !defined(ELF_CLASS) || ELF_CLASS == ELFCLASS32
# define ElfW(type) Elf32_##type
# else
# define ElfW(type) Elf64_##type
# endif
# endif
# endif
# if defined(DGUX) || defined(HURD) || defined(NACL) || defined(SCO_ELF) \
|| defined(SERENITY) \
|| ((defined(ANY_BSD) || defined(LINUX)) && defined(__ELF__))
# ifdef USE_PROC_FOR_LIBRARIES
# include <fcntl.h>
# include <string.h>
# include <sys/stat.h>
# define MAPS_BUF_SIZE (32 * 1024)
/*
* Sort an array of `HeapSects` elements by `hs_start` value.
* Unfortunately, at least some versions of Linux, `qsort` end up calling
* `malloc` by way of `sysconf`, and hence cannot be used in the collector.
* Hence we roll our own. Should be reasonably fast if the array is
* already mostly sorted, as we expect it to be.
*/
static void
sort_heap_sects(struct HeapSect *base, size_t number_of_elements)
{
GC_signed_word n = (GC_signed_word)number_of_elements;
GC_signed_word nsorted = 1;
while (nsorted < n) {
GC_signed_word i;
while (nsorted < n
&& ADDR_LT(base[nsorted - 1].hs_start, base[nsorted].hs_start)) {
++nsorted;
}
if (nsorted == n)
break;
GC_ASSERT(ADDR_LT(base[nsorted].hs_start, base[nsorted - 1].hs_start));
for (i = nsorted - 1;
i >= 0 && ADDR_LT(base[i + 1].hs_start, base[i].hs_start); --i) {
struct HeapSect tmp = base[i];
base[i] = base[i + 1];
base[i + 1] = tmp;
}
GC_ASSERT(ADDR_LT(base[nsorted - 1].hs_start, base[nsorted].hs_start));
++nsorted;
}
}
STATIC void
GC_register_map_entries(const char *maps)
{
const char *prot, *path;
ptr_t my_start, my_end;
ptr_t least_ha, greatest_ha;
unsigned maj_dev;
unsigned i;
GC_ASSERT(I_HOLD_LOCK());
sort_heap_sects(GC_our_memory, GC_n_memory);
least_ha = GC_our_memory[0].hs_start;
greatest_ha = GC_our_memory[GC_n_memory - 1].hs_start
+ GC_our_memory[GC_n_memory - 1].hs_bytes;
for (;;) {
maps
= GC_parse_map_entry(maps, &my_start, &my_end, &prot, &maj_dev, &path);
if (NULL == maps)
break;
if (prot[1] == 'w') {
/*
* This is a writable mapping. Add it to the root set unless
* it is already otherwise accounted for.
*/
# ifndef THREADS
if (ADDR_GE(GC_stackbottom, my_start)
&& ADDR_GE(my_end, GC_stackbottom)) {
/* Stack mapping; discard it. */
continue;
}
# endif
# if defined(E2K) && defined(__ptr64__)
/* TODO: Avoid hard-coded addresses. */
if (ADDR(my_start) == 0xc2fffffff000UL
&& ADDR(my_end) == 0xc30000000000UL && path[0] == '\n') {
/* Discard some special mapping. */
continue;
}
# endif
if (path[0] == '[' && strncmp(path + 1, "heap]", 5) != 0) {
/* Discard a pseudo-file path except for "[heap]". */
continue;
}
# ifdef THREADS
/*
* This may fail, since a thread may already be unregistered, but
* its thread stack may still be there. That can fail because the
* stack may disappear while we are marking. Thus the marker is,
* and has to be prepared to recover from segmentation faults.
*/
if (GC_segment_is_thread_stack(my_start, my_end)) {
continue;
/*
* FIXME: NPTL squirrels away pointers in pieces of the stack
* segment that we do not scan. We work around this by treating
* anything allocated by `libpthread` as uncollectible, as we do
* in some other cases. A specifically identified problem is that
* thread stacks contain pointers to dynamic thread vectors, that
* may be reused due to thread caching. They may not be marked
* if the thread is still live. This specific instance should be
* addressed by `INCLUDE_LINUX_THREAD_DESCR`, but that does not
* quite seem to suffice. We currently trace entire thread stacks,
* if they are are currently cached but unused. This is very
* suboptimal for performance reasons.
*/
}
# endif
/* We no longer exclude the main data segment. */
if (ADDR_GE(least_ha, my_end) || ADDR_GE(my_start, greatest_ha)) {
/* The easy case; just trace the entire segment. */
GC_add_roots_inner(my_start, my_end, TRUE);
continue;
}
/* Add sections that do not belong to us. */
i = 0;
while (ADDR_LT(GC_our_memory[i].hs_start + GC_our_memory[i].hs_bytes,
my_start)) {
++i;
}
GC_ASSERT(i < GC_n_memory);
if (ADDR_GE(my_start, GC_our_memory[i].hs_start)) {
my_start = GC_our_memory[i].hs_start + GC_our_memory[i].hs_bytes;
++i;
}
for (; i < GC_n_memory && ADDR_LT(my_start, my_end)
&& ADDR_LT(GC_our_memory[i].hs_start, my_end);
++i) {
if (ADDR_LT(my_start, GC_our_memory[i].hs_start))
GC_add_roots_inner(my_start, GC_our_memory[i].hs_start, TRUE);
my_start = GC_our_memory[i].hs_start + GC_our_memory[i].hs_bytes;
}
if (ADDR_LT(my_start, my_end))
GC_add_roots_inner(my_start, my_end, TRUE);
} else if (prot[0] == '-' && prot[1] == '-' && prot[2] == '-') {
/*
* Even roots added statically might disappear partially
* (e.g. the roots added by `INCLUDE_LINUX_THREAD_DESCR`).
*/
GC_remove_roots_subregion(my_start, my_end);
}
}
}
GC_INNER void
GC_register_dynamic_libraries(void)
{
GC_register_map_entries(GC_get_maps());
}
GC_INNER GC_bool
GC_register_main_static_data(void)
{
/* We now take care of the main data segment ourselves. */
return FALSE;
}
# define HAVE_REGISTER_MAIN_STATIC_DATA
# else /* !USE_PROC_FOR_LIBRARIES */
/*
* The following is the preferred way to walk dynamic libraries for
* `glibc` 2.2.4+. Unfortunately, it does not work for older versions.
*/
# if GC_GLIBC_PREREQ(2, 3) || defined(HOST_ANDROID)
/* Are others OK here, too? */
# ifndef HAVE_DL_ITERATE_PHDR
# define HAVE_DL_ITERATE_PHDR
# endif
# ifdef HOST_ANDROID
/* Android headers might have no such definition for some targets. */
EXTERN_C_BEGIN
extern int dl_iterate_phdr(int (*cb)(struct dl_phdr_info *, size_t, void *),
void *data);
EXTERN_C_END
# endif
# endif /* __GLIBC__ >= 2 || HOST_ANDROID */
# if defined(__DragonFly__) || defined(__FreeBSD_kernel__) \
|| (defined(FREEBSD) && __FreeBSD__ >= 7)
/*
* On the FreeBSD system, any target system at major version 7 shall have
* `dl_iterate_phdr`; therefore, we need not make it weak as below.
*/
# ifndef HAVE_DL_ITERATE_PHDR
# define HAVE_DL_ITERATE_PHDR
# endif
# define DL_ITERATE_PHDR_STRONG
# elif defined(HAVE_DL_ITERATE_PHDR)
/*
* We have the header files for a `glibc` that includes `dl_iterate_phdr`.
* It may still not be available in the library on the target system.
* Thus we also treat it as a weak symbol.
*/
EXTERN_C_BEGIN
# pragma weak dl_iterate_phdr
EXTERN_C_END
# endif
# ifdef HAVE_DL_ITERATE_PHDR
# ifdef PT_GNU_RELRO
/*
* Instead of registering `PT_LOAD` sections directly, we keep them
* in a temporary list, and filter them by excluding `PT_GNU_RELRO`
* segments. Processing `PT_GNU_RELRO` sections with
* `GC_exclude_static_roots` instead would be superficially cleaner.
* But it runs into trouble if a client registers an overlapping segment,
* which unfortunately seems quite possible.
*/
# define MAX_LOAD_SEGS MAX_ROOT_SETS
static struct load_segment {
ptr_t start;
ptr_t end;
/*
* The room for the second segment if we remove a `PT_GNU_RELRO` segment
* from the middle.
*/
ptr_t start2;
ptr_t end2;
} load_segs[MAX_LOAD_SEGS];
static int n_load_segs;
static GC_bool load_segs_overflow;
# endif /* PT_GNU_RELRO */
STATIC int
GC_register_dynlib_callback(struct dl_phdr_info *info, size_t size, void *ptr)
{
const ElfW(Phdr) * p;
ptr_t load_ptr, my_start, my_end;
int i;
GC_ASSERT(I_HOLD_LOCK());
/* Make sure `dl_phdr_info` structure is at least as big as we need. */
if (size
< offsetof(struct dl_phdr_info, dlpi_phnum) + sizeof(info->dlpi_phnum))
return 1; /*< stop */
load_ptr = (ptr_t)info->dlpi_addr;
p = info->dlpi_phdr;
for (i = 0; i < (int)info->dlpi_phnum; i++, p++) {
if (p->p_type == PT_LOAD) {
GC_has_static_roots_func callback = GC_has_static_roots;
if ((p->p_flags & PF_W) == 0)
continue;
# ifdef CHERI_PURECAP
my_start = load_ptr + p->p_vaddr;
# else
/*
* Prevent "applying nonzero offset to null pointer" compiler warning
* as `load_ptr` could be `NULL`.
*/
my_start = (ptr_t)((GC_uintptr_t)load_ptr + p->p_vaddr);
# endif
my_end = my_start + p->p_memsz;
# ifdef CHERI_PURECAP
my_start = PTR_ALIGN_UP(my_start, ALIGNMENT);
my_end = PTR_ALIGN_DOWN(my_end, ALIGNMENT);
if (!SPANNING_CAPABILITY(load_ptr, ADDR(my_start), ADDR(my_end)))
continue;
my_start = cheri_bounds_set(my_start, (word)(my_end - my_start));
# endif
if (callback != 0 && !callback(info->dlpi_name, my_start, p->p_memsz))
continue;
# ifdef PT_GNU_RELRO
# if CPP_PTRSZ >= 64 && !defined(CHERI_PURECAP)
/*
* TODO: `GC_push_all` eventually does the correct rounding to the
* next multiple of `ALIGNMENT`, so, most probably, we should remove
* the corresponding assertion check in `GC_add_roots_inner` along
* with this code line. `my_start` value may require aligning.
*/
my_start = PTR_ALIGN_DOWN(my_start, ALIGNMENT);
# endif
if (n_load_segs >= MAX_LOAD_SEGS) {
if (!load_segs_overflow) {
WARN("Too many PT_LOAD segments;"
" registering as roots directly...\n",
0);
load_segs_overflow = TRUE;
}
GC_add_roots_inner(my_start, my_end, TRUE);
} else {
load_segs[n_load_segs].start = my_start;
load_segs[n_load_segs].end = my_end;
load_segs[n_load_segs].start2 = NULL;
load_segs[n_load_segs].end2 = NULL;
++n_load_segs;
}
# else
GC_add_roots_inner(my_start, my_end, TRUE);
# endif /* !PT_GNU_RELRO */
}
}
# ifdef PT_GNU_RELRO
p = info->dlpi_phdr;
for (i = 0; i < (int)info->dlpi_phnum; i++, p++) {
if (p->p_type == PT_GNU_RELRO) {
/*
* This entry is known to be constant and will eventually be
* remapped as read-only. However, the address range covered
* by this entry is typically a subset of a previously
* encountered `PT_LOAD` segment, so we need to exclude it.
*/
int j;
# ifdef CHERI_PURECAP
my_start = load_ptr + p->p_vaddr;
# else
my_start = (ptr_t)((GC_uintptr_t)load_ptr + p->p_vaddr);
# endif
my_end = my_start + p->p_memsz;
for (j = n_load_segs; --j >= 0;) {
if (ADDR_INSIDE(my_start, load_segs[j].start, load_segs[j].end)) {
if (load_segs[j].start2 != NULL) {
WARN("More than one GNU_RELRO segment per load one\n", 0);
} else {
GC_ASSERT(
ADDR_GE(PTR_ALIGN_UP(load_segs[j].end, GC_page_size), my_end));
/* Remove it from the existing load segment. */
load_segs[j].end2 = load_segs[j].end;
load_segs[j].end = my_start;
load_segs[j].start2 = my_end;
/*
* Note that `start2` may be greater than `end2` because of
* `p->p_memsz` value is multiple of page size.
*/
}
break;
}
if (0 == j && 0 == GC_has_static_roots)
WARN("Failed to find PT_GNU_RELRO segment"
" inside PT_LOAD region\n",
0);
/*
* No warning reported in case of the callback is present because
* most likely the segment has been excluded.
*/
}
}
}
# endif
/* Signal that we were called. */
*(int *)ptr = 1;
return 0;
}
GC_INNER GC_bool
GC_register_main_static_data(void)
{
# if defined(DL_ITERATE_PHDR_STRONG) && !defined(CPPCHECK)
/*
* If `dl_iterate_phdr` is not a weak symbol, then do not test against
* zero (otherwise a compiler might issue a warning).
*/
return FALSE;
# else
return 0 == COVERT_DATAFLOW(ADDR(dl_iterate_phdr));
# endif
}
# define HAVE_REGISTER_MAIN_STATIC_DATA
/*
* Return `TRUE` if we succeed; return `FALSE` if `dl_iterate_phdr` was
* not there.
*/
STATIC GC_bool
GC_register_dynamic_libraries_dl_iterate_phdr(void)
{
int did_something;
GC_ASSERT(I_HOLD_LOCK());
if (GC_register_main_static_data())
return FALSE;
# ifdef PT_GNU_RELRO
{
static GC_bool excluded_segs = FALSE;
n_load_segs = 0;
load_segs_overflow = FALSE;
if (UNLIKELY(!excluded_segs)) {
GC_exclude_static_roots_inner((ptr_t)load_segs,
(ptr_t)load_segs + sizeof(load_segs));
excluded_segs = TRUE;
}
}
# endif
did_something = 0;
dl_iterate_phdr(GC_register_dynlib_callback, &did_something);
if (did_something) {
# ifdef PT_GNU_RELRO
int i;
for (i = 0; i < n_load_segs; ++i) {
if (ADDR_LT(load_segs[i].start, load_segs[i].end))
GC_add_roots_inner(load_segs[i].start, load_segs[i].end, TRUE);
if (ADDR_LT(load_segs[i].start2, load_segs[i].end2))
GC_add_roots_inner(load_segs[i].start2, load_segs[i].end2, TRUE);
}
# endif
} else {
ptr_t datastart, dataend;
# ifdef DATASTART_USES_XGETDATASTART
static ptr_t datastart_cached = MAKE_CPTR(GC_WORD_MAX);
/* Evaluate `DATASTART` only once. */
if (ADDR(datastart_cached) == GC_WORD_MAX) {
datastart_cached = DATASTART;
}
datastart = datastart_cached;
# else
datastart = DATASTART;
# endif
# ifdef DATAEND_IS_FUNC
{
static ptr_t dataend_cached = 0;
/* Evaluate `DATAEND` only once. */
if (dataend_cached == 0) {
dataend_cached = DATAEND;
}
dataend = dataend_cached;
}
# else
dataend = DATAEND;
# endif
if (NULL == *(char *volatile *)&datastart || ADDR_LT(dataend, datastart))
ABORT_ARG2("Wrong DATASTART/END pair", ": %p .. %p", (void *)datastart,
(void *)dataend);
/*
* `dl_iterate_phdr` may forget the static data segment in
* statically linked executables.
*/
GC_add_roots_inner(datastart, dataend, TRUE);
# ifdef GC_HAVE_DATAREGION2
/* Subtract one to check also for `NULL` without a compiler warning. */
if (ADDR(DATASTART2) - 1U >= ADDR(DATAEND2)) {
ABORT_ARG2("Wrong DATASTART/END2 pair", ": %p .. %p", (void *)DATASTART2,
(void *)DATAEND2);
}
GC_add_roots_inner(DATASTART2, DATAEND2, TRUE);
# endif
}
return TRUE;
}
# else /* !HAVE_DL_ITERATE_PHDR */
# if defined(NETBSD) || defined(OPENBSD)
# include <sys/exec_elf.h>
/* For compatibility with 1.4.x. */
# ifndef DT_DEBUG
# define DT_DEBUG 21
# endif
# ifndef PT_LOAD
# define PT_LOAD 1
# endif
# ifndef PF_W
# define PF_W 2
# endif
# elif !defined(HOST_ANDROID)
# include <elf.h>
# endif
# ifndef HOST_ANDROID
# include <link.h>
# endif
# endif /* !HAVE_DL_ITERATE_PHDR */
# endif /* !USE_PROC_FOR_LIBRARIES */
# endif /* DGUX || HURD || NACL || (ANY_BSD || LINUX) && __ELF__ */
# if (defined(DGUX) || defined(HURD) || defined(NACL) || defined(SCO_ELF) \
|| defined(SERENITY) || defined(SOLARISDL) \
|| ((defined(ANY_BSD) || defined(LINUX)) && defined(__ELF__))) \
&& !defined(USE_PROC_FOR_LIBRARIES)
/* Dynamic loading code for Linux, Solaris or similar OS running ELF. */
/*
* This does not necessarily work in all cases, e.g. with preloaded
* dynamic libraries.
*/
EXTERN_C_BEGIN
# if defined(__GNUC__) && !defined(SOLARISDL)
# pragma weak _DYNAMIC
# endif
extern ElfW(Dyn) _DYNAMIC[];
EXTERN_C_END
STATIC struct link_map *
GC_FirstDLOpenedLinkMap(void)
{
static struct link_map *cachedResult = NULL;
# ifdef SUNOS53_SHARED_LIB
/*
* BTL: Avoid the Solaris 5.3 `ld.so` bug that `_DYNAMIC` is not being
* setup properly in dynamically linked library file. This means we
* have to use its value in the set of original object files loaded at
* the program startup.
*/
static ElfW(Dyn) *dynStructureAddr = NULL;
if (NULL == dynStructureAddr) {
void *startupSyms = dlopen(NULL, RTLD_LAZY);
dynStructureAddr = (ElfW(Dyn) *)dlsym(startupSyms, "_DYNAMIC");
/* Note: `dlclose()` is not called intentionally. */
if (NULL == dynStructureAddr) {
/* `_DYNAMIC` symbol is not resolved. */
return NULL;
}
}
# else
if (0 == COVERT_DATAFLOW(ADDR(_DYNAMIC))) {
/* `_DYNAMIC` symbol is not resolved. */
return NULL;
}
# endif
if (NULL == cachedResult) {
# if defined(NETBSD) && defined(RTLD_DI_LINKMAP)
# if defined(CPPCHECK)
# define GC_RTLD_DI_LINKMAP 2
# else
# define GC_RTLD_DI_LINKMAP RTLD_DI_LINKMAP
# endif
struct link_map *lm = NULL;
if (!dlinfo(RTLD_SELF, GC_RTLD_DI_LINKMAP, &lm) && lm != NULL) {
/*
* Now `lm` points `link_map` object of the collector.
* Since it might not be the first dynamically linked object,
* try to find it (object next to the main object).
*/
while (lm->l_prev != NULL) {
lm = lm->l_prev;
}
cachedResult = lm->l_next;
}
# else
ElfW(Dyn) * dp;
int tag;
for (dp = _DYNAMIC; (tag = dp->d_tag) != 0; dp++) {
if (tag == DT_DEBUG) {
const struct r_debug *rd
= CAST_THRU_UINTPTR(struct r_debug *, dp->d_un.d_ptr);
/* `d_ptr` could be `NULL` if libs are linked statically. */
if (rd != NULL) {
const struct link_map *lm = rd->r_map;
# if defined(CPPCHECK) && defined(LINK_MAP_R_DEBUG_DEFINED)
GC_noop1((word)rd->r_version);
# endif
if (lm != NULL) {
cachedResult = lm->l_next;
/* Might be `NULL`. */
}
}
break;
}
}
# endif
}
return cachedResult;
}
GC_INNER void
GC_register_dynamic_libraries(void)
{
struct link_map *lm;
GC_ASSERT(I_HOLD_LOCK());
# ifdef HAVE_DL_ITERATE_PHDR
if (GC_register_dynamic_libraries_dl_iterate_phdr()) {
return;
}
# endif
for (lm = GC_FirstDLOpenedLinkMap(); lm != NULL; lm = lm->l_next) {
const ElfW(Ehdr) * e;
const ElfW(Phdr) * p;
ptr_t load_ptr = (ptr_t)lm->l_addr;
size_t i;
# ifdef HOST_ANDROID
if (NULL == load_ptr)
continue;
# endif
e = (ElfW(Ehdr) *)load_ptr;
p = (ElfW(Phdr) *)(load_ptr + e->e_phoff);
# ifdef LINT2
/* Workaround tainted e->e_phnum usage as a loop boundary. */
GC_noop1_ptr((/* no const */ void *)&e->e_phnum);
# endif
for (i = 0; i < (size_t)e->e_phnum; i++, p++) {
ptr_t start;
switch (p->p_type) {
case PT_LOAD:
if ((p->p_flags & PF_W) == 0)
break;
start = load_ptr + p->p_vaddr;
GC_add_roots_inner(start, start + p->p_memsz, TRUE);
break;
default:
break;
}
}
# if defined(CPPCHECK) && defined(LINK_MAP_R_DEBUG_DEFINED)
GC_noop1_ptr(lm->l_name);
GC_noop1((word)lm->l_ld);
GC_noop1_ptr(lm->l_prev);
# endif
}
}
# endif /* (SOLARISDL || LINUX && __ELF__) && !USE_PROC_FOR_LIBRARIES */
# if defined(USE_PROC_FOR_LIBRARIES) && !defined(LINUX) || defined(IRIX5)
# include <elf.h>
# include <errno.h>
# include <fcntl.h>
# include <sys/procfs.h>
# include <sys/stat.h>
/* This is included only for the following test. */
# include <signal.h>
# ifndef _sigargs
# define IRIX6
# endif
GC_INNER void
GC_register_dynamic_libraries(void)
{
/*
* We use `/proc` to track down all parts of the address space that
* are mapped by the process, and throw out regions we know we should
* not worry about. This may also work under other SVR4 variants.
*/
static int fd = -1;
static prmap_t *addr_map = 0;
/* Number of records currently in `addr_map`. */
static int current_sz = 0;
char buf[6 + 20 + 1];
/* Required size of `addr_map`. */
int needed_sz = 0;
int i;
long flags;
ptr_t start;
ptr_t limit;
word heap_start = HEAP_START;
word heap_end = heap_start;
# ifdef SOLARISDL
# define MA_PHYS 0
# endif
GC_ASSERT(I_HOLD_LOCK());
if (fd < 0) {
GC_snprintf_s_ld_s(buf, sizeof(buf), "/proc/", (long)getpid(), "");
fd = open(buf, O_RDONLY);
if (fd < 0) {
ABORT("/proc open failed");
}
}
if (ioctl(fd, PIOCNMAP, &needed_sz) < 0) {
ABORT_ARG2("/proc PIOCNMAP ioctl failed", ": fd= %d, errno= %d", fd,
errno);
}
if (needed_sz >= current_sz) {
GC_scratch_recycle_no_gww(addr_map, (size_t)current_sz * sizeof(prmap_t));
/* Expansion, plus room for record 0. */
current_sz = needed_sz * 2 + 1;
addr_map
= (prmap_t *)GC_scratch_alloc((size_t)current_sz * sizeof(prmap_t));
if (NULL == addr_map)
ABORT("Insufficient memory for address map");
}
if (ioctl(fd, PIOCMAP, addr_map) < 0) {
ABORT_ARG3("/proc PIOCMAP ioctl failed",
": errcode= %d, needed_sz= %d, addr_map= %p", errno, needed_sz,
(void *)addr_map);
}
if (GC_n_heap_sects > 0) {
heap_end = ADDR(GC_heap_sects[GC_n_heap_sects - 1].hs_start)
+ GC_heap_sects[GC_n_heap_sects - 1].hs_bytes;
if (heap_end < GC_scratch_last_end_addr)
heap_end = GC_scratch_last_end_addr;
}
for (i = 0; i < needed_sz; i++) {
flags = addr_map[i].pr_mflags;
if ((flags & (MA_BREAK | MA_STACK | MA_PHYS | MA_FETCHOP | MA_NOTCACHED))
!= 0)
goto irrelevant;
if ((flags & (MA_READ | MA_WRITE)) != (MA_READ | MA_WRITE))
goto irrelevant;
/*
* The latter test is empirically useless in very old Irix versions.
* Other than the main data and stack segments, everything appears
* to be mapped readable, writable, executable, and shared(!!).
* Probably this makes no sense.
*/
start = (ptr_t)addr_map[i].pr_vaddr;
if (GC_roots_present(start)
|| (ADDR(start) >= heap_start && ADDR(start) < heap_end))
goto irrelevant;
limit = start + addr_map[i].pr_size;
/*
* The following seemed to be necessary for very old versions of Irix,
* but it has been reported to discard relevant segments under Irix 6.5.
*/
# ifndef IRIX6
if (addr_map[i].pr_off == 0 && strncmp(start, ELFMAG, 4) == 0) {
/*
* Discard text segments, i.e. zero-offset mappings against
* executable files that appear to have ELF headers.
*/
caddr_t arg;
int obj;
# define MAP_IRR_SZ 10
/* Known irrelevant map entries. */
static ptr_t map_irr[MAP_IRR_SZ];
static int n_irr = 0;
struct stat buf;
int j;
for (j = 0; j < n_irr; j++) {
if (map_irr[j] == start)
goto irrelevant;
}
arg = (caddr_t)start;
obj = ioctl(fd, PIOCOPENM, &arg);
if (obj >= 0) {
fstat(obj, &buf);
close(obj);
if ((buf.st_mode & 0111) != 0) {
if (n_irr < MAP_IRR_SZ) {
map_irr[n_irr++] = start;
}
goto irrelevant;
}
}
}
# endif /* !IRIX6 */
GC_add_roots_inner(start, limit, TRUE);
irrelevant:;
}
/*
* Do not keep cached descriptor, for now. Some kernels do not like us
* to keep a `/proc` file descriptor around during `kill -9`. Otherwise,
* it should also require `FD_CLOEXEC` and proper handling at process fork
* (i.e. close the file descriptor because of the `pid` change).
*/
if (close(fd) < 0)
ABORT("Couldn't close /proc file");
fd = -1;
}
# endif /* USE_PROC_FOR_LIBRARIES && !LINUX || IRIX5 */
# ifdef AIX
# include <alloca.h>
# include <sys/errno.h>
# include <sys/ldr.h>
GC_INNER void
GC_register_dynamic_libraries(void)
{
int ldibuflen = 8192;
GC_ASSERT(I_HOLD_LOCK());
for (;;) {
int len;
struct ld_info *ldi;
# if defined(CPPCHECK)
char ldibuf[ldibuflen];
# else
char *ldibuf = alloca(ldibuflen);
# endif
len = loadquery(L_GETINFO, ldibuf, ldibuflen);
if (len < 0) {
if (errno != ENOMEM) {
ABORT("loadquery failed");
}
ldibuflen *= 2;
continue;
}
ldi = (struct ld_info *)ldibuf;
for (;;) {
len = ldi->ldinfo_next;
GC_add_roots_inner((ptr_t)ldi->ldinfo_dataorg,
(ptr_t)ldi->ldinfo_dataorg + ldi->ldinfo_datasize,
TRUE);
if (0 == len)
break;
ldi = (struct ld_info *)((ptr_t)ldi + len);
}
break;
}
}
# endif /* AIX */
# ifdef DARWIN
/* `__private_extern__` hack required for gcc-3.3 and earlier. */
# ifndef __private_extern__
# define __private_extern__ extern
# include <mach-o/dyld.h>
# undef __private_extern__
# else
# include <mach-o/dyld.h>
# endif
# if CPP_WORDSZ == 64
# define GC_MACH_HEADER mach_header_64
# else
# define GC_MACH_HEADER mach_header
# endif
# ifdef MISSING_MACH_O_GETSECT_H
EXTERN_C_BEGIN
extern uint8_t *getsectiondata(const struct GC_MACH_HEADER *, const char *seg,
const char *sect, unsigned long *psz);
EXTERN_C_END
# else
# include <mach-o/getsect.h>
# endif
/* Writable sections generally available on Darwin. */
STATIC const struct dyld_sections_s {
const char *seg;
const char *sect;
} GC_dyld_sections[]
= { { SEG_DATA, SECT_DATA },
/* Used by FSF gcc, but not by OS X system tools, so far. */
{ SEG_DATA, "__static_data" },
{ SEG_DATA, SECT_BSS },
{ SEG_DATA, SECT_COMMON },
/*
* FSF gcc - zero-sized object sections for targets supporting
* section anchors.
*/
{ SEG_DATA, "__zobj_data" },
{ SEG_DATA, "__zobj_bss" } };
/*
* Additional writable sections: gcc on Darwin constructs aligned sections
* "on demand", where the alignment size is embedded in the section name.
* Furthermore, there are distinctions between sections containing private
* vs. public symbols. It also constructs sections specifically for
* zero-sized objects, when the target supports section anchors.
*/
STATIC const char *const GC_dyld_bss_prefixes[]
= { "__bss", "__pu_bss", "__zo_bss", "__zo_pu_bss" };
/*
* Currently, `mach-o` will allow up to the max of `2**15` alignment
* in an object file.
*/
# ifndef L2_MAX_OFILE_ALIGNMENT
# define L2_MAX_OFILE_ALIGNMENT 15
# endif
STATIC const char *
GC_dyld_name_for_hdr(const struct GC_MACH_HEADER *phdr)
{
unsigned long i, count = _dyld_image_count();
for (i = 0; i < count; i++) {
if ((const struct GC_MACH_HEADER *)_dyld_get_image_header(i) == phdr)
return _dyld_get_image_name(i);
}
/* TODO: Probably `ABORT()` in this case? */
return NULL; /*< not found */
}
/*
* `getsectbynamefromheader` is deprecated (first time in macOS 13.0),
* `getsectiondata` (introduced in macOS 10.7) is used instead, if exists.
* Define `USE_GETSECTBYNAME` macro to use the deprecated symbol, if needed.
*/
# if !defined(USE_GETSECTBYNAME) \
&& (MAC_OS_X_VERSION_MIN_REQUIRED \
< 1070 /* `MAC_OS_X_VERSION_10_7` */)
# define USE_GETSECTBYNAME
# endif
static void
dyld_section_add_del(const struct GC_MACH_HEADER *phdr, intptr_t slide,
const char *dlpi_name, GC_has_static_roots_func callback,
const char *seg, const char *secnam, GC_bool is_add)
{
ptr_t start, finish;
unsigned long sec_size;
# ifdef USE_GETSECTBYNAME
# if CPP_WORDSZ == 64
const struct section_64 *sec = getsectbynamefromheader_64(phdr, seg, secnam);
# else
const struct section *sec = getsectbynamefromheader(phdr, seg, secnam);
# endif
if (NULL == sec)
return;
sec_size = sec->size;
start = (ptr_t)slide + sec->addr;
# else
UNUSED_ARG(slide);
sec_size = 0;
start = (ptr_t)getsectiondata(phdr, seg, secnam, &sec_size);
if (NULL == start)
return;
# endif
if (sec_size < sizeof(ptr_t))
return;
finish = start + sec_size;
if (is_add) {
LOCK();
/* The user callback is invoked holding the allocator lock. */
if (UNLIKELY(callback != 0)
&& !callback(dlpi_name, start, (size_t)sec_size)) {
UNLOCK();
return; /*< skip section */
}
GC_add_roots_inner(start, finish, FALSE);
UNLOCK();
} else {
GC_remove_roots(start, finish);
}
# ifdef DARWIN_DEBUG
GC_log_printf("%s section __DATA,%s at %p-%p (%lu bytes) from image %s\n",
is_add ? "Added" : "Removed", secnam, (void *)start,
(void *)finish, sec_size, dlpi_name);
# endif
}
static void
dyld_image_add_del(const struct GC_MACH_HEADER *phdr, intptr_t slide,
GC_has_static_roots_func callback, GC_bool is_add)
{
unsigned i, j;
const char *dlpi_name;
GC_ASSERT(I_DONT_HOLD_LOCK());
# ifndef DARWIN_DEBUG
if (0 == callback) {
dlpi_name = NULL;
} else
# endif
/* else */ {
dlpi_name = GC_dyld_name_for_hdr(phdr);
}
for (i = 0; i < sizeof(GC_dyld_sections) / sizeof(GC_dyld_sections[0]);
i++) {
dyld_section_add_del(phdr, slide, dlpi_name, callback,
GC_dyld_sections[i].seg, GC_dyld_sections[i].sect,
is_add);
}
/* Sections constructed on demand. */
for (j = 0; j < sizeof(GC_dyld_bss_prefixes) / sizeof(char *); j++) {
/* Our manufactured aligned BSS sections. */
for (i = 0; i <= L2_MAX_OFILE_ALIGNMENT; i++) {
char secnam[11 + 20 + 1];
GC_snprintf_s_ld_s(secnam, sizeof(secnam), GC_dyld_bss_prefixes[j],
(long)i, "");
dyld_section_add_del(phdr, slide, dlpi_name, 0 /* `callback` */,
SEG_DATA, secnam, is_add);
}
}
# if defined(DARWIN_DEBUG) && !defined(NO_DEBUGGING)
READER_LOCK();
GC_print_static_roots();
READER_UNLOCK();
# endif
}
STATIC void
GC_dyld_image_add(const struct GC_MACH_HEADER *phdr, intptr_t slide)
{
if (!GC_no_dls)
dyld_image_add_del(phdr, slide, GC_has_static_roots, TRUE);
}
STATIC void
GC_dyld_image_remove(const struct GC_MACH_HEADER *phdr, intptr_t slide)
{
dyld_image_add_del(phdr, slide, 0 /* `callback` */, FALSE);
}
GC_INNER void
GC_register_dynamic_libraries(void)
{
/*
* Currently does nothing. The callbacks are setup by `GC_init_dyld()`.
* The `dyld` library takes it from there.
*/
}
/*
* The `_dyld_*` functions have an internal lock, so none of them can
* be called while the world is stopped without the risk of a deadlock.
* Because of this we must setup callbacks before we ever stop the world.
* This should be called before any thread is created and without the
* allocator lock held.
*/
/*
* `_dyld_bind_fully_image_containing_address` is deprecated, so use
* `dlopen(NULL, RTLD_NOW)` instead; define `USE_DYLD_TO_BIND` macro
* to override this, if needed.
*/
GC_INNER void
GC_init_dyld(void)
{
static GC_bool initialized = FALSE;
GC_ASSERT(I_DONT_HOLD_LOCK());
if (initialized)
return;
# ifdef DARWIN_DEBUG
GC_log_printf("Registering dyld callbacks...\n");
# endif
/*
* From Apple's documentation:
* When you call `_dyld_register_func_for_add_image`, the dynamic linker
* runtime calls the specified callback (func) once for each of the images
* that is currently loaded into the program. When a new image is added to
* the program, your callback is invoked again with the `mach_header` for
* the new image, and the virtual memory slide amount of the new image.
*
* This *will* properly register already linked libraries and libraries
* linked in the future.
*/
/*
* `struct mach_header_64` has the same fields as `struct mach_header`
* except for the reserved one at the end, so these casts are OK.
*/
_dyld_register_func_for_add_image(
(void (*)(const struct mach_header *, intptr_t))GC_dyld_image_add);
_dyld_register_func_for_remove_image(
(void (*)(const struct mach_header *, intptr_t))GC_dyld_image_remove);
/* Set this early to avoid reentrancy issues. */
initialized = TRUE;
# ifndef NO_DYLD_BIND_FULLY_IMAGE
if (GC_no_dls) {
/* Skip main data segment registration. */
return;
}
/*
* When the environment variable is set, the dynamic linker binds
* all undefined symbols the application needs at launch time.
* This includes function symbols that are normally bound lazily at
* the time of their first invocation.
*/
if (GETENV("DYLD_BIND_AT_LAUNCH") != NULL) {
return;
}
/* Else we should bind manually. */
# ifdef DARWIN_DEBUG
GC_log_printf("Forcing full bind of GC code...\n");
# endif
# ifndef USE_DYLD_TO_BIND
{
void *dl_handle = dlopen(NULL, RTLD_NOW);
if (!dl_handle) {
ABORT("dlopen failed (to bind fully image)");
}
/* Note that the handle is never closed. */
# if defined(CPPCHECK) || defined(LINT2)
GC_noop1_ptr(dl_handle);
# endif
}
# else
/* Note: `_dyld_bind_fully_image_containing_address` is deprecated. */
if (!_dyld_bind_fully_image_containing_address((unsigned long *)GC_malloc))
ABORT("_dyld_bind_fully_image_containing_address failed");
# endif
# endif
}
GC_INNER GC_bool
GC_register_main_static_data(void)
{
/* Already done through `dyld` callbacks. */
return FALSE;
}
# define HAVE_REGISTER_MAIN_STATIC_DATA
# endif /* DARWIN */
# if defined(HAIKU)
# include <kernel/image.h>
GC_INNER void
GC_register_dynamic_libraries(void)
{
image_info info;
int32 cookie = 0;
GC_ASSERT(I_HOLD_LOCK());
while (get_next_image_info(B_CURRENT_TEAM, &cookie, &info) == B_OK) {
ptr_t data = (ptr_t)info.data;
GC_add_roots_inner(data, data + info.data_size, TRUE);
}
}
GC_INNER GC_bool
GC_register_main_static_data(void)
{
/*
* On Haiku, the main application binary is also a "shared image" and will
* be reported in an `image_info` same as for dynamically-loaded libraries.
*/
return FALSE;
}
# define HAVE_REGISTER_MAIN_STATIC_DATA
# endif /* HAIKU */
# ifdef HPUX
# include <dl.h>
# include <errno.h>
EXTERN_C_BEGIN
extern char *sys_errlist[];
extern int sys_nerr;
EXTERN_C_END
GC_INNER void
GC_register_dynamic_libraries(void)
{
/* Ordinal position in shared library search list. */
int index = 1;
GC_ASSERT(I_HOLD_LOCK());
/* For each dynamic library loaded. */
for (;;) {
/* Shared library info, see platform `dl.h` file. */
struct shl_descriptor *shl_desc;
/* Get info about next shared library. */
int status = shl_get(index, &shl_desc);
/* Check if this is the end of the list or if some error occurred. */
if (status != 0) {
# ifdef THREADS
/*
* I have seen `errno` values of 0. The man page is not clear as
* to whether `errno` should get set when -1 is returned.
*/
break;
# else
if (errno == EINVAL) {
/* Moved past end of shared library list. Finish. */
break;
} else {
ABORT_ARG3("shl_get failed", ": status= %d, errcode= %d (%s)", status,
errno, errno < sys_nerr ? sys_errlist[errno] : "");
}
# endif
}
# ifdef DL_VERBOSE
GC_log_printf("---Shared library---\n");
GC_log_printf("filename= \"%s\"\n", shl_desc->filename);
GC_log_printf("index= %d\n", index);
GC_log_printf("handle= %08x\n", (unsigned long)shl_desc->handle);
GC_log_printf("text seg.start= %08x\n", shl_desc->tstart);
GC_log_printf("text seg.end= %08x\n", shl_desc->tend);
GC_log_printf("data seg.start= %08x\n", shl_desc->dstart);
GC_log_printf("data seg.end= %08x\n", shl_desc->dend);
GC_log_printf("ref.count= %lu\n", shl_desc->ref_count);
# endif
/*
* Register shared library's data segment as a garbage collection
* root.
*/
GC_add_roots_inner((char *)shl_desc->dstart, (char *)shl_desc->dend, TRUE);
index++;
}
}
# endif /* HPUX */
# ifdef OSF1
# include <loader.h>
EXTERN_C_BEGIN
extern char *sys_errlist[];
extern int sys_nerr, errno;
EXTERN_C_END
GC_INNER void
GC_register_dynamic_libraries(void)
{
ldr_module_t moduleid = LDR_NULL_MODULE;
ldr_process_t mypid;
GC_ASSERT(I_HOLD_LOCK());
/* Obtain id of this process. */
mypid = ldr_my_process();
/* For each module. */
for (;;) {
ldr_module_info_t moduleinfo;
size_t modulereturnsize;
ldr_region_t region;
ldr_region_info_t regioninfo;
size_t regionreturnsize;
/* Get the next (first) module. */
int status = ldr_next_module(mypid, &moduleid);
if (moduleid == LDR_NULL_MODULE) {
/* No more modules. */
break;
}
/*
* Check status *after* checking `moduleid` because of a bug
* in the non-shared `ldr_next_module` stub.
*/
if (status != 0) {
ABORT_ARG3("ldr_next_module failed", ": status= %d, errcode= %d (%s)",
status, errno, errno < sys_nerr ? sys_errlist[errno] : "");
}
/* Get the module information. */
status = ldr_inq_module(mypid, moduleid, &moduleinfo, sizeof(moduleinfo),
&modulereturnsize);
if (status != 0)
ABORT("ldr_inq_module failed");
/* Is module for the main program (i.e. nonshared portion)? */
if ((moduleinfo.lmi_flags & LDR_MAIN) != 0) {
/* Skip the main module. */
continue;
}
# ifdef DL_VERBOSE
GC_log_printf("---Module---\n");
GC_log_printf("Module ID: %ld\n", moduleinfo.lmi_modid);
GC_log_printf("Count of regions: %d\n", moduleinfo.lmi_nregion);
GC_log_printf("Flags for module: %016lx\n", moduleinfo.lmi_flags);
GC_log_printf("Module pathname: \"%s\"\n", moduleinfo.lmi_name);
# endif
/* For each region in this module. */
for (region = 0; region < moduleinfo.lmi_nregion; region++) {
/* Get the region information. */
status = ldr_inq_region(mypid, moduleid, region, &regioninfo,
sizeof(regioninfo), &regionreturnsize);
if (status != 0)
ABORT("ldr_inq_region failed");
/* Only process writable (data) regions. */
if ((regioninfo.lri_prot & LDR_W) == 0)
continue;
# ifdef DL_VERBOSE
GC_log_printf("--- Region ---\n");
GC_log_printf("Region number: %ld\n", regioninfo.lri_region_no);
GC_log_printf("Protection flags: %016x\n", regioninfo.lri_prot);
GC_log_printf("Virtual address: %p\n", regioninfo.lri_vaddr);
GC_log_printf("Mapped address: %p\n", regioninfo.lri_mapaddr);
GC_log_printf("Region size: %ld\n", regioninfo.lri_size);
GC_log_printf("Region name: \"%s\"\n", regioninfo.lri_name);
# endif
/* Register region as a garbage collection root. */
GC_add_roots_inner((char *)regioninfo.lri_mapaddr,
(char *)regioninfo.lri_mapaddr + regioninfo.lri_size,
TRUE);
}
}
}
# endif /* OSF1 */
#endif /* DYNAMIC_LOADING */
#ifdef GC_MUST_RESTORE_REDEFINED_DLOPEN
# define dlopen GC_dlopen
#endif
#if !defined(HAVE_REGISTER_MAIN_STATIC_DATA) && defined(DYNAMIC_LOADING)
GC_INNER GC_bool
GC_register_main_static_data(void)
{
return TRUE;
}
#endif /* HAVE_REGISTER_MAIN_STATIC_DATA */
GC_API void GC_CALL
GC_register_has_static_roots_callback(GC_has_static_roots_func callback)
{
GC_has_static_roots = callback;
}
/*
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1997 by Silicon Graphics. All rights reserved.
* Copyright (c) 2000 by Hewlett-Packard Company. All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* This is located in this file (rather than in `dyn_load.c` file) to
* avoid having to link against `libdl` library if the client does not
* call `dlopen()`. Of course, this is meaningless if the collector is
* in a dynamic library.
*/
#if defined(GC_PTHREADS) && !defined(GC_NO_DLOPEN)
# undef GC_MUST_RESTORE_REDEFINED_DLOPEN
# if defined(dlopen) && !defined(GC_USE_LD_WRAP)
/*
* To support various threads packages, `gc.h` file interposes on `dlopen`
* by defining the latter to be `GC_dlopen`, which is implemented below.
* However, `GC_dlopen()` itself uses the real system `dlopen()`, thus
* we first remove `dlopen` definition by `gc.h` file and restore it later,
* after `GC_dlopen()` definition.
*/
# undef dlopen
# define GC_MUST_RESTORE_REDEFINED_DLOPEN
# endif
/*
* Make sure we are not in the middle of a collection, and make sure we
* do not start any. This is invoked prior to a `dlopen` call to avoid
* synchronization issues. We cannot just acquire the allocator lock,
* since startup code in `dlopen` may try to allocate. This solution
* risks heap growth (or, even, heap overflow) in the presence of many
* `dlopen` calls in either a multi-threaded environment, or if the
* library initialization code allocates substantial amounts of garbage
* collected memory.
*/
# ifndef USE_PROC_FOR_LIBRARIES
static void
disable_gc_for_dlopen(void)
{
LOCK();
while (GC_incremental && GC_collection_in_progress()) {
GC_collect_a_little_inner(1000);
}
++GC_dont_gc;
UNLOCK();
}
# endif
/*
* Redefine `dlopen` to guarantee mutual exclusion with
* `GC_register_dynamic_libraries()`. Should probably happen for
* other operating systems, too.
*/
/* This is similar to `WRAP_FUNC`/`REAL_FUNC` in `pthread_support.c` file. */
# ifdef GC_USE_LD_WRAP
# define WRAP_DLFUNC(f) __wrap_##f
# define REAL_DLFUNC(f) __real_##f
void *REAL_DLFUNC(dlopen)(const char *, int);
# else
# define WRAP_DLFUNC(f) GC_##f
# define REAL_DLFUNC(f) f
# endif
# define GC_wrap_dlopen WRAP_DLFUNC(dlopen)
GC_API void *
GC_wrap_dlopen(const char *path, int mode)
{
void *result;
# ifndef USE_PROC_FOR_LIBRARIES
/*
* Disable collections. This solution risks heap growth (or, even,
* heap overflow) but there seems no better solutions.
*/
disable_gc_for_dlopen();
# endif
result = REAL_DLFUNC(dlopen)(path, mode);
# ifndef USE_PROC_FOR_LIBRARIES
/* This undoes `disable_gc_for_dlopen()`. */
GC_enable();
# endif
return result;
}
# undef GC_wrap_dlopen
/*
* Define `GC_` function as an alias for the plain one, which will be
* intercepted. This allows files that include `gc.h` file, and hence
* generate references to the `GC_` symbol, to see the right symbol.
*/
# ifdef GC_USE_LD_WRAP
GC_API void *
GC_dlopen(const char *path, int mode)
{
return dlopen(path, mode);
}
# endif /* GC_USE_LD_WRAP */
# ifdef GC_MUST_RESTORE_REDEFINED_DLOPEN
# define dlopen GC_dlopen
# endif
#endif /* GC_PTHREADS && !GC_NO_DLOPEN */
#if !defined(PLATFORM_MACH_DEP)
/*
* Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
* Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#if !defined(PLATFORM_MACH_DEP) && !defined(SN_TARGET_PSP2)
# if defined(IA64) && !defined(THREADS)
GC_INNER ptr_t GC_save_regs_ret_val = NULL;
# endif
# if defined(UNIX_LIKE) && !defined(STACK_NOT_SCANNED)
# include <signal.h>
# ifndef NO_GETCONTEXT
# if defined(DARWIN) \
&& (MAC_OS_X_VERSION_MAX_ALLOWED \
>= 1060 /* `MAC_OS_X_VERSION_10_6` */)
# include <sys/ucontext.h>
# else
# include <ucontext.h>
# endif /* !DARWIN */
# ifdef GETCONTEXT_FPU_EXCMASK_BUG
# include <fenv.h>
# endif
# endif /* !NO_GETCONTEXT */
# endif
GC_ATTR_NOINLINE GC_ATTR_NO_SANITIZE_ADDR GC_INNER void
GC_with_callee_saves_pushed(GC_with_callee_saves_func fn, ptr_t arg)
{
volatile int dummy;
volatile ptr_t context = 0;
# if defined(EMSCRIPTEN) || defined(HAVE_BUILTIN_UNWIND_INIT) \
|| defined(STACK_NOT_SCANNED) || (defined(NO_CRT) && defined(MSWIN32)) \
|| !defined(NO_UNDERSCORE_SETJMP)
# define volatile_arg arg
# else
/*
* Note: `volatile` to avoid "arg might be clobbered by setjmp" warning
* produced by some compilers.
*/
volatile ptr_t volatile_arg = arg;
# endif
# if defined(EMSCRIPTEN) || defined(STACK_NOT_SCANNED)
/* No-op, "registers" are pushed in `GC_push_other_roots()`. */
# else
# if defined(UNIX_LIKE) && !defined(NO_GETCONTEXT)
/*
* Older versions of Darwin seem to lack `getcontext()`.
* Linux on ARM and MIPS often does not provide a real `getcontext()`.
*/
/* The variable is set to -1 (means broken) or 1 (means it works). */
static signed char getcontext_works = 0;
ucontext_t ctxt;
# ifdef GETCONTEXT_FPU_EXCMASK_BUG
/*
* Workaround a bug (clearing the FPU exception mask) in `getcontext`
* on Linux/x86_64.
*/
# ifdef X86_64
/*
* We manipulate FPU control word here just not to force the client
* application to use `-lm` linker option.
*/
unsigned short old_fcw;
# if defined(CPPCHECK)
GC_noop1_ptr(&old_fcw);
# endif
__asm__ __volatile__("fstcw %0" : "=m"(*&old_fcw));
# else
int except_mask = fegetexcept();
# endif
# endif
if (getcontext_works >= 0) {
if (getcontext(&ctxt) < 0) {
WARN("getcontext failed:"
" using another register retrieval method...\n",
0);
/*
* `getcontext()` is broken, do not try again.
* E.g., to workaround a bug in Docker `ubuntu_32bit`.
*/
} else {
context = (ptr_t)&ctxt;
}
if (UNLIKELY(0 == getcontext_works))
getcontext_works = context != NULL ? 1 : -1;
}
# ifdef GETCONTEXT_FPU_EXCMASK_BUG
# ifdef X86_64
__asm__ __volatile__("fldcw %0" : : "m"(*&old_fcw));
{
unsigned mxcsr;
/* And now correct the exception mask in SSE `mxcsr`. */
__asm__ __volatile__("stmxcsr %0" : "=m"(*&mxcsr));
mxcsr = (mxcsr & ~(FE_ALL_EXCEPT << 7)) | ((old_fcw & FE_ALL_EXCEPT) << 7);
__asm__ __volatile__("ldmxcsr %0" : : "m"(*&mxcsr));
}
# else /* !X86_64 */
if (feenableexcept(except_mask) < 0)
ABORT("feenableexcept failed");
# endif
# endif /* GETCONTEXT_FPU_EXCMASK_BUG */
# if defined(IA64) || defined(SPARC)
/*
* On a register-window machine, we need to save register contents on
* the stack for this to work. This may already be subsumed by the
* `getcontext()` call.
*/
# if defined(IA64) && !defined(THREADS)
GC_save_regs_ret_val =
# endif
GC_save_regs_in_stack();
# endif
if (NULL == context) /*< `getcontext` failed */
# endif /* !NO_GETCONTEXT */
{
# if defined(HAVE_BUILTIN_UNWIND_INIT)
/*
* This was suggested as the way to force callee-save registers and
* register windows onto the stack.
*/
__builtin_unwind_init();
# elif defined(NO_CRT) && defined(MSWIN32)
CONTEXT ctx;
RtlCaptureContext(&ctx);
# else
/* Generic code. */
jmp_buf regs;
/*
* `setjmp()` does not always clear all of the buffer.
* That tends to preserve garbage. Clear it.
*/
BZERO(regs, sizeof(regs));
# ifdef NO_UNDERSCORE_SETJMP
(void)setjmp(regs);
# else
/*
* We do not want to mess with signals. According to the SUSv3
* (Single UNIX Specification v3), `setjmp` may or may not save
* signal mask. `_setjmp` will not, but is less portable.
*/
(void)_setjmp(regs);
# endif
# endif
}
# endif
/*
* TODO: `context` here is sometimes just zero. At the moment, the
* callees do not really need it. Cast `fn` to a `volatile` type
* to prevent call inlining.
*/
(*(GC_with_callee_saves_func volatile *)&fn)(
volatile_arg, CAST_AWAY_VOLATILE_PVOID(context));
/*
* Strongly discourage the compiler from treating the above as a tail-call,
* since that would pop the register contents before we get a chance to
* look at them.
*/
GC_noop1(COVERT_DATAFLOW(ADDR(&dummy)));
# undef volatile_arg
}
#endif /* !PLATFORM_MACH_DEP && !SN_TARGET_PSP2 */
#endif
#if !defined(PLATFORM_STOP_WORLD)
/*
* Copyright (c) 1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996 by Silicon Graphics. All rights reserved.
* Copyright (c) 1998 by Fergus Henderson. All rights reserved.
* Copyright (c) 2000-2009 by Hewlett-Packard Development Company.
* All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#ifdef PTHREAD_STOP_WORLD_IMPL
# ifdef NACL
# include <sys/time.h>
# else
# include <errno.h>
# include <semaphore.h>
# include <signal.h>
# include <time.h>
# endif /* !NACL */
# ifdef E2K
# include <alloca.h>
# endif
GC_INLINE void
GC_usleep(unsigned us)
{
# if defined(LINT2) || defined(THREAD_SANITIZER)
/*
* Workaround "waiting while holding a lock" static analyzer warning.
* Workaround a rare hang in `usleep()` trying to acquire TSan Lock.
*/
while (us-- > 0) {
/* Sleep for a moment, pretending it takes 1us. */
sched_yield();
}
# elif defined(CPPCHECK) /* `|| _POSIX_C_SOURCE >= 199309L` */
struct timespec ts;
ts.tv_sec = 0;
ts.tv_nsec = (unsigned32)us * 1000;
/* This requires `_POSIX_TIMERS` feature. */
(void)nanosleep(&ts, NULL);
# else
usleep(us);
# endif
}
# ifdef NACL
STATIC int GC_nacl_num_gc_threads = 0;
STATIC volatile int GC_nacl_park_threads_now = 0;
STATIC volatile pthread_t GC_nacl_thread_parker = -1;
STATIC __thread int GC_nacl_thread_idx = -1;
/* TODO: Use `GC_get_tlfs()` instead. */
STATIC __thread GC_thread GC_nacl_gc_thread_self = NULL;
volatile int GC_nacl_thread_parked[MAX_NACL_GC_THREADS];
int GC_nacl_thread_used[MAX_NACL_GC_THREADS];
# else /* !NACL */
# if (!defined(AO_HAVE_load_acquire) || !defined(AO_HAVE_store_release)) \
&& !defined(CPPCHECK)
# error AO_load_acquire and/or AO_store_release are missing;
# error please define AO_REQUIRE_CAS manually
# endif
# ifdef DEBUG_THREADS
/* It is safe to call original `pthread_sigmask()` here. */
# undef pthread_sigmask
# ifndef NSIG
# ifdef CPPCHECK
# define NSIG 32
# elif defined(MAXSIG)
# define NSIG (MAXSIG + 1)
# elif defined(_NSIG)
# define NSIG _NSIG
# elif defined(__SIGRTMAX)
# define NSIG (__SIGRTMAX + 1)
# else
# error define NSIG
# endif
# endif
void
GC_print_sig_mask(void)
{
sigset_t blocked;
int i;
if (pthread_sigmask(SIG_BLOCK, NULL, &blocked) != 0)
ABORT("pthread_sigmask failed");
for (i = 1; i < NSIG; i++) {
if (sigismember(&blocked, i))
GC_printf("Signal blocked: %d\n", i);
}
}
# endif /* DEBUG_THREADS */
/*
* Remove the signals that we want to allow in thread stopping handler
* from a set.
*/
STATIC void
GC_remove_allowed_signals(sigset_t *set)
{
if (sigdelset(set, SIGINT) != 0 || sigdelset(set, SIGQUIT) != 0
|| sigdelset(set, SIGABRT) != 0 || sigdelset(set, SIGTERM) != 0) {
ABORT("sigdelset failed");
}
# ifdef MPROTECT_VDB
/*
* The handlers write to the thread structure, which is in the heap,
* and hence can trigger a protection fault.
*/
if (sigdelset(set, SIGSEGV) != 0
# ifdef HAVE_SIGBUS
|| sigdelset(set, SIGBUS) != 0
# endif
) {
ABORT("sigdelset failed");
}
# endif
}
static sigset_t suspend_handler_mask;
# define THREAD_RESTARTED 0x1
/*
* Incremented (to the nearest even value) at the beginning of
* `GC_stop_world()` (or when a thread is requested to be suspended by
* `GC_suspend_thread`) and once more (to an odd value) at the
* beginning of `GC_start_world`. The lowest bit is `THREAD_RESTARTED`
* one which, if set, means it is safe for threads to restart, i.e.
* they will see another suspend signal before they are expected to
* stop (unless they have stopped voluntarily).
*/
STATIC volatile AO_t GC_stop_count = 0;
STATIC GC_bool GC_retry_signals = FALSE;
/*
* We use signals to stop threads during GC.
*
* Suspended threads wait in signal handler for `SIG_THR_RESTART`.
* That is more portable than semaphores or condition variables.
* (We do use `sem_post()` from a signal handler, but that should be
* portable.)
*
* The thread suspension signal `SIG_SUSPEND` is now defined in `gc_priv.h`
* file. Note that we cannot just stop a thread; we need it to save its
* stack pointers and acknowledge.
*/
# ifndef SIG_THR_RESTART
# if defined(SUSPEND_HANDLER_NO_CONTEXT) || defined(GC_REUSE_SIG_SUSPEND)
/* Reuse the suspend signal. */
# define SIG_THR_RESTART SIG_SUSPEND
# elif defined(HPUX) || defined(NETBSD) || defined(OSF1) \
|| defined(GC_USESIGRT_SIGNALS)
# if defined(_SIGRTMIN) && !defined(CPPCHECK)
# define SIG_THR_RESTART (_SIGRTMIN + 5)
# else
# define SIG_THR_RESTART (SIGRTMIN + 5)
# endif
# elif defined(FREEBSD) && defined(__GLIBC__)
# define SIG_THR_RESTART (32 + 5)
# elif defined(FREEBSD) || defined(HURD) || defined(RTEMS)
# define SIG_THR_RESTART SIGUSR2
# else
# define SIG_THR_RESTART SIGXCPU
# endif
# endif /* !SIG_THR_RESTART */
# define SIGNAL_UNSET (-1)
/*
* Since `SIG_SUSPEND` and/or `SIG_THR_RESTART` could represent
* a non-constant expression (e.g., in case of `SIGRTMIN`), actual
* signal numbers are determined by `GC_stop_init()` unless manually
* set (before the collector initialization). Might be set to the
* same signal number.
*/
STATIC int GC_sig_suspend = SIGNAL_UNSET;
STATIC int GC_sig_thr_restart = SIGNAL_UNSET;
GC_API void GC_CALL
GC_set_suspend_signal(int sig)
{
if (GC_is_initialized)
return;
GC_sig_suspend = sig;
}
GC_API void GC_CALL
GC_set_thr_restart_signal(int sig)
{
if (GC_is_initialized)
return;
GC_sig_thr_restart = sig;
}
GC_API int GC_CALL
GC_get_suspend_signal(void)
{
return GC_sig_suspend != SIGNAL_UNSET ? GC_sig_suspend : SIG_SUSPEND;
}
GC_API int GC_CALL
GC_get_thr_restart_signal(void)
{
return GC_sig_thr_restart != SIGNAL_UNSET ? GC_sig_thr_restart
: SIG_THR_RESTART;
}
# ifdef BASE_ATOMIC_OPS_EMULATED
/*
* The AO primitives emulated with locks cannot be used inside signal
* handlers as this could cause a deadlock or a double lock.
* The following "async" macro definitions are correct only for
* an uniprocessor case and are provided for a test purpose.
*/
# define ao_load_acquire_async(p) (*(p))
# define ao_load_async(p) ao_load_acquire_async(p)
# define ao_store_release_async(p, v) (void)(*(p) = (v))
# define ao_cptr_store_async(p, v) (void)(*(p) = (v))
# else
# define ao_load_acquire_async(p) AO_load_acquire(p)
# define ao_load_async(p) AO_load(p)
# define ao_store_release_async(p, v) AO_store_release(p, v)
# define ao_cptr_store_async(p, v) GC_cptr_store(p, v)
# endif /* !BASE_ATOMIC_OPS_EMULATED */
/* Note: this is also used to acknowledge restart. */
STATIC sem_t GC_suspend_ack_sem;
STATIC void GC_suspend_handler_inner(ptr_t dummy, void *context);
# ifdef SUSPEND_HANDLER_NO_CONTEXT
STATIC void
GC_suspend_handler(int sig)
# else
STATIC void
GC_suspend_sigaction(int sig, siginfo_t *info, void *context)
# endif
{
int old_errno = errno;
if (sig != GC_sig_suspend) {
# ifdef FREEBSD
/* Workaround "deferred signal handling" bug in FreeBSD 9.2. */
if (0 == sig)
return;
# endif
ABORT("Bad signal in suspend_handler");
}
# ifdef SUSPEND_HANDLER_NO_CONTEXT
/* A quick check if the signal is called to restart the world. */
if ((ao_load_async(&GC_stop_count) & THREAD_RESTARTED) != 0)
return;
GC_with_callee_saves_pushed(GC_suspend_handler_inner, NULL);
# else
UNUSED_ARG(info);
/*
* We believe that in this case the full context is already in the
* signal handler frame.
*/
GC_suspend_handler_inner(NULL, context);
# endif
errno = old_errno;
}
/*
* The lookup here is safe, since this is done on behalf of a thread which
* holds the allocator lock in order to stop the world. Thus concurrent
* modification of the data structure is impossible. Unfortunately, we
* have to instruct TSan that the lookup is safe.
*/
# ifdef THREAD_SANITIZER
/*
* Almost same as `GC_self_thread_inner()` except for the `no_sanitize`
* attribute added and the result is never `NULL`.
*/
GC_ATTR_NO_SANITIZE_THREAD
static GC_thread
GC_lookup_self_thread_async(void)
{
thread_id_t self_id = thread_id_self();
GC_thread p = GC_threads[THREAD_TABLE_INDEX(self_id)];
for (;; p = p->tm.next) {
if (THREAD_EQUAL(p->id, self_id))
break;
}
return p;
}
# else
# define GC_lookup_self_thread_async() GC_self_thread_inner()
# endif
GC_INLINE void
GC_store_stack_ptr(GC_stack_context_t crtn)
{
/*
* There is no data race between the suspend handler (storing
* `stack_ptr`) and `GC_push_all_stacks` (fetching `stack_ptr`)
* because `GC_push_all_stacks` is executed after `GC_stop_world`
* exits and the latter runs `sem_wait()` repeatedly waiting for all
* the suspended threads to call `sem_post()`. Nonetheless,
* `stack_ptr` is stored (here) and fetched (by `GC_push_all_stacks`)
* using the atomic primitives to avoid the related TSan warning.
*/
# ifdef SPARC
ao_cptr_store_async(&crtn->stack_ptr, GC_save_regs_in_stack());
/* TODO: Registers saving already done by `GC_with_callee_saves_pushed`. */
# else
# ifdef IA64
crtn->backing_store_ptr = GC_save_regs_in_stack();
# endif
ao_cptr_store_async(&crtn->stack_ptr, GC_approx_sp());
# endif
}
STATIC void
GC_suspend_handler_inner(ptr_t dummy, void *context)
{
GC_thread me;
GC_stack_context_t crtn;
# ifdef E2K
ptr_t bs_lo;
size_t stack_size;
# endif
IF_CANCEL(int cancel_state;)
# ifdef GC_ENABLE_SUSPEND_THREAD
AO_t suspend_cnt;
# endif
AO_t my_stop_count = ao_load_acquire_async(&GC_stop_count);
/*
* After the barrier, this thread should see the actual content of
* `GC_threads`.
*/
UNUSED_ARG(dummy);
UNUSED_ARG(context);
if ((my_stop_count & THREAD_RESTARTED) != 0) {
/* Restarting the world. */
return;
}
/*
* `pthread_setcancelstate()` is not defined to be async-signal-safe.
* But the `glibc` version appears to be defined, in the absence of
* asynchronous cancellation. And since this signal handler to block
* on `sigsuspend`, which is both async-signal-safe and
* a cancellation point, there seems to be no obvious way out of it.
* In fact, it looks like an async-signal-safe cancellation point
* is inherently a problem, unless there is some way to disable
* cancellation in the handler.
*/
DISABLE_CANCEL(cancel_state);
# ifdef DEBUG_THREADS
GC_log_printf("Suspending %p\n", PTHREAD_TO_VPTR(pthread_self()));
# endif
me = GC_lookup_self_thread_async();
if (NULL == me) return; // __v_, make the sanitizers and -cstrict happy
if ((me->last_stop_count & ~(word)THREAD_RESTARTED) == my_stop_count) {
/* Duplicate signal. OK if we are retrying. */
if (!GC_retry_signals) {
WARN("Duplicate suspend signal in thread %p\n", pthread_self());
}
RESTORE_CANCEL(cancel_state);
return;
}
crtn = me->crtn;
GC_store_stack_ptr(crtn);
# ifdef E2K
GC_ASSERT(NULL == crtn->backing_store_end);
GET_PROCEDURE_STACK_LOCAL(crtn->ps_ofs, &bs_lo, &stack_size);
crtn->backing_store_end = bs_lo;
crtn->backing_store_ptr = bs_lo + stack_size;
# endif
# ifdef GC_ENABLE_SUSPEND_THREAD
suspend_cnt = ao_load_async(&me->ext_suspend_cnt);
# endif
/*
* Tell the thread that wants to stop the world that this thread has
* been stopped. Note that `sem_post()` is the only async-signal-safe
* primitive in LinuxThreads.
*/
sem_post(&GC_suspend_ack_sem);
ao_store_release_async(&me->last_stop_count, my_stop_count);
/*
* Wait until that thread tells us to restart by sending this thread
* a `GC_sig_thr_restart` signal (should be masked at this point thus
* there is no race). We do not continue until we receive that signal,
* but we do not take that as authoritative. (We may be accidentally
* restarted by one of the user signals we do not block.)
* After we receive the signal, we use a primitive and an expensive
* mechanism to wait until it is really safe to proceed.
*/
do {
sigsuspend(&suspend_handler_mask);
/* Iterate while not restarting the world or thread is suspended. */
} while (ao_load_acquire_async(&GC_stop_count) == my_stop_count
# ifdef GC_ENABLE_SUSPEND_THREAD
|| ((suspend_cnt & 1) != 0
&& ao_load_async(&me->ext_suspend_cnt) == suspend_cnt)
# endif
);
# ifdef DEBUG_THREADS
GC_log_printf("Resuming %p\n", PTHREAD_TO_VPTR(pthread_self()));
# endif
# ifdef E2K
GC_ASSERT(crtn->backing_store_end == bs_lo);
crtn->backing_store_ptr = NULL;
crtn->backing_store_end = NULL;
# endif
# ifndef GC_NETBSD_THREADS_WORKAROUND
if (GC_retry_signals || GC_sig_suspend == GC_sig_thr_restart)
# endif
{
/*
* If the `SIG_THR_RESTART` signal loss is possible (though it should
* be less likely than losing the `SIG_SUSPEND` signal as we do not
* do much between the first `sem_post` and `sigsuspend` calls), more
* handshaking is provided to work around it.
*/
sem_post(&GC_suspend_ack_sem);
/* Set the flag that the thread has been restarted. */
if (GC_retry_signals)
ao_store_release_async(&me->last_stop_count,
my_stop_count | THREAD_RESTARTED);
}
RESTORE_CANCEL(cancel_state);
}
static void
suspend_restart_barrier(int n_live_threads)
{
int i;
for (i = 0; i < n_live_threads; i++) {
while (sem_wait(&GC_suspend_ack_sem) == -1) {
/*
* On Linux, `sem_wait()` is documented to always return zero.
* But the documentation appears to be incorrect.
* `EINTR` seems to happen with some versions of gdb.
*/
if (errno != EINTR)
ABORT("sem_wait failed");
}
}
# ifdef GC_ASSERTIONS
sem_getvalue(&GC_suspend_ack_sem, &i);
GC_ASSERT(0 == i);
# endif
}
# define WAIT_UNIT 3000 /* us */
static int
resend_lost_signals(int n_live_threads, int (*suspend_restart_all)(void))
{
# define RESEND_SIGNALS_LIMIT 150
# define RETRY_INTERVAL 100000 /* us */
if (n_live_threads > 0) {
unsigned long wait_usecs = 0; /*< total wait since retry */
int retry = 0;
int prev_sent = 0;
for (;;) {
int ack_count;
sem_getvalue(&GC_suspend_ack_sem, &ack_count);
if (ack_count == n_live_threads)
break;
if (wait_usecs > RETRY_INTERVAL) {
int newly_sent = suspend_restart_all();
if (newly_sent != prev_sent) {
/* Restart the counter. */
retry = 0;
} else if (++retry >= RESEND_SIGNALS_LIMIT) {
/* No progress. */
ABORT_ARG1("Signals delivery fails constantly", " at GC #%lu",
(unsigned long)GC_gc_no);
}
GC_COND_LOG_PRINTF("Resent %d signals after timeout, retry: %d\n",
newly_sent, retry);
sem_getvalue(&GC_suspend_ack_sem, &ack_count);
if (newly_sent < n_live_threads - ack_count) {
WARN("Lost some threads while stopping or starting world?!\n", 0);
n_live_threads = ack_count + newly_sent;
}
prev_sent = newly_sent;
wait_usecs = 0;
}
GC_usleep(WAIT_UNIT);
wait_usecs += WAIT_UNIT;
}
}
return n_live_threads;
}
# ifdef HAVE_CLOCK_GETTIME
# define TS_NSEC_ADD(ts, ns) \
(ts.tv_nsec += (ns), \
(void)(ts.tv_nsec >= 1000000L * 1000 \
? (ts.tv_nsec -= 1000000L * 1000, ts.tv_sec++, 0) \
: 0))
# endif
static void
resend_lost_signals_retry(int n_live_threads, int (*suspend_restart_all)(void))
{
# if defined(HAVE_CLOCK_GETTIME) && !defined(DONT_TIMEDWAIT_ACK_SEM)
# define TIMEOUT_BEFORE_RESEND 10000 /* us */
struct timespec ts;
if (n_live_threads > 0 && clock_gettime(CLOCK_REALTIME, &ts) == 0) {
int i;
TS_NSEC_ADD(ts, TIMEOUT_BEFORE_RESEND * (unsigned32)1000);
/*
* First, try to wait for the semaphore with some timeout.
* On failure, fall back to `WAIT_UNIT` pause and resend of the signal.
*/
for (i = 0; i < n_live_threads; i++) {
if (sem_timedwait(&GC_suspend_ack_sem, &ts) == -1)
break; /*< wait timed out or any other error */
}
/* Update the count of threads to wait the acknowledge from. */
n_live_threads -= i;
}
# endif
n_live_threads = resend_lost_signals(n_live_threads, suspend_restart_all);
suspend_restart_barrier(n_live_threads);
}
STATIC void
GC_restart_handler(int sig)
{
# ifdef DEBUG_THREADS
/* Preserve `errno` value. */
int old_errno = errno;
# endif
if (sig != GC_sig_thr_restart) {
ABORT("Bad signal in restart handler");
}
/*
* Note: even if we do not do anything useful here, it would still
* be necessary to have a signal handler, rather than ignoring the
* signals, otherwise the signals will not be delivered at all, and
* will thus not interrupt the `sigsuspend()` above.
*/
# ifdef DEBUG_THREADS
GC_log_printf("In GC_restart_handler for %p\n",
PTHREAD_TO_VPTR(pthread_self()));
errno = old_errno;
# endif
}
# ifdef USE_TKILL_ON_ANDROID
EXTERN_C_BEGIN
extern int tkill(pid_t tid, int sig); /*< from platform `sys/linux-unistd.h` */
EXTERN_C_END
# define THREAD_SYSTEM_ID(t) (t)->kernel_id
# else
# define THREAD_SYSTEM_ID(t) (t)->id
# endif
# ifndef RETRY_TKILL_EAGAIN_LIMIT
# define RETRY_TKILL_EAGAIN_LIMIT 16
# endif
static int
raise_signal(GC_thread p, int sig)
{
int res;
# ifdef RETRY_TKILL_ON_EAGAIN
int retry;
# endif
# if defined(SIMULATE_LOST_SIGNALS) && !defined(GC_ENABLE_SUSPEND_THREAD)
# ifndef LOST_SIGNALS_RATIO
# define LOST_SIGNALS_RATIO 25
# endif
/* Note: race is OK, it is for test purpose only. */
static int signal_cnt;
if (GC_retry_signals && (++signal_cnt) % LOST_SIGNALS_RATIO == 0) {
/* Simulate the signal is sent but lost. */
return 0;
}
# endif
# ifdef RETRY_TKILL_ON_EAGAIN
for (retry = 0;; retry++)
# endif
{
# ifdef USE_TKILL_ON_ANDROID
int old_errno = errno;
res = tkill(THREAD_SYSTEM_ID(p), sig);
if (res < 0) {
res = errno;
errno = old_errno;
}
# else
res = pthread_kill(THREAD_SYSTEM_ID(p), sig);
# endif
# ifdef RETRY_TKILL_ON_EAGAIN
if (res != EAGAIN || retry >= RETRY_TKILL_EAGAIN_LIMIT)
break;
/* A temporal overflow of the real-time signal queue. */
GC_usleep(WAIT_UNIT);
# endif
}
return res;
}
# ifdef GC_ENABLE_SUSPEND_THREAD
# include <sys/time.h>
/* This is to get the prototypes as `extern "C"`. */
STATIC void
GC_brief_async_signal_safe_sleep(void)
{
struct timeval tv;
tv.tv_sec = 0;
# if defined(GC_TIME_LIMIT) && !defined(CPPCHECK)
tv.tv_usec = 1000 * GC_TIME_LIMIT / 2;
# else
tv.tv_usec = 1000 * 15 / 2;
# endif
(void)select(0, 0, 0, 0, &tv);
}
GC_INNER void
GC_suspend_self_inner(GC_thread me, size_t suspend_cnt)
{
IF_CANCEL(int cancel_state;)
GC_ASSERT((suspend_cnt & 1) != 0);
DISABLE_CANCEL(cancel_state);
# ifdef DEBUG_THREADS
GC_log_printf("Suspend self: %p\n", THREAD_ID_TO_VPTR(me->id));
# endif
while (ao_load_acquire_async(&me->ext_suspend_cnt) == suspend_cnt) {
/* TODO: Use `sigsuspend()` even for self-suspended threads. */
GC_brief_async_signal_safe_sleep();
}
# ifdef DEBUG_THREADS
GC_log_printf("Resume self: %p\n", THREAD_ID_TO_VPTR(me->id));
# endif
RESTORE_CANCEL(cancel_state);
}
GC_API void GC_CALL
GC_suspend_thread(GC_SUSPEND_THREAD_ID thread)
{
GC_thread t;
AO_t next_stop_count;
AO_t suspend_cnt;
IF_CANCEL(int cancel_state;)
LOCK();
t = GC_lookup_by_pthread((pthread_t)thread);
if (NULL == t) {
UNLOCK();
return;
}
suspend_cnt = t->ext_suspend_cnt;
if ((suspend_cnt & 1) != 0) /*< already suspended? */ {
GC_ASSERT(!THREAD_EQUAL((pthread_t)thread, pthread_self()));
UNLOCK();
return;
}
if ((t->flags & (FINISHED | DO_BLOCKING)) != 0) {
t->ext_suspend_cnt = suspend_cnt | 1; /*< suspend */
/* Terminated but not joined yet, or in do-blocking state. */
UNLOCK();
return;
}
if (THREAD_EQUAL((pthread_t)thread, pthread_self())) {
t->ext_suspend_cnt = suspend_cnt | 1;
GC_with_callee_saves_pushed(GC_suspend_self_blocked, (ptr_t)t);
UNLOCK();
return;
}
/* `GC_suspend_thread()` is not a cancellation point. */
DISABLE_CANCEL(cancel_state);
# ifdef PARALLEL_MARK
/*
* Ensure we do not suspend a thread while it is rebuilding a free list,
* otherwise such a deadlock is possible:
* - thread 1 is blocked in `GC_wait_for_reclaim` holding the allocator
* lock;
* - thread 2 is suspended in `GC_reclaim_generic` invoked from
* `GC_generic_malloc_many` (with `GC_fl_builder_count` greater than
* zero);
* - thread 3 is blocked acquiring the allocator lock in
* `GC_resume_thread`.
*/
if (GC_parallel)
GC_wait_for_reclaim();
# endif
if (GC_manual_vdb) {
/* See the relevant comment in `GC_stop_world()`. */
GC_acquire_dirty_lock();
}
/*
* Else do not acquire the dirty lock as the write fault handler
* might be trying to acquire it too, and the suspend handler
* execution is deferred until the write fault handler completes.
*/
next_stop_count = GC_stop_count + THREAD_RESTARTED;
GC_ASSERT((next_stop_count & THREAD_RESTARTED) == 0);
AO_store(&GC_stop_count, next_stop_count);
/* Set the flag making the change visible to the signal handler. */
AO_store_release(&t->ext_suspend_cnt, suspend_cnt | 1);
/* TODO: Support `GC_retry_signals` (not needed for TSan). */
switch (raise_signal(t, GC_sig_suspend)) {
/* `ESRCH` cannot happen as terminated threads are handled above. */
case 0:
break;
default:
ABORT("pthread_kill failed");
}
/*
* Wait for the thread to complete threads table lookup and
* `stack_ptr` assignment.
*/
GC_ASSERT(GC_thr_initialized);
suspend_restart_barrier(1);
if (GC_manual_vdb)
GC_release_dirty_lock();
AO_store(&GC_stop_count, next_stop_count | THREAD_RESTARTED);
RESTORE_CANCEL(cancel_state);
UNLOCK();
}
GC_API void GC_CALL
GC_resume_thread(GC_SUSPEND_THREAD_ID thread)
{
GC_thread t;
LOCK();
t = GC_lookup_by_pthread((pthread_t)thread);
if (t != NULL) {
AO_t suspend_cnt = t->ext_suspend_cnt;
if ((suspend_cnt & 1) != 0) { /*< is suspended? */
GC_ASSERT((GC_stop_count & THREAD_RESTARTED) != 0);
/* Mark the thread as not suspended - it will be resumed shortly. */
AO_store(&t->ext_suspend_cnt, suspend_cnt + 1);
if ((t->flags & (FINISHED | DO_BLOCKING)) == 0) {
int result = raise_signal(t, GC_sig_thr_restart);
/* TODO: Support signal resending on `GC_retry_signals`. */
if (result != 0)
ABORT_ARG1("pthread_kill failed in GC_resume_thread",
": errcode= %d", result);
# ifndef GC_NETBSD_THREADS_WORKAROUND
if (GC_retry_signals || GC_sig_suspend == GC_sig_thr_restart)
# endif
{
IF_CANCEL(int cancel_state;)
DISABLE_CANCEL(cancel_state);
suspend_restart_barrier(1);
RESTORE_CANCEL(cancel_state);
}
}
}
}
UNLOCK();
}
GC_API int GC_CALL
GC_is_thread_suspended(GC_SUSPEND_THREAD_ID thread)
{
GC_thread t;
int is_suspended = 0;
READER_LOCK();
t = GC_lookup_by_pthread((pthread_t)thread);
if (t != NULL && (t->ext_suspend_cnt & 1) != 0)
is_suspended = (int)TRUE;
READER_UNLOCK();
return is_suspended;
}
# endif /* GC_ENABLE_SUSPEND_THREAD */
# undef ao_cptr_store_async
# undef ao_load_acquire_async
# undef ao_load_async
# undef ao_store_release_async
# endif /* !NACL */
GC_INNER void
GC_push_all_stacks(void)
{
GC_bool found_me = FALSE;
size_t nthreads = 0;
int i;
GC_thread p;
ptr_t lo; /*< stack top (`sp`) */
ptr_t hi; /*< bottom */
# if defined(E2K) || defined(IA64)
/* We also need to scan the register backing store. */
ptr_t bs_lo, bs_hi;
# endif
struct GC_traced_stack_sect_s *traced_stack_sect;
pthread_t self = pthread_self();
word total_size = 0;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_thr_initialized);
# ifdef DEBUG_THREADS
GC_log_printf("Pushing stacks from thread %p\n", PTHREAD_TO_VPTR(self));
# endif
for (i = 0; i < THREAD_TABLE_SZ; i++) {
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
# if defined(E2K) || defined(IA64)
GC_bool is_self = FALSE;
# endif
GC_stack_context_t crtn = p->crtn;
GC_ASSERT(THREAD_TABLE_INDEX(p->id) == i);
if (KNOWN_FINISHED(p))
continue;
++nthreads;
traced_stack_sect = crtn->traced_stack_sect;
if (THREAD_EQUAL(p->id, self)) {
GC_ASSERT((p->flags & DO_BLOCKING) == 0);
# ifdef SPARC
lo = GC_save_regs_in_stack();
# else
lo = GC_approx_sp();
# ifdef IA64
bs_hi = GC_save_regs_in_stack();
# elif defined(E2K)
{
size_t stack_size;
GC_ASSERT(NULL == crtn->backing_store_end);
GET_PROCEDURE_STACK_LOCAL(crtn->ps_ofs, &bs_lo, &stack_size);
bs_hi = bs_lo + stack_size;
}
# endif
# endif
found_me = TRUE;
# if defined(E2K) || defined(IA64)
is_self = TRUE;
# endif
} else {
lo = GC_cptr_load(&crtn->stack_ptr);
# ifdef IA64
bs_hi = crtn->backing_store_ptr;
# elif defined(E2K)
bs_lo = crtn->backing_store_end;
bs_hi = crtn->backing_store_ptr;
# endif
if (traced_stack_sect != NULL
&& traced_stack_sect->saved_stack_ptr == lo) {
/*
* If the thread has never been stopped since the recent
* `GC_call_with_gc_active` invocation, then skip the top
* "stack section" as `stack_ptr` already points to.
*/
traced_stack_sect = traced_stack_sect->prev;
}
}
hi = crtn->stack_end;
# ifdef IA64
bs_lo = crtn->backing_store_end;
# endif
# ifdef DEBUG_THREADS
# ifdef STACK_GROWS_UP
GC_log_printf("Stack for thread %p is (%p,%p]\n",
THREAD_ID_TO_VPTR(p->id), (void *)hi, (void *)lo);
# else
GC_log_printf("Stack for thread %p is [%p,%p)\n",
THREAD_ID_TO_VPTR(p->id), (void *)lo, (void *)hi);
# endif
# endif
if (NULL == lo)
ABORT("GC_push_all_stacks: sp not set!");
if (crtn->altstack != NULL && ADDR_GE(lo, crtn->altstack)
&& ADDR_GE(crtn->altstack + crtn->altstack_size, lo)) {
# ifdef STACK_GROWS_UP
hi = crtn->altstack;
# else
hi = crtn->altstack + crtn->altstack_size;
# endif
/* FIXME: Need to scan the normal stack too, but how? */
}
# ifdef STACKPTR_CORRECTOR_AVAILABLE
if (GC_sp_corrector != 0)
GC_sp_corrector((void **)&lo, THREAD_ID_TO_VPTR(p->id));
# endif
GC_push_all_stack_sections(lo, hi, traced_stack_sect);
# ifdef STACK_GROWS_UP
total_size += lo - hi;
# else
total_size += hi - lo; /*< `lo` is not greater than `hi` */
# endif
# ifdef NACL
/* Push `reg_storage` as roots, this will cover the reg context. */
GC_push_all_stack((ptr_t)p->reg_storage,
(ptr_t)(p->reg_storage + NACL_GC_REG_STORAGE_SIZE));
total_size += NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t);
# endif
# ifdef E2K
if ((GC_stop_count & THREAD_RESTARTED) != 0
# ifdef GC_ENABLE_SUSPEND_THREAD
&& (p->ext_suspend_cnt & 1) == 0
# endif
&& !is_self && (p->flags & DO_BLOCKING) == 0) {
/* Procedure stack buffer has already been freed. */
continue;
}
# endif
# if defined(E2K) || defined(IA64)
# ifdef DEBUG_THREADS
GC_log_printf("Reg stack for thread %p is [%p,%p)\n",
THREAD_ID_TO_VPTR(p->id), (void *)bs_lo, (void *)bs_hi);
# endif
GC_ASSERT(bs_lo != NULL && bs_hi != NULL);
/*
* FIXME: This (if `is_self`) may add an unbounded number of entries,
* and hence overflow the mark stack, which is bad.
*/
# ifdef IA64
GC_push_all_register_sections(bs_lo, bs_hi, is_self, traced_stack_sect);
# else
if (is_self) {
GC_push_all_eager(bs_lo, bs_hi);
} else {
GC_push_all_stack(bs_lo, bs_hi);
}
# endif
total_size += bs_hi - bs_lo; /*< `bs_lo` is not greater than `bs_hi` */
# endif
}
}
GC_VERBOSE_LOG_PRINTF("Pushed %d thread stacks\n", (int)nthreads);
if (!found_me && !GC_in_thread_creation)
ABORT("Collecting from unknown thread");
GC_total_stacksize = total_size;
}
# ifdef DEBUG_THREADS
/*
* There seems to be a very rare thread stopping problem.
* To help us debug that, we save the ids of the stopping thread.
*/
pthread_t GC_stopping_thread;
int GC_stopping_pid = 0;
# endif
/*
* Suspend all threads that might still be running. Return the number
* of suspend signals that were sent.
*/
STATIC int
GC_suspend_all(void)
{
int n_live_threads = 0;
int i;
# ifndef NACL
GC_thread p;
pthread_t self = pthread_self();
int result;
GC_ASSERT((GC_stop_count & THREAD_RESTARTED) == 0);
GC_ASSERT(I_HOLD_LOCK());
for (i = 0; i < THREAD_TABLE_SZ; i++) {
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
if (!THREAD_EQUAL(p->id, self)) {
if ((p->flags & (FINISHED | DO_BLOCKING)) != 0)
continue;
# ifdef GC_ENABLE_SUSPEND_THREAD
if ((p->ext_suspend_cnt & 1) != 0)
continue;
# endif
if (AO_load(&p->last_stop_count) == GC_stop_count) {
/* Matters only if `GC_retry_signals`. */
continue;
}
n_live_threads++;
# ifdef DEBUG_THREADS
GC_log_printf("Sending suspend signal to %p\n",
THREAD_ID_TO_VPTR(p->id));
# endif
/*
* The synchronization between `GC_dirty` (based on test-and-set)
* and the signal-based thread suspension is performed in
* `GC_stop_world()` because `GC_release_dirty_lock()` cannot be
* called before acknowledging that the thread is really suspended.
*/
result = raise_signal(p, GC_sig_suspend);
switch (result) {
case ESRCH:
/* Not really there anymore. Possible? */
n_live_threads--;
break;
case 0:
if (GC_on_thread_event) {
/* Note: thread id might be truncated. */
GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED,
THREAD_ID_TO_VPTR(THREAD_SYSTEM_ID(p)));
}
break;
default:
ABORT_ARG1("pthread_kill failed at suspend", ": errcode= %d",
result);
}
}
}
}
# else /* NACL */
# ifndef NACL_PARK_WAIT_USEC
# define NACL_PARK_WAIT_USEC 100 /* us */
# endif
unsigned long num_sleeps = 0;
GC_ASSERT(I_HOLD_LOCK());
# ifdef DEBUG_THREADS
GC_log_printf("pthread_stop_world: number of threads: %d\n",
GC_nacl_num_gc_threads - 1);
# endif
GC_nacl_thread_parker = pthread_self();
GC_nacl_park_threads_now = 1;
if (GC_manual_vdb)
GC_acquire_dirty_lock();
for (;;) {
int num_threads_parked = 0;
int num_used = 0;
/* Check the "parked" flag for each thread the collector knows about. */
for (i = 0; i < MAX_NACL_GC_THREADS && num_used < GC_nacl_num_gc_threads;
i++) {
if (GC_nacl_thread_used[i] == 1) {
num_used++;
if (GC_nacl_thread_parked[i] == 1) {
num_threads_parked++;
if (GC_on_thread_event)
GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED, NUMERIC_TO_VPTR(i));
}
}
}
/* Note: -1 for the current thread. */
if (num_threads_parked >= GC_nacl_num_gc_threads - 1)
break;
# ifdef DEBUG_THREADS
GC_log_printf("Sleep waiting for %d threads to park...\n",
GC_nacl_num_gc_threads - num_threads_parked - 1);
# endif
GC_usleep(NACL_PARK_WAIT_USEC);
if (++num_sleeps > (1000 * 1000) / NACL_PARK_WAIT_USEC) {
WARN("GC appears stalled waiting for %" WARN_PRIdPTR
" threads to park...\n",
GC_nacl_num_gc_threads - num_threads_parked - 1);
num_sleeps = 0;
}
}
if (GC_manual_vdb)
GC_release_dirty_lock();
# endif /* NACL */
return n_live_threads;
}
GC_INNER void
GC_stop_world(void)
{
# ifndef NACL
int n_live_threads;
# endif
GC_ASSERT(I_HOLD_LOCK());
/*
* Make sure all free list construction has stopped before we start.
* No new construction can start, since it is required to acquire and
* release the allocator lock before start.
*/
GC_ASSERT(GC_thr_initialized);
# ifdef DEBUG_THREADS
GC_stopping_thread = pthread_self();
GC_stopping_pid = getpid();
GC_log_printf("Stopping the world from %p\n",
PTHREAD_TO_VPTR(GC_stopping_thread));
# endif
# ifdef PARALLEL_MARK
if (GC_parallel) {
GC_acquire_mark_lock();
GC_ASSERT(GC_fl_builder_count == 0);
/* We should have previously waited for it to become zero. */
}
# endif /* PARALLEL_MARK */
# ifdef NACL
(void)GC_suspend_all();
# else
/* Note: only concurrent reads are possible. */
AO_store(&GC_stop_count, GC_stop_count + THREAD_RESTARTED);
if (GC_manual_vdb) {
GC_acquire_dirty_lock();
/*
* The write fault handler cannot be called if `GC_manual_vdb` (thus
* double-locking should not occur in `async_set_pht_entry_from_index`
* based on test-and-set).
*/
}
n_live_threads = GC_suspend_all();
if (GC_retry_signals) {
resend_lost_signals_retry(n_live_threads, GC_suspend_all);
} else {
suspend_restart_barrier(n_live_threads);
}
if (GC_manual_vdb) {
/* Note: cannot be done in `GC_suspend_all`. */
GC_release_dirty_lock();
}
# endif
# ifdef PARALLEL_MARK
if (GC_parallel)
GC_release_mark_lock();
# endif
# ifdef DEBUG_THREADS
GC_log_printf("World stopped from %p\n", PTHREAD_TO_VPTR(pthread_self()));
GC_stopping_thread = 0;
# endif
}
# ifdef NACL
# if defined(__x86_64__)
# define NACL_STORE_REGS() \
do { \
__asm__ __volatile__("push %rbx"); \
__asm__ __volatile__("push %rbp"); \
__asm__ __volatile__("push %r12"); \
__asm__ __volatile__("push %r13"); \
__asm__ __volatile__("push %r14"); \
__asm__ __volatile__("push %r15"); \
__asm__ __volatile__( \
"mov %%esp, %0" \
: "=m"(GC_nacl_gc_thread_self->crtn->stack_ptr)); \
BCOPY(GC_nacl_gc_thread_self->crtn->stack_ptr, \
GC_nacl_gc_thread_self->reg_storage, \
NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t)); \
__asm__ __volatile__("naclasp $48, %r15"); \
} while (0)
# elif defined(__i386__)
# define NACL_STORE_REGS() \
do { \
__asm__ __volatile__("push %ebx"); \
__asm__ __volatile__("push %ebp"); \
__asm__ __volatile__("push %esi"); \
__asm__ __volatile__("push %edi"); \
__asm__ __volatile__( \
"mov %%esp, %0" \
: "=m"(GC_nacl_gc_thread_self->crtn->stack_ptr)); \
BCOPY(GC_nacl_gc_thread_self->crtn->stack_ptr, \
GC_nacl_gc_thread_self->reg_storage, \
NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t)); \
__asm__ __volatile__("add $16, %esp"); \
} while (0)
# elif defined(__arm__)
# define NACL_STORE_REGS() \
do { \
__asm__ __volatile__("push {r4-r8,r10-r12,lr}"); \
__asm__ __volatile__( \
"mov r0, %0" \
: \
: "r"(&GC_nacl_gc_thread_self->crtn->stack_ptr)); \
__asm__ __volatile__("bic r0, r0, #0xc0000000"); \
__asm__ __volatile__("str sp, [r0]"); \
BCOPY(GC_nacl_gc_thread_self->crtn->stack_ptr, \
GC_nacl_gc_thread_self->reg_storage, \
NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t)); \
__asm__ __volatile__("add sp, sp, #40"); \
__asm__ __volatile__("bic sp, sp, #0xc0000000"); \
} while (0)
# else
# error TODO Please port NACL_STORE_REGS
# endif
GC_API_OSCALL void
nacl_pre_syscall_hook(void)
{
if (GC_nacl_thread_idx != -1) {
NACL_STORE_REGS();
GC_nacl_gc_thread_self->crtn->stack_ptr = GC_approx_sp();
GC_nacl_thread_parked[GC_nacl_thread_idx] = 1;
}
}
GC_API_OSCALL void
__nacl_suspend_thread_if_needed(void)
{
if (!GC_nacl_park_threads_now)
return;
/* Do not try to park the thread parker. */
if (GC_nacl_thread_parker == pthread_self())
return;
/*
* This can happen when a thread is created outside of the collector
* system (`wthread` mostly).
*/
if (GC_nacl_thread_idx < 0)
return;
/*
* If it was already "parked", we are returning from a `syscall`,
* so do not bother storing registers again, the collector has a set of.
*/
if (!GC_nacl_thread_parked[GC_nacl_thread_idx]) {
NACL_STORE_REGS();
GC_nacl_gc_thread_self->crtn->stack_ptr = GC_approx_sp();
}
GC_nacl_thread_parked[GC_nacl_thread_idx] = 1;
while (GC_nacl_park_threads_now) {
/* Just spin. */
}
GC_nacl_thread_parked[GC_nacl_thread_idx] = 0;
/* Clear out the `reg_storage` for the next suspend. */
BZERO(GC_nacl_gc_thread_self->reg_storage,
NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t));
}
GC_API_OSCALL void
nacl_post_syscall_hook(void)
{
/*
* Calling `__nacl_suspend_thread_if_needed` right away should guarantee
* we do not mutate the collector set.
*/
__nacl_suspend_thread_if_needed();
if (GC_nacl_thread_idx != -1) {
GC_nacl_thread_parked[GC_nacl_thread_idx] = 0;
}
}
STATIC GC_bool GC_nacl_thread_parking_inited = FALSE;
STATIC pthread_mutex_t GC_nacl_thread_alloc_lock = PTHREAD_MUTEX_INITIALIZER;
struct nacl_irt_blockhook {
int (*register_block_hooks)(void (*pre)(void), void (*post)(void));
};
EXTERN_C_BEGIN
extern size_t nacl_interface_query(const char *interface_ident, void *table,
size_t tablesize);
EXTERN_C_END
GC_INNER void
GC_nacl_initialize_gc_thread(GC_thread me)
{
int i;
static struct nacl_irt_blockhook gc_hook;
GC_ASSERT(NULL == GC_nacl_gc_thread_self);
GC_nacl_gc_thread_self = me;
pthread_mutex_lock(&GC_nacl_thread_alloc_lock);
if (UNLIKELY(!GC_nacl_thread_parking_inited)) {
BZERO(GC_nacl_thread_parked, sizeof(GC_nacl_thread_parked));
BZERO(GC_nacl_thread_used, sizeof(GC_nacl_thread_used));
/*
* TODO: Replace with the public "register hook" function when
* available from `glibc`.
*/
nacl_interface_query("nacl-irt-blockhook-0.1", &gc_hook, sizeof(gc_hook));
gc_hook.register_block_hooks(nacl_pre_syscall_hook,
nacl_post_syscall_hook);
GC_nacl_thread_parking_inited = TRUE;
}
GC_ASSERT(GC_nacl_num_gc_threads <= MAX_NACL_GC_THREADS);
for (i = 0; i < MAX_NACL_GC_THREADS; i++) {
if (GC_nacl_thread_used[i] == 0) {
GC_nacl_thread_used[i] = 1;
GC_nacl_thread_idx = i;
GC_nacl_num_gc_threads++;
break;
}
}
pthread_mutex_unlock(&GC_nacl_thread_alloc_lock);
}
GC_INNER void
GC_nacl_shutdown_gc_thread(void)
{
GC_ASSERT(GC_nacl_gc_thread_self != NULL);
pthread_mutex_lock(&GC_nacl_thread_alloc_lock);
GC_ASSERT(GC_nacl_thread_idx >= 0);
GC_ASSERT(GC_nacl_thread_idx < MAX_NACL_GC_THREADS);
GC_ASSERT(GC_nacl_thread_used[GC_nacl_thread_idx] != 0);
GC_nacl_thread_used[GC_nacl_thread_idx] = 0;
GC_nacl_thread_idx = -1;
GC_nacl_num_gc_threads--;
pthread_mutex_unlock(&GC_nacl_thread_alloc_lock);
GC_nacl_gc_thread_self = NULL;
}
# else /* !NACL */
static GC_bool in_resend_restart_signals;
/*
* Restart all threads that were suspended by the collector.
* Return the number of restart signals that were sent.
*/
STATIC int
GC_restart_all(void)
{
int n_live_threads = 0;
int i;
pthread_t self = pthread_self();
GC_thread p;
int result;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT((GC_stop_count & THREAD_RESTARTED) != 0);
for (i = 0; i < THREAD_TABLE_SZ; i++) {
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
if (!THREAD_EQUAL(p->id, self)) {
if ((p->flags & (FINISHED | DO_BLOCKING)) != 0)
continue;
# ifdef GC_ENABLE_SUSPEND_THREAD
if ((p->ext_suspend_cnt & 1) != 0)
continue;
# endif
if (GC_retry_signals
&& AO_load(&p->last_stop_count) == GC_stop_count) {
/* The thread has been restarted. */
if (!in_resend_restart_signals) {
/*
* Some user signal (which we do not block, e.g. `SIGQUIT`)
* has already restarted the thread, but nonetheless we need to
* count the latter in `n_live_threads`, so that to decrement
* the semaphore's value proper amount of times. (We are also
* sending the restart signal to the thread, it is not needed
* actually but does not hurt.)
*/
} else {
continue;
/*
* FIXME: Still, an extremely low chance exists that the user
* signal restarts the thread after the restart signal has been
* lost (causing `sem_timedwait()` to fail) while retrying,
* causing finally a mismatch between `GC_suspend_ack_sem` and
* `n_live_threads`.
*/
}
}
n_live_threads++;
# ifdef DEBUG_THREADS
GC_log_printf("Sending restart signal to %p\n",
THREAD_ID_TO_VPTR(p->id));
# endif
result = raise_signal(p, GC_sig_thr_restart);
switch (result) {
case ESRCH:
/* Not really there anymore. Possible? */
n_live_threads--;
break;
case 0:
if (GC_on_thread_event)
GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED,
THREAD_ID_TO_VPTR(THREAD_SYSTEM_ID(p)));
break;
default:
ABORT_ARG1("pthread_kill failed at resume", ": errcode= %d", result);
}
}
}
}
return n_live_threads;
}
# endif /* !NACL */
GC_INNER void
GC_start_world(void)
{
# ifndef NACL
int n_live_threads;
/* The allocator lock is held continuously since the world stopped. */
GC_ASSERT(I_HOLD_LOCK());
# ifdef DEBUG_THREADS
GC_log_printf("World starting\n");
# endif
/*
* Note: the updated value should now be visible to the signal handler
* (note that `pthread_kill` is not on the list of functions that
* synchronize memory).
*/
AO_store_release(&GC_stop_count, GC_stop_count + THREAD_RESTARTED);
GC_ASSERT(!in_resend_restart_signals);
n_live_threads = GC_restart_all();
if (GC_retry_signals) {
in_resend_restart_signals = TRUE;
resend_lost_signals_retry(n_live_threads, GC_restart_all);
in_resend_restart_signals = FALSE;
} else {
# ifndef GC_NETBSD_THREADS_WORKAROUND
if (GC_sig_suspend == GC_sig_thr_restart)
# endif
{
suspend_restart_barrier(n_live_threads);
}
}
# ifdef DEBUG_THREADS
GC_log_printf("World started\n");
# endif
# else /* NACL */
# ifdef DEBUG_THREADS
GC_log_printf("World starting...\n");
# endif
GC_nacl_park_threads_now = 0;
if (GC_on_thread_event) {
GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, NULL);
/* TODO: Send event for every unsuspended thread. */
}
# endif
}
GC_INNER void
GC_stop_init(void)
{
# ifndef NACL
struct sigaction act;
char *str;
if (SIGNAL_UNSET == GC_sig_suspend)
GC_sig_suspend = SIG_SUSPEND;
if (SIGNAL_UNSET == GC_sig_thr_restart)
GC_sig_thr_restart = SIG_THR_RESTART;
if (sem_init(&GC_suspend_ack_sem, GC_SEM_INIT_PSHARED, 0) == -1)
ABORT("sem_init failed");
GC_stop_count = THREAD_RESTARTED; /*< i.e. the world is not stopped */
if (sigfillset(&act.sa_mask) != 0) {
ABORT("sigfillset failed");
}
# ifdef RTEMS
if (sigprocmask(SIG_UNBLOCK, &act.sa_mask, NULL) != 0) {
ABORT("sigprocmask failed");
}
# endif
GC_remove_allowed_signals(&act.sa_mask);
/*
* `GC_sig_thr_restart` is set in the resulting mask.
* It is unmasked by the handler when necessary.
*/
# ifdef SA_RESTART
act.sa_flags = SA_RESTART;
# else
act.sa_flags = 0;
# endif
# ifdef SUSPEND_HANDLER_NO_CONTEXT
act.sa_handler = GC_suspend_handler;
# else
act.sa_flags |= SA_SIGINFO;
act.sa_sigaction = GC_suspend_sigaction;
# endif
/* `act.sa_restorer` is deprecated and should not be initialized. */
if (sigaction(GC_sig_suspend, &act, NULL) != 0) {
ABORT("Cannot set SIG_SUSPEND handler");
}
if (GC_sig_suspend != GC_sig_thr_restart) {
# ifndef SUSPEND_HANDLER_NO_CONTEXT
act.sa_flags &= ~SA_SIGINFO;
# endif
act.sa_handler = GC_restart_handler;
if (sigaction(GC_sig_thr_restart, &act, NULL) != 0)
ABORT("Cannot set SIG_THR_RESTART handler");
} else {
GC_COND_LOG_PRINTF("Using same signal for suspend and restart\n");
}
/* Initialize `suspend_handler_mask` (excluding `GC_sig_thr_restart`). */
if (sigfillset(&suspend_handler_mask) != 0)
ABORT("sigfillset failed");
GC_remove_allowed_signals(&suspend_handler_mask);
if (sigdelset(&suspend_handler_mask, GC_sig_thr_restart) != 0)
ABORT("sigdelset failed");
# ifndef NO_RETRY_SIGNALS
/*
* Any platform could lose signals, so let's be conservative and
* always enable signals retry logic.
*/
GC_retry_signals = TRUE;
# endif
/* Override the default value of `GC_retry_signals`. */
str = GETENV("GC_RETRY_SIGNALS");
if (str != NULL) {
/* Do not retry if the environment variable is set to "0". */
GC_retry_signals = *str != '0' || *(str + 1) != '\0';
}
if (GC_retry_signals) {
GC_COND_LOG_PRINTF(
"Will retry suspend and restart signals if necessary\n");
}
# ifndef NO_SIGNALS_UNBLOCK_IN_MAIN
/* Explicitly unblock the signals once before new threads creation. */
GC_unblock_gc_signals();
# endif
# endif /* !NACL */
}
#endif /* PTHREAD_STOP_WORLD_IMPL */
#endif
/*
* Copyright (c) 1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996 by Silicon Graphics. All rights reserved.
* Copyright (c) 1998 by Fergus Henderson. All rights reserved.
* Copyright (c) 2000-2008 by Hewlett-Packard Company. All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* Support code originally for LinuxThreads, the `clone`-based kernel
* thread package for Linux which is included in `libc6`.
*
* This code no doubt makes some assumptions beyond what is guaranteed by
* the `pthreads` standard, though it now does very little of that.
* It now also supports NPTL, and many other POSIX thread implementations.
* We are trying to merge all flavors of `pthreads` support code into this
* file.
*/
#ifdef THREADS
# ifdef GC_PTHREADS
# if defined(DARWIN) \
|| (defined(GC_WIN32_THREADS) && defined(EMULATE_PTHREAD_SEMAPHORE))
/*
* Copyright (c) 1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996 by Silicon Graphics. All rights reserved.
* Copyright (c) 1998 by Fergus Henderson. All rights reserved.
* Copyright (c) 2000-2009 by Hewlett-Packard Development Company.
* All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#ifndef GC_DARWIN_SEMAPHORE_H
#define GC_DARWIN_SEMAPHORE_H
/*
* This is a very simple semaphore implementation based on `pthreads`.
* It is not async-signal safe. But this is not a problem because
* signals are not used to suspend threads on the target.
*/
#if !defined(DARWIN) && !defined(GC_WIN32_THREADS) || !defined(GC_PTHREADS)
# error darwin_semaphore.h included for improper target
#endif
#include <errno.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
pthread_mutex_t mutex;
pthread_cond_t cond;
int value;
} sem_t;
GC_INLINE int
sem_init(sem_t *sem, int pshared, int value)
{
int err;
if (UNLIKELY(pshared != 0)) {
errno = EPERM; /*< unsupported */
return -1;
}
sem->value = value;
err = pthread_mutex_init(&sem->mutex, NULL);
if (UNLIKELY(err != 0)) {
errno = err;
return -1;
}
err = pthread_cond_init(&sem->cond, NULL);
if (UNLIKELY(err != 0)) {
(void)pthread_mutex_destroy(&sem->mutex);
errno = err;
return -1;
}
return 0;
}
GC_INLINE int
sem_post(sem_t *sem)
{
int err = pthread_mutex_lock(&sem->mutex);
if (UNLIKELY(err != 0)) {
errno = err;
return -1;
}
sem->value++;
err = pthread_cond_signal(&sem->cond);
if (UNLIKELY(err != 0)) {
(void)pthread_mutex_unlock(&sem->mutex);
errno = err;
return -1;
}
err = pthread_mutex_unlock(&sem->mutex);
if (UNLIKELY(err != 0)) {
errno = err;
return -1;
}
return 0;
}
GC_INLINE int
sem_wait(sem_t *sem)
{
int err = pthread_mutex_lock(&sem->mutex);
if (UNLIKELY(err != 0)) {
errno = err;
return -1;
}
while (0 == sem->value) {
err = pthread_cond_wait(&sem->cond, &sem->mutex);
if (UNLIKELY(err != 0)) {
(void)pthread_mutex_unlock(&sem->mutex);
errno = err;
return -1;
}
}
sem->value--;
err = pthread_mutex_unlock(&sem->mutex);
if (UNLIKELY(err != 0)) {
errno = err;
return -1;
}
return 0;
}
GC_INLINE int
sem_destroy(sem_t *sem)
{
int err = pthread_cond_destroy(&sem->cond);
if (UNLIKELY(err != 0)) {
errno = err;
return -1;
}
err = pthread_mutex_destroy(&sem->mutex);
if (UNLIKELY(err != 0)) {
errno = err;
return -1;
}
return 0;
}
#ifdef __cplusplus
} /* extern "C" */
#endif
#endif
# elif !defined(PLATFORM_THREADS) && !defined(SN_TARGET_PSP2)
# include <semaphore.h>
# endif
# include <errno.h>
# endif /* GC_PTHREADS */
# if !defined(GC_WIN32_THREADS)
# include <sched.h>
# include <time.h>
# if !defined(PLATFORM_THREADS) && !defined(SN_TARGET_PSP2)
# ifndef RTEMS
# include <sys/mman.h>
# endif
# include <fcntl.h>
# include <sys/stat.h>
# include <sys/time.h>
# endif
# if defined(GC_EXPLICIT_SIGNALS_UNBLOCK) \
|| !defined(GC_NO_PTHREAD_SIGMASK) \
|| (defined(GC_PTHREADS_PARAMARK) \
&& !defined(NO_MARKER_SPECIAL_SIGMASK))
# include <signal.h>
# endif
# endif /* !GC_WIN32_THREADS */
# ifdef E2K
# include <alloca.h>
# endif
# if defined(DARWIN) || defined(ANY_BSD)
# if defined(NETBSD) || defined(OPENBSD)
# include <sys/param.h>
# endif
# include <sys/sysctl.h>
# elif defined(DGUX)
# include <sys/_int_psem.h>
# include <sys/dg_sys_info.h>
/* Note: `sem_t` is `uint` in DG/UX. */
typedef unsigned int sem_t;
# endif
# if defined(GC_PTHREADS) && !defined(PLATFORM_THREADS) \
&& !defined(SN_TARGET_PSP2)
/* Undefine macros used to redirect `pthreads` primitives. */
# undef pthread_create
# ifndef GC_NO_PTHREAD_SIGMASK
# undef pthread_sigmask
# endif
# ifndef GC_NO_PTHREAD_CANCEL
# undef pthread_cancel
# endif
# ifdef GC_HAVE_PTHREAD_EXIT
# undef pthread_exit
# endif
# undef pthread_join
# undef pthread_detach
# if defined(OSF1) && defined(_PTHREAD_USE_MANGLED_NAMES_) \
&& !defined(_PTHREAD_USE_PTDNAM_)
/* Restore the original mangled names on Tru64 UNIX. */
# define pthread_create __pthread_create
# define pthread_join __pthread_join
# define pthread_detach __pthread_detach
# ifndef GC_NO_PTHREAD_CANCEL
# define pthread_cancel __pthread_cancel
# endif
# ifdef GC_HAVE_PTHREAD_EXIT
# define pthread_exit __pthread_exit
# endif
# endif
# endif /* GC_PTHREADS */
# if !defined(GC_WIN32_THREADS) && !defined(PLATFORM_THREADS) \
&& !defined(SN_TARGET_PSP2)
/* TODO: Enable `GC_USE_DLOPEN_WRAP` for Cygwin? */
# ifdef GC_USE_LD_WRAP
# define WRAP_FUNC(f) __wrap_##f
# define REAL_FUNC(f) __real_##f
int REAL_FUNC(pthread_create)(pthread_t *,
GC_PTHREAD_CREATE_CONST pthread_attr_t *,
void *(*start_routine)(void *), void *);
int REAL_FUNC(pthread_join)(pthread_t, void **);
int REAL_FUNC(pthread_detach)(pthread_t);
# ifndef GC_NO_PTHREAD_SIGMASK
int REAL_FUNC(pthread_sigmask)(int, const sigset_t *, sigset_t *);
# endif
# ifndef GC_NO_PTHREAD_CANCEL
int REAL_FUNC(pthread_cancel)(pthread_t);
# endif
# ifdef GC_HAVE_PTHREAD_EXIT
void REAL_FUNC(pthread_exit)(void *) GC_PTHREAD_EXIT_ATTRIBUTE;
# endif
# elif defined(GC_USE_DLOPEN_WRAP)
# include <dlfcn.h>
# define WRAP_FUNC(f) f
# define REAL_FUNC(f) GC_real_##f
/*
* We define both `GC_<fn>` and plain `fn` to be the wrapped function.
* In that way plain calls work, as do calls from files that include
* `gc.h` file which redefines `fn` to `GC_<fn>`.
*/
/* FIXME: Needs work for `DARWIN` and True64 (`OSF1`). */
typedef int (*GC_pthread_create_t)(pthread_t *,
GC_PTHREAD_CREATE_CONST pthread_attr_t *,
void *(*)(void *), void *);
static GC_pthread_create_t REAL_FUNC(pthread_create);
# ifndef GC_NO_PTHREAD_SIGMASK
typedef int (*GC_pthread_sigmask_t)(int, const sigset_t *, sigset_t *);
static GC_pthread_sigmask_t REAL_FUNC(pthread_sigmask);
# endif
typedef int (*GC_pthread_join_t)(pthread_t, void **);
static GC_pthread_join_t REAL_FUNC(pthread_join);
typedef int (*GC_pthread_detach_t)(pthread_t);
static GC_pthread_detach_t REAL_FUNC(pthread_detach);
# ifndef GC_NO_PTHREAD_CANCEL
typedef int (*GC_pthread_cancel_t)(pthread_t);
static GC_pthread_cancel_t REAL_FUNC(pthread_cancel);
# endif
# ifdef GC_HAVE_PTHREAD_EXIT
typedef void (*GC_pthread_exit_t)(void *) GC_PTHREAD_EXIT_ATTRIBUTE;
static GC_pthread_exit_t REAL_FUNC(pthread_exit);
# endif
# else
# define WRAP_FUNC(f) GC_##f
# ifdef DGUX
# define REAL_FUNC(f) __d10_##f
# else
# define REAL_FUNC(f) f
# endif
# endif /* !GC_USE_LD_WRAP && !GC_USE_DLOPEN_WRAP */
/*
* Define `GC_` functions as aliases for the plain ones, which will
* be intercepted. This allows files that include `gc.h` file, and
* hence generate references to the `GC_` symbols, to see the right ones.
*/
# if defined(GC_USE_LD_WRAP) || defined(GC_USE_DLOPEN_WRAP)
GC_API int
GC_pthread_create(pthread_t *t, GC_PTHREAD_CREATE_CONST pthread_attr_t *a,
void *(*fn)(void *), void *arg)
{
return pthread_create(t, a, fn, arg);
}
# ifndef GC_NO_PTHREAD_SIGMASK
GC_API int
GC_pthread_sigmask(int how, const sigset_t *mask, sigset_t *old)
{
return pthread_sigmask(how, mask, old);
}
# endif /* !GC_NO_PTHREAD_SIGMASK */
GC_API int
GC_pthread_join(pthread_t t, void **res)
{
return pthread_join(t, res);
}
GC_API int
GC_pthread_detach(pthread_t t)
{
return pthread_detach(t);
}
# ifndef GC_NO_PTHREAD_CANCEL
GC_API int
GC_pthread_cancel(pthread_t t)
{
return pthread_cancel(t);
}
# endif /* !GC_NO_PTHREAD_CANCEL */
# ifdef GC_HAVE_PTHREAD_EXIT
GC_API GC_PTHREAD_EXIT_ATTRIBUTE void
GC_pthread_exit(void *retval)
{
pthread_exit(retval);
}
# endif
# endif /* GC_USE_LD_WRAP || GC_USE_DLOPEN_WRAP */
# ifdef GC_USE_DLOPEN_WRAP
/*
* Resolve a symbol with name `n` from the dynamic library (given by
* handle `h`) and cast it to the given functional type `fn`.
*/
# define TYPED_DLSYM(fn, h, n) CAST_THRU_UINTPTR(fn, dlsym(h, n))
STATIC void
GC_init_real_syms(void)
{
void *dl_handle;
GC_ASSERT(!GC_syms_wrap_initialized);
# ifdef RTLD_NEXT
dl_handle = RTLD_NEXT;
# else
dl_handle = dlopen("libpthread.so.0", RTLD_LAZY);
if (NULL == dl_handle) {
/* Retry without ".0" suffix. */
dl_handle = dlopen("libpthread.so", RTLD_LAZY);
if (NULL == dl_handle)
ABORT("Couldn't open libpthread");
}
# endif
REAL_FUNC(pthread_create)
= TYPED_DLSYM(GC_pthread_create_t, dl_handle, "pthread_create");
# ifdef RTLD_NEXT
if (REAL_FUNC(pthread_create) == 0)
ABORT("pthread_create not found"
" (probably -lgc is specified after -lpthread)");
# endif
# ifndef GC_NO_PTHREAD_SIGMASK
REAL_FUNC(pthread_sigmask)
= TYPED_DLSYM(GC_pthread_sigmask_t, dl_handle, "pthread_sigmask");
# endif
REAL_FUNC(pthread_join)
= TYPED_DLSYM(GC_pthread_join_t, dl_handle, "pthread_join");
REAL_FUNC(pthread_detach)
= TYPED_DLSYM(GC_pthread_detach_t, dl_handle, "pthread_detach");
# ifndef GC_NO_PTHREAD_CANCEL
REAL_FUNC(pthread_cancel)
= TYPED_DLSYM(GC_pthread_cancel_t, dl_handle, "pthread_cancel");
# endif
# ifdef GC_HAVE_PTHREAD_EXIT
REAL_FUNC(pthread_exit)
= TYPED_DLSYM(GC_pthread_exit_t, dl_handle, "pthread_exit");
# endif
GC_syms_wrap_initialized = TRUE;
}
# define INIT_REAL_SYMS() \
if (LIKELY(GC_syms_wrap_initialized)) { \
} else \
GC_init_real_syms()
# else
# define INIT_REAL_SYMS() (void)0
# endif /* !GC_USE_DLOPEN_WRAP */
# else
# define WRAP_FUNC(f) GC_##f
# define REAL_FUNC(f) f
# define INIT_REAL_SYMS() (void)0
# endif /* GC_WIN32_THREADS */
# if defined(MPROTECT_VDB) && defined(DARWIN)
GC_INNER int
GC_inner_pthread_create(pthread_t *t,
GC_PTHREAD_CREATE_CONST pthread_attr_t *a,
void *(*fn)(void *), void *arg)
{
INIT_REAL_SYMS();
return REAL_FUNC(pthread_create)(t, a, fn, arg);
}
# endif
# ifdef THREAD_LOCAL_ALLOC
GC_INNER void
GC_mark_thread_local_free_lists(void)
{
int i;
GC_thread p;
for (i = 0; i < THREAD_TABLE_SZ; ++i) {
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
if (!KNOWN_FINISHED(p))
GC_mark_thread_local_fls_for(&p->tlfs);
}
}
}
# if defined(GC_ASSERTIONS)
/*
* Check that all thread-local free-lists are completely marked.
* Also check that thread-specific-data structures are marked.
*/
void
GC_check_tls(void)
{
int i;
GC_thread p;
for (i = 0; i < THREAD_TABLE_SZ; ++i) {
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
if (!KNOWN_FINISHED(p))
GC_check_tls_for(&p->tlfs);
}
}
# if defined(USE_CUSTOM_SPECIFIC)
if (GC_thread_key != 0)
GC_check_tsd_marks(GC_thread_key);
# endif
}
# endif
# endif /* THREAD_LOCAL_ALLOC */
# ifdef GC_WIN32_THREADS
/*
* A macro for functions and variables that should be accessible
* from `win32_threads.c` file but otherwise could be `static`.
*/
# define GC_INNER_WIN32THREAD GC_INNER
# else
# define GC_INNER_WIN32THREAD STATIC
# endif
# ifdef PARALLEL_MARK
# if defined(GC_WIN32_THREADS) || defined(USE_PROC_FOR_LIBRARIES) \
|| (defined(IA64) \
&& (defined(HAVE_PTHREAD_ATTR_GET_NP) \
|| defined(HAVE_PTHREAD_GETATTR_NP)))
GC_INNER_WIN32THREAD ptr_t GC_marker_sp[MAX_MARKERS - 1] = { 0 };
# endif /* GC_WIN32_THREADS || USE_PROC_FOR_LIBRARIES */
# if defined(IA64) && defined(USE_PROC_FOR_LIBRARIES)
static ptr_t marker_bsp[MAX_MARKERS - 1] = { 0 };
# endif
# if defined(DARWIN) && !defined(GC_NO_THREADS_DISCOVERY)
static mach_port_t marker_mach_threads[MAX_MARKERS - 1] = { 0 };
GC_INNER GC_bool
GC_is_mach_marker(thread_act_t thread)
{
int i;
for (i = 0; i < GC_markers_m1; i++) {
if (marker_mach_threads[i] == thread)
return TRUE;
}
return FALSE;
}
# endif /* DARWIN && !GC_NO_THREADS_DISCOVERY */
# ifdef HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG
/* For NetBSD. */
static void
set_marker_thread_name(unsigned id)
{
int err = pthread_setname_np(pthread_self(), "GC-marker-%zu",
NUMERIC_TO_VPTR(id));
if (UNLIKELY(err != 0))
WARN("pthread_setname_np failed, errno= %" WARN_PRIdPTR "\n",
(GC_signed_word)err);
}
# elif defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) \
|| defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID) \
|| defined(HAVE_PTHREAD_SET_NAME_NP)
# ifdef HAVE_PTHREAD_SET_NAME_NP
# include <pthread_np.h>
# endif
static void
set_marker_thread_name(unsigned id)
{
/*
* Note: a smaller size of the buffer may result in
* "output may be truncated" compiler warning.
*/
char name_buf[10 + 20 + 1];
GC_snprintf_s_ld_s(name_buf, sizeof(name_buf), "GC-marker-", (long)id, "");
# ifdef HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID
/* The iOS or OS X case. */
(void)pthread_setname_np(name_buf);
# elif defined(HAVE_PTHREAD_SET_NAME_NP)
/* The OpenBSD case. */
pthread_set_name_np(pthread_self(), name_buf);
# else
/* The case of Linux, Solaris, etc. */
GC_ASSERT(strlen(name_buf) < 16);
/* `pthread_setname_np()` may fail for longer names. */
if (UNLIKELY(pthread_setname_np(pthread_self(), name_buf) != 0))
WARN("pthread_setname_np failed\n", 0);
# endif
}
# elif defined(GC_WIN32_THREADS) && !defined(MSWINCE)
/*
* A pointer to `SetThreadDescription()` which is available since Windows 10.
* The function prototype is in the platform `processthreadsapi.h` file.
*/
static FARPROC setThreadDescription_fn;
GC_INNER void
GC_init_win32_thread_naming(HMODULE hK32)
{
if (hK32)
setThreadDescription_fn = GetProcAddress(hK32, "SetThreadDescription");
}
static void
set_marker_thread_name(unsigned id)
{
WCHAR name_buf[16];
int len = sizeof(L"GC-marker-") / sizeof(WCHAR) - 1;
HRESULT hr;
if (!setThreadDescription_fn) {
/* `SetThreadDescription()` is missing. */
return;
}
/* Compose the name manually as `swprintf` may be unavailable. */
BCOPY(L"GC-marker-", name_buf, len * sizeof(WCHAR));
if (id >= 10)
name_buf[len++] = (WCHAR)('0' + (id / 10) % 10);
name_buf[len] = (WCHAR)('0' + id % 10);
name_buf[len + 1] = 0;
/*
* Invoke `SetThreadDescription()`. Cast the function pointer to
* `GC_funcptr_uint` first to avoid "incompatible function types"
* compiler warning.
*/
hr = (*(HRESULT(WINAPI *)(HANDLE, const WCHAR *))(
GC_funcptr_uint)setThreadDescription_fn)(GetCurrentThread(), name_buf);
if (hr < 0)
WARN("SetThreadDescription failed\n", 0);
}
# else
# define set_marker_thread_name(id) (void)(id)
# endif
GC_INNER_WIN32THREAD
# ifdef GC_PTHREADS_PARAMARK
void *
GC_mark_thread(void *id)
# elif defined(MSWINCE)
DWORD WINAPI
GC_mark_thread(LPVOID id)
# else
unsigned __stdcall GC_mark_thread(void *id)
# endif
{
word my_mark_no = 0;
word id_n = (word)(GC_uintptr_t)id;
IF_CANCEL(int cancel_state;)
if (id_n == GC_WORD_MAX)
return 0; /*< to prevent a compiler warning */
/*
* Mark threads are not cancellable; they should be invisible to
* client.
*/
DISABLE_CANCEL(cancel_state);
set_marker_thread_name((unsigned)id_n);
# if defined(GC_WIN32_THREADS) || defined(USE_PROC_FOR_LIBRARIES) \
|| (defined(IA64) \
&& (defined(HAVE_PTHREAD_ATTR_GET_NP) \
|| defined(HAVE_PTHREAD_GETATTR_NP)))
GC_marker_sp[id_n] = GC_approx_sp();
# endif
# if defined(IA64) && defined(USE_PROC_FOR_LIBRARIES)
marker_bsp[id_n] = GC_save_regs_in_stack();
# endif
# if defined(DARWIN) && !defined(GC_NO_THREADS_DISCOVERY)
marker_mach_threads[id_n] = mach_thread_self();
# endif
# if !defined(GC_PTHREADS_PARAMARK)
GC_marker_Id[id_n] = thread_id_self();
# endif
/* Inform `GC_start_mark_threads` about completion of marker data init. */
GC_acquire_mark_lock();
/* Note: the count variable may have a negative value. */
if (0 == --GC_fl_builder_count)
GC_notify_all_builder();
/*
* `GC_mark_no` is passed only to allow `GC_help_marker` to
* terminate promptly. This is important if it were called from the
* signal handler or from the allocator lock acquisition code.
* On Linux, it is not safe to call it from a signal handler, since
* it uses mutex and condition variables. Since it is called only
* here, the argument is unnecessary.
*/
for (;; ++my_mark_no) {
if (my_mark_no - GC_mark_no > (word)2) {
/* Resynchronize if we get far off, e.g. because `GC_mark_no` wrapped. */
my_mark_no = GC_mark_no;
}
# ifdef DEBUG_THREADS
GC_log_printf("Starting helper for mark number %lu (thread %u)\n",
(unsigned long)my_mark_no, (unsigned)id_n);
# endif
GC_help_marker(my_mark_no);
}
}
GC_INNER_WIN32THREAD int GC_available_markers_m1 = 0;
# endif /* PARALLEL_MARK */
# ifdef GC_PTHREADS_PARAMARK
# ifdef GLIBC_2_1_MUTEX_HACK
/*
* Ugly workaround for a Linux threads bug in the final versions
* of `glibc` 2.1. `pthread_mutex_trylock` sets the mutex owner
* field even when it fails to acquire the mutex. This causes
* `pthread_cond_wait` to die. Should not be needed for `glibc` 2.2.
* According to the man page, we should use
* `PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP`, but that is not actually
* defined.
*/
static pthread_mutex_t mark_mutex
= { 0, 0, 0, PTHREAD_MUTEX_ERRORCHECK_NP, { 0, 0 } };
# else
static pthread_mutex_t mark_mutex = PTHREAD_MUTEX_INITIALIZER;
# endif
# ifdef CAN_HANDLE_FORK
/* Note: this is initialized by `GC_start_mark_threads_inner()`. */
static pthread_cond_t mark_cv;
# else
static pthread_cond_t mark_cv = PTHREAD_COND_INITIALIZER;
# endif
GC_INNER void
GC_start_mark_threads_inner(void)
{
int i;
pthread_attr_t attr;
# ifndef NO_MARKER_SPECIAL_SIGMASK
sigset_t set, oldset;
# endif
GC_ASSERT(I_HOLD_LOCK());
ASSERT_CANCEL_DISABLED();
if (GC_available_markers_m1 <= 0 || GC_parallel) {
/* Skip if parallel markers disabled or already started. */
return;
}
GC_wait_for_gc_completion(TRUE);
# ifdef CAN_HANDLE_FORK
/*
* Initialize `mark_cv` (for the first time), or cleanup its value
* after forking in the child process. All the marker threads in the
* parent process were blocked on this variable at process fork, so
* `pthread_cond_wait()` malfunction (hang) is possible in the child
* process without such a cleanup.
*/
/*
* TODO: This is not portable, it is better to shortly unblock all
* marker threads in the parent process at `fork`.
*/
{
pthread_cond_t mark_cv_local = PTHREAD_COND_INITIALIZER;
BCOPY(&mark_cv_local, &mark_cv, sizeof(mark_cv));
}
# endif
GC_ASSERT(0 == GC_fl_builder_count);
INIT_REAL_SYMS(); /*< for `pthread_create` */
if (pthread_attr_init(&attr) != 0)
ABORT("pthread_attr_init failed");
if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0)
ABORT("pthread_attr_setdetachstate failed");
# ifdef DEFAULT_STACK_MAYBE_SMALL
/*
* The default stack size is usually too small: increase it.
* Otherwise marker threads may run out of space.
*/
{
size_t old_size;
if (pthread_attr_getstacksize(&attr, &old_size) != 0)
ABORT("pthread_attr_getstacksize failed");
if (old_size < MIN_STACK_SIZE && old_size != 0 /* stack size is known */) {
if (pthread_attr_setstacksize(&attr, MIN_STACK_SIZE) != 0)
ABORT("pthread_attr_setstacksize failed");
}
}
# endif /* DEFAULT_STACK_MAYBE_SMALL */
# ifndef NO_MARKER_SPECIAL_SIGMASK
/*
* Apply special signal mask to GC marker threads, and do not drop
* user-defined signals by the marker threads.
*/
if (sigfillset(&set) != 0)
ABORT("sigfillset failed");
# ifdef SIGNAL_BASED_STOP_WORLD
/* These are used by GC to stop and restart the world. */
if (sigdelset(&set, GC_get_suspend_signal()) != 0
|| sigdelset(&set, GC_get_thr_restart_signal()) != 0)
ABORT("sigdelset failed");
# endif
if (UNLIKELY(REAL_FUNC(pthread_sigmask)(SIG_BLOCK, &set, &oldset) != 0)) {
WARN("pthread_sigmask set failed, no markers started\n", 0);
GC_markers_m1 = 0;
(void)pthread_attr_destroy(&attr);
return;
}
# endif /* !NO_MARKER_SPECIAL_SIGMASK */
/* To have proper `GC_parallel` value in `GC_help_marker()`. */
GC_markers_m1 = GC_available_markers_m1;
for (i = 0; i < GC_available_markers_m1; ++i) {
pthread_t new_thread;
# ifdef GC_WIN32_THREADS
GC_marker_last_stack_min[i] = ADDR_LIMIT;
# endif
if (UNLIKELY(REAL_FUNC(pthread_create)(&new_thread, &attr, GC_mark_thread,
NUMERIC_TO_VPTR(i))
!= 0)) {
WARN("Marker thread %" WARN_PRIdPTR " creation failed\n",
(GC_signed_word)i);
/* Do not try to create other marker threads. */
GC_markers_m1 = i;
break;
}
}
# ifndef NO_MARKER_SPECIAL_SIGMASK
/* Restore previous signal mask. */
if (UNLIKELY(REAL_FUNC(pthread_sigmask)(SIG_SETMASK, &oldset, NULL) != 0)) {
WARN("pthread_sigmask restore failed\n", 0);
}
# endif
(void)pthread_attr_destroy(&attr);
GC_wait_for_markers_init();
GC_COND_LOG_PRINTF("Started %d mark helper threads\n", GC_markers_m1);
}
# endif /* GC_PTHREADS_PARAMARK */
GC_INNER GC_thread GC_threads[THREAD_TABLE_SZ] = { 0 };
/*
* It may not be safe to allocate when we register the first thread.
* Note that `next` and `status` fields are unused, but there might be
* some other fields (`crtn`) to be pushed.
*/
static struct GC_StackContext_Rep first_crtn;
static struct GC_Thread_Rep first_thread;
# ifndef GC_NO_DEINIT
GC_INNER void
GC_reset_threads(void)
{
BZERO(GC_threads, sizeof(GC_threads));
BZERO(&first_crtn, sizeof(first_crtn));
BZERO(&first_thread, sizeof(first_thread));
}
# endif
/*
* A place to retain a pointer to an allocated object while a thread
* registration is ongoing. Protected by the allocator lock.
*/
static GC_stack_context_t saved_crtn = NULL;
GC_INNER void
GC_push_thread_structures(void)
{
GC_ASSERT(I_HOLD_LOCK());
# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
if (GC_win32_dll_threads) {
/*
* Unlike the other threads implementations, the thread table here
* contains no pointers to the collectible heap (note also that
* `GC_PTHREADS` is incompatible with `DllMain`-based thread
* registration). Thus we have no private structures we need to
* preserve.
*/
} else
# endif
/* else */ {
GC_push_all(&GC_threads, (ptr_t)(&GC_threads) + sizeof(GC_threads));
GC_ASSERT(NULL == first_thread.tm.next);
# ifdef GC_PTHREADS
GC_ASSERT(NULL == first_thread.status);
# endif
GC_PUSH_ALL_SYM(first_thread.crtn);
GC_PUSH_ALL_SYM(saved_crtn);
}
# if defined(THREAD_LOCAL_ALLOC) && defined(USE_CUSTOM_SPECIFIC)
GC_PUSH_ALL_SYM(GC_thread_key);
# endif
}
# if defined(MPROTECT_VDB) && defined(GC_WIN32_THREADS)
GC_INNER void
GC_win32_unprotect_thread(GC_thread t)
{
GC_ASSERT(I_HOLD_LOCK());
if (!GC_win32_dll_threads && GC_auto_incremental) {
GC_stack_context_t crtn = t->crtn;
if (crtn != &first_crtn) {
GC_ASSERT(SMALL_OBJ(GC_size(crtn)));
GC_remove_protection(HBLKPTR(crtn), 1, FALSE);
}
if (t != &first_thread) {
GC_ASSERT(SMALL_OBJ(GC_size(t)));
GC_remove_protection(HBLKPTR(t), 1, FALSE);
}
}
}
# endif /* MPROTECT_VDB && GC_WIN32_THREADS */
# ifdef DEBUG_THREADS
STATIC int
GC_count_threads(void)
{
int i;
int count = 0;
# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
if (GC_win32_dll_threads)
return -1; /*< not implemented */
# endif
GC_ASSERT(I_HOLD_READER_LOCK());
for (i = 0; i < THREAD_TABLE_SZ; ++i) {
GC_thread p;
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
if (!KNOWN_FINISHED(p))
++count;
}
}
return count;
}
# endif /* DEBUG_THREADS */
GC_INNER_WIN32THREAD GC_thread
GC_new_thread(thread_id_t self_id)
{
int hv = THREAD_TABLE_INDEX(self_id);
GC_thread result;
GC_ASSERT(I_HOLD_LOCK());
# ifdef DEBUG_THREADS
GC_log_printf("Creating thread %p\n", THREAD_ID_TO_VPTR(self_id));
for (result = GC_threads[hv]; result != NULL; result = result->tm.next)
if (!THREAD_ID_EQUAL(result->id, self_id)) {
GC_log_printf("Hash collision at GC_threads[%d]\n", hv);
break;
}
# endif
if (UNLIKELY(NULL == first_thread.crtn)) {
result = &first_thread;
first_thread.crtn = &first_crtn;
GC_ASSERT(NULL == GC_threads[hv]);
# if defined(CPPCHECK) && defined(THREAD_SANITIZER) \
&& defined(SIGNAL_BASED_STOP_WORLD)
GC_noop1((unsigned char)first_crtn.dummy[0]);
# endif
} else {
GC_stack_context_t crtn;
GC_ASSERT(!GC_win32_dll_threads);
GC_ASSERT(!GC_in_thread_creation);
GC_in_thread_creation = TRUE; /*< OK to collect from unknown thread */
crtn = (GC_stack_context_t)GC_INTERNAL_MALLOC(
sizeof(struct GC_StackContext_Rep), NORMAL);
/*
* The current stack is not scanned until the thread is registered,
* thus `crtn` pointer is to be retained in the global data roots for
* a while (and pushed explicitly if a collection occurs here).
*/
GC_ASSERT(NULL == saved_crtn);
saved_crtn = crtn;
result
= (GC_thread)GC_INTERNAL_MALLOC(sizeof(struct GC_Thread_Rep), NORMAL);
/* No more collections till thread is registered. */
saved_crtn = NULL;
GC_in_thread_creation = FALSE;
if (NULL == crtn || NULL == result)
ABORT("Failed to allocate memory for thread registering");
result->crtn = crtn;
}
/* The `id` field is not set here. */
# ifdef USE_TKILL_ON_ANDROID
result->kernel_id = gettid();
# endif
result->tm.next = GC_threads[hv];
GC_threads[hv] = result;
# ifdef NACL
GC_nacl_initialize_gc_thread(result);
# endif
GC_ASSERT(0 == result->flags);
if (LIKELY(result != &first_thread))
GC_dirty(result);
return result;
}
GC_INNER_WIN32THREAD void
GC_delete_thread(GC_thread t)
{
# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
if (GC_win32_dll_threads) {
HANDLE handle = t->handle;
GC_cptr_store_release(&t->handle, NULL);
CloseHandle(handle);
/*
* This is intended to be lock-free. It is either called synchronously
* from the thread being deleted, or by the joining thread. In this
* branch asynchronous changes to `*t` are possible. Note that it is
* not allowed to call `GC_printf` (and the friends) here, see
* `GC_stop_world()` in `win32_threads.c` file for the information.
*/
t->crtn->stack_end = NULL;
t->id = 0;
/* The thread is not suspended. */
t->flags = 0;
# ifdef RETRY_GET_THREAD_CONTEXT
t->context_sp = NULL;
# endif
AO_store_release(&t->tm.in_use, FALSE);
} else
# endif
/* else */ {
thread_id_t id = t->id;
int hv = THREAD_TABLE_INDEX(id);
GC_thread p;
GC_thread prev = NULL;
GC_ASSERT(I_HOLD_LOCK());
# if defined(DEBUG_THREADS) && !defined(MSWINCE) \
&& (!defined(MSWIN32) || defined(CONSOLE_LOG))
GC_log_printf("Deleting thread %p, n_threads= %d\n", THREAD_ID_TO_VPTR(id),
GC_count_threads());
# endif
# if defined(GC_WIN32_THREADS) && !defined(MSWINCE)
CloseHandle(t->handle);
# endif
for (p = GC_threads[hv]; p != t; p = p->tm.next) {
prev = p;
}
if (NULL == prev) {
GC_threads[hv] = p->tm.next;
} else {
GC_ASSERT(prev != &first_thread);
prev->tm.next = p->tm.next;
GC_dirty(prev);
}
if (LIKELY(p != &first_thread)) {
# ifdef DARWIN
mach_port_deallocate(mach_task_self(), p->mach_thread);
# endif
GC_ASSERT(p->crtn != &first_crtn);
GC_INTERNAL_FREE(p->crtn);
GC_INTERNAL_FREE(p);
}
}
}
GC_INNER GC_thread
GC_lookup_thread(thread_id_t id)
{
GC_thread p;
# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
if (GC_win32_dll_threads)
return GC_win32_dll_lookup_thread(id);
# endif
for (p = GC_threads[THREAD_TABLE_INDEX(id)]; p != NULL; p = p->tm.next) {
if (LIKELY(THREAD_ID_EQUAL(p->id, id)))
break;
}
return p;
}
/*
* Same as `GC_self_thread_inner()` but acquires the allocator lock (in
* the reader mode).
*/
STATIC GC_thread
GC_self_thread(void)
{
GC_thread p;
READER_LOCK();
p = GC_self_thread_inner();
READER_UNLOCK();
return p;
}
# ifndef GC_NO_FINALIZATION
GC_INNER void
GC_reset_finalizer_nested(void)
{
GC_ASSERT(I_HOLD_LOCK());
GC_thread me;
me = GC_self_thread_inner();
if (NULL == me) return; // __v_, make the sanitizers and -cstrict happy
me->crtn->finalizer_nested = 0;
}
GC_INNER unsigned char *
GC_check_finalizer_nested(void)
{
GC_thread me;
GC_stack_context_t crtn;
unsigned nesting_level;
GC_ASSERT(I_HOLD_LOCK());
me = GC_self_thread_inner();
# if defined(INCLUDE_LINUX_THREAD_DESCR) && defined(REDIRECT_MALLOC)
/*
* As noted in `GC_pthread_start`, an allocation may happen in
* `GC_get_stack_base`, causing `GC_notify_or_invoke_finalizers`
* to be called before the thread gets registered.
*/
if (UNLIKELY(NULL == me))
return NULL;
# endif
if (NULL == me) return NULL; // __v_, make the sanitizers and -cstrict happy
crtn = me->crtn;
nesting_level = crtn->finalizer_nested;
if (nesting_level) {
/*
* We are inside another `GC_invoke_finalizers()`. Skip some
* implicitly-called `GC_invoke_finalizers()` depending on the
* nesting (recursion) level.
*/
if ((unsigned)(++crtn->finalizer_skipped) < (1U << nesting_level))
return NULL;
crtn->finalizer_skipped = 0;
}
crtn->finalizer_nested = (unsigned char)(nesting_level + 1);
return &crtn->finalizer_nested;
}
# endif /* !GC_NO_FINALIZATION */
# define ADDR_INSIDE_OBJ(p, obj) \
ADDR_INSIDE(p, (ptr_t)(&(obj)), (ptr_t)(&(obj)) + sizeof(obj))
# if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
/* This is called from thread-local `GC_malloc()`. */
GC_bool
GC_is_thread_tsd_valid(void *tsd)
{
GC_thread me = GC_self_thread();
return ADDR_INSIDE_OBJ((ptr_t)tsd, me->tlfs);
}
# endif /* GC_ASSERTIONS && THREAD_LOCAL_ALLOC */
GC_API int GC_CALL
GC_thread_is_registered(void)
{
/* TODO: Use `GC_get_tlfs()` instead. */
GC_thread me = GC_self_thread();
return me != NULL && !KNOWN_FINISHED(me);
}
GC_API void GC_CALL
GC_register_altstack(void *normstack, size_t normstack_size, void *altstack,
size_t altstack_size)
{
# ifdef GC_WIN32_THREADS
/* TODO: Implement. */
UNUSED_ARG(normstack);
UNUSED_ARG(normstack_size);
UNUSED_ARG(altstack);
UNUSED_ARG(altstack_size);
# else
GC_thread me;
GC_stack_context_t crtn;
READER_LOCK();
me = GC_self_thread_inner();
if (UNLIKELY(NULL == me)) {
/* We are called before `GC_thr_init()`. */
me = &first_thread;
}
crtn = me->crtn;
crtn->normstack = (ptr_t)normstack;
crtn->normstack_size = normstack_size;
crtn->altstack = (ptr_t)altstack;
crtn->altstack_size = altstack_size;
READER_UNLOCK_RELEASE();
# endif
}
# ifdef USE_PROC_FOR_LIBRARIES
GC_INNER GC_bool
GC_segment_is_thread_stack(ptr_t lo, ptr_t hi)
{
int i;
GC_thread p;
GC_ASSERT(I_HOLD_READER_LOCK());
# ifdef PARALLEL_MARK
for (i = 0; i < GC_markers_m1; ++i) {
if (ADDR_LT(lo, GC_marker_sp[i]) && ADDR_LT(GC_marker_sp[i], hi))
return TRUE;
# ifdef IA64
if (ADDR_LT(lo, marker_bsp[i]) && ADDR_LT(marker_bsp[i], hi))
return TRUE;
# endif
}
# endif
for (i = 0; i < THREAD_TABLE_SZ; i++) {
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
ptr_t stack_end = p->crtn->stack_end;
if (stack_end != NULL) {
# ifdef STACK_GROWS_UP
if (ADDR_INSIDE(stack_end, lo, hi))
return TRUE;
# else
if (ADDR_LT(lo, stack_end) && ADDR_GE(hi, stack_end))
return TRUE;
# endif
}
}
}
return FALSE;
}
# endif /* USE_PROC_FOR_LIBRARIES */
# if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
&& defined(IA64)
GC_INNER ptr_t
GC_greatest_stack_base_below(ptr_t bound)
{
int i;
GC_thread p;
ptr_t result = NULL;
GC_ASSERT(I_HOLD_READER_LOCK());
# ifdef PARALLEL_MARK
for (i = 0; i < GC_markers_m1; ++i) {
if (ADDR_LT(result, GC_marker_sp[i]) && ADDR_LT(GC_marker_sp[i], bound))
result = GC_marker_sp[i];
}
# endif
for (i = 0; i < THREAD_TABLE_SZ; i++) {
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
ptr_t stack_end = p->crtn->stack_end;
if (ADDR_LT(result, stack_end) && ADDR_LT(stack_end, bound))
result = stack_end;
}
}
return result;
}
# endif /* IA64 */
# ifndef STAT_READ
/*
* Note: if `read()` is wrapped, this may need to be redefined to call
* the real one.
*/
# define STAT_READ read
# endif
# ifdef HPUX
# define GC_get_nprocs() pthread_num_processors_np()
# elif defined(AIX) || defined(COSMO) || defined(HAIKU) \
|| defined(HOST_ANDROID) || defined(HURD) || defined(NACL) \
|| defined(OSF1) || defined(SOLARIS)
GC_INLINE int
GC_get_nprocs(void)
{
int nprocs = (int)sysconf(_SC_NPROCESSORS_ONLN);
/* Note: ignore any error silently. */
return nprocs > 0 ? nprocs : 1;
}
# elif defined(IRIX5)
GC_INLINE int
GC_get_nprocs(void)
{
int nprocs = (int)sysconf(_SC_NPROC_ONLN);
/* Note: ignore any error silently. */
return nprocs > 0 ? nprocs : 1;
}
# elif defined(LINUX)
/* Return the number of processors. */
STATIC int
GC_get_nprocs(void)
{
/*
* Should be just `return sysconf(_SC_NPROCESSORS_ONLN)` but that
* appears to be buggy in many cases. We look for lines "cpu<N>" in
* `/proc/stat` pseudo-file. No need to read the entire `/proc/stat`
* pseudo-file to get maximum "cpu<N>" such as:
* - the requested lines are located at the beginning of the file;
* - the lines with "cpu<N>" where `N` is greater than `MAX_MARKERS`
* are not needed.
*/
# define PROC_STAT_BUF_SZ ((1 + MAX_MARKERS) * 100)
char stat_buf[PROC_STAT_BUF_SZ + 1]; /*< the size should be enough */
int f;
int result, i, len;
f = open("/proc/stat", O_RDONLY);
if (f < 0) {
WARN("Could not open /proc/stat\n", 0);
/* Assume an uniprocessor. */
return 1;
}
len = STAT_READ(f, stat_buf, sizeof(stat_buf) - 1);
/* Unlikely that we need to retry because of an incomplete read here. */
if (len < 0) {
WARN("Failed to read /proc/stat, errno= %" WARN_PRIdPTR "\n",
(GC_signed_word)errno);
close(f);
return 1;
}
/* Avoid potential buffer overrun by `atoi()`. */
stat_buf[len] = '\0';
close(f);
/*
* Some old kernels only have a single "cpu nnnn ..." entry in
* `/proc/stat` pseudo-file. We identify those as uniprocessors.
*/
result = 1;
for (i = 0; i < len - 4; ++i) {
if (stat_buf[i] == '\n' && stat_buf[i + 1] == 'c' && stat_buf[i + 2] == 'p'
&& stat_buf[i + 3] == 'u') {
int cpu_no = atoi(&stat_buf[i + 4]);
if (cpu_no >= result)
result = cpu_no + 1;
}
}
return result;
}
# elif defined(DGUX)
/*
* Return the number of processors, or a non-positive value if the number
* cannot be determined.
*/
STATIC int
GC_get_nprocs(void)
{
int numCpus;
struct dg_sys_info_pm_info pm_sysinfo;
int status = 0;
status = dg_sys_info((long int *)&pm_sysinfo, DG_SYS_INFO_PM_INFO_TYPE,
DG_SYS_INFO_PM_CURRENT_VERSION);
if (status < 0) {
/* Set -1 for an error. */
numCpus = -1;
} else {
/* Active CPUs. */
numCpus = pm_sysinfo.idle_vp_count;
}
return numCpus;
}
# elif defined(ANY_BSD) || defined(DARWIN)
STATIC int
GC_get_nprocs(void)
{
int mib[] = { CTL_HW, HW_NCPU };
int res;
size_t len = sizeof(res);
sysctl(mib, sizeof(mib) / sizeof(int), &res, &len, NULL, 0);
return res;
}
# else
/* E.g., RTEMS. */
/* TODO: Implement. */
# define GC_get_nprocs() 1
# endif
# if defined(LINUX) && defined(ARM32)
/*
* Some buggy Linux/arm kernels show only non-sleeping CPUs in
* `/proc/stat` pseudo-file (and in `/proc/cpuinfo` pseudo-file), so
* another data system source is tried first. Returns a non-positive
* value on error.
*/
STATIC int
GC_get_nprocs_present(void)
{
char stat_buf[16];
int f;
int len;
f = open("/sys/devices/system/cpu/present", O_RDONLY);
if (f < 0) {
/* Cannot open the file. */
return -1;
}
len = STAT_READ(f, stat_buf, sizeof(stat_buf));
close(f);
/*
* Recognized file format: "0\n" or "0-<max_cpu_num>\n".
* The file might probably contain a comma-separated list
* but we do not need to handle it (just silently ignore).
*/
if (len < 2 || stat_buf[0] != '0' || stat_buf[len - 1] != '\n') {
/* A read error or an unrecognized content. */
return 0;
} else if (len == 2) {
/* An uniprocessor. */
return 1;
} else if (stat_buf[1] != '-') {
/* An unrecognized content. */
return 0;
}
/* Terminate the string. */
stat_buf[len - 1] = '\0';
/* Skip "0-" and parse `max_cpu_num`. */
return atoi(&stat_buf[2]) + 1;
}
# endif /* LINUX && ARM32 */
# if defined(CAN_HANDLE_FORK) && defined(THREAD_SANITIZER)
/*
* Workaround for TSan which does not notice that the allocator lock
* is acquired in `fork_prepare_proc()`.
*/
GC_ATTR_NO_SANITIZE_THREAD
static GC_bool
collection_in_progress(void)
{
return GC_mark_state != MS_NONE;
}
# else
# define collection_in_progress() GC_collection_in_progress()
# endif
GC_INNER void
GC_wait_for_gc_completion(GC_bool wait_for_all)
{
# if !defined(THREAD_SANITIZER) || !defined(CAN_CALL_ATFORK)
/*
* `GC_lock_holder` is accessed with the allocator lock held, so
* there is no data race actually (unlike what is reported by TSan).
*/
GC_ASSERT(I_HOLD_LOCK());
# endif
ASSERT_CANCEL_DISABLED();
# ifdef GC_DISABLE_INCREMENTAL
(void)wait_for_all;
# else
if (GC_incremental && collection_in_progress()) {
word old_gc_no = GC_gc_no;
/*
* Make sure that no part of our stack is still on the mark stack,
* since it is about to be unmapped.
*/
# ifdef LINT2
/*
* Note: do not transform this `if`-`do`-`while` construction into
* a single while statement because it might cause some static code
* analyzers to report a false positive (FP) code defect about
* missing unlock after lock.
*/
# endif
do {
GC_ASSERT(!GC_in_thread_creation);
GC_in_thread_creation = TRUE;
GC_collect_a_little_inner(1);
GC_in_thread_creation = FALSE;
UNLOCK();
# ifdef GC_WIN32_THREADS
Sleep(0);
# else
sched_yield();
# endif
LOCK();
} while (GC_incremental && collection_in_progress()
&& (wait_for_all || old_gc_no == GC_gc_no));
}
# endif
}
# if defined(GC_ASSERTIONS) && defined(GC_PTHREADS_PARAMARK)
STATIC unsigned long GC_mark_lock_holder = NO_THREAD;
# endif
# ifdef CAN_HANDLE_FORK
/*
* Procedures called before and after a process fork. The goal here is
* to make it safe to call `GC_malloc()` in the forked child process.
* It is unclear that is attainable, since the Single UNIX Specification
* seems to imply that one should only call async-signal-safe functions,
* and we probably cannot quite guarantee that. But we give it our best
* shot. (That same specification also implies that it is not safe to
* call the system `malloc` between `fork` and `exec`. Thus we are doing
* no worse than it.)
*/
IF_CANCEL(static int fork_cancel_state;) /*< protected by the allocator lock */
# ifdef PARALLEL_MARK
# ifdef THREAD_SANITIZER
# if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
STATIC void GC_generic_lock(pthread_mutex_t *);
# endif
GC_ATTR_NO_SANITIZE_THREAD
static void wait_for_reclaim_atfork(void);
# else
# define wait_for_reclaim_atfork() GC_wait_for_reclaim()
# endif
# endif /* PARALLEL_MARK */
/*
* Prevent TSan false positive (FP) about the race during items removal
* from `GC_threads`. (The race cannot happen since only one thread
* survives in the child process.)
*/
# ifdef CAN_CALL_ATFORK
GC_ATTR_NO_SANITIZE_THREAD
# endif
static void
store_to_threads_table(int hv, GC_thread me)
{
GC_threads[hv] = me;
}
/*
* Remove all entries from the `GC_threads` table, except the one for
* the current thread. Also update thread identifiers stored in the
* table for the current thread. We need to do this in the child process
* after a `fork()`, since only the current thread survives in the child
* process.
*/
STATIC void
GC_remove_all_threads_but_me(void)
{
int hv;
GC_thread me = NULL;
# ifndef GC_WIN32_THREADS
# define pthread_id id
# endif
for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
GC_thread p, next;
for (p = GC_threads[hv]; p != NULL; p = next) {
next = p->tm.next;
if (THREAD_EQUAL(p->pthread_id, GC_parent_pthread_self) && me == NULL) {
/* Ignore dead threads with the same id. */
me = p;
p->tm.next = NULL;
} else {
# ifdef THREAD_LOCAL_ALLOC
if (!KNOWN_FINISHED(p)) {
/*
* Cannot call `GC_destroy_thread_local` here. The free
* lists may be in an inconsistent state (as thread `p` may
* be updating one of the lists by `GC_generic_malloc_many()`
* or `GC_FAST_MALLOC_GRANS()` when `fork()` is invoked).
* This should not be a problem because the lost elements
* of the free lists will be collected during GC.
*/
GC_remove_specific_after_fork(GC_thread_key, p->pthread_id);
}
# endif
/*
* TODO: To avoid TSan hang (when updating `GC_bytes_freed`),
* we just skip explicit freeing of `GC_threads` entries.
*/
# if !defined(THREAD_SANITIZER) || !defined(CAN_CALL_ATFORK)
if (p != &first_thread) {
/* TODO: Should call `mach_port_deallocate`? */
GC_ASSERT(p->crtn != &first_crtn);
GC_INTERNAL_FREE(p->crtn);
GC_INTERNAL_FREE(p);
}
# endif
}
}
store_to_threads_table(hv, NULL);
}
# if defined(CPPCHECK) || defined(LINT2)
if (NULL == me)
ABORT("Current thread is not found after fork");
# else
GC_ASSERT(me != NULL);
# endif
/*
* Update `pthreads` id as it is not guaranteed to be the same between
* this (child) process and the parent one.
*/
me->pthread_id = pthread_self();
# ifdef GC_WIN32_THREADS
/*
* Update Win32 thread id and handle. They differ from that in the
* parent process.
*/
me->id = thread_id_self();
# ifndef MSWINCE
if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
GetCurrentProcess(), (HANDLE *)&me->handle,
0 /* `dwDesiredAccess` */, FALSE /* `bInheritHandle` */,
DUPLICATE_SAME_ACCESS))
ABORT("DuplicateHandle failed");
# endif
# endif
# ifdef DARWIN
/*
* Update thread id after process fork (it is OK to call
* `GC_destroy_thread_local()` and `GC_free_inner()` before update).
*/
me->mach_thread = mach_thread_self();
# endif
# ifdef USE_TKILL_ON_ANDROID
me->kernel_id = gettid();
# endif
/* Put `me` back to `GC_threads`. */
store_to_threads_table(THREAD_TABLE_INDEX(me->id), me);
# ifdef THREAD_LOCAL_ALLOC
# ifdef USE_CUSTOM_SPECIFIC
GC_update_specific_after_fork(GC_thread_key);
# else
/*
* Some TLS implementations (e.g., in Cygwin) might be not `fork`-friendly,
* so we re-assign thread-local pointer to `tlfs` for safety instead of the
* assertion check (again, it is OK to call `GC_destroy_thread_local()` and
* `GC_free_inner()` before).
*/
{
int res = GC_setspecific(GC_thread_key, &me->tlfs);
if (COVERT_DATAFLOW(res) != 0)
ABORT("GC_setspecific failed (in child)");
}
# endif
# endif
# undef pthread_id
}
/* Called before a `fork()`. */
# if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
/* `GC_lock_holder` is updated safely (no data race actually). */
GC_ATTR_NO_SANITIZE_THREAD
# endif
static void
fork_prepare_proc(void)
{
# if defined(GC_EXPLICIT_SIGNALS_UNBLOCK) && defined(CAN_CALL_ATFORK)
/*
* The signals might be blocked by `fork()` implementation when the
* at-fork prepare handler is invoked.
*/
if (GC_handle_fork == 1)
GC_unblock_gc_signals();
# endif
/*
* Acquire all relevant locks, so that after releasing the locks the child
* process will see a consistent state in which monitor invariants hold.
* Unfortunately, we cannot acquire `libc` locks we might need, and there
* seems to be no guarantee that `libc` must install a suitable `fork`
* handler. Wait for an ongoing collection to finish, since we cannot
* finish it in the (one remaining thread in) the child process.
*/
LOCK();
DISABLE_CANCEL(fork_cancel_state);
GC_parent_pthread_self = pthread_self();
/* The following waits may include cancellation points. */
# ifdef PARALLEL_MARK
if (GC_parallel)
wait_for_reclaim_atfork();
# endif
GC_wait_for_gc_completion(TRUE);
# ifdef PARALLEL_MARK
if (GC_parallel) {
# if defined(THREAD_SANITIZER) && defined(GC_ASSERTIONS) \
&& defined(CAN_CALL_ATFORK)
/*
* Prevent TSan false positive (FP) about the data race when updating
* `GC_mark_lock_holder`.
*/
GC_generic_lock(&mark_mutex);
# else
GC_acquire_mark_lock();
# endif
}
# endif
GC_acquire_dirty_lock();
}
/*
* Called in the parent process after a `fork()` (even if the latter
* has failed).
*/
# if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
GC_ATTR_NO_SANITIZE_THREAD
# endif
static void
fork_parent_proc(void)
{
GC_release_dirty_lock();
# ifdef PARALLEL_MARK
if (GC_parallel) {
# if defined(THREAD_SANITIZER) && defined(GC_ASSERTIONS) \
&& defined(CAN_CALL_ATFORK)
/* To match that in `fork_prepare_proc`. */
(void)pthread_mutex_unlock(&mark_mutex);
# else
GC_release_mark_lock();
# endif
}
# endif
RESTORE_CANCEL(fork_cancel_state);
# ifdef GC_ASSERTIONS
BZERO(&GC_parent_pthread_self, sizeof(pthread_t));
# endif
UNLOCK();
}
/* Called in the child process after a `fork()`. */
# if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
GC_ATTR_NO_SANITIZE_THREAD
# endif
static void
fork_child_proc(void)
{
# ifdef GC_ASSERTIONS
/*
* Update `GC_lock_holder` as value of `thread_id_self()` might differ
* from that of the parent process.
*/
SET_LOCK_HOLDER();
# endif
GC_release_dirty_lock();
# ifndef GC_DISABLE_INCREMENTAL
GC_dirty_update_child();
# endif
# ifdef PARALLEL_MARK
if (GC_parallel) {
# ifdef GC_WIN32_THREADS
GC_release_mark_lock();
# else
# if !defined(GC_ASSERTIONS) \
|| (defined(THREAD_SANITIZER) && defined(CAN_CALL_ATFORK))
/* Do not change `GC_mark_lock_holder`. */
# else
GC_mark_lock_holder = NO_THREAD;
# endif
/*
* The unlock operation may fail on some targets, just ignore
* the error silently.
*/
(void)pthread_mutex_unlock(&mark_mutex);
/*
* Reinitialize the mark lock. The reason is the same as for
* `GC_allocate_ml` below.
*/
(void)pthread_mutex_destroy(&mark_mutex);
/* TODO: `GLIBC_2_19_TSX_BUG` has no effect. */
if (pthread_mutex_init(&mark_mutex, NULL) != 0)
ABORT("mark_mutex re-init failed in child");
# endif
/*
* Turn off parallel marking in the child process, since we are probably
* just going to exec, and we would have to restart mark threads.
*/
GC_parallel = FALSE;
}
# ifdef THREAD_SANITIZER
/* TSan does not support threads creation in the child process. */
GC_available_markers_m1 = 0;
# endif
# endif
/* Clean up the thread table, so that just our thread is left. */
GC_remove_all_threads_but_me();
GC_stackbase_info_update_after_fork();
RESTORE_CANCEL(fork_cancel_state);
# ifdef GC_ASSERTIONS
BZERO(&GC_parent_pthread_self, sizeof(pthread_t));
# endif
UNLOCK();
/*
* Even though after a `fork()` the child process only inherits the
* single thread that called the `fork()`, if another thread in the
* parent process was attempting to lock the mutex while being held
* in `fork_child_prepare()`, the mutex will be left in
* an inconsistent state in the child process after the `UNLOCK()`.
* This is the case, at least, in Mac OS X and leads to an unusable
* collector in the child process which will block when attempting
* to perform any GC operation that acquires the allocator lock.
*/
# if defined(USE_PTHREAD_LOCKS) && !defined(GC_WIN32_THREADS)
GC_ASSERT(I_DONT_HOLD_LOCK());
/*
* Reinitialize the mutex. It should be safe since we are running
* this in the child process which only inherits a single thread.
* `pthread_mutex_destroy()` and `pthread_rwlock_destroy()` may
* return `EBUSY`, which makes no sense, but that is the reason for
* the need of the reinitialization.
* Note: excluded for Cygwin as does not seem to be needed.
*/
# ifdef USE_RWLOCK
(void)pthread_rwlock_destroy(&GC_allocate_ml);
# ifdef DARWIN
/* A workaround for `pthread_rwlock_init()` fail with `EBUSY`. */
{
pthread_rwlock_t rwlock_local = PTHREAD_RWLOCK_INITIALIZER;
BCOPY(&rwlock_local, &GC_allocate_ml, sizeof(GC_allocate_ml));
}
# else
if (pthread_rwlock_init(&GC_allocate_ml, NULL) != 0)
ABORT("pthread_rwlock_init failed (in child)");
# endif
# else
(void)pthread_mutex_destroy(&GC_allocate_ml);
/*
* TODO: Probably some targets (e.g. with `GLIBC_2_19_TSX_BUG`) might
* need the default mutex attribute to be passed instead of `NULL`.
*/
if (pthread_mutex_init(&GC_allocate_ml, NULL) != 0)
ABORT("pthread_mutex_init failed (in child)");
# endif
# endif
}
/*
* Routines for `fork()` handling by client (no-op if `pthread_atfork`
* works).
*/
GC_API void GC_CALL
GC_atfork_prepare(void)
{
if (UNLIKELY(!GC_is_initialized))
GC_init();
if (GC_handle_fork <= 0)
fork_prepare_proc();
}
GC_API void GC_CALL
GC_atfork_parent(void)
{
if (GC_handle_fork <= 0)
fork_parent_proc();
}
GC_API void GC_CALL
GC_atfork_child(void)
{
if (GC_handle_fork <= 0)
fork_child_proc();
}
GC_INNER_WIN32THREAD void
GC_setup_atfork(void)
{
if (GC_handle_fork) {
# ifdef CAN_CALL_ATFORK
if (pthread_atfork(fork_prepare_proc, fork_parent_proc, fork_child_proc)
== 0) {
/* Handlers successfully registered. */
GC_handle_fork = 1;
} else
# endif
/* else */ {
if (GC_handle_fork != -1)
ABORT("pthread_atfork failed");
}
}
}
# endif /* CAN_HANDLE_FORK */
# ifdef INCLUDE_LINUX_THREAD_DESCR
__thread int GC_dummy_thread_local;
# endif
# ifdef PARALLEL_MARK
# ifndef GC_WIN32_THREADS
static void setup_mark_lock(void);
# endif
GC_INNER_WIN32THREAD unsigned GC_required_markers_cnt = 0;
GC_API void GC_CALL
GC_set_markers_count(unsigned markers)
{
GC_required_markers_cnt = markers < MAX_MARKERS ? markers : MAX_MARKERS;
}
# endif /* PARALLEL_MARK */
GC_INNER GC_bool GC_in_thread_creation = FALSE;
GC_INNER_WIN32THREAD void
GC_record_stack_base(GC_stack_context_t crtn, const struct GC_stack_base *sb)
{
# if !defined(DARWIN) && !defined(GC_WIN32_THREADS)
crtn->stack_ptr = (ptr_t)sb->mem_base;
# endif
if ((crtn->stack_end = (ptr_t)sb->mem_base) == NULL)
ABORT("Bad stack base in GC_register_my_thread");
# ifdef E2K
crtn->ps_ofs = (size_t)(GC_uintptr_t)sb->reg_base;
# elif defined(IA64)
crtn->backing_store_end = (ptr_t)sb->reg_base;
# elif defined(I386) && defined(GC_WIN32_THREADS)
crtn->initial_stack_base = (ptr_t)sb->mem_base;
# endif
}
# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS) \
|| !defined(DONT_USE_ATEXIT)
GC_INNER_WIN32THREAD thread_id_t GC_main_thread_id;
# endif
# ifndef DONT_USE_ATEXIT
GC_INNER GC_bool
GC_is_main_thread(void)
{
GC_ASSERT(GC_thr_initialized);
return THREAD_ID_EQUAL(GC_main_thread_id, thread_id_self());
}
# endif /* !DONT_USE_ATEXIT */
# ifndef GC_WIN32_THREADS
STATIC GC_thread
GC_register_my_thread_inner(const struct GC_stack_base *sb,
thread_id_t self_id)
{
GC_thread me;
GC_ASSERT(I_HOLD_LOCK());
me = GC_new_thread(self_id);
me->id = self_id;
# ifdef DARWIN
me->mach_thread = mach_thread_self();
# endif
GC_record_stack_base(me->crtn, sb);
return me;
}
/*
* Number of processors. We may not have access to all of them, but
* this is as good a guess as any...
*/
STATIC int GC_nprocs = 1;
GC_INNER void
GC_thr_init(void)
{
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(!GC_thr_initialized);
GC_ASSERT(ADDR(&GC_threads) % ALIGNMENT == 0);
# ifdef GC_ASSERTIONS
GC_thr_initialized = TRUE;
# endif
# ifdef CAN_HANDLE_FORK
GC_setup_atfork();
# endif
# ifdef INCLUDE_LINUX_THREAD_DESCR
/*
* Explicitly register the region including the address of a thread-local
* variable. This should include thread locals for the main thread,
* except for those allocated in response to `dlopen()` calls.
*/
{
ptr_t thread_local_addr = (ptr_t)(&GC_dummy_thread_local);
ptr_t main_thread_start, main_thread_end;
if (!GC_enclosing_writable_mapping(thread_local_addr, &main_thread_start,
&main_thread_end)) {
ABORT("Failed to find TLS mapping for the primordial thread");
} else {
/* `main_thread_start` and `main_thread_end` are initialized. */
GC_add_roots_inner(main_thread_start, main_thread_end, FALSE);
}
}
# endif
/* Set `GC_nprocs` and `GC_available_markers_m1` variables. */
{
const char *nprocs_string = GETENV("GC_NPROCS");
GC_nprocs = -1;
if (nprocs_string != NULL)
GC_nprocs = atoi(nprocs_string);
}
if (GC_nprocs <= 0
# if defined(LINUX) && defined(ARM32)
/* Workaround for some Linux/arm kernels. */
&& (GC_nprocs = GC_get_nprocs_present()) <= 1
# endif
) {
GC_nprocs = GC_get_nprocs();
}
if (GC_nprocs <= 0) {
WARN("GC_get_nprocs() returned %" WARN_PRIdPTR "\n",
(GC_signed_word)GC_nprocs);
/* Assume a dual-core CPU. */
GC_nprocs = 2;
# ifdef PARALLEL_MARK
/* But use only one marker. */
GC_available_markers_m1 = 0;
# endif
} else {
# ifdef PARALLEL_MARK
{
const char *markers_string = GETENV("GC_MARKERS");
int markers = GC_required_markers_cnt;
if (markers_string != NULL) {
markers = atoi(markers_string);
if (markers <= 0 || markers > MAX_MARKERS) {
WARN("Too big or invalid number of mark threads: %" WARN_PRIdPTR
"; using maximum threads\n",
(GC_signed_word)markers);
markers = MAX_MARKERS;
}
} else if (0 == markers) {
/*
* Unless the client sets the desired number of parallel markers,
* it is determined based on the number of CPU cores.
*/
markers = GC_nprocs;
# if defined(GC_MIN_MARKERS) && !defined(CPPCHECK)
/* This is primarily for targets without `getenv()`. */
if (markers < GC_MIN_MARKERS)
markers = GC_MIN_MARKERS;
# endif
if (markers > MAX_MARKERS) {
/* Silently limit the amount of markers. */
markers = MAX_MARKERS;
}
}
GC_available_markers_m1 = markers - 1;
}
# endif
}
GC_COND_LOG_PRINTF("Number of processors: %d\n", GC_nprocs);
# if defined(BASE_ATOMIC_OPS_EMULATED) && defined(SIGNAL_BASED_STOP_WORLD)
/*
* Ensure the process is running on just one CPU core. This is needed
* because the AO primitives emulated with locks cannot be used inside
* signal handlers.
*/
{
cpu_set_t mask;
int cpu_set_cnt = 0;
int cpu_lowest_set = 0;
# ifdef RANDOM_ONE_CPU_CORE
int cpu_highest_set = 0;
# endif
/* Ensure at least 2 cores. */
int i = GC_nprocs > 1 ? GC_nprocs : 2;
if (sched_getaffinity(0 /* current process */, sizeof(mask), &mask) == -1)
ABORT_ARG1("sched_getaffinity failed", ": errno= %d", errno);
while (i-- > 0)
if (CPU_ISSET(i, &mask)) {
# ifdef RANDOM_ONE_CPU_CORE
if (i + 1 != cpu_lowest_set)
cpu_highest_set = i;
# endif
cpu_lowest_set = i;
cpu_set_cnt++;
}
if (0 == cpu_set_cnt)
ABORT("sched_getaffinity returned empty mask");
if (cpu_set_cnt > 1) {
# ifdef RANDOM_ONE_CPU_CORE
if (cpu_lowest_set < cpu_highest_set) {
/* Pseudo-randomly adjust the bit to set among valid ones. */
cpu_lowest_set
+= (unsigned)getpid() % (cpu_highest_set - cpu_lowest_set + 1);
}
# endif
CPU_ZERO(&mask);
/* Select just one CPU. */
CPU_SET(cpu_lowest_set, &mask);
if (sched_setaffinity(0, sizeof(mask), &mask) == -1)
ABORT_ARG1("sched_setaffinity failed", ": errno= %d", errno);
WARN("CPU affinity mask is set to %p\n", (word)1 << cpu_lowest_set);
}
}
# endif /* BASE_ATOMIC_OPS_EMULATED */
# ifndef DARWIN
GC_stop_init();
# endif
# ifdef PARALLEL_MARK
if (GC_available_markers_m1 <= 0) {
/* Disable parallel marking. */
GC_parallel = FALSE;
GC_COND_LOG_PRINTF("Single marker thread, turning off parallel marking\n");
} else {
setup_mark_lock();
}
# endif
/* Add the initial thread, so we can stop it. */
{
struct GC_stack_base sb;
GC_thread me;
thread_id_t self_id = thread_id_self();
sb.mem_base = GC_stackbottom;
GC_ASSERT(sb.mem_base != NULL);
# if defined(E2K) || defined(IA64)
sb.reg_base = GC_register_stackbottom;
# endif
GC_ASSERT(NULL == GC_self_thread_inner());
me = GC_register_my_thread_inner(&sb, self_id);
# ifndef DONT_USE_ATEXIT
GC_main_thread_id = self_id;
# endif
me->flags = DETACHED;
}
}
# endif /* !GC_WIN32_THREADS */
GC_INNER void
GC_init_parallel(void)
{
# ifdef THREAD_LOCAL_ALLOC
GC_thread me;
GC_ASSERT(GC_is_initialized);
LOCK();
me = GC_self_thread_inner();
GC_init_thread_local(&me->tlfs);
UNLOCK();
# endif
# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
if (GC_win32_dll_threads) {
/*
* Cannot intercept thread creation. Hence we do not know if other
* threads exist. However, client is not allowed to create other threads
* before collector initialization. Thus it is OK not to lock before
* this.
*/
set_need_to_lock();
}
# endif
}
# if !defined(GC_NO_PTHREAD_SIGMASK) && defined(GC_PTHREADS)
# define GC_wrap_pthread_sigmask WRAP_FUNC(pthread_sigmask)
GC_API int
GC_wrap_pthread_sigmask(int how, const sigset_t *set, sigset_t *oset)
{
# ifdef GC_WIN32_THREADS
/*
* `pthreads-win32` library does not support `sigmask`.
* So, nothing is required here...
*/
# else
sigset_t fudged_set;
INIT_REAL_SYMS();
if (LIKELY(set != NULL) && (how == SIG_BLOCK || how == SIG_SETMASK)) {
int sig_suspend = GC_get_suspend_signal();
fudged_set = *set;
GC_ASSERT(sig_suspend >= 0);
if (sigdelset(&fudged_set, sig_suspend) != 0)
ABORT("sigdelset failed");
set = &fudged_set;
}
# endif
return REAL_FUNC(pthread_sigmask)(how, set, oset);
}
# undef GC_wrap_pthread_sigmask
# endif /* !GC_NO_PTHREAD_SIGMASK */
/*
* Wrapper for functions that are likely to block for an appreciable
* length of time.
*/
# ifdef E2K
/*
* Cannot be defined as a function because the stack-allocated buffer
* (pointed to by `bs_lo`) should be preserved till completion of
* `GC_do_blocking_inner` (or `GC_suspend_self_blocked`).
*/
# define do_blocking_enter(pTopOfStackUnset, me) \
do { \
ptr_t bs_lo; \
size_t stack_size; \
GC_stack_context_t crtn = (me)->crtn; \
\
*(pTopOfStackUnset) = FALSE; \
crtn->stack_ptr = GC_approx_sp(); \
GC_ASSERT(NULL == crtn->backing_store_end); \
GET_PROCEDURE_STACK_LOCAL(crtn->ps_ofs, &bs_lo, &stack_size); \
crtn->backing_store_end = bs_lo; \
crtn->backing_store_ptr = bs_lo + stack_size; \
(me)->flags |= DO_BLOCKING; \
} while (0)
# else /* !E2K */
static void
do_blocking_enter(GC_bool *pTopOfStackUnset, GC_thread me)
{
# if defined(SPARC) || defined(IA64)
ptr_t bs_hi = GC_save_regs_in_stack();
/* TODO: Registers saving already done by `GC_with_callee_saves_pushed`. */
# endif
GC_stack_context_t crtn = me->crtn;
GC_ASSERT(I_HOLD_READER_LOCK());
GC_ASSERT((me->flags & DO_BLOCKING) == 0);
*pTopOfStackUnset = FALSE;
# ifdef SPARC
crtn->stack_ptr = bs_hi;
# else
crtn->stack_ptr = GC_approx_sp();
# endif
# if defined(DARWIN) && !defined(DARWIN_DONT_PARSE_STACK)
if (NULL == crtn->topOfStack) {
/*
* `GC_do_blocking_inner` is not called recursively, so `topOfStack`
* should be computed now.
*/
*pTopOfStackUnset = TRUE;
crtn->topOfStack = GC_FindTopOfStack(0);
}
# endif
# ifdef IA64
crtn->backing_store_ptr = bs_hi;
# endif
me->flags |= DO_BLOCKING;
/* Save context here if we want to support precise stack marking. */
}
# endif /* !E2K */
static void
do_blocking_leave(GC_thread me, GC_bool topOfStackUnset)
{
GC_ASSERT(I_HOLD_READER_LOCK());
me->flags &= (unsigned char)~DO_BLOCKING;
# ifdef E2K
{
GC_stack_context_t crtn = me->crtn;
GC_ASSERT(crtn->backing_store_end != NULL);
crtn->backing_store_ptr = NULL;
crtn->backing_store_end = NULL;
}
# endif
# if defined(DARWIN) && !defined(DARWIN_DONT_PARSE_STACK)
if (topOfStackUnset) {
/* Make it unset again. */
me->crtn->topOfStack = NULL;
}
# else
(void)topOfStackUnset;
# endif
}
GC_INNER void
GC_do_blocking_inner(ptr_t data, void *context)
{
GC_thread me;
GC_bool topOfStackUnset;
UNUSED_ARG(context);
READER_LOCK();
me = GC_self_thread_inner();
do_blocking_enter(&topOfStackUnset, me);
READER_UNLOCK_RELEASE();
((struct blocking_data *)data)->client_data /*< result */
= ((struct blocking_data *)data)
->fn(((struct blocking_data *)data)->client_data);
/* This will block if the world is stopped. */
READER_LOCK();
# ifdef LINT2
{
# ifdef GC_ASSERTIONS
GC_thread saved_me = me;
# endif
/*
* The pointer to the GC thread descriptor should not be changed while
* the thread is registered but a static analysis tool might complain
* that this pointer value (obtained in the first locked section) is
* unreliable in the second locked section.
*/
me = GC_self_thread_inner();
GC_ASSERT(me == saved_me);
}
# endif
# if defined(GC_ENABLE_SUSPEND_THREAD) && defined(SIGNAL_BASED_STOP_WORLD)
/*
* Note: this code cannot be moved into `do_blocking_leave()` otherwise
* there could be a static analysis tool warning (false positive) about
* unlock without a matching lock.
*/
while (UNLIKELY((me->ext_suspend_cnt & 1) != 0)) {
/* Read suspend counter (number) before unlocking. */
size_t suspend_cnt = me->ext_suspend_cnt;
READER_UNLOCK_RELEASE();
GC_suspend_self_inner(me, suspend_cnt);
READER_LOCK();
}
# endif
do_blocking_leave(me, topOfStackUnset);
READER_UNLOCK_RELEASE();
}
# if defined(GC_ENABLE_SUSPEND_THREAD) && defined(SIGNAL_BASED_STOP_WORLD)
GC_INNER void
GC_suspend_self_blocked(ptr_t thread_me, void *context)
{
GC_thread me = (GC_thread)thread_me;
GC_bool topOfStackUnset;
UNUSED_ARG(context);
/*
* The caller holds the allocator lock in the exclusive mode, thus
* we require and restore it to the same mode upon return from the
* function.
*/
GC_ASSERT(I_HOLD_LOCK());
do_blocking_enter(&topOfStackUnset, me);
while ((me->ext_suspend_cnt & 1) != 0) {
size_t suspend_cnt = me->ext_suspend_cnt;
UNLOCK();
GC_suspend_self_inner(me, suspend_cnt);
LOCK();
}
do_blocking_leave(me, topOfStackUnset);
}
# endif /* GC_ENABLE_SUSPEND_THREAD */
GC_API void GC_CALL
GC_set_stackbottom(void *gc_thread_handle, const struct GC_stack_base *sb)
{
GC_thread t = (GC_thread)gc_thread_handle;
GC_stack_context_t crtn;
GC_ASSERT(sb->mem_base != NULL);
if (UNLIKELY(!GC_is_initialized)) {
GC_ASSERT(NULL == t);
/* Alter the stack bottom of the primordial thread. */
GC_stackbottom = (char *)sb->mem_base;
# if defined(E2K) || defined(IA64)
GC_register_stackbottom = (ptr_t)sb->reg_base;
# endif
return;
}
GC_ASSERT(I_HOLD_READER_LOCK());
if (NULL == t) {
/* The current thread. */
t = GC_self_thread_inner();
}
GC_ASSERT(!KNOWN_FINISHED(t));
crtn = t->crtn;
GC_ASSERT((t->flags & DO_BLOCKING) == 0
&& NULL == crtn->traced_stack_sect); /*< for now */
crtn->stack_end = (ptr_t)sb->mem_base;
# ifdef E2K
crtn->ps_ofs = (size_t)(GC_uintptr_t)sb->reg_base;
# elif defined(IA64)
crtn->backing_store_end = (ptr_t)sb->reg_base;
# endif
# ifdef GC_WIN32_THREADS
/* Reset the known minimum (hottest address in the stack). */
crtn->last_stack_min = ADDR_LIMIT;
# endif
}
GC_API void *GC_CALL
GC_get_my_stackbottom(struct GC_stack_base *sb)
{
GC_thread me;
GC_stack_context_t crtn;
READER_LOCK();
me = GC_self_thread_inner();
/* The thread is assumed to be registered. */
crtn = me->crtn;
sb->mem_base = crtn->stack_end;
# ifdef E2K
/* Store the offset in the procedure stack, not address. */
sb->reg_base = NUMERIC_TO_VPTR(crtn->ps_ofs);
# elif defined(IA64)
sb->reg_base = crtn->backing_store_end;
# endif
READER_UNLOCK();
return me; /*< `gc_thread_handle` */
}
GC_ATTR_NOINLINE
GC_API void *GC_CALL
GC_call_with_gc_active(GC_fn_type fn, void *client_data)
{
struct GC_traced_stack_sect_s stacksect;
GC_thread me;
GC_stack_context_t crtn;
ptr_t stack_end;
# ifdef E2K
ptr_t saved_bs_ptr, saved_bs_end;
size_t saved_ps_ofs;
# endif
/* This will block if the world is stopped. */
READER_LOCK();
me = GC_self_thread_inner();
crtn = me->crtn;
/*
* Adjust our stack bottom value (this could happen unless
* `GC_get_stack_base()` was used which returned `GC_SUCCESS`).
*/
stack_end = crtn->stack_end; /*< read of a `volatile` field */
GC_ASSERT(stack_end != NULL);
STORE_APPROX_SP_TO(*(volatile ptr_t *)&stacksect.saved_stack_ptr);
if (HOTTER_THAN(stack_end, stacksect.saved_stack_ptr)) {
crtn->stack_end = stacksect.saved_stack_ptr;
# if defined(I386) && defined(GC_WIN32_THREADS)
crtn->initial_stack_base = stacksect.saved_stack_ptr;
# endif
}
if ((me->flags & DO_BLOCKING) == 0) {
/* We are not inside `GC_do_blocking()` - do nothing more. */
READER_UNLOCK_RELEASE();
/* Cast `fn` to a `volatile` type to prevent its call inlining. */
client_data = (*(GC_fn_type volatile *)&fn)(client_data);
/* Prevent treating the above as a tail call. */
GC_noop1(COVERT_DATAFLOW(ADDR(&stacksect)));
return client_data; /*< result */
}
# if defined(GC_ENABLE_SUSPEND_THREAD) && defined(SIGNAL_BASED_STOP_WORLD)
while (UNLIKELY((me->ext_suspend_cnt & 1) != 0)) {
size_t suspend_cnt = me->ext_suspend_cnt;
READER_UNLOCK_RELEASE();
GC_suspend_self_inner(me, suspend_cnt);
READER_LOCK();
GC_ASSERT(me->crtn == crtn);
}
# endif
/* Setup new "stack section". */
stacksect.saved_stack_ptr = crtn->stack_ptr;
# ifdef E2K
GC_ASSERT(crtn->backing_store_end != NULL);
{
unsigned long long sz_ull;
GET_PROCEDURE_STACK_SIZE_INNER(&sz_ull);
saved_ps_ofs = crtn->ps_ofs;
GC_ASSERT(saved_ps_ofs <= (size_t)sz_ull);
crtn->ps_ofs = (size_t)sz_ull;
}
saved_bs_end = crtn->backing_store_end;
saved_bs_ptr = crtn->backing_store_ptr;
crtn->backing_store_ptr = NULL;
crtn->backing_store_end = NULL;
# elif defined(IA64)
/* This is the same as in `GC_call_with_stack_base()`. */
stacksect.backing_store_end = GC_save_regs_in_stack();
/*
* Unnecessarily flushes the register stack, but that probably does
* not hurt.
*/
stacksect.saved_backing_store_ptr = crtn->backing_store_ptr;
# endif
stacksect.prev = crtn->traced_stack_sect;
me->flags &= (unsigned char)~DO_BLOCKING;
crtn->traced_stack_sect = &stacksect;
READER_UNLOCK_RELEASE();
client_data = (*(GC_fn_type volatile *)&fn)(client_data);
GC_ASSERT((me->flags & DO_BLOCKING) == 0);
/* Restore original "stack section". */
READER_LOCK();
GC_ASSERT(me->crtn == crtn);
GC_ASSERT(crtn->traced_stack_sect == &stacksect);
# ifdef CPPCHECK
GC_noop1_ptr(crtn->traced_stack_sect);
# endif
crtn->traced_stack_sect = stacksect.prev;
# ifdef E2K
GC_ASSERT(NULL == crtn->backing_store_end);
crtn->backing_store_end = saved_bs_end;
crtn->backing_store_ptr = saved_bs_ptr;
crtn->ps_ofs = saved_ps_ofs;
# elif defined(IA64)
crtn->backing_store_ptr = stacksect.saved_backing_store_ptr;
# endif
me->flags |= DO_BLOCKING;
crtn->stack_ptr = stacksect.saved_stack_ptr;
READER_UNLOCK_RELEASE();
return client_data; /*< result */
}
STATIC void
GC_unregister_my_thread_inner(GC_thread me)
{
GC_ASSERT(I_HOLD_LOCK());
# ifdef DEBUG_THREADS
GC_log_printf("Unregistering thread %p, gc_thread= %p, n_threads= %d\n",
THREAD_ID_TO_VPTR(me->id), (void *)me, GC_count_threads());
# endif
GC_ASSERT(!KNOWN_FINISHED(me));
# if defined(THREAD_LOCAL_ALLOC)
GC_destroy_thread_local(&me->tlfs);
# endif
# ifdef NACL
GC_nacl_shutdown_gc_thread();
# endif
# ifdef GC_PTHREADS
# if defined(GC_HAVE_PTHREAD_EXIT) || !defined(GC_NO_PTHREAD_CANCEL)
/*
* Handle `DISABLED_GC` flag which is set by the intercepted
* `pthread_cancel()` or `pthread_exit()`.
*/
if ((me->flags & DISABLED_GC) != 0) {
GC_dont_gc--;
}
# endif
if ((me->flags & DETACHED) == 0) {
me->flags |= FINISHED;
} else
# endif
/* else */ {
GC_delete_thread(me);
}
# if defined(THREAD_LOCAL_ALLOC)
/*
* It is required to call `GC_remove_specific()` defined in
* `specific.c` file.
*/
GC_remove_specific(GC_thread_key);
# endif
}
GC_API int GC_CALL
GC_unregister_my_thread(void)
{
GC_thread me;
IF_CANCEL(int cancel_state;)
/*
* Client should not unregister the thread explicitly
* if it is registered by `DllMain`, except for the main thread.
*/
# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
GC_ASSERT(!GC_win32_dll_threads
|| THREAD_ID_EQUAL(GC_main_thread_id, thread_id_self()));
# endif
LOCK();
DISABLE_CANCEL(cancel_state);
/*
* Wait for any collection that may be marking from our stack to complete
* before we remove this thread.
*/
GC_wait_for_gc_completion(FALSE);
me = GC_self_thread_inner();
GC_ASSERT(THREAD_ID_EQUAL(me->id, thread_id_self()));
GC_unregister_my_thread_inner(me);
RESTORE_CANCEL(cancel_state);
UNLOCK();
return GC_SUCCESS;
}
# if !defined(GC_NO_PTHREAD_CANCEL) && defined(GC_PTHREADS)
/*
* We should deal with the fact that apparently on Solaris and, probably,
* on some Linux we cannot collect while a thread is exiting, since
* signals are not handled properly. This currently gives rise to deadlocks.
* The only workaround seen is to intercept `pthread_cancel()` and
* `pthread_exit()`, and disable the collections until the thread exit
* handler is called. That is ugly, because we risk growing the heap
* unnecessarily. But it seems that we do not really have an option in that
* the process is not in a fully functional state while a thread is exiting.
*/
# define GC_wrap_pthread_cancel WRAP_FUNC(pthread_cancel)
GC_API int
GC_wrap_pthread_cancel(pthread_t thread)
{
# ifdef CANCEL_SAFE
GC_thread t;
# endif
INIT_REAL_SYMS();
# ifdef CANCEL_SAFE
LOCK();
t = GC_lookup_by_pthread(thread);
/*
* We test `DISABLED_GC` because `pthread_exit` could be called at
* the same time. (If `t` is `NULL`, then `pthread_cancel()` should
* return `ESRCH`.)
*/
if (t != NULL && (t->flags & DISABLED_GC) == 0) {
t->flags |= DISABLED_GC;
GC_dont_gc++;
}
UNLOCK();
# endif
return REAL_FUNC(pthread_cancel)(thread);
}
# undef GC_wrap_pthread_cancel
# endif /* !GC_NO_PTHREAD_CANCEL */
# ifdef GC_HAVE_PTHREAD_EXIT
# define GC_wrap_pthread_exit WRAP_FUNC(pthread_exit)
GC_API GC_PTHREAD_EXIT_ATTRIBUTE void
GC_wrap_pthread_exit(void *retval)
{
GC_thread me;
INIT_REAL_SYMS();
LOCK();
me = GC_self_thread_inner();
/*
* We test `DISABLED_GC` because someone else could call `pthread_cancel()`
* at the same time.
*/
if (me != NULL && (me->flags & DISABLED_GC) == 0) {
me->flags |= DISABLED_GC;
GC_dont_gc++;
}
UNLOCK();
REAL_FUNC(pthread_exit)(retval);
}
# undef GC_wrap_pthread_exit
# endif /* GC_HAVE_PTHREAD_EXIT */
GC_API void GC_CALL
GC_allow_register_threads(void)
{
/*
* Check the collector is initialized and the current thread is
* registered.
*/
GC_ASSERT(GC_self_thread() != NULL);
/* Initialize symbols while still single-threaded. */
INIT_REAL_SYMS();
GC_init_lib_bounds();
GC_start_mark_threads();
set_need_to_lock();
}
# if defined(PTHREAD_STOP_WORLD_IMPL) \
&& !defined(NO_SIGNALS_UNBLOCK_IN_MAIN) \
|| defined(GC_EXPLICIT_SIGNALS_UNBLOCK)
GC_INNER void
GC_unblock_gc_signals(void)
{
sigset_t set;
/* This is for `pthread_sigmask`. */
INIT_REAL_SYMS();
sigemptyset(&set);
sigaddset(&set, GC_get_suspend_signal());
sigaddset(&set, GC_get_thr_restart_signal());
if (REAL_FUNC(pthread_sigmask)(SIG_UNBLOCK, &set, NULL) != 0)
ABORT("pthread_sigmask failed");
}
# endif /* PTHREAD_STOP_WORLD_IMPL || GC_EXPLICIT_SIGNALS_UNBLOCK */
GC_API int GC_CALL
GC_register_my_thread(const struct GC_stack_base *sb)
{
GC_thread me;
if (!GC_need_to_lock)
ABORT("Threads explicit registering is not previously enabled");
/* We lock here, since we want to wait for an ongoing GC. */
LOCK();
me = GC_self_thread_inner();
if (LIKELY(NULL == me)) {
me = GC_register_my_thread_inner(sb, thread_id_self());
# ifdef GC_PTHREADS
# ifdef CPPCHECK
GC_noop1(me->flags);
# endif
/*
* Treat as detached, since we do not need to worry about pointer
* results.
*/
me->flags |= DETACHED;
# else
(void)me;
# endif
} else {
# ifdef GC_PTHREADS
if (KNOWN_FINISHED(me)) {
/*
* This code is executed when a thread is registered from the destructor
* of the client thread key.
*/
# ifdef NACL
GC_nacl_initialize_gc_thread(me);
# endif
# ifdef DARWIN
/*
* Reinitialize `mach_thread` to avoid `thread_suspend()` fail
* with `MACH_SEND_INVALID_DEST` error.
*/
me->mach_thread = mach_thread_self();
# endif
GC_record_stack_base(me->crtn, sb);
me->flags &= (unsigned char)~FINISHED; /*< but not `DETACHED` */
} else
# endif
/* else */ {
UNLOCK();
return GC_DUPLICATE;
}
}
# ifdef THREAD_LOCAL_ALLOC
GC_init_thread_local(&me->tlfs);
# endif
# ifdef GC_EXPLICIT_SIGNALS_UNBLOCK
/*
* Since this could be executed from a thread destructor,
* our signals might already be blocked.
*/
GC_unblock_gc_signals();
# endif
# if defined(GC_ENABLE_SUSPEND_THREAD) && defined(SIGNAL_BASED_STOP_WORLD)
if (UNLIKELY((me->ext_suspend_cnt & 1) != 0)) {
GC_with_callee_saves_pushed(GC_suspend_self_blocked, (ptr_t)me);
}
# endif
UNLOCK();
return GC_SUCCESS;
}
# if defined(GC_PTHREADS) && !defined(PLATFORM_THREADS) \
&& !defined(SN_TARGET_PSP2)
GC_INNER_PTHRSTART void
GC_thread_exit_proc(void *arg)
{
GC_thread me = (GC_thread)arg;
IF_CANCEL(int cancel_state;)
# ifdef DEBUG_THREADS
GC_log_printf("Called GC_thread_exit_proc on %p, gc_thread= %p\n",
THREAD_ID_TO_VPTR(me->id), (void *)me);
# endif
LOCK();
DISABLE_CANCEL(cancel_state);
GC_wait_for_gc_completion(FALSE);
GC_unregister_my_thread_inner(me);
RESTORE_CANCEL(cancel_state);
UNLOCK();
}
# define GC_wrap_pthread_join WRAP_FUNC(pthread_join)
GC_API int
GC_wrap_pthread_join(pthread_t thread, void **retval)
{
int result;
GC_thread t;
INIT_REAL_SYMS();
# ifdef DEBUG_THREADS
GC_log_printf("thread %p is joining thread %p\n",
PTHREAD_TO_VPTR(pthread_self()), PTHREAD_TO_VPTR(thread));
# endif
/* After the join, thread id may have been recycled. */
READER_LOCK();
t = (GC_thread)COVERT_DATAFLOW_P(GC_lookup_by_pthread(thread));
/*
* This is guaranteed to be the intended one, since the thread id
* cannot have been recycled by `pthreads`.
*/
READER_UNLOCK();
result = REAL_FUNC(pthread_join)(thread, retval);
# ifdef FREEBSD
/*
* On FreeBSD, the wrapped `pthread_join()` sometimes returns (what
* appears to be) a spurious `EINTR` which caused the test and real code
* to fail gratuitously. Having looked at system `pthreads` library
* source code, I see how such return code value may be generated:
* in one path of the code, `pthread_join()` just returns the `errno`
* setting of the thread being joined - this does not match the POSIX
* specification or the local man pages. Thus, I have taken the liberty
* to catch this one spurious return value.
*/
if (UNLIKELY(result == EINTR))
result = 0;
# endif
if (LIKELY(0 == result)) {
LOCK();
/*
* Here the `pthread_id` may have been recycled. Delete the thread from
*`GC_threads` (unless it has been registered again from the destructor
* of the client thread key).
*/
if (KNOWN_FINISHED(t)) {
GC_delete_thread(t);
}
UNLOCK();
}
# ifdef DEBUG_THREADS
GC_log_printf("thread %p join with thread %p %s\n",
PTHREAD_TO_VPTR(pthread_self()), PTHREAD_TO_VPTR(thread),
result != 0 ? "failed" : "succeeded");
# endif
return result;
}
# undef GC_wrap_pthread_join
# define GC_wrap_pthread_detach WRAP_FUNC(pthread_detach)
GC_API int
GC_wrap_pthread_detach(pthread_t thread)
{
int result;
GC_thread t;
INIT_REAL_SYMS();
READER_LOCK();
t = (GC_thread)COVERT_DATAFLOW_P(GC_lookup_by_pthread(thread));
READER_UNLOCK();
result = REAL_FUNC(pthread_detach)(thread);
if (LIKELY(0 == result)) {
LOCK();
/* Here the `pthread_id` may have been recycled. */
if (KNOWN_FINISHED(t)) {
GC_delete_thread(t);
} else {
t->flags |= DETACHED;
}
UNLOCK();
}
return result;
}
# undef GC_wrap_pthread_detach
struct start_info {
void *(*start_routine)(void *);
void *arg;
/*
* Note: a value of 1 means the thread is in our thread table, but
* the parent process has not noticed it yet.
*/
sem_t registered;
unsigned char flags;
};
GC_INNER_PTHRSTART GC_thread
GC_start_rtn_prepare_thread(void *(**pstart)(void *), void **pstart_arg,
struct GC_stack_base *sb, void *arg)
{
struct start_info *psi = (struct start_info *)arg;
thread_id_t self_id = thread_id_self();
GC_thread me;
# ifdef DEBUG_THREADS
GC_log_printf("Starting thread %p, sp= %p\n",
PTHREAD_TO_VPTR(pthread_self()), (void *)GC_approx_sp());
# endif
/*
* If a collection occurs before the thread is registered, that collection
* will ignore this thread. That is fine, since it will block trying to
* acquire the allocator lock, and will not yet hold interesting pointers.
*/
LOCK();
/*
* We register the thread here instead of in the parent process, so that
* we do not need to hold the allocator lock during `pthread_create()`
* call.
*/
me = GC_register_my_thread_inner(sb, self_id);
GC_ASSERT(me != &first_thread);
me->flags = psi->flags;
# ifdef GC_WIN32_THREADS
GC_win32_cache_self_pthread(self_id);
# endif
# ifdef THREAD_LOCAL_ALLOC
GC_init_thread_local(&me->tlfs);
# endif
UNLOCK();
*pstart = psi->start_routine;
*pstart_arg = psi->arg;
# if defined(DEBUG_THREADS) && defined(FUNCPTR_IS_DATAPTR)
GC_log_printf("start_routine= %p\n", CAST_THRU_UINTPTR(void *, *pstart));
# endif
sem_post(&psi->registered);
/* This was the last action on `*psi`; OK to deallocate. */
return me;
}
STATIC void *
GC_pthread_start(void *arg)
{
# ifdef INCLUDE_LINUX_THREAD_DESCR
struct GC_stack_base sb;
# ifdef REDIRECT_MALLOC
/*
* `GC_get_stack_base()` may call `pthread_getattr_np()`, which
* can unfortunately call `realloc()`, which may allocate from
* an unregistered thread. This is unpleasant, since it might
* force heap growth (or, even, heap overflow).
*/
GC_disable();
# endif
if (GC_get_stack_base(&sb) != GC_SUCCESS)
ABORT("Failed to get thread stack base");
# ifdef REDIRECT_MALLOC
GC_enable();
# endif
return GC_pthread_start_inner(&sb, arg);
# else
return GC_call_with_stack_base(GC_pthread_start_inner, arg);
# endif
}
# define GC_wrap_pthread_create WRAP_FUNC(pthread_create)
GC_API int
GC_wrap_pthread_create(pthread_t *new_thread,
GC_PTHREAD_CREATE_CONST pthread_attr_t *attr,
void *(*start_routine)(void *), void *arg)
{
int result;
struct start_info si;
GC_ASSERT(I_DONT_HOLD_LOCK());
INIT_REAL_SYMS();
if (UNLIKELY(!GC_is_initialized))
GC_init();
GC_ASSERT(GC_thr_initialized);
GC_init_lib_bounds();
if (sem_init(&si.registered, GC_SEM_INIT_PSHARED, 0) == -1)
ABORT("sem_init failed");
si.flags = 0;
si.start_routine = start_routine;
si.arg = arg;
/*
* We resist the temptation to muck with the stack size here, even if the
* default is unreasonably small. That is the client's responsibility.
*/
# ifdef GC_ASSERTIONS
{
size_t stack_size = 0;
if (NULL != attr) {
if (pthread_attr_getstacksize(attr, &stack_size) != 0)
ABORT("pthread_attr_getstacksize failed");
}
if (0 == stack_size) {
pthread_attr_t my_attr;
if (pthread_attr_init(&my_attr) != 0)
ABORT("pthread_attr_init failed");
if (pthread_attr_getstacksize(&my_attr, &stack_size) != 0)
ABORT("pthread_attr_getstacksize failed");
(void)pthread_attr_destroy(&my_attr);
}
/*
* On Solaris 10 and on Win32 with `winpthreads` library, with the
* default `attr` initialization, `stack_size` remains 0; fudge it.
*/
if (UNLIKELY(0 == stack_size)) {
# if !defined(SOLARIS) && !defined(GC_WIN32_PTHREADS)
WARN("Failed to get stack size for assertion checking\n", 0);
# endif
stack_size = 1000000;
}
GC_ASSERT(stack_size >= 65536);
/*
* Our threads may need to do some work for the GC.
* Ridiculously small threads will not work, and they
* probably would not work anyway.
*/
}
# endif
if (attr != NULL) {
int detachstate;
if (pthread_attr_getdetachstate(attr, &detachstate) != 0)
ABORT("pthread_attr_getdetachstate failed");
if (PTHREAD_CREATE_DETACHED == detachstate)
si.flags |= DETACHED;
}
# ifdef PARALLEL_MARK
if (!GC_parallel && UNLIKELY(GC_available_markers_m1 > 0))
GC_start_mark_threads();
# endif
# ifdef DEBUG_THREADS
GC_log_printf("About to start new thread from thread %p\n",
PTHREAD_TO_VPTR(pthread_self()));
# endif
set_need_to_lock();
result = REAL_FUNC(pthread_create)(new_thread, attr, GC_pthread_start, &si);
/*
* Wait until child has been added to the thread table.
* This also ensures that we hold onto the stack-allocated `si`
* until the child is done with it.
*/
if (LIKELY(0 == result)) {
IF_CANCEL(int cancel_state;)
/* `pthread_create()` is not a cancellation point. */
DISABLE_CANCEL(cancel_state);
while (sem_wait(&si.registered) == -1) {
# ifdef HAIKU
/* To workaround some bug in Haiku semaphores. */
if (EACCES == errno)
continue;
# endif
if (errno != EINTR)
ABORT("sem_wait failed");
}
RESTORE_CANCEL(cancel_state);
}
sem_destroy(&si.registered);
return result;
}
# undef GC_wrap_pthread_create
# endif /* GC_PTHREADS && !PLATFORM_THREADS && !SN_TARGET_PSP2 */
# if ((defined(GC_PTHREADS_PARAMARK) || defined(USE_PTHREAD_LOCKS)) \
&& !defined(NO_PTHREAD_TRYLOCK)) \
|| defined(USE_SPIN_LOCK)
/*
* Spend a few cycles in a way that cannot introduce contention with
* other threads.
*/
# define GC_PAUSE_SPIN_CYCLES 10
STATIC void
GC_pause(void)
{
int i;
for (i = 0; i < GC_PAUSE_SPIN_CYCLES; ++i) {
/* Something that is unlikely to be optimized away. */
# if defined(AO_HAVE_compiler_barrier) && !defined(BASE_ATOMIC_OPS_EMULATED)
AO_compiler_barrier();
# else
GC_noop1(i);
# endif
}
}
# endif /* USE_SPIN_LOCK || !NO_PTHREAD_TRYLOCK */
# ifndef SPIN_MAX
/* Maximum number of calls to `GC_pause()` before give up. */
# define SPIN_MAX 128
# endif
# if (!defined(USE_SPIN_LOCK) && !defined(NO_PTHREAD_TRYLOCK) \
&& defined(USE_PTHREAD_LOCKS)) \
|| defined(GC_PTHREADS_PARAMARK)
/*
* If we do not want to use the below spinlock implementation, either
* because we do not have a `GC_test_and_set` implementation, or because
* we do not want to risk sleeping, we can still try spinning
* on `pthread_mutex_trylock` for a while. This appears to be very
* beneficial in many cases.
* I suspect that under high contention this is nearly always better
* than the spin lock. But it is a bit slower on a uniprocessor.
* Hence we still default to the spin lock. This is also used to
* acquire the mark lock for the parallel marker.
*
* Here we use a strict exponential back-off scheme. I do not know
* whether that is better or worse than the above. We eventually
* yield by calling `pthread_mutex_lock()`; it never makes sense to
* explicitly sleep.
*/
# ifdef LOCK_STATS
/* Note that `LOCK_STATS` requires `AO_HAVE_test_and_set`. */
volatile AO_t GC_spin_count = 0;
volatile AO_t GC_block_count = 0;
volatile AO_t GC_unlocked_count = 0;
# endif
STATIC void
GC_generic_lock(pthread_mutex_t *lock)
{
# ifndef NO_PTHREAD_TRYLOCK
unsigned pause_length = 1;
unsigned i;
if (LIKELY(0 == pthread_mutex_trylock(lock))) {
# ifdef LOCK_STATS
(void)AO_fetch_and_add1(&GC_unlocked_count);
# endif
return;
}
for (; pause_length <= (unsigned)SPIN_MAX; pause_length <<= 1) {
for (i = 0; i < pause_length; ++i) {
GC_pause();
}
switch (pthread_mutex_trylock(lock)) {
case 0:
# ifdef LOCK_STATS
(void)AO_fetch_and_add1(&GC_spin_count);
# endif
return;
case EBUSY:
break;
default:
ABORT("Unexpected error from pthread_mutex_trylock");
}
}
# endif /* !NO_PTHREAD_TRYLOCK */
# ifdef LOCK_STATS
(void)AO_fetch_and_add1(&GC_block_count);
# endif
pthread_mutex_lock(lock);
}
# endif /* !USE_SPIN_LOCK || ... */
# if defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS)
GC_INNER volatile unsigned char GC_collecting = FALSE;
# if defined(AO_HAVE_char_load) && !defined(BASE_ATOMIC_OPS_EMULATED)
# define is_collecting() ((GC_bool)AO_char_load(&GC_collecting))
# else
/*
* `GC_collecting` is a hint, a potential data race between `GC_lock()`
* and `ENTER_GC()`/`EXIT_GC()` is OK to ignore.
*/
# define is_collecting() ((GC_bool)GC_collecting)
# endif
# endif /* GC_PTHREADS && !GC_WIN32_THREADS */
# ifdef GC_ASSERTIONS
GC_INNER unsigned long GC_lock_holder = NO_THREAD;
# endif
# if defined(USE_SPIN_LOCK)
/*
* Reasonably fast spin locks. Basically the same implementation as
* in STL `alloc.h` file. This is not really the right way to do this
* but until the POSIX scheduling mess gets straightened out...
*/
/* Spin cycles if we suspect we are running on an uniprocessor. */
# define low_spin_max 30
/* Spin cycles for a multiprocessor. */
# define high_spin_max SPIN_MAX
static volatile AO_t spin_max = low_spin_max;
/*
* A potential data race between threads invoking `GC_lock` which reads
* and updates `spin_max` and `last_spins` could be ignored because these
* variables are hints only.
*/
static volatile AO_t last_spins = 0;
GC_INNER void
GC_lock(void)
{
AO_t my_spin_max, my_last_spins_half;
size_t i;
if (LIKELY(AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR))
return;
my_spin_max = AO_load(&spin_max);
my_last_spins_half = AO_load(&last_spins) / 2;
for (i = 0; i < my_spin_max; i++) {
if (is_collecting() || GC_nprocs == 1)
goto yield;
if (i < my_last_spins_half) {
GC_pause();
continue;
}
if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
/*
* Got it, spinning worked! Thus we are probably not being
* scheduled against the other process with which we were
* contending. Thus it makes sense to spin longer the next time.
*/
AO_store(&last_spins, i);
AO_store(&spin_max, high_spin_max);
return;
}
}
/* We are probably being scheduled against the other process. Sleep. */
AO_store(&spin_max, low_spin_max);
yield:
for (i = 0;; ++i) {
if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
return;
}
/*
* Under Linux, very short sleeps tend to wait until the current time
* quantum expires. On old Linux kernels, `nanosleep()` (<= 2 ms) just
* spins. (Under Linux 2.4, this happens only for real-time processes.)
* We want to minimize both behaviors here.
*/
# define SLEEP_THRESHOLD 12
if (i < SLEEP_THRESHOLD) {
sched_yield();
} else {
struct timespec ts;
/*
* Do not wait for more than about 15 ms, even under extreme
* contention.
*/
if (i > 24)
i = 24;
ts.tv_sec = 0;
ts.tv_nsec = (unsigned32)1 << i;
nanosleep(&ts, 0);
}
}
}
# elif defined(USE_PTHREAD_LOCKS)
# ifdef USE_RWLOCK
GC_INNER pthread_rwlock_t GC_allocate_ml = PTHREAD_RWLOCK_INITIALIZER;
# else
GC_INNER pthread_mutex_t GC_allocate_ml = PTHREAD_MUTEX_INITIALIZER;
# endif
# ifndef NO_PTHREAD_TRYLOCK
GC_INNER void
GC_lock(void)
{
if (1 == GC_nprocs || is_collecting()) {
pthread_mutex_lock(&GC_allocate_ml);
} else {
GC_generic_lock(&GC_allocate_ml);
}
}
# elif defined(GC_ASSERTIONS)
GC_INNER void
GC_lock(void)
{
# ifdef USE_RWLOCK
(void)pthread_rwlock_wrlock(&GC_allocate_ml); /*< exclusive */
# else
pthread_mutex_lock(&GC_allocate_ml);
# endif
}
# endif /* NO_PTHREAD_TRYLOCK && GC_ASSERTIONS */
# endif /* !USE_SPIN_LOCK && USE_PTHREAD_LOCKS */
# ifdef GC_PTHREADS_PARAMARK
# if defined(GC_ASSERTIONS) && defined(GC_WIN32_THREADS) \
&& !defined(USE_PTHREAD_LOCKS)
/* Note: result is not guaranteed to be unique. */
# define NUMERIC_THREAD_ID(id) ((unsigned long)ADDR(PTHREAD_TO_VPTR(id)))
# endif
# ifdef GC_ASSERTIONS
# define SET_MARK_LOCK_HOLDER \
(void)(GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self()))
# define UNSET_MARK_LOCK_HOLDER \
do { \
GC_ASSERT(GC_mark_lock_holder \
== NUMERIC_THREAD_ID(pthread_self())); \
GC_mark_lock_holder = NO_THREAD; \
} while (0)
# else
# define SET_MARK_LOCK_HOLDER (void)0
# define UNSET_MARK_LOCK_HOLDER (void)0
# endif /* !GC_ASSERTIONS */
static pthread_cond_t builder_cv = PTHREAD_COND_INITIALIZER;
# ifndef GC_WIN32_THREADS
static void
setup_mark_lock(void)
{
# ifdef GLIBC_2_19_TSX_BUG
pthread_mutexattr_t mattr;
int glibc_minor = -1;
int glibc_major = GC_parse_version(&glibc_minor, gnu_get_libc_version());
if (glibc_major > 2 || (glibc_major == 2 && glibc_minor >= 19)) {
/* TODO: Disable this workaround for `glibc` with fixed TSX. */
/* This disables lock elision to workaround a bug in `glibc` 2.19+. */
if (pthread_mutexattr_init(&mattr) != 0)
ABORT("pthread_mutexattr_init failed");
if (pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_NORMAL) != 0)
ABORT("pthread_mutexattr_settype failed");
if (pthread_mutex_init(&mark_mutex, &mattr) != 0)
ABORT("pthread_mutex_init failed");
(void)pthread_mutexattr_destroy(&mattr);
}
# endif
}
# endif /* !GC_WIN32_THREADS */
GC_INNER void
GC_acquire_mark_lock(void)
{
# if defined(NUMERIC_THREAD_ID_UNIQUE) && !defined(THREAD_SANITIZER)
GC_ASSERT(GC_mark_lock_holder != NUMERIC_THREAD_ID(pthread_self()));
# endif
GC_generic_lock(&mark_mutex);
SET_MARK_LOCK_HOLDER;
}
GC_INNER void
GC_release_mark_lock(void)
{
UNSET_MARK_LOCK_HOLDER;
if (pthread_mutex_unlock(&mark_mutex) != 0)
ABORT("pthread_mutex_unlock failed");
}
/*
* Collector must wait for free-list builders for 2 reasons:
* - Mark bits may still be getting examined without lock;
* - Partial free lists referenced only by locals may not be scanned
* correctly, e.g. if they contain "pointer-free" objects, since the
* free-list link may be ignored.
*/
STATIC void
GC_wait_builder(void)
{
ASSERT_CANCEL_DISABLED();
UNSET_MARK_LOCK_HOLDER;
if (pthread_cond_wait(&builder_cv, &mark_mutex) != 0)
ABORT("pthread_cond_wait failed");
GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
SET_MARK_LOCK_HOLDER;
}
GC_INNER void
GC_wait_for_reclaim(void)
{
GC_acquire_mark_lock();
while (GC_fl_builder_count > 0) {
GC_wait_builder();
}
GC_release_mark_lock();
}
# if defined(CAN_HANDLE_FORK) && defined(THREAD_SANITIZER)
/*
* Identical to `GC_wait_for_reclaim()` but with the `no_sanitize`
* attribute as a workaround for TSan which does not notice that the
* allocator lock is acquired in `fork_prepare_proc()`.
*/
GC_ATTR_NO_SANITIZE_THREAD
static void
wait_for_reclaim_atfork(void)
{
GC_acquire_mark_lock();
while (GC_fl_builder_count > 0)
GC_wait_builder();
GC_release_mark_lock();
}
# endif /* CAN_HANDLE_FORK && THREAD_SANITIZER */
GC_INNER void
GC_notify_all_builder(void)
{
GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
if (pthread_cond_broadcast(&builder_cv) != 0)
ABORT("pthread_cond_broadcast failed");
}
GC_INNER void
GC_wait_marker(void)
{
ASSERT_CANCEL_DISABLED();
GC_ASSERT(GC_parallel);
UNSET_MARK_LOCK_HOLDER;
if (pthread_cond_wait(&mark_cv, &mark_mutex) != 0)
ABORT("pthread_cond_wait failed");
GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
SET_MARK_LOCK_HOLDER;
}
GC_INNER void
GC_notify_all_marker(void)
{
GC_ASSERT(GC_parallel);
if (pthread_cond_broadcast(&mark_cv) != 0)
ABORT("pthread_cond_broadcast failed");
}
# endif /* GC_PTHREADS_PARAMARK */
GC_INNER GC_on_thread_event_proc GC_on_thread_event = 0;
GC_API void GC_CALL
GC_set_on_thread_event(GC_on_thread_event_proc fn)
{
/* Note: `fn` may be 0 (means no event notifier). */
LOCK();
GC_on_thread_event = fn;
UNLOCK();
}
GC_API GC_on_thread_event_proc GC_CALL
GC_get_on_thread_event(void)
{
GC_on_thread_event_proc fn;
READER_LOCK();
fn = GC_on_thread_event;
READER_UNLOCK();
return fn;
}
# ifdef STACKPTR_CORRECTOR_AVAILABLE
GC_INNER GC_sp_corrector_proc GC_sp_corrector = 0;
# endif
GC_API void GC_CALL
GC_set_sp_corrector(GC_sp_corrector_proc fn)
{
# ifdef STACKPTR_CORRECTOR_AVAILABLE
LOCK();
GC_sp_corrector = fn;
UNLOCK();
# else
UNUSED_ARG(fn);
# endif
}
GC_API GC_sp_corrector_proc GC_CALL
GC_get_sp_corrector(void)
{
# ifdef STACKPTR_CORRECTOR_AVAILABLE
GC_sp_corrector_proc fn;
READER_LOCK();
fn = GC_sp_corrector;
READER_UNLOCK();
return fn;
# else
return 0; /*< unsupported */
# endif
}
# ifdef PTHREAD_REGISTER_CANCEL_WEAK_STUBS
/* Workaround "undefined reference" linkage errors on some targets. */
EXTERN_C_BEGIN
extern void __pthread_register_cancel(void) __attribute__((__weak__));
extern void __pthread_unregister_cancel(void) __attribute__((__weak__));
EXTERN_C_END
void
__pthread_register_cancel(void)
{
}
void
__pthread_unregister_cancel(void)
{
}
# endif
# undef do_blocking_enter
#endif /* THREADS */
/*
* Copyright (c) 2000 by Hewlett-Packard Company. All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* To determine type of `tsd` implementation; includes `private/specific.h`
* file if needed.
*/
#if defined(USE_CUSTOM_SPECIFIC)
/*
* A thread-specific data entry which will never appear valid to a reader.
* Used to fill in empty cache entries to avoid a check for 0.
*/
static const tse invalid_tse = { INVALID_QTID, 0, 0, INVALID_THREADID };
GC_INNER int
GC_key_create_inner(tsd **key_ptr)
{
int i;
int ret;
tsd *result;
GC_ASSERT(I_HOLD_LOCK());
/* A quick alignment check, since we need atomic stores. */
GC_ASSERT(ADDR(&invalid_tse.next) % ALIGNMENT == 0);
result = (tsd *)MALLOC_CLEAR(sizeof(tsd));
if (NULL == result)
return ENOMEM;
ret = pthread_mutex_init(&result->lock, NULL);
if (ret != 0)
return ret;
for (i = 0; i < TS_CACHE_SIZE; ++i) {
result->cache[i] = (tse *)GC_CAST_AWAY_CONST_PVOID(&invalid_tse);
}
# ifdef GC_ASSERTIONS
for (i = 0; i < TS_HASH_SIZE; ++i) {
GC_ASSERT(NULL == result->hash[i]);
}
# endif
*key_ptr = result;
return 0;
}
GC_INNER int
GC_setspecific(tsd *key, void *value)
{
pthread_t self = pthread_self();
unsigned hash_val = TS_HASH(self);
volatile tse *entry;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(self != INVALID_THREADID);
/* Disable garbage collection during `GC_malloc`. */
GC_dont_gc++;
entry = (volatile tse *)MALLOC_CLEAR(sizeof(tse));
GC_dont_gc--;
if (UNLIKELY(NULL == entry))
return ENOMEM;
pthread_mutex_lock(&key->lock);
entry->next = key->hash[hash_val];
# ifdef GC_ASSERTIONS
{
tse *p;
/* Ensure no existing entry. */
for (p = entry->next; p != NULL; p = p->next) {
GC_ASSERT(!THREAD_EQUAL(p->thread, self));
}
}
# endif
entry->thread = self;
entry->value = TS_HIDE_VALUE(value);
GC_ASSERT(entry->qtid == INVALID_QTID);
/*
* There can only be one writer at a time, but this needs to be atomic
* with respect to concurrent readers.
*/
GC_cptr_store_release((volatile ptr_t *)&key->hash[hash_val],
(ptr_t)CAST_AWAY_VOLATILE_PVOID(entry));
GC_dirty(CAST_AWAY_VOLATILE_PVOID(entry));
GC_dirty(key->hash + hash_val);
if (pthread_mutex_unlock(&key->lock) != 0)
ABORT("pthread_mutex_unlock failed (setspecific)");
return 0;
}
GC_INNER void
GC_remove_specific_after_fork(tsd *key, pthread_t t)
{
unsigned hash_val = TS_HASH(t);
tse *entry;
tse *prev = NULL;
# ifdef CAN_HANDLE_FORK
/*
* Both `GC_setspecific` and `GC_remove_specific` should be called with
* the allocator lock held to ensure the consistency of the hash table
* in the forked child process.
*/
GC_ASSERT(I_HOLD_LOCK());
# endif
pthread_mutex_lock(&key->lock);
for (entry = key->hash[hash_val];
entry != NULL && !THREAD_EQUAL(entry->thread, t); entry = entry->next) {
prev = entry;
}
/*
* Invalidate `qtid` field, since `qtid` values may be reused, and
* a later cache lookup could otherwise find this `entry`.
*/
if (entry != NULL) {
entry->qtid = INVALID_QTID;
if (NULL == prev) {
key->hash[hash_val] = entry->next;
GC_dirty(key->hash + hash_val);
} else {
prev->next = entry->next;
GC_dirty(prev);
}
/*
* Atomic! Concurrent accesses still work. They must, since readers
* do not lock. We should not need a `volatile` access here, since
* both this and the preceding write should become visible no later
* than the `pthread_mutex_unlock()` call.
*/
}
/*
* If we wanted to deallocate the entry, we would first have to clear
* any cache entries pointing to it. That probably requires additional
* synchronization, since we cannot prevent a concurrent cache lookup,
* which should still be examining deallocated memory. This can only
* happen if the concurrent access is from another thread, and hence
* has missed the cache, but still...
*/
# ifdef LINT2
GC_noop1_ptr(entry);
# endif
/*
* With GC, we are done, since the pointers from the cache will be
* overwritten, all local pointers to the entries will be dropped,
* and the entry will then be reclaimed.
*/
if (pthread_mutex_unlock(&key->lock) != 0)
ABORT("pthread_mutex_unlock failed (remove_specific after fork)");
}
# ifdef CAN_HANDLE_FORK
GC_INNER void
GC_update_specific_after_fork(tsd *key)
{
unsigned hash_val = TS_HASH(GC_parent_pthread_self);
tse *entry;
GC_ASSERT(I_HOLD_LOCK());
# ifdef LINT2
pthread_mutex_lock(&key->lock);
# endif
entry = key->hash[hash_val];
if (LIKELY(entry != NULL)) {
GC_ASSERT(THREAD_EQUAL(entry->thread, GC_parent_pthread_self));
GC_ASSERT(NULL == entry->next);
/* Remove the `entry` from the table. */
key->hash[hash_val] = NULL;
entry->thread = pthread_self();
/* Then put the `entry` back to the table (based on new hash value). */
key->hash[TS_HASH(entry->thread)] = entry;
}
# ifdef LINT2
(void)pthread_mutex_unlock(&key->lock);
# endif
}
# endif
GC_INNER void *
GC_slow_getspecific(tsd *key, size_t qtid, tse *volatile *cache_ptr)
{
pthread_t self = pthread_self();
tse *entry = key->hash[TS_HASH(self)];
GC_ASSERT(qtid != INVALID_QTID);
while (entry != NULL && !THREAD_EQUAL(entry->thread, self)) {
entry = entry->next;
}
if (entry == NULL)
return NULL;
/*
* Set the cache `entry`. It is safe to do this asynchronously.
* Either value is safe, though may produce spurious misses.
* We are replacing one `qtid` with another one for the same thread.
*/
AO_store(&entry->qtid, qtid);
GC_cptr_store((volatile ptr_t *)cache_ptr, (ptr_t)entry);
return TS_REVEAL_PTR(entry->value);
}
# ifdef GC_ASSERTIONS
/*
* Check that that all elements of the data structure associated with
* `key` are marked.
*/
void
GC_check_tsd_marks(tsd *key)
{
int i;
tse *p;
if (!GC_is_marked(GC_base(key))) {
ABORT("Unmarked thread-specific-data table");
}
for (i = 0; i < TS_HASH_SIZE; ++i) {
for (p = key->hash[i]; p != NULL; p = p->next) {
if (!GC_is_marked(GC_base(p))) {
ABORT_ARG1("Unmarked thread-specific-data entry", " at %p", (void *)p);
}
}
}
for (i = 0; i < TS_CACHE_SIZE; ++i) {
p = key->cache[i];
if (p != &invalid_tse && !GC_is_marked(GC_base(p))) {
ABORT_ARG1("Unmarked cached thread-specific-data entry", " at %p",
(void *)p);
}
}
}
# endif /* GC_ASSERTIONS */
#endif /* USE_CUSTOM_SPECIFIC */
/*
* Copyright (c) 1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996 by Silicon Graphics. All rights reserved.
* Copyright (c) 1998 by Fergus Henderson. All rights reserved.
* Copyright (c) 2000-2008 by Hewlett-Packard Development Company.
* All rights reserved.
* Copyright (c) 2008-2025 Ivan Maidanski
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
#if defined(GC_WIN32_THREADS)
/* The allocator lock definition. */
# ifndef USE_PTHREAD_LOCKS
# ifdef USE_RWLOCK
GC_INNER SRWLOCK GC_allocate_ml;
# else
GC_INNER CRITICAL_SECTION GC_allocate_ml;
# endif
# endif /* !USE_PTHREAD_LOCKS */
# undef CreateThread
# undef ExitThread
# undef _beginthreadex
# undef _endthreadex
# if !defined(GC_PTHREADS) && !defined(MSWINCE)
# include <errno.h>
# include <process.h> /*< for `_beginthreadex`, `_endthreadex` */
# endif
static ptr_t copy_ptr_regs(word *regs, const CONTEXT *pcontext);
# ifndef GC_NO_THREADS_DISCOVERY
/*
* This code operates in two distinct modes, depending on the setting
* of `GC_win32_dll_threads`. If `GC_win32_dll_threads`, then all
* threads in the process are implicitly registered with the collector
* by `DllMain()`. No explicit registration is required, and attempts
* at explicit registration are ignored. This mode is very different
* from the POSIX operation of the collector. In this mode access to
* the thread table is lock-free. Hence there is a static limit on the
* number of threads.
*/
/*
* `GC_DISCOVER_TASK_THREADS` should be used if `DllMain`-based thread
* registration is required but it is impossible to call
* `GC_use_threads_discovery()` before other GC routines.
*/
# ifndef GC_DISCOVER_TASK_THREADS
GC_INNER GC_bool GC_win32_dll_threads = FALSE;
# endif
# else
/*
* If not `GC_win32_dll_threads` (or the collector is built without `GC_DLL`
* macro defined), things operate in a way that is very similar to POSIX
* platforms, and new threads must be registered with the collector,
* e.g. by using preprocessor-based interception of the thread primitives.
* In this case, we use a real data structure for the thread table.
* Note that there is no equivalent of linker-based call interception,
* since we do not have ELF-like facilities. The Windows analog appears
* to be "API hooking", which really seems to be a standard way to do minor
* binary rewriting (?). I would prefer not to have the basic collector
* rely on such facilities, but an optional package that intercepts thread
* calls this way would probably be nice.
*/
# undef MAX_THREADS
/* `dll_thread_table[]` is always empty. */
# define MAX_THREADS 1
# endif /* GC_NO_THREADS_DISCOVERY */
/*
* We have two variants of the thread table. Which one we use depends
* on whether `GC_win32_dll_threads` is set. Note that before the
* initialization, we do not add any entries to either table, even
* if `DllMain()` is called. The main thread will be added on the
* collector initialization.
*/
GC_API void GC_CALL
GC_use_threads_discovery(void)
{
# ifdef GC_NO_THREADS_DISCOVERY
/*
* `GC_use_threads_discovery()` is currently incompatible with
* `pthreads` and WinCE. It might be possible to get `DllMain`-based
* thread registration to work with Cygwin, but if you try it then
* you are on your own.
*/
ABORT("GC DllMain-based thread registration unsupported");
# else
/* Turn on `GC_win32_dll_threads`. */
GC_ASSERT(!GC_is_initialized);
/*
* Note that `GC_use_threads_discovery()` is expected to be called by
* the client application (not from `DllMain()`) at start-up.
*/
# ifndef GC_DISCOVER_TASK_THREADS
GC_win32_dll_threads = TRUE;
# endif
GC_init();
# ifdef CPPCHECK
GC_noop1((word)(GC_funcptr_uint)(&GC_DllMain));
# endif
# endif
}
# ifndef GC_NO_THREADS_DISCOVERY
/*
* We track thread attachments while the world is supposed to be stopped.
* Unfortunately, we cannot stop them from starting, since blocking in
* `DllMain` seems to cause the world to deadlock. Thus, we have to
* recover if we notice this in the middle of marking.
*/
STATIC volatile AO_t GC_attached_thread = FALSE;
/*
* We assume that `volatile` implies memory ordering, at least among
* `volatile` variables. This code should consistently use `libatomic_ops`
* package or gcc atomic intrinsics.
*/
STATIC volatile GC_bool GC_please_stop = FALSE;
# elif defined(GC_ASSERTIONS)
STATIC GC_bool GC_please_stop = FALSE;
# endif /* GC_NO_THREADS_DISCOVERY && GC_ASSERTIONS */
# if defined(WRAP_MARK_SOME) && !defined(GC_PTHREADS)
GC_INNER GC_bool
GC_started_thread_while_stopped(void)
{
# ifndef GC_NO_THREADS_DISCOVERY
if (GC_win32_dll_threads) {
# ifdef AO_HAVE_compare_and_swap_release
if (AO_compare_and_swap_release(&GC_attached_thread, TRUE,
FALSE /* stored */))
return TRUE;
# else
/* Prior heap reads need to complete earlier. */
AO_nop_full();
if (AO_load(&GC_attached_thread)) {
AO_store(&GC_attached_thread, FALSE);
return TRUE;
}
# endif
}
# endif
return FALSE;
}
# endif /* WRAP_MARK_SOME */
/*
* The thread table used if `GC_win32_dll_threads`. This is a fixed-size
* array. Since we use runtime conditionals, both variants are always
* defined.
*/
# ifndef MAX_THREADS
# define MAX_THREADS 512
# endif
/*
* Things may get quite slow for large numbers of threads,
* since we look them up with the sequential search.
*/
static volatile struct GC_Thread_Rep dll_thread_table[MAX_THREADS];
# ifndef GC_NO_THREADS_DISCOVERY
static struct GC_StackContext_Rep dll_crtn_table[MAX_THREADS];
# endif
GC_INNER GC_thread
GC_register_my_thread_inner(const struct GC_stack_base *sb,
thread_id_t self_id)
{
GC_thread me;
# ifdef GC_NO_THREADS_DISCOVERY
GC_ASSERT(I_HOLD_LOCK());
# endif
/*
* The following should be a no-op according to the Win32 documentation.
* There is an empirical evidence that it is not.
*/
# if defined(MPROTECT_VDB) && !defined(CYGWIN32)
if (GC_auto_incremental
# ifdef GWW_VDB
&& !GC_gww_dirty_init()
# endif
)
GC_set_write_fault_handler();
# endif
# ifndef GC_NO_THREADS_DISCOVERY
if (GC_win32_dll_threads) {
int i;
/*
* It appears to be unsafe to acquire a lock here, since this code
* is apparently not preemptible on some systems. (This is based on
* complaints, not on Microsoft's official documentation, which says
* this should perform "only simple initialization tasks".)
* Hence we make it does with a nonblocking synchronization.
* It has been claimed that `DllMain` is really only executed with
* a particular system lock held, and thus careful use of locking
* around code that does not call back into the system libraries might
* be OK. (But this has not been tested across all Win32 versions.)
*/
for (i = 0;
InterlockedExchange(&dll_thread_table[i].tm.long_in_use, 1) != 0;
i++) {
/*
* Compare-and-swap would make this cleaner, but that is not
* supported before Windows 98 and NT 4.0. In Windows 2000,
* `InterlockedExchange` is supposed to be replaced by
* `InterlockedExchangePointer`, but that is not really what
* is needed here.
*/
/*
* FIXME: We should eventually declare Windows 95 dead and use
* AO primitives here.
*/
if (i == MAX_THREADS - 1)
ABORT("Too many threads");
}
/*
* Update `GC_max_thread_index` if necessary. The following is safe,
* and unlike `CompareExchange`-based solutions seems to work on all
* Windows 95 and later platforms. Note that `GC_max_thread_index`
* may be temporarily out of bounds, so readers have to compensate.
*/
while (i > GC_max_thread_index) {
InterlockedIncrement((LONG *)&GC_max_thread_index);
/* Cast away `volatile` for older versions of Win32 headers. */
}
if (UNLIKELY(GC_max_thread_index >= MAX_THREADS)) {
/*
* We overshot due to simultaneous increments.
* Setting it to `MAX_THREADS - 1` is always safe.
*/
GC_max_thread_index = MAX_THREADS - 1;
}
me = (GC_thread)(dll_thread_table + i);
me->crtn = &dll_crtn_table[i];
} else
# endif
/* else */ {
/* Not using `DllMain`. */
me = GC_new_thread(self_id);
}
# ifdef GC_PTHREADS
me->pthread_id = pthread_self();
# endif
# ifndef MSWINCE
/* `GetCurrentThread()` returns a pseudohandle (a constant value). */
if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
GetCurrentProcess(), (HANDLE *)&me->handle,
0 /* `dwDesiredAccess` */, FALSE /* `bInheritHandle` */,
DUPLICATE_SAME_ACCESS)) {
ABORT_ARG1("DuplicateHandle failed", ": errcode= 0x%X",
(unsigned)GetLastError());
}
# endif
# if defined(WOW64_THREAD_CONTEXT_WORKAROUND) && defined(MSWINRT_FLAVOR)
/*
* Lookup TIB value via a call to `NtCurrentTeb()` on thread registration
* rather than calling `GetThreadSelectorEntry()` which is not available
* on UWP.
*/
me->tib = (GC_NT_TIB *)NtCurrentTeb();
# endif
me->crtn->last_stack_min = ADDR_LIMIT;
GC_record_stack_base(me->crtn, sb);
/*
* Up until this point, `GC_push_all_stacks` considers this thread
* invalid. And, up until this point, the entry is viewed by
* `GC_win32_dll_lookup_thread` as reserved but invalid.
*/
((volatile struct GC_Thread_Rep *)me)->id = self_id;
# ifndef GC_NO_THREADS_DISCOVERY
if (GC_win32_dll_threads) {
if (GC_please_stop) {
AO_store(&GC_attached_thread, TRUE);
AO_nop_full(); /*< later updates must become visible after this */
}
/*
* We would like to wait here, but cannot, since waiting in `DllMain()`
* provokes deadlocks. Thus we force marking to be restarted instead.
*/
} else
# endif
/* else */ {
/*
* `GC_please_stop` is `FALSE`, otherwise both we and the thread-stopping
* code would be holding the allocator lock.
*/
GC_ASSERT(!GC_please_stop);
}
return me;
}
/*
* `GC_max_thread_index` may temporarily be larger than `MAX_THREADS`.
* To avoid subscript errors, we check it on access.
*/
GC_INLINE LONG
GC_get_max_thread_index(void)
{
LONG my_max = GC_max_thread_index;
if (UNLIKELY(my_max >= MAX_THREADS))
return MAX_THREADS - 1;
return my_max;
}
# ifndef GC_NO_THREADS_DISCOVERY
GC_INNER GC_thread
GC_win32_dll_lookup_thread(thread_id_t id)
{
int i;
LONG my_max = GC_get_max_thread_index();
GC_ASSERT(GC_win32_dll_threads);
for (i = 0; i <= my_max; i++) {
if (AO_load_acquire(&dll_thread_table[i].tm.in_use)
&& dll_thread_table[i].id == id) {
/*
* Must still be in use, since nobody else can store our
* thread `id`.
*/
break;
}
}
return i <= my_max ? (GC_thread)(dll_thread_table + i) : NULL;
}
# endif /* !GC_NO_THREADS_DISCOVERY */
# ifdef GC_PTHREADS
/*
* A quick-and-dirty cache of the mapping between `pthread_t` and
* Win32 thread id.
*/
# define PTHREAD_MAP_SIZE 512
thread_id_t GC_pthread_map_cache[PTHREAD_MAP_SIZE] = { 0 };
/* It appears `pthread_t` is really a pointer type... */
# define PTHREAD_MAP_INDEX(pthread_id) \
((NUMERIC_THREAD_ID(pthread_id) >> 5) % PTHREAD_MAP_SIZE)
# define SET_PTHREAD_MAP_CACHE(pthread_id, win32_id) \
(void)(GC_pthread_map_cache[PTHREAD_MAP_INDEX(pthread_id)] = (win32_id))
# define GET_PTHREAD_MAP_CACHE(pthread_id) \
GC_pthread_map_cache[PTHREAD_MAP_INDEX(pthread_id)]
GC_INNER void
GC_win32_cache_self_pthread(thread_id_t self_id)
{
pthread_t self = pthread_self();
GC_ASSERT(I_HOLD_LOCK());
SET_PTHREAD_MAP_CACHE(self, self_id);
}
GC_INNER GC_thread
GC_lookup_by_pthread(pthread_t thread)
{
/*
* TODO: Search in `dll_thread_table` instead when `DllMain`-based thread
* registration is made compatible with `pthreads` (and turned on).
*/
thread_id_t id;
GC_thread p;
int hv;
GC_ASSERT(I_HOLD_READER_LOCK());
id = GET_PTHREAD_MAP_CACHE(thread);
/* We first try the cache. */
for (p = GC_threads[THREAD_TABLE_INDEX(id)]; p != NULL; p = p->tm.next) {
if (LIKELY(THREAD_EQUAL(p->pthread_id, thread)))
return p;
}
/* If that fails, we use a very slow approach. */
for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
for (p = GC_threads[hv]; p != NULL; p = p->tm.next) {
if (THREAD_EQUAL(p->pthread_id, thread))
return p;
}
}
return NULL;
}
# endif /* GC_PTHREADS */
# ifdef WOW64_THREAD_CONTEXT_WORKAROUND
# ifndef CONTEXT_EXCEPTION_ACTIVE
# define CONTEXT_EXCEPTION_ACTIVE 0x08000000
# define CONTEXT_EXCEPTION_REQUEST 0x40000000
# define CONTEXT_EXCEPTION_REPORTING 0x80000000
# endif
/* Is 32-bit code running on Win64? */
static GC_bool isWow64;
# define GET_THREAD_CONTEXT_FLAGS \
(isWow64 ? CONTEXT_INTEGER | CONTEXT_CONTROL \
| CONTEXT_EXCEPTION_REQUEST | CONTEXT_SEGMENTS \
: CONTEXT_INTEGER | CONTEXT_CONTROL)
# elif defined(I386) || defined(XMM_CANT_STORE_PTRS)
# define GET_THREAD_CONTEXT_FLAGS (CONTEXT_INTEGER | CONTEXT_CONTROL)
# else
# define GET_THREAD_CONTEXT_FLAGS \
(CONTEXT_INTEGER | CONTEXT_CONTROL | CONTEXT_FLOATING_POINT)
# endif /* !WOW64_THREAD_CONTEXT_WORKAROUND && !I386 */
/* Suspend the given thread, if it is still active. */
STATIC void
GC_suspend(GC_thread t)
{
# ifndef MSWINCE
DWORD exitCode;
# ifdef RETRY_GET_THREAD_CONTEXT
int retry_cnt;
# define MAX_SUSPEND_THREAD_RETRIES (1000 * 1000)
# endif
# endif
GC_ASSERT(I_HOLD_LOCK());
# ifndef GC_NO_THREADS_DISCOVERY
if (NULL == GC_cptr_load_acquire(&t->handle))
return;
# endif
# if defined(DEBUG_THREADS) && !defined(MSWINCE) \
&& (!defined(MSWIN32) || defined(CONSOLE_LOG))
GC_log_printf("Suspending 0x%x\n", (int)t->id);
# endif
GC_win32_unprotect_thread(t);
GC_acquire_dirty_lock();
# ifdef MSWINCE
/* `SuspendThread()` will fail if thread is running kernel code. */
while (SuspendThread(THREAD_HANDLE(t)) == (DWORD)-1) {
GC_release_dirty_lock();
Sleep(10); /*< in millis */
GC_acquire_dirty_lock();
}
# elif defined(RETRY_GET_THREAD_CONTEXT)
for (retry_cnt = 0;;) {
/*
* Apparently the Windows 95 `GetOpenFileName()` creates a thread
* that does not properly get cleaned up, and `SuspendThread()` on
* its descriptor may provoke a crash. This reduces the probability
* of that event, though it still appears there is a race here.
*/
if (GetExitCodeThread(t->handle, &exitCode) && exitCode != STILL_ACTIVE) {
GC_release_dirty_lock();
# ifdef GC_PTHREADS
/* Prevent stack from being pushed. */
t->crtn->stack_end = NULL;
# else
/*
* This breaks `pthread_join` on Cygwin, which is guaranteed to
* only see user threads.
*/
GC_delete_thread(t);
# endif
return;
}
if (SuspendThread(t->handle) != (DWORD)-1) {
CONTEXT context;
/*
* Calls to `GetThreadContext()` may fail. Work around this by
* putting access in suspend/resume loop to advance thread past
* problematic areas where suspend fails. Capture the `context`
* in per-thread structure at the suspend time rather than at
* retrieving the `context` during the push logic.
*/
context.ContextFlags = GET_THREAD_CONTEXT_FLAGS;
if (GetThreadContext(t->handle, &context)) {
/*
* TODO: WoW64 extra workaround: if `CONTEXT_EXCEPTION_ACTIVE`
* then `Sleep(1)` and retry.
*/
t->context_sp = copy_ptr_regs(t->context_regs, &context);
/* Success; the context pointer registers are saved. */
break;
}
/* Resume the thread, try to suspend it in a better location. */
if (ResumeThread(t->handle) == (DWORD)-1) {
# ifndef GC_NO_THREADS_DISCOVERY
if (NULL == GC_cptr_load_acquire(&t->handle)) {
/*
* It might be the scenario like this:
* 1. `GC_suspend` calls `SuspendThread` on a valid handle;
* 2. Within the `SuspendThread` call a context switch
* occurs to `DllMain` (before the thread has actually
* been suspended);
* 3. `DllMain` sets `t->handle` to `NULL`, but does not
* yet close the handle;
* 4. A context switch occurs returning to `SuspendThread`
* which completes on the handle that was originally
* passed into it;
* 5. Then `ResumeThread` attempts to run on `t->handle`
* which is now `NULL`.
*/
GC_release_dirty_lock();
/*
* FIXME: The thread seems to be suspended forever (causing
* a resource leak).
*/
WARN("ResumeThread failed (async CloseHandle by DllMain)\n", 0);
return;
}
# endif
ABORT("ResumeThread failed in suspend loop");
}
} else {
# ifndef GC_NO_THREADS_DISCOVERY
if (NULL == GC_cptr_load_acquire(&t->handle)) {
/* The thread handle is closed asynchronously by `GC_DllMain`. */
GC_release_dirty_lock();
return;
}
# endif
}
if (retry_cnt > 1) {
GC_release_dirty_lock();
Sleep(0); /*< yield */
# ifndef GC_NO_THREADS_DISCOVERY
if (NULL == GC_cptr_load_acquire(&t->handle))
return;
# endif
GC_acquire_dirty_lock();
}
if (++retry_cnt >= MAX_SUSPEND_THREAD_RETRIES) {
/* Something must be wrong. */
ABORT("SuspendThread loop failed");
}
}
# else
if (GetExitCodeThread(t->handle, &exitCode) && exitCode != STILL_ACTIVE) {
GC_release_dirty_lock();
# ifdef GC_PTHREADS
/* Prevent stack from being pushed. */
t->crtn->stack_end = NULL;
# else
GC_delete_thread(t);
# endif
return;
}
if (SuspendThread(t->handle) == (DWORD)-1) {
# ifndef GC_NO_THREADS_DISCOVERY
if (NULL == GC_cptr_load_acquire(&t->handle)) {
GC_release_dirty_lock();
return;
}
# endif
ABORT("SuspendThread failed");
}
# endif
t->flags |= IS_SUSPENDED;
GC_release_dirty_lock();
if (GC_on_thread_event)
GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED, THREAD_HANDLE(t));
}
# if defined(GC_ASSERTIONS) \
&& ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE))
GC_INNER GC_bool GC_write_disabled = FALSE;
# endif
GC_INNER void
GC_stop_world(void)
{
thread_id_t self_id = GetCurrentThreadId();
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_thr_initialized);
/* This code is the same as in `pthread_stop_world.c` file. */
# ifdef PARALLEL_MARK
if (GC_parallel) {
GC_acquire_mark_lock();
/* We should have previously waited for the count to become zero. */
GC_ASSERT(0 == GC_fl_builder_count);
}
# endif /* PARALLEL_MARK */
# if !defined(GC_NO_THREADS_DISCOVERY) || defined(GC_ASSERTIONS)
GC_please_stop = TRUE;
# endif
# if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
GC_ASSERT(!GC_write_disabled);
EnterCriticalSection(&GC_write_cs);
/*
* It is not allowed to call `GC_printf()` (and friends) here down to
* `LeaveCriticalSection` (same applies recursively to `GC_suspend`,
* `GC_delete_thread`, `GC_get_max_thread_index`, `GC_size` and
* `GC_remove_protection`).
*/
# ifdef GC_ASSERTIONS
GC_write_disabled = TRUE;
# endif
# endif
# ifndef GC_NO_THREADS_DISCOVERY
if (GC_win32_dll_threads) {
int i;
int my_max;
/*
* Any threads being created during this loop will end up setting
* `GC_attached_thread` when they start. This will force marking
* to restart. This is not ideal, but hopefully correct.
*/
AO_store(&GC_attached_thread, FALSE);
my_max = (int)GC_get_max_thread_index();
for (i = 0; i <= my_max; i++) {
GC_thread p = (GC_thread)(dll_thread_table + i);
if (p->crtn->stack_end != NULL && (p->flags & DO_BLOCKING) == 0
&& p->id != self_id) {
GC_suspend(p);
}
}
} else
# endif
/* else */ {
GC_thread p;
int i;
for (i = 0; i < THREAD_TABLE_SZ; i++) {
for (p = GC_threads[i]; p != NULL; p = p->tm.next)
if (p->crtn->stack_end != NULL && p->id != self_id
&& (p->flags & (FINISHED | DO_BLOCKING)) == 0)
GC_suspend(p);
}
}
# if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
# ifdef GC_ASSERTIONS
GC_write_disabled = FALSE;
# endif
LeaveCriticalSection(&GC_write_cs);
# endif
# ifdef PARALLEL_MARK
if (GC_parallel)
GC_release_mark_lock();
# endif
}
GC_INNER void
GC_start_world(void)
{
# ifdef GC_ASSERTIONS
thread_id_t self_id = GetCurrentThreadId();
# endif
GC_ASSERT(I_HOLD_LOCK());
# ifndef GC_NO_THREADS_DISCOVERY
if (GC_win32_dll_threads) {
LONG my_max = GC_get_max_thread_index();
int i;
for (i = 0; i <= my_max; i++) {
GC_thread p = (GC_thread)(dll_thread_table + i);
if ((p->flags & IS_SUSPENDED) != 0) {
# ifdef DEBUG_THREADS
GC_log_printf("Resuming 0x%x\n", (int)p->id);
# endif
GC_ASSERT(p->id != self_id);
GC_ASSERT(*(ptr_t *)CAST_AWAY_VOLATILE_PVOID(&p->crtn->stack_end)
!= NULL);
if (ResumeThread(p->handle) == (DWORD)-1) {
if (NULL == GC_cptr_load_acquire(&p->handle)) {
/* FIXME: See the same issue in `GC_suspend`. */
WARN("ResumeThread failed (async CloseHandle by DllMain)\n", 0);
} else {
ABORT("ResumeThread failed");
}
}
p->flags &= (unsigned char)~IS_SUSPENDED;
if (GC_on_thread_event)
GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, p->handle);
} else {
/* The thread is unregistered or not suspended. */
}
}
} else
# endif
/* else */ {
GC_thread p;
int i;
for (i = 0; i < THREAD_TABLE_SZ; i++) {
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
if ((p->flags & IS_SUSPENDED) != 0) {
# ifdef DEBUG_THREADS
GC_log_printf("Resuming 0x%x\n", (int)p->id);
# endif
GC_ASSERT(p->id != self_id && *(ptr_t *)&p->crtn->stack_end != NULL);
if (ResumeThread(THREAD_HANDLE(p)) == (DWORD)-1)
ABORT("ResumeThread failed");
GC_win32_unprotect_thread(p);
p->flags &= (unsigned char)~IS_SUSPENDED;
if (GC_on_thread_event)
GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, THREAD_HANDLE(p));
} else {
# ifdef DEBUG_THREADS
GC_log_printf("Not resuming thread 0x%x as it is not suspended\n",
(int)p->id);
# endif
}
}
}
}
# if !defined(GC_NO_THREADS_DISCOVERY) || defined(GC_ASSERTIONS)
GC_please_stop = FALSE;
# endif
}
# ifdef MSWINCE
/*
* The `VirtualQuery()` calls below will not work properly on some old
* WinCE versions, but since each stack is restricted to an aligned
* 64 KiB region of virtual memory we can just take the next lowest
* multiple of 64 KiB. The result of this macro must not be used as
* its argument later and must not be used as the lower bound for `sp`
* check (since the stack may be bigger than 64 KiB).
*/
# define GC_wince_evaluate_stack_min(s) \
(ptr_t)(((word)(s) - (word)1) & ~(word)0xFFFF)
# elif defined(GC_ASSERTIONS)
# define GC_dont_query_stack_min FALSE
# endif
/*
* A cache holding the results of the recent `VirtualQuery()` call.
* Protected by the allocator lock.
*/
static ptr_t last_address = 0;
static MEMORY_BASIC_INFORMATION last_info;
/*
* Probe stack memory region (starting at `s`) to find out its lowest
* address (i.e. stack top). `s` must be a mapped address inside the
* region, *not* the first unmapped address.
*/
STATIC ptr_t
GC_get_stack_min(ptr_t s)
{
ptr_t bottom;
GC_ASSERT(I_HOLD_LOCK());
if (s != last_address) {
VirtualQuery(s, &last_info, sizeof(last_info));
last_address = s;
}
do {
bottom = (ptr_t)last_info.BaseAddress;
VirtualQuery(bottom - 1, &last_info, sizeof(last_info));
last_address = bottom - 1;
} while ((last_info.Protect & PAGE_READWRITE)
&& !(last_info.Protect & PAGE_GUARD));
return bottom;
}
/*
* Return `TRUE` if the page at `s` has protections appropriate for
* a stack page.
*/
static GC_bool
may_be_in_stack(ptr_t s)
{
GC_ASSERT(I_HOLD_LOCK());
if (s != last_address) {
VirtualQuery(s, &last_info, sizeof(last_info));
last_address = s;
}
return (last_info.Protect & PAGE_READWRITE) != 0
&& (last_info.Protect & PAGE_GUARD) == 0;
}
/*
* Copy all registers that might point into the heap. Frame pointer
* registers are included in case client code was compiled with the
* "omit frame pointer" optimization. The context register values are
* stored to `regs` argument which is expected to be of `PUSHED_REGS_COUNT`
* length exactly. Returns the context stack pointer value.
*/
static ptr_t
copy_ptr_regs(word *regs, const CONTEXT *pcontext)
{
ptr_t sp;
int cnt = 0;
# define context (*pcontext)
# define PUSH1(reg) (regs[cnt++] = (word)pcontext->reg)
# define PUSH2(r1, r2) (PUSH1(r1), PUSH1(r2))
# define PUSH4(r1, r2, r3, r4) (PUSH2(r1, r2), PUSH2(r3, r4))
# define PUSH8_LH(r1, r2, r3, r4) \
(PUSH4(r1.Low, r1.High, r2.Low, r2.High), \
PUSH4(r3.Low, r3.High, r4.Low, r4.High))
# if defined(I386)
# ifdef WOW64_THREAD_CONTEXT_WORKAROUND
/*
* Notes: these should be the first "pushed" registers, exactly in this
* order, see the WoW64 logic in `GC_push_stack_for()`; these registers
* do not contain pointers.
*/
PUSH2(ContextFlags, SegFs);
# endif
PUSH4(Edi, Esi, Ebx, Edx), PUSH2(Ecx, Eax), PUSH1(Ebp);
sp = (ptr_t)context.Esp;
# elif defined(X86_64)
PUSH4(Rax, Rcx, Rdx, Rbx);
PUSH2(Rbp, Rsi);
PUSH1(Rdi);
PUSH4(R8, R9, R10, R11);
PUSH4(R12, R13, R14, R15);
# ifndef XMM_CANT_STORE_PTRS
PUSH8_LH(Xmm0, Xmm1, Xmm2, Xmm3);
PUSH8_LH(Xmm4, Xmm5, Xmm6, Xmm7);
PUSH8_LH(Xmm8, Xmm9, Xmm10, Xmm11);
PUSH8_LH(Xmm12, Xmm13, Xmm14, Xmm15);
# endif
sp = (ptr_t)context.Rsp;
# elif defined(ARM32)
PUSH4(R0, R1, R2, R3), PUSH4(R4, R5, R6, R7), PUSH4(R8, R9, R10, R11);
PUSH1(R12);
sp = (ptr_t)context.Sp;
# elif defined(AARCH64)
PUSH4(X0, X1, X2, X3), PUSH4(X4, X5, X6, X7), PUSH4(X8, X9, X10, X11);
PUSH4(X12, X13, X14, X15), PUSH4(X16, X17, X18, X19),
PUSH4(X20, X21, X22, X23);
PUSH4(X24, X25, X26, X27), PUSH1(X28);
PUSH1(Lr);
sp = (ptr_t)context.Sp;
# elif defined(SHx)
PUSH4(R0, R1, R2, R3), PUSH4(R4, R5, R6, R7), PUSH4(R8, R9, R10, R11);
PUSH2(R12, R13), PUSH1(R14);
sp = (ptr_t)context.R15;
# elif defined(MIPS)
PUSH4(IntAt, IntV0, IntV1, IntA0), PUSH4(IntA1, IntA2, IntA3, IntT0);
PUSH4(IntT1, IntT2, IntT3, IntT4), PUSH4(IntT5, IntT6, IntT7, IntS0);
PUSH4(IntS1, IntS2, IntS3, IntS4), PUSH4(IntS5, IntS6, IntS7, IntT8);
PUSH4(IntT9, IntK0, IntK1, IntS8);
sp = (ptr_t)context.IntSp;
# elif defined(PPC)
PUSH4(Gpr0, Gpr3, Gpr4, Gpr5), PUSH4(Gpr6, Gpr7, Gpr8, Gpr9);
PUSH4(Gpr10, Gpr11, Gpr12, Gpr14), PUSH4(Gpr15, Gpr16, Gpr17, Gpr18);
PUSH4(Gpr19, Gpr20, Gpr21, Gpr22), PUSH4(Gpr23, Gpr24, Gpr25, Gpr26);
PUSH4(Gpr27, Gpr28, Gpr29, Gpr30), PUSH1(Gpr31);
sp = (ptr_t)context.Gpr1;
# elif defined(ALPHA)
PUSH4(IntV0, IntT0, IntT1, IntT2), PUSH4(IntT3, IntT4, IntT5, IntT6);
PUSH4(IntT7, IntS0, IntS1, IntS2), PUSH4(IntS3, IntS4, IntS5, IntFp);
PUSH4(IntA0, IntA1, IntA2, IntA3), PUSH4(IntA4, IntA5, IntT8, IntT9);
PUSH4(IntT10, IntT11, IntT12, IntAt);
sp = (ptr_t)context.IntSp;
# elif defined(CPPCHECK)
GC_noop1_ptr(regs);
sp = (ptr_t)(word)cnt; /*< to workaround "cnt not used" false positive */
# else
# error Architecture is not supported
# endif
# undef context
# undef PUSH1
# undef PUSH2
# undef PUSH4
# undef PUSH8_LH
GC_ASSERT(cnt == PUSHED_REGS_COUNT);
return sp;
}
STATIC word
GC_push_stack_for(GC_thread thread, thread_id_t self_id, GC_bool *pfound_me)
{
GC_bool is_self = FALSE;
ptr_t sp, stack_min;
GC_stack_context_t crtn = thread->crtn;
ptr_t stack_end = crtn->stack_end;
struct GC_traced_stack_sect_s *traced_stack_sect = crtn->traced_stack_sect;
GC_ASSERT(I_HOLD_LOCK());
if (UNLIKELY(NULL == stack_end))
return 0;
if (thread->id == self_id) {
GC_ASSERT((thread->flags & DO_BLOCKING) == 0);
sp = GC_approx_sp();
is_self = TRUE;
*pfound_me = TRUE;
} else if ((thread->flags & DO_BLOCKING) != 0) {
/* Use saved `sp` value for blocked threads. */
sp = crtn->stack_ptr;
} else {
# ifdef RETRY_GET_THREAD_CONTEXT
/*
* We cache context when suspending the thread since it may require
* looping.
*/
word *regs = thread->context_regs;
if ((thread->flags & IS_SUSPENDED) != 0) {
sp = thread->context_sp;
} else
# else
word regs[PUSHED_REGS_COUNT];
# endif
/* else */ {
CONTEXT context;
/* For unblocked threads call `GetThreadContext()`. */
context.ContextFlags = GET_THREAD_CONTEXT_FLAGS;
if (GetThreadContext(THREAD_HANDLE(thread), &context)) {
sp = copy_ptr_regs(regs, &context);
} else {
# ifndef GC_NO_THREADS_DISCOVERY
if (NULL == GC_cptr_load_acquire(&thread->handle))
return 0;
# endif
# ifdef RETRY_GET_THREAD_CONTEXT
/* At least, try to use the stale context if saved. */
sp = thread->context_sp;
if (NULL == sp) {
/*
* Skip the current thread; anyway its stack will be pushed
* when the world is stopped.
*/
return 0;
}
# else
/* This is to avoid "might be uninitialized" compiler warning. */
*(volatile ptr_t *)&sp = NULL;
ABORT("GetThreadContext failed");
# endif
}
}
# ifdef THREAD_LOCAL_ALLOC
GC_ASSERT((thread->flags & IS_SUSPENDED) != 0 || !GC_world_stopped);
# endif
# ifndef WOW64_THREAD_CONTEXT_WORKAROUND
GC_push_many_regs(regs, PUSHED_REGS_COUNT);
# else
GC_push_many_regs(regs + 2, PUSHED_REGS_COUNT - 2);
/* Skip `ContextFlags` and `SegFs`. */
/* WoW64 workaround. */
if (isWow64) {
DWORD ContextFlags = (DWORD)regs[0];
if ((ContextFlags & CONTEXT_EXCEPTION_REPORTING) != 0
&& (ContextFlags
& (CONTEXT_EXCEPTION_ACTIVE
/* `| CONTEXT_SERVICE_ACTIVE` */))
!= 0) {
GC_NT_TIB *tib;
# ifdef MSWINRT_FLAVOR
tib = thread->tib;
# else
WORD SegFs = (WORD)regs[1];
LDT_ENTRY selector;
if (!GetThreadSelectorEntry(THREAD_HANDLE(thread), SegFs, &selector))
ABORT("GetThreadSelectorEntry failed");
tib = (GC_NT_TIB *)(selector.BaseLow
| (selector.HighWord.Bits.BaseMid << 16)
| (selector.HighWord.Bits.BaseHi << 24));
# endif
# ifdef DEBUG_THREADS
GC_log_printf("TIB stack limit/base: %p .. %p\n",
(void *)tib->StackLimit, (void *)tib->StackBase);
# endif
GC_ASSERT(!HOTTER_THAN((ptr_t)tib->StackBase, stack_end));
if (stack_end != crtn->initial_stack_base
/* We are in a coroutine (old-style way of the support). */
&& (ADDR(stack_end) <= (word)tib->StackLimit
|| (word)tib->StackBase < ADDR(stack_end))) {
/* The coroutine stack is not within TIB stack. */
WARN("GetThreadContext might return stale register values"
" including ESP= %p\n",
sp);
/*
* TODO: Because of WoW64 bug, there is no guarantee that `sp`
* really points to the stack top but, for now, we do our best
* as the TIB stack limit/base cannot be used while we are
* inside a coroutine.
*/
} else {
/*
* `GetThreadContext()` might return stale register values,
* so we scan the entire stack region (down to the stack limit).
* There is no 100% guarantee that all the registers are pushed
* but we do our best (the proper solution would be to fix it
* inside Windows).
*/
sp = (ptr_t)tib->StackLimit;
}
} /* else */
# ifdef DEBUG_THREADS
else {
static GC_bool logged;
if (!logged && (ContextFlags & CONTEXT_EXCEPTION_REPORTING) == 0) {
GC_log_printf("CONTEXT_EXCEPTION_REQUEST not supported\n");
logged = TRUE;
}
}
# endif
}
# endif /* WOW64_THREAD_CONTEXT_WORKAROUND */
}
# if defined(STACKPTR_CORRECTOR_AVAILABLE) && defined(GC_PTHREADS)
if (GC_sp_corrector != 0)
GC_sp_corrector((void **)&sp, PTHREAD_TO_VPTR(thread->pthread_id));
# endif
/*
* Set `stack_min` to the lowest address in the thread stack, or to
* an address in the thread stack not bigger than `sp`, taking advantage
* of the old value to avoid slow traversals of large stacks.
*/
if (crtn->last_stack_min == ADDR_LIMIT) {
# ifdef MSWINCE
if (GC_dont_query_stack_min) {
stack_min = GC_wince_evaluate_stack_min(
traced_stack_sect != NULL ? (ptr_t)traced_stack_sect : stack_end);
/* Keep `last_stack_min` value unmodified. */
} else
# endif
/* else */ {
stack_min = GC_get_stack_min(
traced_stack_sect != NULL ? (ptr_t)traced_stack_sect : stack_end);
GC_win32_unprotect_thread(thread);
crtn->last_stack_min = stack_min;
}
} else {
/*
* First, adjust the latest known minimum stack address if we are
* inside `GC_call_with_gc_active`.
*/
if (traced_stack_sect != NULL
&& ADDR_LT((ptr_t)traced_stack_sect, crtn->last_stack_min)) {
GC_win32_unprotect_thread(thread);
crtn->last_stack_min = (ptr_t)traced_stack_sect;
}
if (ADDR_INSIDE(sp, crtn->last_stack_min, stack_end)) {
stack_min = sp;
} else {
/* In the current thread it is always safe to use `sp` value. */
if (may_be_in_stack(is_self && ADDR_LT(sp, crtn->last_stack_min)
? sp
: crtn->last_stack_min)) {
stack_min = (ptr_t)last_info.BaseAddress;
/* Do not probe rest of the stack if `sp` is correct. */
if (!ADDR_INSIDE(sp, stack_min, stack_end))
stack_min = GC_get_stack_min(crtn->last_stack_min);
} else {
/* Stack shrunk? Is this possible? */
stack_min = GC_get_stack_min(stack_end);
}
GC_win32_unprotect_thread(thread);
crtn->last_stack_min = stack_min;
}
}
GC_ASSERT(GC_dont_query_stack_min || stack_min == GC_get_stack_min(stack_end)
|| (ADDR_GE(sp, stack_min) && ADDR_LT(stack_min, stack_end)
&& ADDR_LT(GC_get_stack_min(stack_end), stack_min)));
if (ADDR_INSIDE(sp, stack_min, stack_end)) {
# ifdef DEBUG_THREADS
GC_log_printf("Pushing stack for 0x%x from sp %p to %p from 0x%x\n",
(int)thread->id, (void *)sp, (void *)stack_end,
(int)self_id);
# endif
GC_push_all_stack_sections(sp, stack_end, traced_stack_sect);
} else {
/*
* If not the current thread, then it is possible for `sp` to point to
* the guarded (untouched yet) page just below the current `stack_min`
* of the thread.
*/
if (is_self || ADDR_GE(sp, stack_end)
|| ADDR_LT(sp + GC_page_size, stack_min))
WARN("Thread stack pointer %p out of range, pushing everything\n", sp);
# ifdef DEBUG_THREADS
GC_log_printf("Pushing stack for 0x%x from (min) %p to %p from 0x%x\n",
(int)thread->id, (void *)stack_min, (void *)stack_end,
(int)self_id);
# endif
/* Push everything - ignore "traced stack section" data. */
GC_push_all_stack(stack_min, stack_end);
}
/* Note: stack grows down. */
return stack_end - sp;
}
GC_INNER void
GC_push_all_stacks(void)
{
thread_id_t self_id = GetCurrentThreadId();
GC_bool found_me = FALSE;
# ifndef SMALL_CONFIG
unsigned nthreads = 0;
# endif
word total_size = 0;
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(GC_thr_initialized);
# ifndef GC_NO_THREADS_DISCOVERY
if (GC_win32_dll_threads) {
int i;
LONG my_max = GC_get_max_thread_index();
for (i = 0; i <= my_max; i++) {
GC_thread p = (GC_thread)(dll_thread_table + i);
if (p->tm.in_use) {
# ifndef SMALL_CONFIG
++nthreads;
# endif
total_size += GC_push_stack_for(p, self_id, &found_me);
}
}
} else
# endif
/* else */ {
int i;
for (i = 0; i < THREAD_TABLE_SZ; i++) {
GC_thread p;
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
GC_ASSERT(THREAD_TABLE_INDEX(p->id) == i);
if (!KNOWN_FINISHED(p)) {
# ifndef SMALL_CONFIG
++nthreads;
# endif
total_size += GC_push_stack_for(p, self_id, &found_me);
}
}
}
}
# ifndef SMALL_CONFIG
GC_VERBOSE_LOG_PRINTF(
"Pushed %d thread stacks%s\n", nthreads,
GC_win32_dll_threads ? " based on DllMain thread tracking" : "");
# endif
if (!found_me && !GC_in_thread_creation)
ABORT("Collecting from unknown thread");
GC_total_stacksize = total_size;
}
# ifdef PARALLEL_MARK
GC_INNER ptr_t GC_marker_last_stack_min[MAX_MARKERS - 1] = { 0 };
# endif
GC_INNER void
GC_get_next_stack(ptr_t start, ptr_t limit, ptr_t *plo, ptr_t *phi)
{
int i;
/* Least in-range stack base. */
ptr_t current_min = ADDR_LIMIT;
/*
* Address of `last_stack_min` field for thread corresponding to
* `current_min`.
*/
ptr_t *plast_stack_min = NULL;
/*
* Either `NULL` or points to the thread's hash table entry
* containing `*plast_stack_min`.
*/
GC_thread thread = NULL;
GC_ASSERT(I_HOLD_LOCK());
/* First set `current_min`, ignoring `limit`. */
if (GC_win32_dll_threads) {
LONG my_max = GC_get_max_thread_index();
for (i = 0; i <= my_max; i++) {
ptr_t stack_end = (ptr_t)dll_thread_table[i].crtn->stack_end;
if (ADDR_LT(start, stack_end) && ADDR_LT(stack_end, current_min)) {
/* Update address of `last_stack_min`. */
plast_stack_min = &dll_thread_table[i].crtn->last_stack_min;
current_min = stack_end;
# ifdef CPPCHECK
/* To avoid a warning that thread is always null. */
thread = (GC_thread)&dll_thread_table[i];
# endif
}
}
} else {
for (i = 0; i < THREAD_TABLE_SZ; i++) {
GC_thread p;
for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
GC_stack_context_t crtn = p->crtn;
/* Note: the following is read of a `volatile` field. */
ptr_t stack_end = crtn->stack_end;
if (ADDR_LT(start, stack_end) && ADDR_LT(stack_end, current_min)) {
/* Update value of `*plast_stack_min`. */
plast_stack_min = &crtn->last_stack_min;
/* Remember current thread to unprotect. */
thread = p;
current_min = stack_end;
}
}
}
# ifdef PARALLEL_MARK
for (i = 0; i < GC_markers_m1; ++i) {
ptr_t s = GC_marker_sp[i];
# ifdef IA64
/* FIXME: Not implemented. */
# endif
if (ADDR_LT(start, s) && ADDR_LT(s, current_min)) {
GC_ASSERT(GC_marker_last_stack_min[i] != NULL);
plast_stack_min = &GC_marker_last_stack_min[i];
current_min = s;
/* Not a thread's hash table entry. */
thread = NULL;
}
}
# endif
}
*phi = current_min;
if (current_min == ADDR_LIMIT) {
*plo = ADDR_LIMIT;
return;
}
GC_ASSERT(ADDR_LT(start, current_min) && plast_stack_min != NULL);
# ifdef MSWINCE
if (GC_dont_query_stack_min) {
*plo = GC_wince_evaluate_stack_min(current_min);
/* Keep `last_stack_min` value unmodified. */
return;
}
# endif
if (ADDR_LT(limit, current_min) && !may_be_in_stack(limit)) {
/*
* Skip the rest since the memory region at `limit` address is not
* a stack (so the lowest address of the found stack would be above
* the `limit` value anyway).
*/
*plo = ADDR_LIMIT;
return;
}
/*
* Get the minimum address of the found stack by probing its memory
* region starting from the recent known minimum (if set).
*/
if (*plast_stack_min == ADDR_LIMIT || !may_be_in_stack(*plast_stack_min)) {
/* Unsafe to start from `last_stack_min` value. */
*plo = GC_get_stack_min(current_min);
} else {
/* Use the recent value to optimize search for minimum address. */
*plo = GC_get_stack_min(*plast_stack_min);
}
/* Remember current `stack_min` value. */
if (thread != NULL)
GC_win32_unprotect_thread(thread);
*plast_stack_min = *plo;
}
# if defined(PARALLEL_MARK) && !defined(GC_PTHREADS_PARAMARK)
# ifndef MARK_THREAD_STACK_SIZE
/* The default size of the marker's thread stack. */
# define MARK_THREAD_STACK_SIZE 0
# endif
/* Events with manual reset (one for each mark helper). */
STATIC HANDLE GC_marker_cv[MAX_MARKERS - 1] = { 0 };
GC_INNER thread_id_t GC_marker_Id[MAX_MARKERS - 1] = { 0 };
/*
* `mark_mutex_event`, `builder_cv`, `mark_cv` are initialized in
* `GC_thr_init()`.
*/
/* Note: this event should be with auto-reset. */
static HANDLE mark_mutex_event = (HANDLE)0;
/* Note: these events are with manual reset. */
static HANDLE builder_cv = (HANDLE)0;
static HANDLE mark_cv = (HANDLE)0;
GC_INNER void
GC_start_mark_threads_inner(void)
{
int i;
GC_ASSERT(I_HOLD_LOCK());
ASSERT_CANCEL_DISABLED();
if (GC_available_markers_m1 <= 0 || GC_parallel)
return;
GC_wait_for_gc_completion(TRUE);
GC_ASSERT(0 == GC_fl_builder_count);
/*
* Initialize `GC_marker_cv[]` fully before starting the first helper
* thread.
*/
GC_markers_m1 = GC_available_markers_m1;
for (i = 0; i < GC_markers_m1; ++i) {
if ((GC_marker_cv[i]
= CreateEvent(NULL /* `attrs` */, TRUE /* `isManualReset` */,
FALSE /* `initialState` */, NULL /* `name` (A/W) */))
== (HANDLE)0)
ABORT("CreateEvent failed");
}
for (i = 0; i < GC_markers_m1; ++i) {
# if defined(MSWINCE) || defined(MSWIN_XBOX1)
HANDLE handle;
DWORD thread_id;
GC_marker_last_stack_min[i] = ADDR_LIMIT;
/* There is no `_beginthreadex()` in WinCE. */
handle = CreateThread(NULL /* `lpsa` */,
MARK_THREAD_STACK_SIZE /* ignored */, GC_mark_thread,
NUMERIC_TO_VPTR(i), 0 /* `fdwCreate` */, &thread_id);
if (UNLIKELY(NULL == handle)) {
WARN("Marker thread %" WARN_PRIdPTR " creation failed\n",
(GC_signed_word)i);
/*
* The most probable failure reason is "not enough memory".
* Do not try to create other marker threads.
*/
break;
}
/* It is safe to detach the thread. */
CloseHandle(handle);
# else
GC_uintptr_t handle;
unsigned thread_id;
GC_marker_last_stack_min[i] = ADDR_LIMIT;
handle = _beginthreadex(NULL /* `security_attr` */, MARK_THREAD_STACK_SIZE,
GC_mark_thread, NUMERIC_TO_VPTR(i),
0 /* `flags` */, &thread_id);
if (UNLIKELY(!handle || handle == ~(GC_uintptr_t)0)) {
WARN("Marker thread %" WARN_PRIdPTR " creation failed\n",
(GC_signed_word)i);
/* Do not try to create other marker threads. */
break;
} else {
/* We may detach the thread (if `handle` is of `HANDLE` type). */
/* `CloseHandle((HANDLE)handle);` */
}
# endif
}
/* Adjust `GC_markers_m1` (and free unused resources) if failed. */
while (GC_markers_m1 > i) {
GC_markers_m1--;
CloseHandle(GC_marker_cv[GC_markers_m1]);
}
GC_wait_for_markers_init();
GC_COND_LOG_PRINTF("Started %d mark helper threads\n", GC_markers_m1);
if (UNLIKELY(0 == i)) {
CloseHandle(mark_cv);
CloseHandle(builder_cv);
CloseHandle(mark_mutex_event);
}
}
# ifdef GC_ASSERTIONS
STATIC unsigned long GC_mark_lock_holder = NO_THREAD;
# define SET_MARK_LOCK_HOLDER \
(void)(GC_mark_lock_holder = GetCurrentThreadId())
# define UNSET_MARK_LOCK_HOLDER \
do { \
GC_ASSERT(GC_mark_lock_holder == GetCurrentThreadId()); \
GC_mark_lock_holder = NO_THREAD; \
} while (0)
# else
# define SET_MARK_LOCK_HOLDER (void)0
# define UNSET_MARK_LOCK_HOLDER (void)0
# endif /* !GC_ASSERTIONS */
/*
* Allowed values for `GC_mark_mutex_state`.
* `MARK_MUTEX_LOCKED` means "locked but no other waiters";
* `MARK_MUTEX_WAITERS_EXIST` means "locked and waiters may exist".
*/
# define MARK_MUTEX_UNLOCKED 0
# define MARK_MUTEX_LOCKED 1
# define MARK_MUTEX_WAITERS_EXIST (-1)
/* The mutex state. Accessed using `InterlockedExchange()`. */
STATIC /* `volatile` */ LONG GC_mark_mutex_state = MARK_MUTEX_UNLOCKED;
# ifdef LOCK_STATS
volatile AO_t GC_block_count = 0;
volatile AO_t GC_unlocked_count = 0;
# endif
GC_INNER void
GC_acquire_mark_lock(void)
{
GC_ASSERT(GC_mark_lock_holder != GetCurrentThreadId());
if (UNLIKELY(InterlockedExchange(&GC_mark_mutex_state, MARK_MUTEX_LOCKED)
!= 0)) {
# ifdef LOCK_STATS
(void)AO_fetch_and_add1(&GC_block_count);
# endif
/* Repeatedly reset the state and wait until we acquire the mark lock. */
while (InterlockedExchange(&GC_mark_mutex_state, MARK_MUTEX_WAITERS_EXIST)
!= 0) {
if (WaitForSingleObject(mark_mutex_event, INFINITE) == WAIT_FAILED)
ABORT("WaitForSingleObject failed");
}
}
# ifdef LOCK_STATS
else {
(void)AO_fetch_and_add1(&GC_unlocked_count);
}
# endif
GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
SET_MARK_LOCK_HOLDER;
}
GC_INNER void
GC_release_mark_lock(void)
{
UNSET_MARK_LOCK_HOLDER;
if (UNLIKELY(InterlockedExchange(&GC_mark_mutex_state, MARK_MUTEX_UNLOCKED)
< 0)) {
/* Wake a waiter. */
if (!SetEvent(mark_mutex_event))
ABORT("SetEvent failed");
}
}
/*
* In `GC_wait_for_reclaim()`/`GC_notify_all_builder()` we emulate
* `pthread_cond_wait()`/`pthread_cond_broadcast()` primitives with
* Win32 API event object (working in the "manual reset" mode).
* This works here because `GC_notify_all_builder()` is always called
* holding the mark lock and the checked condition
* (`GC_fl_builder_count` is zero) is the only one for which
* broadcasting on `builder_cv` is performed.
*/
GC_INNER void
GC_wait_for_reclaim(void)
{
GC_ASSERT(builder_cv != 0);
for (;;) {
GC_acquire_mark_lock();
if (0 == GC_fl_builder_count)
break;
if (!ResetEvent(builder_cv))
ABORT("ResetEvent failed");
GC_release_mark_lock();
if (WaitForSingleObject(builder_cv, INFINITE) == WAIT_FAILED)
ABORT("WaitForSingleObject failed");
}
GC_release_mark_lock();
}
GC_INNER void
GC_notify_all_builder(void)
{
GC_ASSERT(GC_mark_lock_holder == GetCurrentThreadId());
GC_ASSERT(builder_cv != 0);
GC_ASSERT(0 == GC_fl_builder_count);
if (!SetEvent(builder_cv))
ABORT("SetEvent failed");
}
/* `mark_cv` is used (for waiting) by a non-helper thread. */
GC_INNER void
GC_wait_marker(void)
{
HANDLE event = mark_cv;
thread_id_t self_id = GetCurrentThreadId();
int i = GC_markers_m1;
while (i-- > 0) {
if (GC_marker_Id[i] == self_id) {
event = GC_marker_cv[i];
break;
}
}
if (!ResetEvent(event))
ABORT("ResetEvent failed");
GC_release_mark_lock();
if (WaitForSingleObject(event, INFINITE) == WAIT_FAILED)
ABORT("WaitForSingleObject failed");
GC_acquire_mark_lock();
}
GC_INNER void
GC_notify_all_marker(void)
{
thread_id_t self_id = GetCurrentThreadId();
int i = GC_markers_m1;
while (i-- > 0) {
/* Notify every marker ignoring self (for efficiency). */
if (!SetEvent(GC_marker_Id[i] != self_id ? GC_marker_cv[i] : mark_cv))
ABORT("SetEvent failed");
}
}
# endif /* PARALLEL_MARK && !GC_PTHREADS_PARAMARK */
/*
* We have no `DllMain` to take care of new threads. Thus, we must properly
* intercept thread creation.
*/
struct win32_start_info {
LPTHREAD_START_ROUTINE start_routine;
LPVOID arg;
};
STATIC void *GC_CALLBACK
GC_win32_start_inner(struct GC_stack_base *sb, void *arg)
{
void *ret;
LPTHREAD_START_ROUTINE start_routine
= ((struct win32_start_info *)arg)->start_routine;
LPVOID start_arg = ((struct win32_start_info *)arg)->arg;
GC_ASSERT(!GC_win32_dll_threads);
/* This waits for an in-progress garbage collection. */
GC_register_my_thread(sb);
# ifdef DEBUG_THREADS
GC_log_printf("thread 0x%lx starting...\n", (long)GetCurrentThreadId());
# endif
GC_free(arg);
/*
* Clear the thread entry even if we exit with an exception.
* This is probably pointless, since an uncaught exception is
* supposed to result in the process being killed.
*/
# ifndef NO_SEH_AVAILABLE
ret = NULL; /*< to avoid "might be uninitialized" compiler warning */
__try
# endif
{
ret = NUMERIC_TO_VPTR(start_routine(start_arg));
}
# ifndef NO_SEH_AVAILABLE
__finally
# endif
{
(void)GC_unregister_my_thread();
}
# ifdef DEBUG_THREADS
GC_log_printf("thread 0x%lx returned from start routine\n",
(long)GetCurrentThreadId());
# endif
# if defined(CPPCHECK)
GC_noop1_ptr(sb);
# endif
return ret;
}
STATIC DWORD WINAPI
GC_win32_start(LPVOID arg)
{
return (DWORD)(GC_uintptr_t)GC_call_with_stack_base(GC_win32_start_inner,
arg);
}
GC_API HANDLE WINAPI
GC_CreateThread(LPSECURITY_ATTRIBUTES lpThreadAttributes,
GC_WIN32_SIZE_T dwStackSize,
LPTHREAD_START_ROUTINE lpStartAddress, LPVOID lpParameter,
DWORD dwCreationFlags, LPDWORD lpThreadId)
{
/*
* Make sure the collector is initialized (i.e. main thread is attached,
* TLS is initialized). This is redundant when `GC_win32_dll_threads`
* is set by `GC_use_threads_discovery()`.
*/
if (UNLIKELY(!GC_is_initialized))
GC_init();
GC_ASSERT(GC_thr_initialized);
# ifdef DEBUG_THREADS
GC_log_printf("About to create a thread from 0x%lx\n",
(long)GetCurrentThreadId());
# endif
if (GC_win32_dll_threads) {
return CreateThread(lpThreadAttributes, dwStackSize, lpStartAddress,
lpParameter, dwCreationFlags, lpThreadId);
} else {
/* Note: this is handed off to and deallocated by child thread. */
struct win32_start_info *psi
= (struct win32_start_info *)GC_malloc_uncollectable(
sizeof(struct win32_start_info));
HANDLE thread_h;
if (UNLIKELY(NULL == psi)) {
SetLastError(ERROR_NOT_ENOUGH_MEMORY);
return NULL;
}
/* Set up the thread arguments. */
psi->start_routine = lpStartAddress;
psi->arg = lpParameter;
GC_dirty(psi);
REACHABLE_AFTER_DIRTY(lpParameter);
# ifdef PARALLEL_MARK
if (!GC_parallel && UNLIKELY(GC_available_markers_m1 > 0))
GC_start_mark_threads();
# endif
set_need_to_lock();
thread_h = CreateThread(lpThreadAttributes, dwStackSize, GC_win32_start,
psi, dwCreationFlags, lpThreadId);
if (UNLIKELY(0 == thread_h))
GC_free(psi);
return thread_h;
}
}
GC_API DECLSPEC_NORETURN void WINAPI
GC_ExitThread(DWORD dwExitCode)
{
if (!GC_win32_dll_threads)
(void)GC_unregister_my_thread();
ExitThread(dwExitCode);
}
# if defined(MSWIN32) && !defined(NO_CRT)
GC_API GC_uintptr_t GC_CALL
GC_beginthreadex(void *security, unsigned stack_size,
unsigned(__stdcall *start_address)(void *), void *arglist,
unsigned initflag, unsigned *thrdaddr)
{
if (UNLIKELY(!GC_is_initialized))
GC_init();
GC_ASSERT(GC_thr_initialized);
# ifdef DEBUG_THREADS
GC_log_printf("About to create a thread from 0x%lx\n",
(long)GetCurrentThreadId());
# endif
if (GC_win32_dll_threads) {
return _beginthreadex(security, stack_size, start_address, arglist,
initflag, thrdaddr);
} else {
GC_uintptr_t thread_h;
/* Note: this is handed off to and deallocated by child thread. */
struct win32_start_info *psi
= (struct win32_start_info *)GC_malloc_uncollectable(
sizeof(struct win32_start_info));
if (UNLIKELY(NULL == psi)) {
/*
* MSDN docs say `_beginthreadex()` returns 0 on error and sets
* `errno` to either `EAGAIN` (too many threads) or `EINVAL` (the
* argument is invalid or the stack size is incorrect), so we set
* `errno` to `EAGAIN` on "not enough memory".
*/
errno = EAGAIN;
return 0;
}
/* Set up the thread arguments. */
psi->start_routine = (LPTHREAD_START_ROUTINE)start_address;
psi->arg = arglist;
GC_dirty(psi);
REACHABLE_AFTER_DIRTY(arglist);
# ifdef PARALLEL_MARK
if (!GC_parallel && UNLIKELY(GC_available_markers_m1 > 0))
GC_start_mark_threads();
# endif
set_need_to_lock();
thread_h = _beginthreadex(security, stack_size,
(unsigned(__stdcall *)(void *))GC_win32_start,
psi, initflag, thrdaddr);
if (UNLIKELY(0 == thread_h))
GC_free(psi);
return thread_h;
}
}
GC_API void GC_CALL
GC_endthreadex(unsigned retval)
{
if (!GC_win32_dll_threads)
(void)GC_unregister_my_thread();
_endthreadex(retval);
}
# endif /* MSWIN32 && !NO_CRT */
# ifdef GC_WINMAIN_REDIRECT
/* This might be useful on WinCE. Should not be used with `GC_DLL`. */
# if defined(MSWINCE) && defined(UNDER_CE)
# define WINMAIN_LPTSTR LPWSTR
# else
# define WINMAIN_LPTSTR LPSTR
# endif
/* This is defined in `gc.h` file. */
# undef WinMain
/* Defined outside the collector by an application. */
int WINAPI GC_WinMain(HINSTANCE, HINSTANCE, WINMAIN_LPTSTR, int);
typedef struct {
HINSTANCE hInstance;
HINSTANCE hPrevInstance;
WINMAIN_LPTSTR lpCmdLine;
int nShowCmd;
} main_thread_args;
static DWORD WINAPI
main_thread_start(LPVOID arg)
{
main_thread_args *main_args = (main_thread_args *)arg;
return (DWORD)GC_WinMain(main_args->hInstance, main_args->hPrevInstance,
main_args->lpCmdLine, main_args->nShowCmd);
}
STATIC void *GC_CALLBACK
GC_waitForSingleObjectInfinite(void *handle)
{
return NUMERIC_TO_VPTR(WaitForSingleObject((HANDLE)handle, INFINITE));
}
# ifndef WINMAIN_THREAD_STACK_SIZE
/* The default size of the `WinMain`'s thread stack. */
# define WINMAIN_THREAD_STACK_SIZE 0
# endif
int WINAPI
WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, WINMAIN_LPTSTR lpCmdLine,
int nShowCmd)
{
DWORD exit_code = 1;
main_thread_args args = { hInstance, hPrevInstance, lpCmdLine, nShowCmd };
HANDLE thread_h;
DWORD thread_id;
/* Initialize everything. */
GC_INIT();
/* Start the main thread. */
thread_h = GC_CreateThread(
NULL /* `lpsa` */, WINMAIN_THREAD_STACK_SIZE /* ignored on WinCE */,
main_thread_start, &args, 0 /* `fdwCreate` */, &thread_id);
if (NULL == thread_h)
ABORT("GC_CreateThread(main_thread) failed");
if ((DWORD)(GC_uintptr_t)GC_do_blocking(GC_waitForSingleObjectInfinite,
(void *)thread_h)
== WAIT_FAILED)
ABORT("WaitForSingleObject(main_thread) failed");
GetExitCodeThread(thread_h, &exit_code);
CloseHandle(thread_h);
# if defined(MSWINCE) && !defined(GC_NO_DEINIT)
GC_deinit();
# endif
return (int)exit_code;
}
# endif /* GC_WINMAIN_REDIRECT */
# ifdef WOW64_THREAD_CONTEXT_WORKAROUND
# ifdef MSWINRT_FLAVOR
/* Available on WinRT but we have to declare it manually. */
__declspec(dllimport) HMODULE WINAPI GetModuleHandleW(LPCWSTR);
# endif
static GC_bool
is_wow64_process(HMODULE hK32)
{
BOOL is_wow64;
# ifdef MSWINRT_FLAVOR
/* Try to use `IsWow64Process2()` as it handles different WoW64 cases. */
HMODULE hWow64 = GetModuleHandleW(L"api-ms-win-core-wow64-l1-1-1.dll");
UNUSED_ARG(hK32);
if (hWow64) {
FARPROC pfn2 = GetProcAddress(hWow64, "IsWow64Process2");
USHORT process_machine, native_machine;
if (pfn2
&& (*(BOOL(WINAPI *)(HANDLE, USHORT *, USHORT *))(GC_funcptr_uint)
pfn2)(GetCurrentProcess(), &process_machine, &native_machine))
return process_machine != native_machine;
}
if (IsWow64Process(GetCurrentProcess(), &is_wow64))
return (GC_bool)is_wow64;
# else
if (hK32) {
FARPROC pfn = GetProcAddress(hK32, "IsWow64Process");
if (pfn
&& (*(BOOL(WINAPI *)(HANDLE, BOOL *))(GC_funcptr_uint)pfn)(
GetCurrentProcess(), &is_wow64))
return (GC_bool)is_wow64;
}
# endif
/* `IsWow64Process()` failed. */
return FALSE;
}
# endif /* WOW64_THREAD_CONTEXT_WORKAROUND */
GC_INNER void
GC_thr_init(void)
{
struct GC_stack_base sb;
thread_id_t self_id = GetCurrentThreadId();
# if (!defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) && !defined(MSWINCE) \
&& defined(PARALLEL_MARK)) \
|| defined(WOW64_THREAD_CONTEXT_WORKAROUND)
HMODULE hK32;
# if defined(MSWINRT_FLAVOR) && defined(FUNCPTR_IS_DATAPTR)
MEMORY_BASIC_INFORMATION memInfo;
if (VirtualQuery(CAST_THRU_UINTPTR(void *, GetProcAddress), &memInfo,
sizeof(memInfo))
!= sizeof(memInfo))
ABORT("Weird VirtualQuery result");
hK32 = (HMODULE)memInfo.AllocationBase;
# else
hK32 = GetModuleHandle(TEXT("kernel32.dll"));
# endif
# endif
GC_ASSERT(I_HOLD_LOCK());
GC_ASSERT(!GC_thr_initialized);
GC_ASSERT(ADDR(&GC_threads) % ALIGNMENT == 0);
# ifdef GC_ASSERTIONS
GC_thr_initialized = TRUE;
# endif
# if !defined(DONT_USE_ATEXIT) || !defined(GC_NO_THREADS_DISCOVERY)
GC_main_thread_id = self_id;
# endif
# ifdef CAN_HANDLE_FORK
GC_setup_atfork();
# endif
# ifdef WOW64_THREAD_CONTEXT_WORKAROUND
/* Set `isWow64` flag. */
isWow64 = is_wow64_process(hK32);
# endif
/* Add the initial thread, so we can stop it. */
sb.mem_base = GC_stackbottom;
GC_ASSERT(sb.mem_base != NULL);
# ifdef IA64
sb.reg_base = GC_register_stackbottom;
# endif
# if defined(PARALLEL_MARK)
{
const char *markers_string = GETENV("GC_MARKERS");
int markers = GC_required_markers_cnt;
if (markers_string != NULL) {
markers = atoi(markers_string);
if (markers <= 0 || markers > MAX_MARKERS) {
WARN("Too big or invalid number of mark threads: %" WARN_PRIdPTR
"; using maximum threads\n",
(GC_signed_word)markers);
markers = MAX_MARKERS;
}
} else if (0 == markers) {
/*
* Unless the client sets the desired number of parallel markers,
* it is determined based on the number of CPU cores.
*/
# ifdef MSWINCE
/*
* There is no `GetProcessAffinityMask()` in WinCE.
* `GC_sysinfo` is already initialized.
*/
markers = (int)GC_sysinfo.dwNumberOfProcessors;
# else
# ifdef _WIN64
DWORD_PTR procMask = 0;
DWORD_PTR sysMask;
# else
DWORD procMask = 0;
DWORD sysMask;
# endif
int ncpu = 0;
if (
# ifdef __cplusplus
GetProcessAffinityMask(GetCurrentProcess(), &procMask, &sysMask)
# else
/*
* Cast the mask arguments to `void *` for compatibility with
* some old SDKs.
*/
GetProcessAffinityMask(GetCurrentProcess(), (void *)&procMask,
(void *)&sysMask)
# endif
&& procMask) {
do {
ncpu++;
} while ((procMask &= procMask - 1) != 0);
}
markers = ncpu;
# endif
# if defined(GC_MIN_MARKERS) && !defined(CPPCHECK)
/* This is primarily for testing on systems without `getenv()`. */
if (markers < GC_MIN_MARKERS)
markers = GC_MIN_MARKERS;
# endif
if (markers > MAX_MARKERS) {
/* Silently limit the amount of markers. */
markers = MAX_MARKERS;
}
}
GC_available_markers_m1 = markers - 1;
}
/* Check whether parallel mode could be enabled. */
if (GC_win32_dll_threads || GC_available_markers_m1 <= 0) {
/* Disable parallel marking. */
GC_parallel = FALSE;
GC_COND_LOG_PRINTF("Single marker thread, turning off parallel marking\n");
} else {
# ifndef GC_PTHREADS_PARAMARK
/* Initialize Win32 event objects for parallel marking. */
mark_mutex_event
= CreateEvent(NULL /* `attrs` */, FALSE /* `isManualReset` */,
FALSE /* `initialState` */, NULL /* `name` */);
builder_cv = CreateEvent(NULL /* `attrs` */, TRUE /* `isManualReset` */,
FALSE /* `initialState` */, NULL /* `name` */);
mark_cv = CreateEvent(NULL /* `attrs` */, TRUE /* `isManualReset` */,
FALSE /* `initialState` */, NULL /* `name` */);
if (mark_mutex_event == (HANDLE)0 || builder_cv == (HANDLE)0
|| mark_cv == (HANDLE)0)
ABORT("CreateEvent failed");
# endif
# if !defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) && !defined(MSWINCE)
GC_init_win32_thread_naming(hK32);
# endif
}
# endif /* PARALLEL_MARK */
GC_register_my_thread_inner(&sb, self_id);
}
# ifndef GC_NO_THREADS_DISCOVERY
/*
* We avoid acquiring locks here, since this does not seem to be preemptible.
* This may run with an uninitialized collector, in which case we do not
* do much. This implies that no threads other than the main one should be
* created with an uninitialized collector. (The alternative of initializing
* the collector here seems dangerous, since `DllMain` is limited in what it
* can do.)
*/
# ifdef GC_INSIDE_DLL
/* Export only if needed by client. */
GC_API
# else
# define GC_DllMain DllMain
# endif
BOOL WINAPI
GC_DllMain(HINSTANCE inst, ULONG reason, LPVOID reserved)
{
thread_id_t self_id;
UNUSED_ARG(inst);
UNUSED_ARG(reserved);
/*
* Note that `GC_use_threads_discovery` should be called by the client
* application at start-up to activate automatic thread registration
* (it is the default collector behavior); to always have automatic
* thread registration turned on, the collector should be compiled with
* `-D GC_DISCOVER_TASK_THREADS` option.
*/
if (!GC_win32_dll_threads && GC_is_initialized)
return TRUE;
switch (reason) {
case DLL_THREAD_ATTACH:
/* This is invoked for threads other than main one. */
# ifdef PARALLEL_MARK
/* Do not register marker threads. */
if (GC_parallel) {
/*
* We could reach here only if the collector is not initialized.
* Because `GC_thr_init()` sets `GC_parallel` to `FALSE`.
*/
break;
}
# endif
/* FALLTHRU */
case DLL_PROCESS_ATTACH:
/* This may run with the collector uninitialized. */
self_id = GetCurrentThreadId();
if (GC_is_initialized && GC_main_thread_id != self_id) {
struct GC_stack_base sb;
/* Do not lock here. */
# ifdef GC_ASSERTIONS
int sb_result =
# endif
GC_get_stack_base(&sb);
GC_ASSERT(sb_result == GC_SUCCESS);
GC_register_my_thread_inner(&sb, self_id);
} else {
/* We already did it during `GC_thr_init()`, called by `GC_init()`. */
}
break;
case DLL_THREAD_DETACH:
/* We are hopefully running in the context of the exiting thread. */
if (GC_win32_dll_threads) {
GC_thread t = GC_win32_dll_lookup_thread(GetCurrentThreadId());
if (LIKELY(t != NULL))
GC_delete_thread(t);
}
break;
case DLL_PROCESS_DETACH:
if (GC_win32_dll_threads) {
int i;
int my_max = (int)GC_get_max_thread_index();
for (i = 0; i <= my_max; ++i) {
if (AO_load(&dll_thread_table[i].tm.in_use))
GC_delete_thread((GC_thread)&dll_thread_table[i]);
}
# ifndef GC_NO_DEINIT
GC_deinit();
# endif
}
break;
}
return TRUE;
}
# endif /* !GC_NO_THREADS_DISCOVERY */
# ifndef GC_NO_THREAD_REDIRECTS
/* Restore thread calls redirection. */
# define CreateThread GC_CreateThread
# define ExitThread GC_ExitThread
# undef _beginthreadex
# define _beginthreadex GC_beginthreadex
# undef _endthreadex
# define _endthreadex GC_endthreadex
# endif /* !GC_NO_THREAD_REDIRECTS */
#endif /* GC_WIN32_THREADS */
#ifndef GC_PTHREAD_START_STANDALONE
/*
* Copyright (c) 1994 by Xerox Corporation. All rights reserved.
* Copyright (c) 1996 by Silicon Graphics. All rights reserved.
* Copyright (c) 1998 by Fergus Henderson. All rights reserved.
* Copyright (c) 2000-2010 by Hewlett-Packard Development Company.
* All rights reserved.
*
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
*
* Permission is hereby granted to use or copy this program
* for any purpose, provided the above notices are retained on all copies.
* Permission to modify the code and to distribute modified code is granted,
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
/*
* We want to make sure that `GC_thread_exit_proc()` is unconditionally
* invoked, even if the client is not compiled with `-fexceptions` option,
* but the collector is. The workaround is to put `GC_pthread_start_inner()`
* in its own file (this one), and undefine `__EXCEPTIONS` in the gcc case
* at the top of the file.
* FIXME: It is still unclear whether this will actually cause the
* `exit` handler to be invoked last when `thread_exit` is called (and
* if `-fexceptions` option is used).
*/
#if !defined(DONT_UNDEF_EXCEPTIONS) && defined(__GNUC__) && defined(__linux__)
/*
* We undefine `__EXCEPTIONS` to avoid using gcc `__cleanup__` attribute.
* The current NPTL implementation of `pthread_cleanup_push` uses
* `__cleanup__` attribute when `__EXCEPTIONS` is defined (`-fexceptions`
* option is given). The stack unwinding and cleanup with `__cleanup__`
* attribute work correctly when everything is compiled with `-fexceptions`
* option, but it is not the requirement for this library clients to use
* `-fexceptions` option everywhere. With `__EXCEPTIONS` undefined, the
* cleanup routines are registered with `__pthread_register_cancel` thus
* should work anyway.
*/
# undef __EXCEPTIONS
#endif
#if defined(GC_PTHREADS) && !defined(PLATFORM_THREADS) \
&& !defined(SN_TARGET_PSP2)
/* Invoked from `GC_pthread_start()`. */
GC_INNER_PTHRSTART void *GC_CALLBACK
GC_pthread_start_inner(struct GC_stack_base *sb, void *arg)
{
void *(*start)(void *);
void *start_arg;
void *result;
volatile GC_thread me
= GC_start_rtn_prepare_thread(&start, &start_arg, sb, arg);
# ifndef NACL
pthread_cleanup_push(GC_thread_exit_proc, (/* no volatile */ void *)me);
# endif
result = (*start)(start_arg);
# if defined(DEBUG_THREADS) && !defined(GC_PTHREAD_START_STANDALONE)
GC_log_printf("Finishing thread %p\n", PTHREAD_TO_VPTR(pthread_self()));
# endif
me->status = result;
/* Note: we cannot use `GC_dirty()` instead. */
GC_end_stubborn_change(me);
/*
* Cleanup acquires the allocator lock, ensuring that we cannot exit
* while a collection that thinks we are alive is trying to stop us.
*/
# ifdef NACL
GC_thread_exit_proc((/* no volatile */ void *)me);
# else
pthread_cleanup_pop(1);
# endif
return result;
}
#endif /* GC_PTHREADS */
#endif
/*
* Restore `pthreads` calls redirection (if altered in `pthread_stop_world.c`
* or `pthread_support.c` file). This is only useful if directly included
* from client code (instead of linking with `gc.o` file).
*/
#if !defined(GC_NO_THREAD_REDIRECTS) && defined(GC_PTHREADS)
# define GC_PTHREAD_REDIRECTS_ONLY
# include "gc/gc_pthread_redirects.h"
#endif
/* Note: the files from `extra` folder are not included. */
Reference in new issue View Git Blame Copy Permalink