Thanks for using Compiler Explorer
Sponsors
Jakt
C++
Ada
Analysis
Android Java
Android Kotlin
Assembly
C
C3
Carbon
C++ (Circle)
CIRCT
Clean
CMake
CMakeScript
COBOL
C++ for OpenCL
MLIR
Cppx
Cppx-Blue
Cppx-Gold
Cpp2-cppfront
Crystal
C#
CUDA C++
D
Dart
Elixir
Erlang
Fortran
F#
GLSL
Go
Haskell
HLSL
Hook
Hylo
IL
ispc
Java
Julia
Kotlin
LLVM IR
LLVM MIR
Modula-2
Nim
Objective-C
Objective-C++
OCaml
OpenCL C
Pascal
Pony
Python
Racket
Ruby
Rust
Snowball
Scala
Solidity
Spice
SPIR-V
Swift
LLVM TableGen
Toit
TypeScript Native
V
Vala
Visual Basic
WASM
Zig
Javascript
GIMPLE
Ygen
c source #1
Output
Compile to binary object
Link to binary
Execute the code
Intel asm syntax
Demangle identifiers
Verbose demangling
Filters
Unused labels
Library functions
Directives
Comments
Horizontal whitespace
Debug intrinsics
Compiler
6502 cc65 2.17
6502 cc65 2.18
6502 cc65 2.19
6502 cc65 trunk
ARM GCC 10.2.0 (linux)
ARM GCC 10.2.1 (none)
ARM GCC 10.3.0 (linux)
ARM GCC 10.3.1 (2021.07 none)
ARM GCC 10.3.1 (2021.10 none)
ARM GCC 10.5.0
ARM GCC 11.1.0 (linux)
ARM GCC 11.2.0 (linux)
ARM GCC 11.2.1 (none)
ARM GCC 11.3.0 (linux)
ARM GCC 11.4.0
ARM GCC 12.1.0 (linux)
ARM GCC 12.2.0 (linux)
ARM GCC 12.3.0
ARM GCC 12.4.0
ARM GCC 13.1.0 (linux)
ARM GCC 13.2.0
ARM GCC 13.2.0 (unknown-eabi)
ARM GCC 13.3.0
ARM GCC 13.3.0 (unknown-eabi)
ARM GCC 14.1.0
ARM GCC 14.1.0 (unknown-eabi)
ARM GCC 14.2.0
ARM GCC 14.2.0 (unknown-eabi)
ARM GCC 4.5.4 (linux)
ARM GCC 4.6.4 (linux)
ARM GCC 5.4 (linux)
ARM GCC 5.4.1 (none)
ARM GCC 6.3.0 (linux)
ARM GCC 6.4.0 (linux)
ARM GCC 7.2.1 (none)
ARM GCC 7.3.0 (linux)
ARM GCC 7.5.0 (linux)
ARM GCC 8.2.0 (WinCE)
ARM GCC 8.2.0 (linux)
ARM GCC 8.3.1 (none)
ARM GCC 8.5.0 (linux)
ARM GCC 9.2.1 (none)
ARM GCC 9.3.0 (linux)
ARM GCC trunk (linux)
ARM msvc v19.0 (WINE)
ARM msvc v19.10 (WINE)
ARM msvc v19.14 (WINE)
ARM64 GCC 10.2.0
ARM64 GCC 10.3.0
ARM64 GCC 10.4.0
ARM64 GCC 10.5.0
ARM64 GCC 11.1.0
ARM64 GCC 11.2.0
ARM64 GCC 11.3.0
ARM64 GCC 11.4.0
ARM64 GCC 12.1.0
ARM64 GCC 12.2.0
ARM64 GCC 12.3.0
ARM64 GCC 12.4.0
ARM64 GCC 13.1.0
ARM64 GCC 13.2.0
ARM64 GCC 13.3.0
ARM64 GCC 14.1.0
ARM64 GCC 14.2.0
ARM64 GCC 4.9.4
ARM64 GCC 5.4
ARM64 GCC 5.5.0
ARM64 GCC 6.3
ARM64 GCC 6.4.0
ARM64 GCC 7.3.0
ARM64 GCC 7.5.0
ARM64 GCC 8.2.0
ARM64 GCC 8.5.0
ARM64 GCC 9.3.0
ARM64 GCC 9.4.0
ARM64 GCC 9.5.0
ARM64 GCC trunk
ARM64 Morello GCC 10.1.0 Alpha 1
ARM64 Morello GCC 10.1.2 Alpha 2
ARM64 msvc v19.14 (WINE)
AVR gcc 10.3.0
AVR gcc 11.1.0
AVR gcc 12.1.0
AVR gcc 12.2.0
AVR gcc 12.3.0
AVR gcc 12.4.0
AVR gcc 13.1.0
AVR gcc 13.2.0
AVR gcc 13.3.0
AVR gcc 14.1.0
AVR gcc 14.2.0
AVR gcc 4.5.4
AVR gcc 4.6.4
AVR gcc 5.4.0
AVR gcc 9.2.0
AVR gcc 9.3.0
Arduino Mega (1.8.9)
Arduino Uno (1.8.9)
BPF clang (trunk)
BPF clang 13.0.0
BPF clang 14.0.0
BPF clang 15.0.0
BPF clang 16.0.0
BPF clang 17.0.1
BPF clang 18.1.0
BPF gcc 13.1.0
BPF gcc 13.2.0
BPF gcc 13.3.0
BPF gcc 14.1.0
BPF gcc 14.2.0
BPF gcc trunk
Chibicc 2020-12-07
FRC 2019
FRC 2020
FRC 2023
HPPA gcc 14.2.0
K1C gcc 7.4
K1C gcc 7.5
KVX ACB 4.1.0 (GCC 7.5.0)
KVX ACB 4.1.0-cd1 (GCC 7.5.0)
KVX ACB 4.10.0 (GCC 10.3.1)
KVX ACB 4.11.1 (GCC 10.3.1)
KVX ACB 4.12.0 (GCC 11.3.0)
KVX ACB 4.2.0 (GCC 7.5.0)
KVX ACB 4.3.0 (GCC 7.5.0)
KVX ACB 4.4.0 (GCC 7.5.0)
KVX ACB 4.6.0 (GCC 9.4.1)
KVX ACB 4.8.0 (GCC 9.4.1)
KVX ACB 4.9.0 (GCC 9.4.1)
KVX ACB 5.0.0 (GCC 12.2.1)
KVX ACB 5.2.0 (GCC 13.2.1)
LC3 (trunk)
M68K clang (trunk)
M68K gcc 13.1.0
M68K gcc 13.2.0
M68K gcc 13.3.0
M68K gcc 14.1.0
M68K gcc 14.2.0
MRISC32 gcc (trunk)
MSP430 gcc 12.1.0
MSP430 gcc 12.2.0
MSP430 gcc 12.3.0
MSP430 gcc 12.4.0
MSP430 gcc 13.1.0
MSP430 gcc 13.2.0
MSP430 gcc 13.3.0
MSP430 gcc 14.1.0
MSP430 gcc 14.2.0
MSP430 gcc 4.5.3
MSP430 gcc 5.3.0
MSP430 gcc 6.2.1
MinGW clang 14.0.3
MinGW clang 14.0.6
MinGW clang 15.0.7
MinGW clang 16.0.0
MinGW clang 16.0.2
MinGW gcc 11.3.0
MinGW gcc 12.1.0
MinGW gcc 12.2.0
MinGW gcc 13.1.0
POWER64 gcc 11.2.0
POWER64 gcc 12.1.0
POWER64 gcc 12.2.0
POWER64 gcc 12.3.0
POWER64 gcc 12.4.0
POWER64 gcc 13.1.0
POWER64 gcc 13.2.0
POWER64 gcc 13.3.0
POWER64 gcc 14.1.0
POWER64 gcc 14.2.0
POWER64 gcc trunk
RISC-V (32-bits) gcc (trunk)
RISC-V (32-bits) gcc 10.2.0
RISC-V (32-bits) gcc 10.3.0
RISC-V (32-bits) gcc 11.2.0
RISC-V (32-bits) gcc 11.3.0
RISC-V (32-bits) gcc 11.4.0
RISC-V (32-bits) gcc 12.1.0
RISC-V (32-bits) gcc 12.2.0
RISC-V (32-bits) gcc 12.3.0
RISC-V (32-bits) gcc 12.4.0
RISC-V (32-bits) gcc 13.1.0
RISC-V (32-bits) gcc 13.2.0
RISC-V (32-bits) gcc 13.3.0
RISC-V (32-bits) gcc 14.1.0
RISC-V (32-bits) gcc 14.2.0
RISC-V (32-bits) gcc 8.2.0
RISC-V (32-bits) gcc 8.5.0
RISC-V (32-bits) gcc 9.4.0
RISC-V (64-bits) gcc (trunk)
RISC-V (64-bits) gcc 10.2.0
RISC-V (64-bits) gcc 10.3.0
RISC-V (64-bits) gcc 11.2.0
RISC-V (64-bits) gcc 11.3.0
RISC-V (64-bits) gcc 11.4.0
RISC-V (64-bits) gcc 12.1.0
RISC-V (64-bits) gcc 12.2.0
RISC-V (64-bits) gcc 12.3.0
RISC-V (64-bits) gcc 12.4.0
RISC-V (64-bits) gcc 13.1.0
RISC-V (64-bits) gcc 13.2.0
RISC-V (64-bits) gcc 13.3.0
RISC-V (64-bits) gcc 14.1.0
RISC-V (64-bits) gcc 14.2.0
RISC-V (64-bits) gcc 8.2.0
RISC-V (64-bits) gcc 8.5.0
RISC-V (64-bits) gcc 9.4.0
RISC-V rv32gc clang (trunk)
RISC-V rv32gc clang 10.0.0
RISC-V rv32gc clang 10.0.1
RISC-V rv32gc clang 11.0.0
RISC-V rv32gc clang 11.0.1
RISC-V rv32gc clang 12.0.0
RISC-V rv32gc clang 12.0.1
RISC-V rv32gc clang 13.0.0
RISC-V rv32gc clang 13.0.1
RISC-V rv32gc clang 14.0.0
RISC-V rv32gc clang 15.0.0
RISC-V rv32gc clang 16.0.0
RISC-V rv32gc clang 17.0.1
RISC-V rv32gc clang 18.1.0
RISC-V rv32gc clang 9.0.0
RISC-V rv32gc clang 9.0.1
RISC-V rv64gc clang (trunk)
RISC-V rv64gc clang 10.0.0
RISC-V rv64gc clang 10.0.1
RISC-V rv64gc clang 11.0.0
RISC-V rv64gc clang 11.0.1
RISC-V rv64gc clang 12.0.0
RISC-V rv64gc clang 12.0.1
RISC-V rv64gc clang 13.0.0
RISC-V rv64gc clang 13.0.1
RISC-V rv64gc clang 14.0.0
RISC-V rv64gc clang 15.0.0
RISC-V rv64gc clang 16.0.0
RISC-V rv64gc clang 17.0.1
RISC-V rv64gc clang 18.1.0
RISC-V rv64gc clang 9.0.0
RISC-V rv64gc clang 9.0.1
Raspbian Buster
Raspbian Stretch
SDCC 4.0.0
SDCC 4.1.0
SDCC 4.2.0
SDCC 4.3.0
SDCC 4.4.0
SPARC LEON gcc 12.2.0
SPARC LEON gcc 12.3.0
SPARC LEON gcc 12.4.0
SPARC LEON gcc 13.1.0
SPARC LEON gcc 13.2.0
SPARC LEON gcc 13.3.0
SPARC LEON gcc 14.1.0
SPARC LEON gcc 14.2.0
SPARC gcc 12.2.0
SPARC gcc 12.3.0
SPARC gcc 12.4.0
SPARC gcc 13.1.0
SPARC gcc 13.2.0
SPARC gcc 13.3.0
SPARC gcc 14.1.0
SPARC gcc 14.2.0
SPARC64 gcc 12.2.0
SPARC64 gcc 12.3.0
SPARC64 gcc 12.4.0
SPARC64 gcc 13.1.0
SPARC64 gcc 13.2.0
SPARC64 gcc 13.3.0
SPARC64 gcc 14.1.0
SPARC64 gcc 14.2.0
TCC (trunk)
TCC 0.9.27
TI C6x gcc 12.2.0
TI C6x gcc 12.3.0
TI C6x gcc 12.4.0
TI C6x gcc 13.1.0
TI C6x gcc 13.2.0
TI C6x gcc 13.3.0
TI C6x gcc 14.1.0
TI C6x gcc 14.2.0
TI CL430 21.6.1
VAX gcc NetBSDELF 10.4.0
VAX gcc NetBSDELF 10.5.0 (Nov 15 03:50:22 2023)
WebAssembly clang (trunk)
Xtensa ESP32 gcc 11.2.0 (2022r1)
Xtensa ESP32 gcc 12.2.0 (20230208)
Xtensa ESP32 gcc 8.2.0 (2019r2)
Xtensa ESP32 gcc 8.2.0 (2020r1)
Xtensa ESP32 gcc 8.2.0 (2020r2)
Xtensa ESP32 gcc 8.4.0 (2020r3)
Xtensa ESP32 gcc 8.4.0 (2021r1)
Xtensa ESP32 gcc 8.4.0 (2021r2)
Xtensa ESP32-S2 gcc 11.2.0 (2022r1)
Xtensa ESP32-S2 gcc 12.2.0 (20230208)
Xtensa ESP32-S2 gcc 8.2.0 (2019r2)
Xtensa ESP32-S2 gcc 8.2.0 (2020r1)
Xtensa ESP32-S2 gcc 8.2.0 (2020r2)
Xtensa ESP32-S2 gcc 8.4.0 (2020r3)
Xtensa ESP32-S2 gcc 8.4.0 (2021r1)
Xtensa ESP32-S2 gcc 8.4.0 (2021r2)
Xtensa ESP32-S3 gcc 11.2.0 (2022r1)
Xtensa ESP32-S3 gcc 12.2.0 (20230208)
Xtensa ESP32-S3 gcc 8.4.0 (2020r3)
Xtensa ESP32-S3 gcc 8.4.0 (2021r1)
Xtensa ESP32-S3 gcc 8.4.0 (2021r2)
arm64 msvc v19.20 VS16.0
arm64 msvc v19.21 VS16.1
arm64 msvc v19.22 VS16.2
arm64 msvc v19.23 VS16.3
arm64 msvc v19.24 VS16.4
arm64 msvc v19.25 VS16.5
arm64 msvc v19.27 VS16.7
arm64 msvc v19.28 VS16.8
arm64 msvc v19.28 VS16.9
arm64 msvc v19.29 VS16.10
arm64 msvc v19.29 VS16.11
arm64 msvc v19.30 VS17.0
arm64 msvc v19.31 VS17.1
arm64 msvc v19.32 VS17.2
arm64 msvc v19.33 VS17.3
arm64 msvc v19.34 VS17.4
arm64 msvc v19.35 VS17.5
arm64 msvc v19.36 VS17.6
arm64 msvc v19.37 VS17.7
arm64 msvc v19.38 VS17.8
arm64 msvc v19.39 VS17.9
arm64 msvc v19.40 VS17.10
arm64 msvc v19.latest
armv7-a clang (trunk)
armv7-a clang 10.0.0
armv7-a clang 10.0.1
armv7-a clang 11.0.0
armv7-a clang 11.0.1
armv7-a clang 12.0.0
armv7-a clang 12.0.1
armv7-a clang 13.0.0
armv7-a clang 13.0.1
armv7-a clang 14.0.0
armv7-a clang 15.0.0
armv7-a clang 16.0.0
armv7-a clang 17.0.1
armv7-a clang 18.1.0
armv7-a clang 9.0.0
armv7-a clang 9.0.1
armv8-a clang (all architectural features, trunk)
armv8-a clang (trunk)
armv8-a clang 10.0.0
armv8-a clang 10.0.1
armv8-a clang 11.0.0
armv8-a clang 11.0.1
armv8-a clang 12.0.0
armv8-a clang 12.0.1
armv8-a clang 13.0.0
armv8-a clang 13.0.1
armv8-a clang 14.0.0
armv8-a clang 15.0.0
armv8-a clang 16.0.0
armv8-a clang 17.0.1
armv8-a clang 18.1.0
armv8-a clang 9.0.0
armv8-a clang 9.0.1
clang 12 for DPU (rel 2023.2.0)
cproc-master
llvm-mos commander X16
llvm-mos commodore 64
llvm-mos mega65
llvm-mos nes-cnrom
llvm-mos nes-mmc1
llvm-mos nes-mmc3
llvm-mos nes-nrom
llvm-mos osi-c1p
loongarch64 gcc 12.2.0
loongarch64 gcc 12.3.0
loongarch64 gcc 12.4.0
loongarch64 gcc 13.1.0
loongarch64 gcc 13.2.0
loongarch64 gcc 13.3.0
loongarch64 gcc 14.1.0
loongarch64 gcc 14.2.0
mips (el) gcc 12.1.0
mips (el) gcc 12.2.0
mips (el) gcc 12.3.0
mips (el) gcc 12.4.0
mips (el) gcc 13.1.0
mips (el) gcc 13.2.0
mips (el) gcc 13.3.0
mips (el) gcc 14.1.0
mips (el) gcc 14.2.0
mips (el) gcc 4.9.4
mips (el) gcc 5.4
mips (el) gcc 5.5.0
mips (el) gcc 9.5.0
mips clang 13.0.0
mips clang 14.0.0
mips clang 15.0.0
mips clang 16.0.0
mips clang 17.0.1
mips clang 18.1.0
mips gcc 11.2.0
mips gcc 12.1.0
mips gcc 12.2.0
mips gcc 12.3.0
mips gcc 12.4.0
mips gcc 13.1.0
mips gcc 13.2.0
mips gcc 13.3.0
mips gcc 14.1.0
mips gcc 14.2.0
mips gcc 4.9.4
mips gcc 5.4
mips gcc 5.5.0
mips gcc 9.3.0 (codescape)
mips gcc 9.5.0
mips64 (el) gcc 12.1.0
mips64 (el) gcc 12.2.0
mips64 (el) gcc 12.3.0
mips64 (el) gcc 12.4.0
mips64 (el) gcc 13.1.0
mips64 (el) gcc 13.2.0
mips64 (el) gcc 13.3.0
mips64 (el) gcc 14.1.0
mips64 (el) gcc 14.2.0
mips64 (el) gcc 4.9.4
mips64 (el) gcc 5.4.0
mips64 (el) gcc 5.5.0
mips64 (el) gcc 9.5.0
mips64 clang 13.0.0
mips64 clang 14.0.0
mips64 clang 15.0.0
mips64 clang 16.0.0
mips64 clang 17.0.1
mips64 clang 18.1.0
mips64 gcc 11.2.0
mips64 gcc 12.1.0
mips64 gcc 12.2.0
mips64 gcc 12.3.0
mips64 gcc 12.4.0
mips64 gcc 13.1.0
mips64 gcc 13.2.0
mips64 gcc 13.3.0
mips64 gcc 14.1.0
mips64 gcc 14.2.0
mips64 gcc 4.9.4
mips64 gcc 5.4
mips64 gcc 5.5.0
mips64 gcc 9.5.0
mips64el clang 13.0.0
mips64el clang 14.0.0
mips64el clang 15.0.0
mips64el clang 16.0.0
mips64el clang 17.0.1
mips64el clang 18.1.0
mipsel clang 13.0.0
mipsel clang 14.0.0
mipsel clang 15.0.0
mipsel clang 16.0.0
mipsel clang 17.0.1
mipsel clang 18.1.0
movfuscator (trunk)
nanoMIPS gcc 6.3.0
power gcc 11.2.0
power gcc 12.1.0
power gcc 12.2.0
power gcc 12.3.0
power gcc 12.4.0
power gcc 13.1.0
power gcc 13.2.0
power gcc 13.3.0
power gcc 14.1.0
power gcc 14.2.0
power gcc 4.8.5
power64 AT12.0 (gcc8)
power64 AT13.0 (gcc9)
power64le AT12.0 (gcc8)
power64le AT13.0 (gcc9)
power64le clang (trunk)
power64le gcc 11.2.0
power64le gcc 12.1.0
power64le gcc 12.2.0
power64le gcc 12.3.0
power64le gcc 12.4.0
power64le gcc 13.1.0
power64le gcc 13.2.0
power64le gcc 13.3.0
power64le gcc 14.1.0
power64le gcc 14.2.0
power64le gcc 6.3.0
power64le gcc trunk
powerpc64 clang (trunk)
ppci 0.5.5
s390x gcc 11.2.0
s390x gcc 12.1.0
s390x gcc 12.2.0
s390x gcc 12.3.0
s390x gcc 12.4.0
s390x gcc 13.1.0
s390x gcc 13.2.0
s390x gcc 13.3.0
s390x gcc 14.1.0
s390x gcc 14.2.0
sh gcc 12.2.0
sh gcc 12.3.0
sh gcc 12.4.0
sh gcc 13.1.0
sh gcc 13.2.0
sh gcc 13.3.0
sh gcc 14.1.0
sh gcc 14.2.0
sh gcc 4.9.4
sh gcc 9.5.0
vast (trunk)
x64 msvc v19.0 (WINE)
x64 msvc v19.10 (WINE)
x64 msvc v19.14 (WINE)
x64 msvc v19.20 VS16.0
x64 msvc v19.21 VS16.1
x64 msvc v19.22 VS16.2
x64 msvc v19.23 VS16.3
x64 msvc v19.24 VS16.4
x64 msvc v19.25 VS16.5
x64 msvc v19.27 VS16.7
x64 msvc v19.28 VS16.8
x64 msvc v19.28 VS16.9
x64 msvc v19.29 VS16.10
x64 msvc v19.29 VS16.11
x64 msvc v19.30 VS17.0
x64 msvc v19.31 VS17.1
x64 msvc v19.32 VS17.2
x64 msvc v19.33 VS17.3
x64 msvc v19.34 VS17.4
x64 msvc v19.35 VS17.5
x64 msvc v19.36 VS17.6
x64 msvc v19.37 VS17.7
x64 msvc v19.38 VS17.8
x64 msvc v19.39 VS17.9
x64 msvc v19.40 VS17.10
x64 msvc v19.latest
x86 CompCert 3.10
x86 CompCert 3.11
x86 CompCert 3.12
x86 CompCert 3.9
x86 gcc 1.27
x86 msvc v19.0 (WINE)
x86 msvc v19.10 (WINE)
x86 msvc v19.14 (WINE)
x86 msvc v19.20 VS16.0
x86 msvc v19.21 VS16.1
x86 msvc v19.22 VS16.2
x86 msvc v19.23 VS16.3
x86 msvc v19.24 VS16.4
x86 msvc v19.25 VS16.5
x86 msvc v19.27 VS16.7
x86 msvc v19.28 VS16.8
x86 msvc v19.28 VS16.9
x86 msvc v19.29 VS16.10
x86 msvc v19.29 VS16.11
x86 msvc v19.30 VS17.0
x86 msvc v19.31 VS17.1
x86 msvc v19.32 VS17.2
x86 msvc v19.33 VS17.3
x86 msvc v19.34 VS17.4
x86 msvc v19.35 VS17.5
x86 msvc v19.36 VS17.6
x86 msvc v19.37 VS17.7
x86 msvc v19.38 VS17.8
x86 msvc v19.39 VS17.9
x86 msvc v19.40 VS17.10
x86 msvc v19.latest
x86 nvc 24.9
x86 tendra (trunk)
x86-64 clang (assertions trunk)
x86-64 clang (thephd.dev)
x86-64 clang (trunk)
x86-64 clang (widberg)
x86-64 clang 10.0.0
x86-64 clang 10.0.1
x86-64 clang 11.0.0
x86-64 clang 11.0.1
x86-64 clang 12.0.0
x86-64 clang 12.0.1
x86-64 clang 13.0.0
x86-64 clang 13.0.1
x86-64 clang 14.0.0
x86-64 clang 15.0.0
x86-64 clang 16.0.0
x86-64 clang 17.0.1
x86-64 clang 18.1.0
x86-64 clang 19.1.0
x86-64 clang 3.0.0
x86-64 clang 3.1
x86-64 clang 3.2
x86-64 clang 3.3
x86-64 clang 3.4.1
x86-64 clang 3.5
x86-64 clang 3.5.1
x86-64 clang 3.5.2
x86-64 clang 3.6
x86-64 clang 3.7
x86-64 clang 3.7.1
x86-64 clang 3.8
x86-64 clang 3.8.1
x86-64 clang 3.9.0
x86-64 clang 3.9.1
x86-64 clang 4.0.0
x86-64 clang 4.0.1
x86-64 clang 5.0.0
x86-64 clang 5.0.1
x86-64 clang 5.0.2
x86-64 clang 6.0.0
x86-64 clang 6.0.1
x86-64 clang 7.0.0
x86-64 clang 7.0.1
x86-64 clang 7.1.0
x86-64 clang 8.0.0
x86-64 clang 8.0.1
x86-64 clang 9.0.0
x86-64 clang 9.0.1
x86-64 gcc (trunk)
x86-64 gcc 10.1
x86-64 gcc 10.2
x86-64 gcc 10.3
x86-64 gcc 10.4
x86-64 gcc 10.5
x86-64 gcc 11.1
x86-64 gcc 11.2
x86-64 gcc 11.3
x86-64 gcc 11.4
x86-64 gcc 12.1
x86-64 gcc 12.2
x86-64 gcc 12.3
x86-64 gcc 12.4
x86-64 gcc 13.1
x86-64 gcc 13.2
x86-64 gcc 13.3
x86-64 gcc 14.1
x86-64 gcc 14.2
x86-64 gcc 3.4.6
x86-64 gcc 4.0.4
x86-64 gcc 4.1.2
x86-64 gcc 4.4.7
x86-64 gcc 4.5.3
x86-64 gcc 4.6.4
x86-64 gcc 4.7.1
x86-64 gcc 4.7.2
x86-64 gcc 4.7.3
x86-64 gcc 4.7.4
x86-64 gcc 4.8.1
x86-64 gcc 4.8.2
x86-64 gcc 4.8.3
x86-64 gcc 4.8.4
x86-64 gcc 4.8.5
x86-64 gcc 4.9.0
x86-64 gcc 4.9.1
x86-64 gcc 4.9.2
x86-64 gcc 4.9.3
x86-64 gcc 4.9.4
x86-64 gcc 5.1
x86-64 gcc 5.2
x86-64 gcc 5.3
x86-64 gcc 5.4
x86-64 gcc 6.1
x86-64 gcc 6.2
x86-64 gcc 6.3
x86-64 gcc 6.5
x86-64 gcc 7.1
x86-64 gcc 7.2
x86-64 gcc 7.3
x86-64 gcc 7.4
x86-64 gcc 7.5
x86-64 gcc 8.1
x86-64 gcc 8.2
x86-64 gcc 8.3
x86-64 gcc 8.4
x86-64 gcc 8.5
x86-64 gcc 9.1
x86-64 gcc 9.2
x86-64 gcc 9.3
x86-64 gcc 9.4
x86-64 gcc 9.5
x86-64 icc 13.0.1
x86-64 icc 16.0.3
x86-64 icc 17.0.0
x86-64 icc 18.0.0
x86-64 icc 19.0.0
x86-64 icc 19.0.1
x86-64 icc 2021.1.2
x86-64 icc 2021.10.0
x86-64 icc 2021.2.0
x86-64 icc 2021.3.0
x86-64 icc 2021.4.0
x86-64 icc 2021.5.0
x86-64 icc 2021.6.0
x86-64 icc 2021.7.0
x86-64 icc 2021.7.1
x86-64 icc 2021.8.0
x86-64 icc 2021.9.0
x86-64 icx (latest)
x86-64 icx 2021.1.2
x86-64 icx 2021.2.0
x86-64 icx 2021.3.0
x86-64 icx 2021.4.0
x86-64 icx 2022.0.0
x86-64 icx 2022.1.0
x86-64 icx 2022.2.0
x86-64 icx 2022.2.1
x86-64 icx 2023.0.0
x86-64 icx 2023.1.0
x86-64 icx 2024.0.0
x86_64 CompCert 3.10
x86_64 CompCert 3.11
x86_64 CompCert 3.12
x86_64 CompCert 3.9
z88dk 2.2
zig cc 0.10.0
zig cc 0.11.0
zig cc 0.12.0
zig cc 0.12.1
zig cc 0.13.0
zig cc 0.6.0
zig cc 0.7.0
zig cc 0.7.1
zig cc 0.8.0
zig cc 0.9.0
zig cc trunk
Options
Source code
#define __ATOMIC_RELAXED 0 #define __ATOMIC_ACQUIRE 2 #define __ATOMIC_RELEASE 3 #define __PTRDIFF_TYPE__ long int #define __SIZE_TYPE__ long unsigned int #define MODULE_libc 18 #define PASTE_NAME1(a,b) a##b #define PASTE_NAME(a,b) PASTE_NAME1 (a,b) #define IN_MODULE PASTE_NAME (MODULE_, MODULE_NAME) #define IS_IN(lib) (IN_MODULE == MODULE_##lib) #define attribute_hidden __attribute__ ((visibility ("hidden"))) #define void void typedef __PTRDIFF_TYPE__ ptrdiff_t; typedef __SIZE_TYPE__ size_t; #define offsetof(t, d) __builtin_offsetof(t, d) #define bool _Bool #define true 1 #define false 0 #define _Static_assert(expr, diagnostic) _Static_assert (expr, diagnostic) #define __LEAF #define __THROW __attribute__ ((__nothrow__ __LEAF)) #define __always_inline __inline __attribute__ ((__always_inline__)) #define __glibc_unlikely(cond) __builtin_expect ((cond), 0) #define __glibc_likely(cond) __builtin_expect ((cond), 1) #define __nonnull(params) #define __WORDSIZE 64 #define __SYSCALL_WORDSIZE 64 #define __TIMESIZE __WORDSIZE typedef unsigned char __uint8_t; typedef unsigned short int __uint16_t; typedef signed int __int32_t; typedef unsigned int __uint32_t; typedef signed long int __int64_t; typedef long int __intmax_t; #define __S32_TYPE int #define __SLONGWORD_TYPE long int #define __SWORD_TYPE long int #define __STD_TYPE typedef #define __SYSCALL_SLONG_TYPE __SLONGWORD_TYPE #define __OFF_T_TYPE __SYSCALL_SLONG_TYPE #define __TIME_T_TYPE __SYSCALL_SLONG_TYPE #define __CLOCKID_T_TYPE __S32_TYPE #define __SSIZE_T_TYPE __SWORD_TYPE __STD_TYPE __OFF_T_TYPE __off_t; __STD_TYPE __TIME_T_TYPE __time_t; __STD_TYPE __CLOCKID_T_TYPE __clockid_t; __STD_TYPE __SSIZE_T_TYPE __ssize_t; __STD_TYPE __SYSCALL_SLONG_TYPE __syscall_slong_t; typedef __ssize_t ssize_t; typedef __clockid_t clockid_t; typedef __int32_t int32_t; typedef __int64_t int64_t; struct timespec { #ifdef __USE_TIME_BITS64 __time64_t tv_sec; /* Seconds. */ #else __time_t tv_sec; /* Seconds. */ #endif #if __WORDSIZE == 64 \ || (defined __SYSCALL_WORDSIZE && __SYSCALL_WORDSIZE == 64) \ || (__TIMESIZE == 32 && !defined __USE_TIME_BITS64) __syscall_slong_t tv_nsec; /* Nanoseconds. */ #else # if __BYTE_ORDER == __BIG_ENDIAN int: 32; /* Padding. */ long int tv_nsec; /* Nanoseconds. */ # else long int tv_nsec; /* Nanoseconds. */ int: 32; /* Padding. */ # endif #endif }; typedef __uint8_t uint8_t; typedef __uint16_t uint16_t; typedef __uint32_t uint32_t; typedef long int intptr_t; typedef unsigned long int uintptr_t; typedef __intmax_t intmax_t; #define UINT16_MAX (65535) #define PTRDIFF_MAX (9223372036854775807L) #define SIZE_MAX (18446744073709551615UL) #define __pointer_type(type) (__builtin_classify_type ((type) 0) == 5) #define __integer_if_pointer_type_sub(T, P) \ __typeof__ (*(0 ? (__typeof__ (0 ? (T *) 0 : (void *) (P))) 0 \ : (__typeof__ (0 ? (intptr_t *) 0 : (void *) (!(P)))) 0)) #define __integer_if_pointer_type(expr) \ __integer_if_pointer_type_sub(__typeof__ ((__typeof__ (expr)) 0), \ __pointer_type (__typeof__ (expr))) #define cast_to_integer(val) ((__integer_if_pointer_type (val)) (val)) #define ALIGN_DOWN(base, size) ((base) & -((__typeof__ (base)) (size))) #define ALIGN_UP(base, size) ALIGN_DOWN ((base) + (size) - 1, (size)) #define PTR_ALIGN_DOWN(base, size) \ ((__typeof__ (base)) ALIGN_DOWN ((uintptr_t) (base), (size))) #define PTR_IS_ALIGNED(base, size) \ ((((uintptr_t) (base)) & (size - 1)) == 0) #define PTR_DIFF(p1, p2) \ ((ptrdiff_t)((uintptr_t)(p1) - (uintptr_t)(p2))) extern void abort (void) __THROW __attribute__ ((__noreturn__)); #define __timespec64 timespec extern void abort(void) __attribute__((visibility("hidden"))); #undef __STD_TYPE enum { _SC_ARG_MAX, _SC_CHILD_MAX, _SC_CLK_TCK, _SC_NGROUPS_MAX, _SC_OPEN_MAX, _SC_STREAM_MAX, _SC_TZNAME_MAX, _SC_JOB_CONTROL, _SC_SAVED_IDS, _SC_REALTIME_SIGNALS, _SC_PRIORITY_SCHEDULING, _SC_TIMERS, _SC_ASYNCHRONOUS_IO, _SC_PRIORITIZED_IO, _SC_SYNCHRONIZED_IO, _SC_FSYNC, _SC_MAPPED_FILES, _SC_MEMLOCK, _SC_MEMLOCK_RANGE, _SC_MEMORY_PROTECTION, _SC_MESSAGE_PASSING, _SC_SEMAPHORES, _SC_SHARED_MEMORY_OBJECTS, _SC_AIO_LISTIO_MAX, _SC_AIO_MAX, _SC_AIO_PRIO_DELTA_MAX, _SC_DELAYTIMER_MAX, _SC_MQ_OPEN_MAX, _SC_MQ_PRIO_MAX, _SC_VERSION, _SC_PAGESIZE, _SC_RTSIG_MAX, _SC_SEM_NSEMS_MAX, _SC_SEM_VALUE_MAX, _SC_SIGQUEUE_MAX, _SC_TIMER_MAX, _SC_BC_BASE_MAX, _SC_BC_DIM_MAX, _SC_BC_SCALE_MAX, _SC_BC_STRING_MAX, _SC_COLL_WEIGHTS_MAX, _SC_EQUIV_CLASS_MAX, _SC_EXPR_NEST_MAX, _SC_LINE_MAX, _SC_RE_DUP_MAX, _SC_CHARCLASS_NAME_MAX, _SC_2_VERSION, _SC_2_C_BIND, _SC_2_C_DEV, _SC_2_FORT_DEV, _SC_2_FORT_RUN, _SC_2_SW_DEV, _SC_2_LOCALEDEF, _SC_PII, _SC_PII_XTI, _SC_PII_SOCKET, _SC_PII_INTERNET, _SC_PII_OSI, _SC_POLL, _SC_SELECT, _SC_UIO_MAXIOV, _SC_IOV_MAX = _SC_UIO_MAXIOV, _SC_PII_INTERNET_STREAM, _SC_PII_INTERNET_DGRAM, _SC_PII_OSI_COTS, _SC_PII_OSI_CLTS, _SC_PII_OSI_M, _SC_T_IOV_MAX, _SC_THREADS, _SC_THREAD_SAFE_FUNCTIONS, _SC_GETGR_R_SIZE_MAX, _SC_GETPW_R_SIZE_MAX, _SC_LOGIN_NAME_MAX, _SC_TTY_NAME_MAX, _SC_THREAD_DESTRUCTOR_ITERATIONS, _SC_THREAD_KEYS_MAX, _SC_THREAD_STACK_MIN, _SC_THREAD_THREADS_MAX, _SC_THREAD_ATTR_STACKADDR, _SC_THREAD_ATTR_STACKSIZE, _SC_THREAD_PRIORITY_SCHEDULING, _SC_THREAD_PRIO_INHERIT, _SC_THREAD_PRIO_PROTECT, _SC_THREAD_PROCESS_SHARED, _SC_NPROCESSORS_CONF, _SC_NPROCESSORS_ONLN, _SC_PHYS_PAGES, _SC_AVPHYS_PAGES, _SC_ATEXIT_MAX, _SC_PASS_MAX, _SC_XOPEN_VERSION, _SC_XOPEN_XCU_VERSION, _SC_XOPEN_UNIX, _SC_XOPEN_CRYPT, _SC_XOPEN_ENH_I18N, _SC_XOPEN_SHM, _SC_2_CHAR_TERM, _SC_2_C_VERSION, _SC_2_UPE, _SC_XOPEN_XPG2, _SC_XOPEN_XPG3, _SC_XOPEN_XPG4, _SC_CHAR_BIT, _SC_CHAR_MAX, _SC_CHAR_MIN, _SC_INT_MAX, _SC_INT_MIN, _SC_LONG_BIT, _SC_WORD_BIT, _SC_MB_LEN_MAX, _SC_NZERO, _SC_SSIZE_MAX, _SC_SCHAR_MAX, _SC_SCHAR_MIN, _SC_SHRT_MAX, _SC_SHRT_MIN, _SC_UCHAR_MAX, _SC_UINT_MAX, _SC_ULONG_MAX, _SC_USHRT_MAX, _SC_NL_ARGMAX, _SC_NL_LANGMAX, _SC_NL_MSGMAX, _SC_NL_NMAX, _SC_NL_SETMAX, _SC_NL_TEXTMAX, _SC_XBS5_ILP32_OFF32, _SC_XBS5_ILP32_OFFBIG, _SC_XBS5_LP64_OFF64, _SC_XBS5_LPBIG_OFFBIG, _SC_XOPEN_LEGACY, _SC_XOPEN_REALTIME, _SC_XOPEN_REALTIME_THREADS, _SC_ADVISORY_INFO, _SC_BARRIERS, _SC_BASE, _SC_C_LANG_SUPPORT, _SC_C_LANG_SUPPORT_R, _SC_CLOCK_SELECTION, _SC_CPUTIME, _SC_THREAD_CPUTIME, _SC_DEVICE_IO, _SC_DEVICE_SPECIFIC, _SC_DEVICE_SPECIFIC_R, _SC_FD_MGMT, _SC_FIFO, _SC_PIPE, _SC_FILE_ATTRIBUTES, _SC_FILE_LOCKING, _SC_FILE_SYSTEM, _SC_MONOTONIC_CLOCK, _SC_MULTI_PROCESS, _SC_SINGLE_PROCESS, _SC_NETWORKING, _SC_READER_WRITER_LOCKS, _SC_SPIN_LOCKS, _SC_REGEXP, _SC_REGEX_VERSION, _SC_SHELL, _SC_SIGNALS, _SC_SPAWN, _SC_SPORADIC_SERVER, _SC_THREAD_SPORADIC_SERVER, _SC_SYSTEM_DATABASE, _SC_SYSTEM_DATABASE_R, _SC_TIMEOUTS, _SC_TYPED_MEMORY_OBJECTS, _SC_USER_GROUPS, _SC_USER_GROUPS_R, _SC_2_PBS, _SC_2_PBS_ACCOUNTING, _SC_2_PBS_LOCATE, _SC_2_PBS_MESSAGE, _SC_2_PBS_TRACK, _SC_SYMLOOP_MAX, _SC_STREAMS, _SC_2_PBS_CHECKPOINT, _SC_V6_ILP32_OFF32, _SC_V6_ILP32_OFFBIG, _SC_V6_LP64_OFF64, _SC_V6_LPBIG_OFFBIG, _SC_HOST_NAME_MAX, _SC_TRACE, _SC_TRACE_EVENT_FILTER, _SC_TRACE_INHERIT, _SC_TRACE_LOG, _SC_LEVEL1_ICACHE_SIZE, _SC_LEVEL1_ICACHE_ASSOC, _SC_LEVEL1_ICACHE_LINESIZE, _SC_LEVEL1_DCACHE_SIZE, _SC_LEVEL1_DCACHE_ASSOC, _SC_LEVEL1_DCACHE_LINESIZE, _SC_LEVEL2_CACHE_SIZE, _SC_LEVEL2_CACHE_ASSOC, _SC_LEVEL2_CACHE_LINESIZE, _SC_LEVEL3_CACHE_SIZE, _SC_LEVEL3_CACHE_ASSOC, _SC_LEVEL3_CACHE_LINESIZE, _SC_LEVEL4_CACHE_SIZE, _SC_LEVEL4_CACHE_ASSOC, _SC_LEVEL4_CACHE_LINESIZE, _SC_IPV6 = _SC_LEVEL1_ICACHE_SIZE + 50, _SC_RAW_SOCKETS, _SC_V7_ILP32_OFF32, _SC_V7_ILP32_OFFBIG, _SC_V7_LP64_OFF64, _SC_V7_LPBIG_OFFBIG, _SC_SS_REPL_MAX, _SC_TRACE_EVENT_NAME_MAX, _SC_TRACE_NAME_MAX, _SC_TRACE_SYS_MAX, _SC_TRACE_USER_EVENT_MAX, _SC_XOPEN_STREAMS, _SC_THREAD_ROBUST_PRIO_INHERIT, _SC_THREAD_ROBUST_PRIO_PROTECT, _SC_MINSIGSTKSZ, _SC_SIGSTKSZ }; extern int __close (int __fd); extern int __close(int) __attribute__((visibility("hidden"))); extern ssize_t __read (int __fd, void *__buf, size_t __nbytes); extern ssize_t __read(int, void *, size_t) __attribute__((visibility("hidden"))); extern void *__sbrk (intptr_t __delta); extern void *__sbrk(intptr_t) __attribute__((visibility("hidden"))); extern int __libc_enable_secure; #define ENOMEM 12 #define errno __libc_errno extern __thread int errno; #define __set_errno(val) (errno = (val)) extern char __libc_single_threaded; #define __libc_single_threaded_internal __libc_single_threaded #define SINGLE_THREAD_P (__libc_single_threaded_internal != 0) #define __NR_getrandom 318 #define __SYSCALL_CONCAT_X(a,b) a##b #define __SYSCALL_CONCAT(a,b) __SYSCALL_CONCAT_X (a, b) #define __INTERNAL_SYSCALL3(name, a1, a2, a3) \ INTERNAL_SYSCALL (name, 3, a1, a2, a3) #define __INTERNAL_SYSCALL_NARGS_X(a,b,c,d,e,f,g,h,n,...) n #define __INTERNAL_SYSCALL_NARGS(...) \ __INTERNAL_SYSCALL_NARGS_X (__VA_ARGS__,7,6,5,4,3,2,1,0,) #define __INTERNAL_SYSCALL_DISP(b,...) \ __SYSCALL_CONCAT (b,__INTERNAL_SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__) #define INTERNAL_SYSCALL_CALL(...) \ __INTERNAL_SYSCALL_DISP (__INTERNAL_SYSCALL, __VA_ARGS__) #define SYS_ify(syscall_name) __NR_##syscall_name #define REGISTERS_CLOBBERED_BY_SYSCALL "cc", "r11", "cx" #define TYPEFY1(X) __typeof__ ((X) - (X)) #define ARGIFY(X) ((TYPEFY1 (X)) (X)) #define TYPEFY(X, name) __typeof__ (ARGIFY (X)) name #define INTERNAL_SYSCALL(name, nr, args...) \ internal_syscall##nr (SYS_ify (name), args) #define internal_syscall3(number, arg1, arg2, arg3) \ ({ \ unsigned long int resultvar; \ TYPEFY (arg3, __arg3) = ARGIFY (arg3); \ TYPEFY (arg2, __arg2) = ARGIFY (arg2); \ TYPEFY (arg1, __arg1) = ARGIFY (arg1); \ register TYPEFY (arg3, _a3) asm ("rdx") = __arg3; \ register TYPEFY (arg2, _a2) asm ("rsi") = __arg2; \ register TYPEFY (arg1, _a1) asm ("rdi") = __arg1; \ asm volatile ( \ "syscall\n\t" \ : "=a" (resultvar) \ : "0" (number), "r" (_a1), "r" (_a2), "r" (_a3) \ : "memory", REGISTERS_CLOBBERED_BY_SYSCALL); \ (long int) resultvar; \ }) struct dtv_pointer { void *val; /* Pointer to data, or TLS_DTV_UNALLOCATED. */ void *to_free; /* Unaligned pointer, for deallocation. */ }; typedef union dtv { size_t counter; struct dtv_pointer pointer; } dtv_t; typedef struct { int i[4]; } __128bits; typedef struct { void *tcb; /* Pointer to the TCB. Not necessarily the thread descriptor used by libpthread. */ dtv_t *dtv; void *self; /* Pointer to the thread descriptor. */ int multiple_threads; int gscope_flag; uintptr_t sysinfo; uintptr_t stack_guard; uintptr_t pointer_guard; unsigned long int unused_vgetcpu_cache[2]; /* Bit 0: X86_FEATURE_1_IBT. Bit 1: X86_FEATURE_1_SHSTK. */ unsigned int feature_1; int __glibc_unused1; /* Reservation of some values for the TM ABI. */ void *__private_tm[4]; /* GCC split stack support. */ void *__private_ss; /* The lowest address of shadow stack, */ unsigned long long int ssp_base; /* Must be kept even if it is no longer used by glibc since programs, like AddressSanitizer, depend on the size of tcbhead_t. */ __128bits __glibc_unused2[8][4] __attribute__ ((aligned (32))); void *__padding[8]; } tcbhead_t; extern void *memset (void *__s, int __c, size_t __n) __THROW; extern void *memset(void *, int, size_t) __attribute__((visibility("hidden"))); #define powerof2(x) ((((x) - 1) & (x)) == 0) #define CLOCK_MONOTONIC 1 extern int clock_gettime (clockid_t __clock_id, struct timespec *__tp) __THROW; extern __typeof (clock_gettime) __clock_gettime; extern int __clock_gettime(clockid_t, struct timespec *) __attribute__((visibility("hidden"))); #define __clock_gettime64 __clock_gettime #define LLL_PRIVATE 0 #define __lll_trylock(lock) \ __glibc_unlikely (atomic_compare_and_exchange_bool_acq ((lock), 1, 0)) #define lll_trylock(lock) \ __lll_trylock (&(lock)) extern void __lll_lock_wait_private (int *futex); extern void __lll_lock_wait_private(int *) __attribute__((visibility("hidden"))); extern void __lll_lock_wait (int *futex, int private); extern void __lll_lock_wait(int *, int) __attribute__((visibility("hidden"))); #define __lll_lock(futex, private) \ ((void) \ ({ \ int *__futex = (futex); \ if (__glibc_unlikely \ (atomic_compare_and_exchange_bool_acq (__futex, 1, 0))) \ { \ if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ __lll_lock_wait_private (__futex); \ else \ __lll_lock_wait (__futex, private); \ } \ })) #define lll_lock(futex, private) \ __lll_lock (&(futex), private) extern void __lll_lock_wake_private (int *futex); extern void __lll_lock_wake_private(int *) __attribute__((visibility("hidden"))); extern void __lll_lock_wake (int *futex, int private); extern void __lll_lock_wake(int *, int) __attribute__((visibility("hidden"))); #define __lll_unlock(futex, private) \ ((void) \ ({ \ int *__futex = (futex); \ int __private = (private); \ int __oldval = atomic_exchange_release (__futex, 0); \ if (__glibc_unlikely (__oldval > 1)) \ { \ if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ __lll_lock_wake_private (__futex); \ else \ __lll_lock_wake (__futex, __private); \ } \ })) #define lll_unlock(futex, private) \ __lll_unlock (&(futex), private) #define LLL_LOCK_INITIALIZER (0) enum { MSG_OOB = 1, MSG_PEEK = 2, MSG_DONTROUTE = 4, MSG_TRYHARD = MSG_DONTROUTE, MSG_CTRUNC = 8, MSG_PROXY = 16, MSG_TRUNC = 32, MSG_DONTWAIT = 64, MSG_EOR = 128, MSG_WAITALL = 256, MSG_FIN = 512, MSG_SYN = 1024, MSG_CONFIRM = 2048, MSG_RST = 4096, MSG_ERRQUEUE = 8192, MSG_NOSIGNAL = 16384, MSG_MORE = 32768, MSG_WAITFORONE = 65536, MSG_BATCH = 262144, MSG_ZEROCOPY = 67108864, MSG_FASTOPEN = 536870912, MSG_CMSG_CLOEXEC = 1073741824 }; #define __HAVE_64B_ATOMICS 1 #define SEG_REG "fs" #define BR_CONSTRAINT "q" #define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ (! __sync_bool_compare_and_swap (mem, oldval, newval)) #define __arch_c_compare_and_exchange_val_8_acq(mem, newval, oldval) \ ({ __typeof (*mem) ret; \ __asm __volatile ("cmpl $0, %%" SEG_REG ":%P5\n\t" \ "je 0f\n\t" \ "lock\n" \ "0:\tcmpxchgb %b2, %1" \ : "=a" (ret), "=m" (*mem) \ : BR_CONSTRAINT (newval), "m" (*mem), "0" (oldval), \ "i" (offsetof (tcbhead_t, multiple_threads))); \ ret; }) #define __arch_c_compare_and_exchange_val_16_acq(mem, newval, oldval) \ ({ __typeof (*mem) ret; \ __asm __volatile ("cmpl $0, %%" SEG_REG ":%P5\n\t" \ "je 0f\n\t" \ "lock\n" \ "0:\tcmpxchgw %w2, %1" \ : "=a" (ret), "=m" (*mem) \ : BR_CONSTRAINT (newval), "m" (*mem), "0" (oldval), \ "i" (offsetof (tcbhead_t, multiple_threads))); \ ret; }) #define __arch_c_compare_and_exchange_val_32_acq(mem, newval, oldval) \ ({ __typeof (*mem) ret; \ __asm __volatile ("cmpl $0, %%" SEG_REG ":%P5\n\t" \ "je 0f\n\t" \ "lock\n" \ "0:\tcmpxchgl %2, %1" \ : "=a" (ret), "=m" (*mem) \ : BR_CONSTRAINT (newval), "m" (*mem), "0" (oldval), \ "i" (offsetof (tcbhead_t, multiple_threads))); \ ret; }) #define __arch_c_compare_and_exchange_val_64_acq(mem, newval, oldval) \ ({ __typeof (*mem) ret; \ __asm __volatile ("cmpl $0, %%fs:%P5\n\t" \ "je 0f\n\t" \ "lock\n" \ "0:\tcmpxchgq %q2, %1" \ : "=a" (ret), "=m" (*mem) \ : "q" ((int64_t) cast_to_integer (newval)), \ "m" (*mem), \ "0" ((int64_t) cast_to_integer (oldval)), \ "i" (offsetof (tcbhead_t, multiple_threads))); \ ret; }) #define do_add_val_64_acq(pfx, mem, value) do { } while (0) #define __arch_decrement_body(lock, pfx, mem) \ do { \ if (sizeof (*mem) == 1) \ __asm __volatile (lock "decb %b0" \ : "=m" (*mem) \ : "m" (*mem), \ "i" (offsetof (tcbhead_t, multiple_threads))); \ else if (sizeof (*mem) == 2) \ __asm __volatile (lock "decw %w0" \ : "=m" (*mem) \ : "m" (*mem), \ "i" (offsetof (tcbhead_t, multiple_threads))); \ else if (sizeof (*mem) == 4) \ __asm __volatile (lock "decl %0" \ : "=m" (*mem) \ : "m" (*mem), \ "i" (offsetof (tcbhead_t, multiple_threads))); \ else if (__HAVE_64B_ATOMICS) \ __asm __volatile (lock "decq %q0" \ : "=m" (*mem) \ : "m" (*mem), \ "i" (offsetof (tcbhead_t, multiple_threads))); \ else \ do_add_val_64_acq (pfx, mem, -1); \ } while (0) #define __arch_decrement_cprefix \ "cmpl $0, %%" SEG_REG ":%P2\n\tje 0f\n\tlock\n0:\t" #define catomic_decrement(mem) \ __arch_decrement_body (__arch_decrement_cprefix, __arch_c, mem) #define atomic_write_barrier() __asm ("" ::: "memory") #define __atomic_val_bysize(pre, post, mem, ...) \ ({ \ __typeof ((__typeof (*(mem))) *(mem)) __atg1_result; \ if (sizeof (*mem) == 1) \ __atg1_result = pre##_8_##post (mem, __VA_ARGS__); \ else if (sizeof (*mem) == 2) \ __atg1_result = pre##_16_##post (mem, __VA_ARGS__); \ else if (sizeof (*mem) == 4) \ __atg1_result = pre##_32_##post (mem, __VA_ARGS__); \ else if (sizeof (*mem) == 8) \ __atg1_result = pre##_64_##post (mem, __VA_ARGS__); \ else \ abort (); \ __atg1_result; \ }) #define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \ __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq, \ mem, newval, oldval) #define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \ catomic_compare_and_exchange_val_acq (mem, newval, oldval) #define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ ({ /* Cannot use __oldval here, because macros later in this file might \ call this macro with __oldval argument. */ \ __typeof (oldval) __atg4_old = (oldval); \ catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old) \ != __atg4_old; \ }) #define atomic_max(mem, value) \ do { \ __typeof (*(mem)) __atg8_oldval; \ __typeof (mem) __atg8_memp = (mem); \ __typeof (*(mem)) __atg8_value = (value); \ do { \ __atg8_oldval = *__atg8_memp; \ if (__atg8_oldval >= __atg8_value) \ break; \ } while (__builtin_expect \ (atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\ __atg8_oldval), 0)); \ } while (0) void __atomic_link_error (void); #define __atomic_check_size(mem) \ if ((sizeof (*mem) != 4) && (sizeof (*mem) != 8)) \ __atomic_link_error (); #define __atomic_check_size_ls(mem) \ if ((sizeof (*mem) != 1) && (sizeof (*mem) != 2) && (sizeof (*mem) != 4) \ && (sizeof (*mem) != 8)) \ __atomic_link_error (); #define atomic_load_relaxed(mem) \ ({ __atomic_check_size_ls((mem)); \ __atomic_load_n ((mem), __ATOMIC_RELAXED); }) #define atomic_store_relaxed(mem, val) \ do { \ __atomic_check_size_ls((mem)); \ __atomic_store_n ((mem), (val), __ATOMIC_RELAXED); \ } while (0) #define atomic_exchange_acquire(mem, desired) \ ({ __atomic_check_size((mem)); \ __atomic_exchange_n ((mem), (desired), __ATOMIC_ACQUIRE); }) #define atomic_exchange_release(mem, desired) \ ({ __atomic_check_size((mem)); \ __atomic_exchange_n ((mem), (desired), __ATOMIC_RELEASE); }) #define atomic_fetch_add_relaxed(mem, operand) \ ({ __atomic_check_size((mem)); \ __atomic_fetch_add ((mem), (operand), __ATOMIC_RELAXED); }) extern int __get_nprocs_sched (void) attribute_hidden; enum { PREFERRED_FEATURE_INDEX_1 = 0, PREFERRED_FEATURE_INDEX_MAX }; #define __WORDSIZE 64 typedef int __libc_lock_t; #define __libc_lock_define(CLASS,NAME) \ CLASS __libc_lock_t NAME; #define _LIBC_LOCK_INITIALIZER LLL_LOCK_INITIALIZER #define __libc_lock_init(NAME) ((void) ((NAME) = LLL_LOCK_INITIALIZER)) #define __libc_lock_lock(NAME) ({ lll_lock (NAME, LLL_PRIVATE); 0; }) #define __libc_lock_trylock(NAME) lll_trylock (NAME) #define __libc_lock_unlock(NAME) lll_unlock (NAME, LLL_PRIVATE) #define MAP_HUGETLB 0x40000 #define PROT_READ 0x1 #define PROT_WRITE 0x2 #define PROT_NONE 0x0 #define MAP_PRIVATE 0x02 #define MAP_FIXED 0x10 #define MAP_ANONYMOUS 0x20 #define MADV_DONTNEED 4 #define MADV_HUGEPAGE 14 #define MAP_FAILED ((void *) -1) extern void *__mmap (void *__addr, size_t __len, int __prot, int __flags, int __fd, __off_t __offset); extern void *__mmap(void *, size_t, int, int, int, __off_t) __attribute__((visibility("hidden"))); extern int __munmap (void *__addr, size_t __len); extern int __munmap(void *, size_t) __attribute__((visibility("hidden"))); extern int __mprotect (void *__addr, size_t __len, int __prot); extern int __mprotect(void *, size_t, int) __attribute__((visibility("hidden"))); extern int __madvise (void *__addr, size_t __len, int __advice); extern int __madvise(void *, size_t, int) __attribute__((visibility("hidden"))); #define EXTERN extern #define GLRO(name) _##name EXTERN size_t _dl_pagesize; #undef EXTERN #define NULL ((void*)0) _Noreturn void __libc_message (const char *__fnt, ...) attribute_hidden; extern void __assert_fail (const char *__assertion, const char *__file, unsigned int __line, const char *__function) __THROW __attribute__ ((__noreturn__)); #define assert(expr) \ ((void) sizeof ((expr) ? 1 : 0), __extension__ ({ \ if (expr) \ ; /* empty */ \ else \ __assert_fail (#expr, __FILE__, __LINE__, __ASSERT_FUNCTION); \ })) #define __ASSERT_FUNCTION __extension__ __PRETTY_FUNCTION__ extern void __assert_fail (const char *__assertion, const char *__file, unsigned int __line, const char *__function) __THROW __attribute__ ((__noreturn__)); extern _Noreturn __typeof (__assert_fail) __libc_assert_fail attribute_hidden; #define __assert_fail __libc_assert_fail #define DIAG_PUSH_NEEDS_COMMENT _Pragma ("GCC diagnostic push") #define DIAG_POP_NEEDS_COMMENT _Pragma ("GCC diagnostic pop") #define __MTAG_GRANULE_SIZE 1 void __libc_mtag_link_error (void); static inline void * __libc_mtag_tag_region (void *p, size_t n) { __libc_mtag_link_error (); return p; } static inline void * __libc_mtag_address_get_tag (void *p) { __libc_mtag_link_error (); return p; } static inline void * __libc_mtag_new_tag (void *p) { __libc_mtag_link_error (); return p; } #define DEFAULT_TOP_PAD 131072 #define O_RDONLY 00 #define __O_CLOEXEC 02000000 #define O_CLOEXEC __O_CLOEXEC extern int open (const char *__file, int __oflag, ...); __typeof (open) __open_nocancel; __typeof (__read) __read_nocancel; __typeof (__close) __close_nocancel; extern int __open_nocancel(const char *, int, ...) __attribute__((visibility("hidden"))); extern ssize_t __read_nocancel(int, void *, size_t) __attribute__((visibility("hidden"))); extern int __close_nocancel(int) __attribute__((visibility("hidden"))); static inline void __close_nocancel_nostatus (int fd) { __close_nocancel (fd); } static inline ssize_t __getrandom_nocancel (void *buf, size_t buflen, unsigned int flags) { return INTERNAL_SYSCALL_CALL (getrandom, buf, buflen, flags); } static inline bool check_may_shrink_heap (void) { static int may_shrink_heap = -1; if (__builtin_expect (may_shrink_heap >= 0, 1)) return may_shrink_heap; may_shrink_heap = __libc_enable_secure; if (__builtin_expect (may_shrink_heap == 0, 1)) { int fd = __open_nocancel ("/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC); if (fd >= 0) { char val; ssize_t n = __read_nocancel (fd, &val, 1); may_shrink_heap = n > 0 && val == '2'; __close_nocancel_nostatus (fd); } } return may_shrink_heap; } #define INTERNAL_SIZE_T size_t #define SIZE_SZ (sizeof (INTERNAL_SIZE_T)) #define MALLOC_ALIGNMENT (2 * SIZE_SZ < __alignof__ (long double) \ ? __alignof__ (long double) : 2 * SIZE_SZ) #define MALLOC_ALIGN_MASK (MALLOC_ALIGNMENT - 1) unsigned long int __malloc_default_thp_pagesize (void) attribute_hidden; enum malloc_thp_mode_t { malloc_thp_mode_always, malloc_thp_mode_madvise, malloc_thp_mode_never, malloc_thp_mode_not_supported }; enum malloc_thp_mode_t __malloc_thp_mode (void) attribute_hidden; void __malloc_hugepage_config (size_t requested, size_t *pagesize, int *flags) attribute_hidden; static inline uint32_t random_bits (void) { struct __timespec64 tv; __clock_gettime64 (CLOCK_MONOTONIC, &tv); /* Shuffle the lower bits to minimize the clock bias. */ uint32_t ret = tv.tv_nsec ^ tv.tv_sec; ret ^= (ret << 24) | (ret >> 8); return ret; } #define GRND_NONBLOCK 0x01 #define TCACHE_MAX_BINS 64 #define MAX_TCACHE_SIZE tidx2usize (TCACHE_MAX_BINS-1) #define tidx2usize(idx) (((size_t) idx) * MALLOC_ALIGNMENT + MINSIZE - SIZE_SZ) #define csize2tidx(x) (((x) - MINSIZE + MALLOC_ALIGNMENT - 1) / MALLOC_ALIGNMENT) #define TCACHE_FILL_COUNT 7 #define MAX_TCACHE_COUNT UINT16_MAX #define PROTECT_PTR(pos, ptr) \ ((__typeof (ptr)) ((((size_t) pos) >> 12) ^ ((size_t) ptr))) #define REVEAL_PTR(ptr) PROTECT_PTR (&ptr, ptr) #define TRIM_FASTBINS 0 static bool __always_fail_morecore = false; void * __glibc_morecore (ptrdiff_t increment) { if (__always_fail_morecore) return NULL; void *result = (void *) __sbrk (increment); if (result == (void *) -1) return NULL; return result; } #define MORECORE (*__glibc_morecore) #define MORECORE_FAILURE 0 #define MODULE_NAME libc #define USE_TCACHE 1 #define TOP_NAMESPACE glibc #define mtag_enabled false #define mtag_mmap_flags 0 static void * tag_new_usable (void *ptr); static __always_inline void * tag_at (void *ptr) { if (__glibc_unlikely (mtag_enabled)) return __libc_mtag_address_get_tag (ptr); return ptr; } #define MORECORE_CONTIGUOUS 1 #define MMAP_AS_MORECORE_SIZE (1024 * 1024) void* __libc_malloc(size_t); extern void *__libc_malloc(size_t) __attribute__((visibility("hidden"))); #define DEFAULT_MXFAST (64 * SIZE_SZ / 4) #define DEFAULT_TRIM_THRESHOLD (128 * 1024) #define DEFAULT_MMAP_THRESHOLD_MIN (128 * 1024) #define DEFAULT_MMAP_THRESHOLD_MAX (4 * 1024 * 1024 * sizeof(long)) #define DEFAULT_MMAP_THRESHOLD DEFAULT_MMAP_THRESHOLD_MIN #define DEFAULT_MMAP_MAX (65536) struct malloc_state; typedef struct malloc_state *mstate; struct malloc_chunk; typedef struct malloc_chunk* mchunkptr; static void* _int_malloc(mstate, size_t); static void _int_free(mstate, mchunkptr, int); static void malloc_printerr(const char *str) __attribute__ ((noreturn)); static void munmap_chunk(mchunkptr p); #define MMAP(addr, size, prot, flags) \ __mmap((addr), (size), (prot), (flags)|MAP_ANONYMOUS|MAP_PRIVATE, -1, 0) struct malloc_chunk { INTERNAL_SIZE_T mchunk_prev_size; /* Size of previous chunk (if free). */ INTERNAL_SIZE_T mchunk_size; /* Size in bytes, including overhead. */ struct malloc_chunk* fd; /* double links -- used only if free. */ struct malloc_chunk* bk; /* Only used for large blocks: pointer to next larger size. */ struct malloc_chunk* fd_nextsize; /* double links -- used only if free. */ struct malloc_chunk* bk_nextsize; }; #define CHUNK_HDR_SZ (2 * SIZE_SZ) #define chunk2mem(p) ((void*)((char*)(p) + CHUNK_HDR_SZ)) #define mem2chunk(mem) ((mchunkptr)tag_at (((char*)(mem) - CHUNK_HDR_SZ))) #define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize)) #define MINSIZE \ (unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)) #define aligned_OK(m) (((unsigned long)(m) & MALLOC_ALIGN_MASK) == 0) #define misaligned_chunk(p) \ ((uintptr_t)(MALLOC_ALIGNMENT == CHUNK_HDR_SZ ? (p) : chunk2mem (p)) \ & MALLOC_ALIGN_MASK) #define request2size(req) \ (((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \ MINSIZE : \ ((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK) static inline size_t checked_request2size (size_t req) __nonnull (1) { if (__glibc_unlikely (req > PTRDIFF_MAX)) return 0; /* When using tagged memory, we cannot share the end of the user block with the header for the next chunk, so ensure that we allocate blocks that are rounded up to the granule size. Take care not to overflow from close to MAX_SIZE_T to a small number. Ideally, this would be part of request2size(), but that must be a macro that produces a compile time constant if passed a constant literal. */ if (__glibc_unlikely (mtag_enabled)) { /* Ensure this is not evaluated if !mtag_enabled, see gcc PR 99551. */ asm (""); req = (req + (__MTAG_GRANULE_SIZE - 1)) & ~(size_t)(__MTAG_GRANULE_SIZE - 1); } return request2size (req); } #define PREV_INUSE 0x1 #define prev_inuse(p) ((p)->mchunk_size & PREV_INUSE) #define IS_MMAPPED 0x2 #define chunk_is_mmapped(p) ((p)->mchunk_size & IS_MMAPPED) #define NON_MAIN_ARENA 0x4 #define chunk_main_arena(p) (((p)->mchunk_size & NON_MAIN_ARENA) == 0) #define set_non_main_arena(p) ((p)->mchunk_size |= NON_MAIN_ARENA) #define SIZE_BITS (PREV_INUSE | IS_MMAPPED | NON_MAIN_ARENA) #define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS)) #define chunksize_nomask(p) ((p)->mchunk_size) #define next_chunk(p) ((mchunkptr) (((char *) (p)) + chunksize (p))) #define prev_size(p) ((p)->mchunk_prev_size) #define set_prev_size(p, sz) ((p)->mchunk_prev_size = (sz)) #define prev_chunk(p) ((mchunkptr) (((char *) (p)) - prev_size (p))) #define chunk_at_offset(p, s) ((mchunkptr) (((char *) (p)) + (s))) #define inuse_bit_at_offset(p, s) \ (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size & PREV_INUSE) #define set_inuse_bit_at_offset(p, s) \ (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size |= PREV_INUSE) #define clear_inuse_bit_at_offset(p, s) \ (((mchunkptr) (((char *) (p)) + (s)))->mchunk_size &= ~(PREV_INUSE)) #define set_head(p, s) ((p)->mchunk_size = (s)) #define set_foot(p, s) (((mchunkptr) ((char *) (p) + (s)))->mchunk_prev_size = (s)) #define memsize(p) \ (__MTAG_GRANULE_SIZE > SIZE_SZ && __glibc_unlikely (mtag_enabled) ? \ chunksize (p) - CHUNK_HDR_SZ : \ chunksize (p) - CHUNK_HDR_SZ + (chunk_is_mmapped (p) ? 0 : SIZE_SZ)) static __always_inline void * tag_new_usable (void *ptr) { if (__glibc_unlikely (mtag_enabled) && ptr) { mchunkptr cp = mem2chunk(ptr); ptr = __libc_mtag_tag_region (__libc_mtag_new_tag (ptr), memsize (cp)); } return ptr; } typedef struct malloc_chunk *mbinptr; #define bin_at(m, i) \ (mbinptr) (((char *) &((m)->bins[((i) - 1) * 2])) \ - offsetof (struct malloc_chunk, fd)) #define next_bin(b) ((mbinptr) ((char *) (b) + (sizeof (mchunkptr) << 1))) #define first(b) ((b)->fd) #define last(b) ((b)->bk) #define NBINS 128 #define NSMALLBINS 64 #define SMALLBIN_WIDTH MALLOC_ALIGNMENT #define SMALLBIN_CORRECTION (MALLOC_ALIGNMENT > CHUNK_HDR_SZ) #define MIN_LARGE_SIZE ((NSMALLBINS - SMALLBIN_CORRECTION) * SMALLBIN_WIDTH) #define in_smallbin_range(sz) \ ((unsigned long) (sz) < (unsigned long) MIN_LARGE_SIZE) #define smallbin_index(sz) \ ((SMALLBIN_WIDTH == 16 ? (((unsigned) (sz)) >> 4) : (((unsigned) (sz)) >> 3))\ + SMALLBIN_CORRECTION) #define largebin_index_32(sz) \ (((((unsigned long) (sz)) >> 6) <= 38) ? 56 + (((unsigned long) (sz)) >> 6) :\ ((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\ ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\ ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\ ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\ 126) #define largebin_index_32_big(sz) \ (((((unsigned long) (sz)) >> 6) <= 45) ? 49 + (((unsigned long) (sz)) >> 6) :\ ((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\ ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\ ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\ ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\ 126) #define largebin_index_64(sz) \ (((((unsigned long) (sz)) >> 6) <= 48) ? 48 + (((unsigned long) (sz)) >> 6) :\ ((((unsigned long) (sz)) >> 9) <= 20) ? 91 + (((unsigned long) (sz)) >> 9) :\ ((((unsigned long) (sz)) >> 12) <= 10) ? 110 + (((unsigned long) (sz)) >> 12) :\ ((((unsigned long) (sz)) >> 15) <= 4) ? 119 + (((unsigned long) (sz)) >> 15) :\ ((((unsigned long) (sz)) >> 18) <= 2) ? 124 + (((unsigned long) (sz)) >> 18) :\ 126) #define largebin_index(sz) \ (SIZE_SZ == 8 ? largebin_index_64 (sz) \ : MALLOC_ALIGNMENT == 16 ? largebin_index_32_big (sz) \ : largebin_index_32 (sz)) static void unlink_chunk (mstate av, mchunkptr p) { if (chunksize (p) != prev_size (next_chunk (p))) malloc_printerr ("corrupted size vs. prev_size"); mchunkptr fd = p->fd; mchunkptr bk = p->bk; if (__builtin_expect (fd->bk != p || bk->fd != p, 0)) malloc_printerr ("corrupted double-linked list"); fd->bk = bk; bk->fd = fd; if (!in_smallbin_range (chunksize_nomask (p)) && p->fd_nextsize != NULL) { if (p->fd_nextsize->bk_nextsize != p || p->bk_nextsize->fd_nextsize != p) malloc_printerr ("corrupted double-linked list (not small)"); if (fd->fd_nextsize == NULL) { if (p->fd_nextsize == p) fd->fd_nextsize = fd->bk_nextsize = fd; else { fd->fd_nextsize = p->fd_nextsize; fd->bk_nextsize = p->bk_nextsize; p->fd_nextsize->bk_nextsize = fd; p->bk_nextsize->fd_nextsize = fd; } } else { p->fd_nextsize->bk_nextsize = p->bk_nextsize; p->bk_nextsize->fd_nextsize = p->fd_nextsize; } } } #define unsorted_chunks(M) (bin_at (M, 1)) #define initial_top(M) (unsorted_chunks (M)) #define BINMAPSHIFT 5 #define BITSPERMAP (1U << BINMAPSHIFT) #define BINMAPSIZE (NBINS / BITSPERMAP) #define idx2block(i) ((i) >> BINMAPSHIFT) #define idx2bit(i) ((1U << ((i) & ((1U << BINMAPSHIFT) - 1)))) #define mark_bin(m, i) ((m)->binmap[idx2block (i)] |= idx2bit (i)) typedef struct malloc_chunk *mfastbinptr; #define fastbin(ar_ptr, idx) ((ar_ptr)->fastbinsY[idx]) #define fastbin_index(sz) \ ((((unsigned int) (sz)) >> (SIZE_SZ == 8 ? 4 : 3)) - 2) #define MAX_FAST_SIZE (80 * SIZE_SZ / 4) #define NFASTBINS (fastbin_index (request2size (MAX_FAST_SIZE)) + 1) #define FASTBIN_CONSOLIDATION_THRESHOLD (65536UL) #define NONCONTIGUOUS_BIT (2U) #define contiguous(M) (((M)->flags & NONCONTIGUOUS_BIT) == 0) #define set_noncontiguous(M) ((M)->flags |= NONCONTIGUOUS_BIT) static uint8_t global_max_fast; #define set_max_fast(s) \ global_max_fast = (((size_t) (s) <= MALLOC_ALIGN_MASK - SIZE_SZ) \ ? MIN_CHUNK_SIZE / 2 : ((s + SIZE_SZ) & ~MALLOC_ALIGN_MASK)) static inline INTERNAL_SIZE_T get_max_fast (void) { /* Tell the GCC optimizers that global_max_fast is never larger than MAX_FAST_SIZE. This avoids out-of-bounds array accesses in _int_malloc after constant propagation of the size parameter. (The code never executes because malloc preserves the global_max_fast invariant, but the optimizers may not recognize this.) */ if (global_max_fast > MAX_FAST_SIZE) __builtin_unreachable (); return global_max_fast; } struct malloc_state { /* Serialize access. */ __libc_lock_define (, mutex); /* Flags (formerly in max_fast). */ int flags; /* Set if the fastbin chunks contain recently inserted free blocks. */ /* Note this is a bool but not all targets support atomics on booleans. */ int have_fastchunks; /* Fastbins */ mfastbinptr fastbinsY[NFASTBINS]; /* Base of the topmost chunk -- not otherwise kept in a bin */ mchunkptr top; /* The remainder from the most recent split of a small request */ mchunkptr last_remainder; /* Normal bins packed as described above */ mchunkptr bins[NBINS * 2 - 2]; /* Bitmap of bins */ unsigned int binmap[BINMAPSIZE]; /* Linked list */ struct malloc_state *next; /* Linked list for free arenas. Access to this field is serialized by free_list_lock in arena.c. */ struct malloc_state *next_free; /* Number of threads attached to this arena. 0 if the arena is on the free list. Access to this field is serialized by free_list_lock in arena.c. */ INTERNAL_SIZE_T attached_threads; /* Memory allocated from the system in this arena. */ INTERNAL_SIZE_T system_mem; INTERNAL_SIZE_T max_system_mem; }; struct malloc_par { /* Tunable parameters */ unsigned long trim_threshold; INTERNAL_SIZE_T top_pad; INTERNAL_SIZE_T mmap_threshold; INTERNAL_SIZE_T arena_test; INTERNAL_SIZE_T arena_max; /* Transparent Large Page support. */ INTERNAL_SIZE_T thp_pagesize; /* A value different than 0 means to align mmap allocation to hp_pagesize add hp_flags on flags. */ INTERNAL_SIZE_T hp_pagesize; int hp_flags; /* Memory map support */ int n_mmaps; int n_mmaps_max; int max_n_mmaps; /* the mmap_threshold is dynamic, until the user sets it manually, at which point we need to disable any dynamic behavior. */ int no_dyn_threshold; /* Statistics */ INTERNAL_SIZE_T mmapped_mem; INTERNAL_SIZE_T max_mmapped_mem; /* First address handed out by MORECORE/sbrk. */ char *sbrk_base; #if USE_TCACHE /* Maximum number of buckets to use. */ size_t tcache_bins; size_t tcache_max_bytes; /* Maximum number of chunks in each bucket. */ size_t tcache_count; /* Maximum number of chunks to remove from the unsorted list, which aren't used to prefill the cache. */ size_t tcache_unsorted_limit; #endif }; static struct malloc_state main_arena = { .mutex = _LIBC_LOCK_INITIALIZER, .next = &main_arena, .attached_threads = 1 }; static struct malloc_par mp_ = { .top_pad = DEFAULT_TOP_PAD, .n_mmaps_max = DEFAULT_MMAP_MAX, .mmap_threshold = DEFAULT_MMAP_THRESHOLD, .trim_threshold = DEFAULT_TRIM_THRESHOLD, #define NARENAS_FROM_NCORES(n) ((n) * (sizeof (long) == 4 ? 2 : 8)) .arena_test = NARENAS_FROM_NCORES (1) #if USE_TCACHE , .tcache_count = TCACHE_FILL_COUNT, .tcache_bins = TCACHE_MAX_BINS, .tcache_max_bytes = tidx2usize (TCACHE_MAX_BINS-1), .tcache_unsorted_limit = 0 /* No limit. */ #endif }; static void malloc_init_state (mstate av) { int i; mbinptr bin; /* Establish circular links for normal bins */ for (i = 1; i < NBINS; ++i) { bin = bin_at (av, i); bin->fd = bin->bk = bin; } #if MORECORE_CONTIGUOUS if (av != &main_arena) #endif set_noncontiguous (av); if (av == &main_arena) set_max_fast (DEFAULT_MXFAST); atomic_store_relaxed (&av->have_fastchunks, false); av->top = initial_top (av); } static void *sysmalloc (INTERNAL_SIZE_T, mstate); static int systrim (size_t, mstate); static void malloc_consolidate (mstate); static int perturb_byte; static void alloc_perturb (char *p, size_t n) { if (__glibc_unlikely (perturb_byte)) memset (p, perturb_byte ^ 0xff, n); } static void free_perturb (char *p, size_t n) { if (__glibc_unlikely (perturb_byte)) memset (p, perturb_byte, n); } #define LIBC_PROBE(name, n, ...) STAP_PROBE##n (__VA_ARGS__) #define STAP_PROBE1(a1) do {} while (0) #define STAP_PROBE2(a1, a2) do {} while (0) #define STAP_PROBE3(a1, a2, a3) do {} while (0) static inline void madvise_thp (void *p, INTERNAL_SIZE_T size) { #ifdef MADV_HUGEPAGE /* Do not consider areas smaller than a huge page or if the tunable is not active. */ if (mp_.thp_pagesize == 0 || size < mp_.thp_pagesize) return; /* Linux requires the input address to be page-aligned, and unaligned inputs happens only for initial data segment. */ if (__glibc_unlikely (!PTR_IS_ALIGNED (p, GLRO (dl_pagesize)))) { void *q = PTR_ALIGN_DOWN (p, GLRO (dl_pagesize)); size += PTR_DIFF (p, q); p = q; } __madvise (p, size, MADV_HUGEPAGE); #endif } #define TUNABLE_NAMESPACE malloc typedef intmax_t tunable_num_t; typedef union { tunable_num_t numval; const char *strval; } tunable_val_t; typedef void (*tunable_callback_t) (tunable_val_t *); #define TUNABLE_ENUM_NAME(__top,__ns,__id) TUNABLE_ENUM_NAME1 (__top,__ns,__id) #define TUNABLE_ENUM_NAME1(__top,__ns,__id) __top ## _ ## __ns ## _ ## __id typedef enum { TUNABLE_ENUM_NAME(glibc, malloc, mxfast), TUNABLE_ENUM_NAME(glibc, malloc, mmap_threshold), TUNABLE_ENUM_NAME(glibc, malloc, tcache_max), TUNABLE_ENUM_NAME(glibc, pthread, mutex_spin_count), TUNABLE_ENUM_NAME(glibc, cpu, x86_rep_movsb_threshold), TUNABLE_ENUM_NAME(glibc, malloc, tcache_unsorted_limit), TUNABLE_ENUM_NAME(glibc, elision, tries), TUNABLE_ENUM_NAME(glibc, cpu, x86_non_temporal_threshold), TUNABLE_ENUM_NAME(glibc, cpu, x86_data_cache_size), TUNABLE_ENUM_NAME(glibc, cpu, x86_shared_cache_size), TUNABLE_ENUM_NAME(glibc, elision, skip_trylock_internal_abort), TUNABLE_ENUM_NAME(glibc, rtld, dynamic_sort), TUNABLE_ENUM_NAME(glibc, malloc, mmap_max), TUNABLE_ENUM_NAME(glibc, cpu, prefer_map_32bit_exec), TUNABLE_ENUM_NAME(glibc, malloc, trim_threshold), TUNABLE_ENUM_NAME(glibc, elision, skip_lock_after_retries), TUNABLE_ENUM_NAME(glibc, rtld, optional_static_tls), TUNABLE_ENUM_NAME(glibc, cpu, hwcap_mask), TUNABLE_ENUM_NAME(glibc, cpu, x86_ibt), TUNABLE_ENUM_NAME(glibc, pthread, stack_cache_size), TUNABLE_ENUM_NAME(glibc, rtld, nns), TUNABLE_ENUM_NAME(glibc, malloc, arena_test), TUNABLE_ENUM_NAME(glibc, malloc, hugetlb), TUNABLE_ENUM_NAME(glibc, cpu, x86_rep_stosb_threshold), TUNABLE_ENUM_NAME(glibc, malloc, perturb), TUNABLE_ENUM_NAME(glibc, malloc, tcache_count), TUNABLE_ENUM_NAME(glibc, elision, enable), TUNABLE_ENUM_NAME(glibc, cpu, x86_shstk), TUNABLE_ENUM_NAME(glibc, gmon, maxarcs), TUNABLE_ENUM_NAME(glibc, malloc, check), TUNABLE_ENUM_NAME(glibc, gmon, minarcs), TUNABLE_ENUM_NAME(glibc, malloc, top_pad), TUNABLE_ENUM_NAME(glibc, pthread, stack_hugetlb), TUNABLE_ENUM_NAME(glibc, mem, tagging), TUNABLE_ENUM_NAME(glibc, cpu, hwcaps), TUNABLE_ENUM_NAME(glibc, elision, skip_lock_internal_abort), TUNABLE_ENUM_NAME(glibc, malloc, arena_max), TUNABLE_ENUM_NAME(glibc, elision, skip_lock_busy), TUNABLE_ENUM_NAME(glibc, pthread, rseq), } tunable_id_t; extern void __tunable_get_val (tunable_id_t, void *, tunable_callback_t); #define TUNABLE_GET(__id, __type, __cb) \ TUNABLE_GET_FULL (TOP_NAMESPACE, TUNABLE_NAMESPACE, __id, __type, __cb) #define TUNABLE_GET_FULL(__top, __ns, __id, __type, __cb) \ ({ \ tunable_id_t id = TUNABLE_ENUM_NAME (__top, __ns, __id); \ __type ret; \ __tunable_get_val (id, &ret, __cb); \ ret; \ }) #define TUNABLE_CALLBACK(__name) _dl_tunable_ ## __name #define HEAP_MIN_SIZE (32 * 1024) #define HEAP_MAX_SIZE (2 * DEFAULT_MMAP_THRESHOLD_MAX) static inline size_t heap_min_size (void) { return mp_.hp_pagesize == 0 ? HEAP_MIN_SIZE : mp_.hp_pagesize; } static inline size_t heap_max_size (void) { return mp_.hp_pagesize == 0 ? HEAP_MAX_SIZE : mp_.hp_pagesize * 4; } #define top(ar_ptr) ((ar_ptr)->top) typedef struct _heap_info { mstate ar_ptr; /* Arena for this heap. */ struct _heap_info *prev; /* Previous heap. */ size_t size; /* Current size in bytes. */ size_t mprotect_size; /* Size in bytes that has been mprotected PROT_READ|PROT_WRITE. */ size_t pagesize; /* Page size used when allocating the arena. */ /* Make sure the following data is properly aligned, particularly that sizeof (heap_info) + 2 * SIZE_SZ is a multiple of MALLOC_ALIGNMENT. */ char pad[-3 * SIZE_SZ & MALLOC_ALIGN_MASK]; } heap_info; static __thread mstate thread_arena; static __libc_lock_t free_list_lock; static size_t narenas = 1; static mstate free_list; static __libc_lock_t list_lock; static bool __malloc_initialized = false; #define arena_get(ptr, size) do { \ ptr = thread_arena; \ arena_lock (ptr, size); \ } while (0) #define arena_lock(ptr, size) do { \ if (ptr) \ __libc_lock_lock (ptr->mutex); \ else \ ptr = arena_get2 ((size), NULL); \ } while (0) static inline heap_info * heap_for_ptr (void *ptr) { size_t max_size = heap_max_size (); return PTR_ALIGN_DOWN (ptr, max_size); } static inline struct malloc_state * arena_for_chunk (mchunkptr ptr) { return chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr; } static inline int do_set_mmap_threshold(size_t value); static void _dl_tunable_set_mmap_threshold(tunable_val_t *valp) { size_t value = (size_t)(valp)->numval; do_set_mmap_threshold(value); } static inline int do_set_mmaps_max(int32_t value); static void _dl_tunable_set_mmaps_max(tunable_val_t *valp) { int32_t value = (int32_t)(valp)->numval; do_set_mmaps_max(value); } static inline int do_set_top_pad(size_t value); static void _dl_tunable_set_top_pad(tunable_val_t *valp) { size_t value = (size_t)(valp)->numval; do_set_top_pad(value); } static inline int do_set_perturb_byte(int32_t value); static void _dl_tunable_set_perturb_byte(tunable_val_t *valp) { int32_t value = (int32_t)(valp)->numval; do_set_perturb_byte(value); } static inline int do_set_trim_threshold(size_t value); static void _dl_tunable_set_trim_threshold(tunable_val_t *valp) { size_t value = (size_t)(valp)->numval; do_set_trim_threshold(value); } static inline int do_set_arena_max(size_t value); static void _dl_tunable_set_arena_max(tunable_val_t *valp) { size_t value = (size_t)(valp)->numval; do_set_arena_max(value); } static inline int do_set_arena_test(size_t value); static void _dl_tunable_set_arena_test(tunable_val_t *valp) { size_t value = (size_t)(valp)->numval; do_set_arena_test(value); } static inline int do_set_tcache_max(size_t value); static void _dl_tunable_set_tcache_max(tunable_val_t *valp) { size_t value = (size_t)(valp)->numval; do_set_tcache_max(value); } static inline int do_set_tcache_count(size_t value); static void _dl_tunable_set_tcache_count(tunable_val_t *valp) { size_t value = (size_t)(valp)->numval; do_set_tcache_count(value); } static inline int do_set_tcache_unsorted_limit(size_t value); static void _dl_tunable_set_tcache_unsorted_limit(tunable_val_t *valp) { size_t value = (size_t)(valp)->numval; do_set_tcache_unsorted_limit(value); } static inline int do_set_mxfast(size_t value); static void _dl_tunable_set_mxfast(tunable_val_t *valp) { size_t value = (size_t)(valp)->numval; do_set_mxfast(value); } static inline int do_set_hugetlb(size_t value); static void _dl_tunable_set_hugetlb(tunable_val_t *valp) { size_t value = (size_t)(valp)->numval; do_set_hugetlb(value); } static void tcache_key_initialize (void); static void ptmalloc_init (void) { if (__malloc_initialized) return; __malloc_initialized = true; #if USE_TCACHE tcache_key_initialize (); #endif #ifdef USE_MTAG if ((TUNABLE_GET_FULL (glibc, mem, tagging, int32_t, NULL) & 1) != 0) { /* If the tunable says that we should be using tagged memory and that morecore does not support tagged regions, then disable it. */ if (__MTAG_SBRK_UNTAGGED) __always_fail_morecore = true; mtag_enabled = true; mtag_mmap_flags = __MTAG_MMAP_FLAGS; } #endif #if defined SHARED && IS_IN (libc) /* In case this libc copy is in a non-default namespace, never use brk. Likewise if dlopened from statically linked program. The generic sbrk implementation also enforces this, but it is not used on Hurd. */ if (!__libc_initial) __always_fail_morecore = true; #endif thread_arena = &main_arena; malloc_init_state (&main_arena); TUNABLE_GET (top_pad, size_t, TUNABLE_CALLBACK (set_top_pad)); TUNABLE_GET (perturb, int32_t, TUNABLE_CALLBACK (set_perturb_byte)); TUNABLE_GET (mmap_threshold, size_t, TUNABLE_CALLBACK (set_mmap_threshold)); TUNABLE_GET (trim_threshold, size_t, TUNABLE_CALLBACK (set_trim_threshold)); TUNABLE_GET (mmap_max, int32_t, TUNABLE_CALLBACK (set_mmaps_max)); TUNABLE_GET (arena_max, size_t, TUNABLE_CALLBACK (set_arena_max)); TUNABLE_GET (arena_test, size_t, TUNABLE_CALLBACK (set_arena_test)); # if USE_TCACHE TUNABLE_GET (tcache_max, size_t, TUNABLE_CALLBACK (set_tcache_max)); TUNABLE_GET (tcache_count, size_t, TUNABLE_CALLBACK (set_tcache_count)); TUNABLE_GET (tcache_unsorted_limit, size_t, TUNABLE_CALLBACK (set_tcache_unsorted_limit)); # endif TUNABLE_GET (mxfast, size_t, TUNABLE_CALLBACK (set_mxfast)); TUNABLE_GET (hugetlb, size_t, TUNABLE_CALLBACK (set_hugetlb)); if (mp_.hp_pagesize > 0) /* Force mmap for main arena instead of sbrk, so hugepages are explicitly used. */ __always_fail_morecore = true; } static char *aligned_heap_area; static heap_info * alloc_new_heap (size_t size, size_t top_pad, size_t pagesize, int mmap_flags) { char *p1, *p2; unsigned long ul; heap_info *h; size_t min_size = heap_min_size (); size_t max_size = heap_max_size (); if (size + top_pad < min_size) size = min_size; else if (size + top_pad <= max_size) size += top_pad; else if (size > max_size) return 0; else size = max_size; size = ALIGN_UP (size, pagesize); /* A memory region aligned to a multiple of max_size is needed. No swap space needs to be reserved for the following large mapping (on Linux, this is the case for all non-writable mappings anyway). */ p2 = MAP_FAILED; if (aligned_heap_area) { p2 = (char *) MMAP (aligned_heap_area, max_size, PROT_NONE, mmap_flags); aligned_heap_area = NULL; if (p2 != MAP_FAILED && ((unsigned long) p2 & (max_size - 1))) { __munmap (p2, max_size); p2 = MAP_FAILED; } } if (p2 == MAP_FAILED) { p1 = (char *) MMAP (0, max_size << 1, PROT_NONE, mmap_flags); if (p1 != MAP_FAILED) { p2 = (char *) (((uintptr_t) p1 + (max_size - 1)) & ~(max_size - 1)); ul = p2 - p1; if (ul) __munmap (p1, ul); else aligned_heap_area = p2 + max_size; __munmap (p2 + max_size, max_size - ul); } else { /* Try to take the chance that an allocation of only max_size is already aligned. */ p2 = (char *) MMAP (0, max_size, PROT_NONE, mmap_flags); if (p2 == MAP_FAILED) return 0; if ((unsigned long) p2 & (max_size - 1)) { __munmap (p2, max_size); return 0; } } } if (__mprotect (p2, size, mtag_mmap_flags | PROT_READ | PROT_WRITE) != 0) { __munmap (p2, max_size); return 0; } madvise_thp (p2, size); h = (heap_info *) p2; h->size = size; h->mprotect_size = size; h->pagesize = pagesize; LIBC_PROBE (memory_heap_new, 2, h, h->size); return h; } static heap_info * new_heap (size_t size, size_t top_pad) { if (__glibc_unlikely (mp_.hp_pagesize != 0)) { heap_info *h = alloc_new_heap (size, top_pad, mp_.hp_pagesize, mp_.hp_flags); if (h != NULL) return h; } return alloc_new_heap (size, top_pad, GLRO (dl_pagesize), 0); } static int grow_heap (heap_info *h, long diff) { size_t pagesize = h->pagesize; size_t max_size = heap_max_size (); long new_size; diff = ALIGN_UP (diff, pagesize); new_size = (long) h->size + diff; if ((unsigned long) new_size > (unsigned long) max_size) return -1; if ((unsigned long) new_size > h->mprotect_size) { if (__mprotect ((char *) h + h->mprotect_size, (unsigned long) new_size - h->mprotect_size, mtag_mmap_flags | PROT_READ | PROT_WRITE) != 0) return -2; h->mprotect_size = new_size; } h->size = new_size; LIBC_PROBE (memory_heap_more, 2, h, h->size); return 0; } static int shrink_heap (heap_info *h, long diff) { long new_size; new_size = (long) h->size - diff; if (new_size < (long) sizeof (*h)) return -1; /* Try to re-map the extra heap space freshly to save memory, and make it inaccessible. See malloc-sysdep.h to know when this is true. */ if (__glibc_unlikely (check_may_shrink_heap ())) { if ((char *) MMAP ((char *) h + new_size, diff, PROT_NONE, MAP_FIXED) == (char *) MAP_FAILED) return -2; h->mprotect_size = new_size; } else __madvise ((char *) h + new_size, diff, MADV_DONTNEED); /*fprintf(stderr, "shrink %p %08lx\n", h, new_size);*/ h->size = new_size; LIBC_PROBE (memory_heap_less, 2, h, h->size); return 0; } static int heap_trim (heap_info *heap, size_t pad) { mstate ar_ptr = heap->ar_ptr; mchunkptr top_chunk = top (ar_ptr), p; heap_info *prev_heap; long new_size, top_size, top_area, extra, prev_size, misalign; size_t max_size = heap_max_size (); /* Can this heap go away completely? */ while (top_chunk == chunk_at_offset (heap, sizeof (*heap))) { prev_heap = heap->prev; prev_size = prev_heap->size - (MINSIZE - 2 * SIZE_SZ); p = chunk_at_offset (prev_heap, prev_size); /* fencepost must be properly aligned. */ misalign = ((long) p) & MALLOC_ALIGN_MASK; p = chunk_at_offset (prev_heap, prev_size - misalign); assert (chunksize_nomask (p) == (0 | PREV_INUSE)); /* must be fencepost */ p = prev_chunk (p); new_size = chunksize (p) + (MINSIZE - 2 * SIZE_SZ) + misalign; assert (new_size > 0 && new_size < (long) (2 * MINSIZE)); if (!prev_inuse (p)) new_size += prev_size (p); assert (new_size > 0 && new_size < max_size); if (new_size + (max_size - prev_heap->size) < pad + MINSIZE + heap->pagesize) break; ar_ptr->system_mem -= heap->size; LIBC_PROBE (memory_heap_free, 2, heap, heap->size); if ((char *) heap + max_size == aligned_heap_area) aligned_heap_area = NULL; __munmap (heap, max_size); heap = prev_heap; if (!prev_inuse (p)) /* consolidate backward */ { p = prev_chunk (p); unlink_chunk (ar_ptr, p); } assert (((unsigned long) ((char *) p + new_size) & (heap->pagesize - 1)) == 0); assert (((char *) p + new_size) == ((char *) heap + heap->size)); top (ar_ptr) = top_chunk = p; set_head (top_chunk, new_size | PREV_INUSE); /*check_chunk(ar_ptr, top_chunk);*/ } /* Uses similar logic for per-thread arenas as the main arena with systrim and _int_free by preserving the top pad and rounding down to the nearest page. */ top_size = chunksize (top_chunk); if ((unsigned long)(top_size) < (unsigned long)(mp_.trim_threshold)) return 0; top_area = top_size - MINSIZE - 1; if (top_area < 0 || (size_t) top_area <= pad) return 0; /* Release in pagesize units and round down to the nearest page. */ extra = ALIGN_DOWN(top_area - pad, heap->pagesize); if (extra == 0) return 0; /* Try to shrink. */ if (shrink_heap (heap, extra) != 0) return 0; ar_ptr->system_mem -= extra; /* Success. Adjust top accordingly. */ set_head (top_chunk, (top_size - extra) | PREV_INUSE); /*check_chunk(ar_ptr, top_chunk);*/ return 1; } static void detach_arena (mstate replaced_arena) { if (replaced_arena != NULL) { assert (replaced_arena->attached_threads > 0); /* The current implementation only detaches from main_arena in case of allocation failure. This means that it is likely not beneficial to put the arena on free_list even if the reference count reaches zero. */ --replaced_arena->attached_threads; } } static mstate _int_new_arena (size_t size) { mstate a; heap_info *h; char *ptr; unsigned long misalign; h = new_heap (size + (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT), mp_.top_pad); if (!h) { /* Maybe size is too large to fit in a single heap. So, just try to create a minimally-sized arena and let _int_malloc() attempt to deal with the large request via mmap_chunk(). */ h = new_heap (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT, mp_.top_pad); if (!h) return 0; } a = h->ar_ptr = (mstate) (h + 1); malloc_init_state (a); a->attached_threads = 1; /*a->next = NULL;*/ a->system_mem = a->max_system_mem = h->size; /* Set up the top chunk, with proper alignment. */ ptr = (char *) (a + 1); misalign = (uintptr_t) chunk2mem (ptr) & MALLOC_ALIGN_MASK; if (misalign > 0) ptr += MALLOC_ALIGNMENT - misalign; top (a) = (mchunkptr) ptr; set_head (top (a), (((char *) h + h->size) - ptr) | PREV_INUSE); LIBC_PROBE (memory_arena_new, 2, a, size); mstate replaced_arena = thread_arena; thread_arena = a; __libc_lock_init (a->mutex); __libc_lock_lock (list_lock); /* Add the new arena to the global list. */ a->next = main_arena.next; /* FIXME: The barrier is an attempt to synchronize with read access in reused_arena, which does not acquire list_lock while traversing the list. */ atomic_write_barrier (); main_arena.next = a; __libc_lock_unlock (list_lock); __libc_lock_lock (free_list_lock); detach_arena (replaced_arena); __libc_lock_unlock (free_list_lock); /* Lock this arena. NB: Another thread may have been attached to this arena because the arena is now accessible from the main_arena.next list and could have been picked by reused_arena. This can only happen for the last arena created (before the arena limit is reached). At this point, some arena has to be attached to two threads. We could acquire the arena lock before list_lock to make it less likely that reused_arena picks this new arena, but this could result in a deadlock with __malloc_fork_lock_parent. */ __libc_lock_lock (a->mutex); return a; } static mstate get_free_list (void) { mstate replaced_arena = thread_arena; mstate result = free_list; if (result != NULL) { __libc_lock_lock (free_list_lock); result = free_list; if (result != NULL) { free_list = result->next_free; /* The arena will be attached to this thread. */ assert (result->attached_threads == 0); result->attached_threads = 1; detach_arena (replaced_arena); } __libc_lock_unlock (free_list_lock); if (result != NULL) { LIBC_PROBE (memory_arena_reuse_free_list, 1, result); __libc_lock_lock (result->mutex); thread_arena = result; } } return result; } static void remove_from_free_list (mstate arena) { mstate *previous = &free_list; for (mstate p = free_list; p != NULL; p = p->next_free) { assert (p->attached_threads == 0); if (p == arena) { /* Remove the requested arena from the list. */ *previous = p->next_free; break; } else previous = &p->next_free; } } static mstate reused_arena (mstate avoid_arena) { mstate result; /* FIXME: Access to next_to_use suffers from data races. */ static mstate next_to_use; if (next_to_use == NULL) next_to_use = &main_arena; /* Iterate over all arenas (including those linked from free_list). */ result = next_to_use; do { if (!__libc_lock_trylock (result->mutex)) goto out; /* FIXME: This is a data race, see _int_new_arena. */ result = result->next; } while (result != next_to_use); /* Avoid AVOID_ARENA as we have already failed to allocate memory in that arena and it is currently locked. */ if (result == avoid_arena) result = result->next; /* No arena available without contention. Wait for the next in line. */ LIBC_PROBE (memory_arena_reuse_wait, 3, &result->mutex, result, avoid_arena); __libc_lock_lock (result->mutex); out: /* Attach the arena to the current thread. */ { /* Update the arena thread attachment counters. */ mstate replaced_arena = thread_arena; __libc_lock_lock (free_list_lock); detach_arena (replaced_arena); /* We may have picked up an arena on the free list. We need to preserve the invariant that no arena on the free list has a positive attached_threads counter (otherwise, arena_thread_freeres cannot use the counter to determine if the arena needs to be put on the free list). We unconditionally remove the selected arena from the free list. The caller of reused_arena checked the free list and observed it to be empty, so the list is very short. */ remove_from_free_list (result); ++result->attached_threads; __libc_lock_unlock (free_list_lock); } LIBC_PROBE (memory_arena_reuse, 2, result, avoid_arena); thread_arena = result; next_to_use = result->next; return result; } static mstate arena_get2 (size_t size, mstate avoid_arena) { mstate a; static size_t narenas_limit; a = get_free_list (); if (a == NULL) { /* Nothing immediately available, so generate a new arena. */ if (narenas_limit == 0) { if (mp_.arena_max != 0) narenas_limit = mp_.arena_max; else if (narenas > mp_.arena_test) { int n = __get_nprocs_sched (); if (n >= 1) narenas_limit = NARENAS_FROM_NCORES (n); else /* We have no information about the system. Assume two cores. */ narenas_limit = NARENAS_FROM_NCORES (2); } } repeat:; size_t n = narenas; /* NB: the following depends on the fact that (size_t)0 - 1 is a very large number and that the underflow is OK. If arena_max is set the value of arena_test is irrelevant. If arena_test is set but narenas is not yet larger or equal to arena_test narenas_limit is 0. There is no possibility for narenas to be too big for the test to always fail since there is not enough address space to create that many arenas. */ if (__glibc_unlikely (n <= narenas_limit - 1)) { if (catomic_compare_and_exchange_bool_acq (&narenas, n + 1, n)) goto repeat; a = _int_new_arena (size); if (__glibc_unlikely (a == NULL)) catomic_decrement (&narenas); } else a = reused_arena (avoid_arena); } return a; } static mstate arena_get_retry (mstate ar_ptr, size_t bytes) { LIBC_PROBE (memory_arena_retry, 2, bytes, ar_ptr); if (ar_ptr != &main_arena) { __libc_lock_unlock (ar_ptr->mutex); ar_ptr = &main_arena; __libc_lock_lock (ar_ptr->mutex); } else { __libc_lock_unlock (ar_ptr->mutex); ar_ptr = arena_get2 (bytes, ar_ptr); } return ar_ptr; } #define check_chunk(A, P) #define check_free_chunk(A, P) #define check_inuse_chunk(A, P) #define check_remalloced_chunk(A, P, N) #define check_malloced_chunk(A, P, N) #define check_malloc_state(A) static void * sysmalloc_mmap (INTERNAL_SIZE_T nb, size_t pagesize, int extra_flags, mstate av) { long int size; /* Round up size to nearest page. For mmapped chunks, the overhead is one SIZE_SZ unit larger than for normal chunks, because there is no following chunk whose prev_size field could be used. See the front_misalign handling below, for glibc there is no need for further alignments unless we have have high alignment. */ if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ) size = ALIGN_UP (nb + SIZE_SZ, pagesize); else size = ALIGN_UP (nb + SIZE_SZ + MALLOC_ALIGN_MASK, pagesize); /* Don't try if size wraps around 0. */ if ((unsigned long) (size) <= (unsigned long) (nb)) return MAP_FAILED; char *mm = (char *) MMAP (0, size, mtag_mmap_flags | PROT_READ | PROT_WRITE, extra_flags); if (mm == MAP_FAILED) return mm; #ifdef MAP_HUGETLB if (!(extra_flags & MAP_HUGETLB)) madvise_thp (mm, size); #endif /* The offset to the start of the mmapped region is stored in the prev_size field of the chunk. This allows us to adjust returned start address to meet alignment requirements here and in memalign(), and still be able to compute proper address argument for later munmap in free() and realloc(). */ INTERNAL_SIZE_T front_misalign; /* unusable bytes at front of new space */ if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ) { /* For glibc, chunk2mem increases the address by CHUNK_HDR_SZ and MALLOC_ALIGN_MASK is CHUNK_HDR_SZ-1. Each mmap'ed area is page aligned and therefore definitely MALLOC_ALIGN_MASK-aligned. */ assert (((INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK) == 0); front_misalign = 0; } else front_misalign = (INTERNAL_SIZE_T) chunk2mem (mm) & MALLOC_ALIGN_MASK; mchunkptr p; /* the allocated/returned chunk */ if (front_misalign > 0) { ptrdiff_t correction = MALLOC_ALIGNMENT - front_misalign; p = (mchunkptr) (mm + correction); set_prev_size (p, correction); set_head (p, (size - correction) | IS_MMAPPED); } else { p = (mchunkptr) mm; set_prev_size (p, 0); set_head (p, size | IS_MMAPPED); } /* update statistics */ int new = atomic_fetch_add_relaxed (&mp_.n_mmaps, 1) + 1; atomic_max (&mp_.max_n_mmaps, new); unsigned long sum; sum = atomic_fetch_add_relaxed (&mp_.mmapped_mem, size) + size; atomic_max (&mp_.max_mmapped_mem, sum); check_chunk (av, p); return chunk2mem (p); } static void * sysmalloc_mmap_fallback (long int *s, INTERNAL_SIZE_T nb, INTERNAL_SIZE_T old_size, size_t minsize, size_t pagesize, int extra_flags, mstate av) { long int size = *s; /* Cannot merge with old top, so add its size back in */ if (contiguous (av)) size = ALIGN_UP (size + old_size, pagesize); /* If we are relying on mmap as backup, then use larger units */ if ((unsigned long) (size) < minsize) size = minsize; /* Don't try if size wraps around 0 */ if ((unsigned long) (size) <= (unsigned long) (nb)) return MORECORE_FAILURE; char *mbrk = (char *) (MMAP (0, size, mtag_mmap_flags | PROT_READ | PROT_WRITE, extra_flags)); if (mbrk == MAP_FAILED) return MAP_FAILED; #ifdef MAP_HUGETLB if (!(extra_flags & MAP_HUGETLB)) madvise_thp (mbrk, size); #endif /* Record that we no longer have a contiguous sbrk region. After the first time mmap is used as backup, we do not ever rely on contiguous space since this could incorrectly bridge regions. */ set_noncontiguous (av); *s = size; return mbrk; } static void * sysmalloc (INTERNAL_SIZE_T nb, mstate av) { mchunkptr old_top; /* incoming value of av->top */ INTERNAL_SIZE_T old_size; /* its size */ char *old_end; /* its end address */ long size; /* arg to first MORECORE or mmap call */ char *brk; /* return value from MORECORE */ long correction; /* arg to 2nd MORECORE call */ char *snd_brk; /* 2nd return val */ INTERNAL_SIZE_T front_misalign; /* unusable bytes at front of new space */ INTERNAL_SIZE_T end_misalign; /* partial page left at end of new space */ char *aligned_brk; /* aligned offset into brk */ mchunkptr p; /* the allocated/returned chunk */ mchunkptr remainder; /* remainder from allocation */ unsigned long remainder_size; /* its size */ size_t pagesize = GLRO (dl_pagesize); bool tried_mmap = false; /* If have mmap, and the request size meets the mmap threshold, and the system supports mmap, and there are few enough currently allocated mmapped regions, try to directly map this request rather than expanding top. */ if (av == NULL || ((unsigned long) (nb) >= (unsigned long) (mp_.mmap_threshold) && (mp_.n_mmaps < mp_.n_mmaps_max))) { char *mm; if (mp_.hp_pagesize > 0 && nb >= mp_.hp_pagesize) { /* There is no need to issue the THP madvise call if Huge Pages are used directly. */ mm = sysmalloc_mmap (nb, mp_.hp_pagesize, mp_.hp_flags, av); if (mm != MAP_FAILED) return mm; } mm = sysmalloc_mmap (nb, pagesize, 0, av); if (mm != MAP_FAILED) return mm; tried_mmap = true; } /* There are no usable arenas and mmap also failed. */ if (av == NULL) return 0; /* Record incoming configuration of top */ old_top = av->top; old_size = chunksize (old_top); old_end = (char *) (chunk_at_offset (old_top, old_size)); brk = snd_brk = (char *) (MORECORE_FAILURE); /* If not the first time through, we require old_size to be at least MINSIZE and to have prev_inuse set. */ assert ((old_top == initial_top (av) && old_size == 0) || ((unsigned long) (old_size) >= MINSIZE && prev_inuse (old_top) && ((unsigned long) old_end & (pagesize - 1)) == 0)); /* Precondition: not enough current space to satisfy nb request */ assert ((unsigned long) (old_size) < (unsigned long) (nb + MINSIZE)); if (av != &main_arena) { heap_info *old_heap, *heap; size_t old_heap_size; /* First try to extend the current heap. */ old_heap = heap_for_ptr (old_top); old_heap_size = old_heap->size; if ((long) (MINSIZE + nb - old_size) > 0 && grow_heap (old_heap, MINSIZE + nb - old_size) == 0) { av->system_mem += old_heap->size - old_heap_size; set_head (old_top, (((char *) old_heap + old_heap->size) - (char *) old_top) | PREV_INUSE); } else if ((heap = new_heap (nb + (MINSIZE + sizeof (*heap)), mp_.top_pad))) { /* Use a newly allocated heap. */ heap->ar_ptr = av; heap->prev = old_heap; av->system_mem += heap->size; /* Set up the new top. */ top (av) = chunk_at_offset (heap, sizeof (*heap)); set_head (top (av), (heap->size - sizeof (*heap)) | PREV_INUSE); /* Setup fencepost and free the old top chunk with a multiple of MALLOC_ALIGNMENT in size. */ /* The fencepost takes at least MINSIZE bytes, because it might become the top chunk again later. Note that a footer is set up, too, although the chunk is marked in use. */ old_size = (old_size - MINSIZE) & ~MALLOC_ALIGN_MASK; set_head (chunk_at_offset (old_top, old_size + CHUNK_HDR_SZ), 0 | PREV_INUSE); if (old_size >= MINSIZE) { set_head (chunk_at_offset (old_top, old_size), CHUNK_HDR_SZ | PREV_INUSE); set_foot (chunk_at_offset (old_top, old_size), CHUNK_HDR_SZ); set_head (old_top, old_size | PREV_INUSE | NON_MAIN_ARENA); _int_free (av, old_top, 1); } else { set_head (old_top, (old_size + CHUNK_HDR_SZ) | PREV_INUSE); set_foot (old_top, (old_size + CHUNK_HDR_SZ)); } } else if (!tried_mmap) { /* We can at least try to use to mmap memory. If new_heap fails it is unlikely that trying to allocate huge pages will succeed. */ char *mm = sysmalloc_mmap (nb, pagesize, 0, av); if (mm != MAP_FAILED) return mm; } } else /* av == main_arena */ { /* Request enough space for nb + pad + overhead */ size = nb + mp_.top_pad + MINSIZE; /* If contiguous, we can subtract out existing space that we hope to combine with new space. We add it back later only if we don't actually get contiguous space. */ if (contiguous (av)) size -= old_size; /* Round to a multiple of page size or huge page size. If MORECORE is not contiguous, this ensures that we only call it with whole-page arguments. And if MORECORE is contiguous and this is not first time through, this preserves page-alignment of previous calls. Otherwise, we correct to page-align below. */ #ifdef MADV_HUGEPAGE /* Defined in brk.c. */ extern void *__curbrk; if (__glibc_unlikely (mp_.thp_pagesize != 0)) { uintptr_t top = ALIGN_UP ((uintptr_t) __curbrk + size, mp_.thp_pagesize); size = top - (uintptr_t) __curbrk; } else #endif size = ALIGN_UP (size, GLRO(dl_pagesize)); /* Don't try to call MORECORE if argument is so big as to appear negative. Note that since mmap takes size_t arg, it may succeed below even if we cannot call MORECORE. */ if (size > 0) { brk = (char *) (MORECORE (size)); if (brk != (char *) (MORECORE_FAILURE)) madvise_thp (brk, size); LIBC_PROBE (memory_sbrk_more, 2, brk, size); } if (brk == (char *) (MORECORE_FAILURE)) { /* If have mmap, try using it as a backup when MORECORE fails or cannot be used. This is worth doing on systems that have "holes" in address space, so sbrk cannot extend to give contiguous space, but space is available elsewhere. Note that we ignore mmap max count and threshold limits, since the space will not be used as a segregated mmap region. */ char *mbrk = MAP_FAILED; if (mp_.hp_pagesize > 0) mbrk = sysmalloc_mmap_fallback (&size, nb, old_size, mp_.hp_pagesize, mp_.hp_pagesize, mp_.hp_flags, av); if (mbrk == MAP_FAILED) mbrk = sysmalloc_mmap_fallback (&size, nb, old_size, MMAP_AS_MORECORE_SIZE, pagesize, 0, av); if (mbrk != MAP_FAILED) { /* We do not need, and cannot use, another sbrk call to find end */ brk = mbrk; snd_brk = brk + size; } } if (brk != (char *) (MORECORE_FAILURE)) { if (mp_.sbrk_base == 0) mp_.sbrk_base = brk; av->system_mem += size; /* If MORECORE extends previous space, we can likewise extend top size. */ if (brk == old_end && snd_brk == (char *) (MORECORE_FAILURE)) set_head (old_top, (size + old_size) | PREV_INUSE); else if (contiguous (av) && old_size && brk < old_end) /* Oops! Someone else killed our space.. Can't touch anything. */ malloc_printerr ("break adjusted to free malloc space"); /* Otherwise, make adjustments: * If the first time through or noncontiguous, we need to call sbrk just to find out where the end of memory lies. * We need to ensure that all returned chunks from malloc will meet MALLOC_ALIGNMENT * If there was an intervening foreign sbrk, we need to adjust sbrk request size to account for fact that we will not be able to combine new space with existing space in old_top. * Almost all systems internally allocate whole pages at a time, in which case we might as well use the whole last page of request. So we allocate enough more memory to hit a page boundary now, which in turn causes future contiguous calls to page-align. */ else { front_misalign = 0; end_misalign = 0; correction = 0; aligned_brk = brk; /* handle contiguous cases */ if (contiguous (av)) { /* Count foreign sbrk as system_mem. */ if (old_size) av->system_mem += brk - old_end; /* Guarantee alignment of first new chunk made from this space */ front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK; if (front_misalign > 0) { /* Skip over some bytes to arrive at an aligned position. We don't need to specially mark these wasted front bytes. They will never be accessed anyway because prev_inuse of av->top (and any chunk created from its start) is always true after initialization. */ correction = MALLOC_ALIGNMENT - front_misalign; aligned_brk += correction; } /* If this isn't adjacent to existing space, then we will not be able to merge with old_top space, so must add to 2nd request. */ correction += old_size; /* Extend the end address to hit a page boundary */ end_misalign = (INTERNAL_SIZE_T) (brk + size + correction); correction += (ALIGN_UP (end_misalign, pagesize)) - end_misalign; assert (correction >= 0); snd_brk = (char *) (MORECORE (correction)); /* If can't allocate correction, try to at least find out current brk. It might be enough to proceed without failing. Note that if second sbrk did NOT fail, we assume that space is contiguous with first sbrk. This is a safe assumption unless program is multithreaded but doesn't use locks and a foreign sbrk occurred between our first and second calls. */ if (snd_brk == (char *) (MORECORE_FAILURE)) { correction = 0; snd_brk = (char *) (MORECORE (0)); } else madvise_thp (snd_brk, correction); } /* handle non-contiguous cases */ else { if (MALLOC_ALIGNMENT == CHUNK_HDR_SZ) /* MORECORE/mmap must correctly align */ assert (((unsigned long) chunk2mem (brk) & MALLOC_ALIGN_MASK) == 0); else { front_misalign = (INTERNAL_SIZE_T) chunk2mem (brk) & MALLOC_ALIGN_MASK; if (front_misalign > 0) { /* Skip over some bytes to arrive at an aligned position. We don't need to specially mark these wasted front bytes. They will never be accessed anyway because prev_inuse of av->top (and any chunk created from its start) is always true after initialization. */ aligned_brk += MALLOC_ALIGNMENT - front_misalign; } } /* Find out current end of memory */ if (snd_brk == (char *) (MORECORE_FAILURE)) { snd_brk = (char *) (MORECORE (0)); } } /* Adjust top based on results of second sbrk */ if (snd_brk != (char *) (MORECORE_FAILURE)) { av->top = (mchunkptr) aligned_brk; set_head (av->top, (snd_brk - aligned_brk + correction) | PREV_INUSE); av->system_mem += correction; /* If not the first time through, we either have a gap due to foreign sbrk or a non-contiguous region. Insert a double fencepost at old_top to prevent consolidation with space we don't own. These fenceposts are artificial chunks that are marked as inuse and are in any case too small to use. We need two to make sizes and alignments work out. */ if (old_size != 0) { /* Shrink old_top to insert fenceposts, keeping size a multiple of MALLOC_ALIGNMENT. We know there is at least enough space in old_top to do this. */ old_size = (old_size - 2 * CHUNK_HDR_SZ) & ~MALLOC_ALIGN_MASK; set_head (old_top, old_size | PREV_INUSE); /* Note that the following assignments completely overwrite old_top when old_size was previously MINSIZE. This is intentional. We need the fencepost, even if old_top otherwise gets lost. */ set_head (chunk_at_offset (old_top, old_size), CHUNK_HDR_SZ | PREV_INUSE); set_head (chunk_at_offset (old_top, old_size + CHUNK_HDR_SZ), CHUNK_HDR_SZ | PREV_INUSE); /* If possible, release the rest. */ if (old_size >= MINSIZE) { _int_free (av, old_top, 1); } } } } } } /* if (av != &main_arena) */ if ((unsigned long) av->system_mem > (unsigned long) (av->max_system_mem)) av->max_system_mem = av->system_mem; check_malloc_state (av); /* finally, do the allocation */ p = av->top; size = chunksize (p); /* check that one of the above allocation paths succeeded */ if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE)) { remainder_size = size - nb; remainder = chunk_at_offset (p, nb); av->top = remainder; set_head (p, nb | PREV_INUSE | (av != &main_arena ? NON_MAIN_ARENA : 0)); set_head (remainder, remainder_size | PREV_INUSE); check_malloced_chunk (av, p, nb); return chunk2mem (p); } /* catch all failure paths */ __set_errno (ENOMEM); return 0; } static int systrim (size_t pad, mstate av) { long top_size; /* Amount of top-most memory */ long extra; /* Amount to release */ long released; /* Amount actually released */ char *current_brk; /* address returned by pre-check sbrk call */ char *new_brk; /* address returned by post-check sbrk call */ long top_area; top_size = chunksize (av->top); top_area = top_size - MINSIZE - 1; if (top_area <= pad) return 0; /* Release in pagesize units and round down to the nearest page. */ #ifdef MADV_HUGEPAGE if (__glibc_unlikely (mp_.thp_pagesize != 0)) extra = ALIGN_DOWN (top_area - pad, mp_.thp_pagesize); else #endif extra = ALIGN_DOWN (top_area - pad, GLRO(dl_pagesize)); if (extra == 0) return 0; /* Only proceed if end of memory is where we last set it. This avoids problems if there were foreign sbrk calls. */ current_brk = (char *) (MORECORE (0)); if (current_brk == (char *) (av->top) + top_size) { /* Attempt to release memory. We ignore MORECORE return value, and instead call again to find out where new end of memory is. This avoids problems if first call releases less than we asked, of if failure somehow altered brk value. (We could still encounter problems if it altered brk in some very bad way, but the only thing we can do is adjust anyway, which will cause some downstream failure.) */ MORECORE (-extra); new_brk = (char *) (MORECORE (0)); LIBC_PROBE (memory_sbrk_less, 2, new_brk, extra); if (new_brk != (char *) MORECORE_FAILURE) { released = (long) (current_brk - new_brk); if (released != 0) { /* Success. Adjust top. */ av->system_mem -= released; set_head (av->top, (top_size - released) | PREV_INUSE); check_malloc_state (av); return 1; } } } return 0; } static void munmap_chunk (mchunkptr p) { size_t pagesize = GLRO (dl_pagesize); INTERNAL_SIZE_T size = chunksize (p); assert (chunk_is_mmapped (p)); uintptr_t mem = (uintptr_t) chunk2mem (p); uintptr_t block = (uintptr_t) p - prev_size (p); size_t total_size = prev_size (p) + size; /* Unfortunately we have to do the compilers job by hand here. Normally we would test BLOCK and TOTAL-SIZE separately for compliance with the page size. But gcc does not recognize the optimization possibility (in the moment at least) so we combine the two values into one before the bit test. */ if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0 || __glibc_unlikely (!powerof2 (mem & (pagesize - 1)))) malloc_printerr ("munmap_chunk(): invalid pointer"); atomic_fetch_add_relaxed (&mp_.n_mmaps, -1); atomic_fetch_add_relaxed (&mp_.mmapped_mem, -total_size); /* If munmap failed the process virtual memory address space is in a bad shape. Just leave the block hanging around, the process will terminate shortly anyway since not much can be done. */ __munmap ((char *) block, total_size); } typedef struct tcache_entry { struct tcache_entry *next; /* This field exists to detect double frees. */ uintptr_t key; } tcache_entry; typedef struct tcache_perthread_struct { uint16_t counts[TCACHE_MAX_BINS]; tcache_entry *entries[TCACHE_MAX_BINS]; } tcache_perthread_struct; static __thread bool tcache_shutting_down = false; static __thread tcache_perthread_struct *tcache = NULL; static uintptr_t tcache_key; static void tcache_key_initialize (void) { if (__getrandom_nocancel (&tcache_key, sizeof(tcache_key), GRND_NONBLOCK) != sizeof (tcache_key)) { tcache_key = random_bits (); #if __WORDSIZE == 64 tcache_key = (tcache_key << 32) | random_bits (); #endif } } static __always_inline void tcache_put (mchunkptr chunk, size_t tc_idx) { tcache_entry *e = (tcache_entry *) chunk2mem (chunk); /* Mark this chunk as "in the tcache" so the test in _int_free will detect a double free. */ e->key = tcache_key; e->next = PROTECT_PTR (&e->next, tcache->entries[tc_idx]); tcache->entries[tc_idx] = e; ++(tcache->counts[tc_idx]); } static __always_inline void * tcache_get_n (size_t tc_idx, tcache_entry **ep) { tcache_entry *e; if (ep == &(tcache->entries[tc_idx])) e = *ep; else e = REVEAL_PTR (*ep); if (__glibc_unlikely (!aligned_OK (e))) malloc_printerr ("malloc(): unaligned tcache chunk detected"); if (ep == &(tcache->entries[tc_idx])) *ep = REVEAL_PTR (e->next); else *ep = PROTECT_PTR (ep, REVEAL_PTR (e->next)); --(tcache->counts[tc_idx]); e->key = 0; return (void *) e; } static __always_inline void * tcache_get (size_t tc_idx) { return tcache_get_n (tc_idx, & tcache->entries[tc_idx]); } static void tcache_init(void) { mstate ar_ptr; void *victim = 0; const size_t bytes = sizeof (tcache_perthread_struct); if (tcache_shutting_down) return; arena_get (ar_ptr, bytes); victim = _int_malloc (ar_ptr, bytes); if (!victim && ar_ptr != NULL) { ar_ptr = arena_get_retry (ar_ptr, bytes); victim = _int_malloc (ar_ptr, bytes); } if (ar_ptr != NULL) __libc_lock_unlock (ar_ptr->mutex); /* In a low memory situation, we may not be able to allocate memory - in which case, we just keep trying later. However, we typically do this very early, so either there is sufficient memory, or there isn't enough memory to do non-trivial allocations anyway. */ if (victim) { tcache = (tcache_perthread_struct *) victim; memset (tcache, 0, sizeof (tcache_perthread_struct)); } } #define MAYBE_INIT_TCACHE() \ if (__glibc_unlikely (tcache == NULL)) \ tcache_init(); void * __libc_malloc (size_t bytes) { mstate ar_ptr; void *victim; _Static_assert (PTRDIFF_MAX <= SIZE_MAX / 2, "PTRDIFF_MAX is not more than half of SIZE_MAX"); if (!__malloc_initialized) ptmalloc_init (); #if USE_TCACHE /* int_free also calls request2size, be careful to not pad twice. */ size_t tbytes = checked_request2size (bytes); if (tbytes == 0) { __set_errno (ENOMEM); return NULL; } size_t tc_idx = csize2tidx (tbytes); MAYBE_INIT_TCACHE (); DIAG_PUSH_NEEDS_COMMENT; if (tc_idx < mp_.tcache_bins && tcache != NULL && tcache->counts[tc_idx] > 0) { victim = tcache_get (tc_idx); return tag_new_usable (victim); } DIAG_POP_NEEDS_COMMENT; #endif if (SINGLE_THREAD_P) { victim = tag_new_usable (_int_malloc (&main_arena, bytes)); assert (!victim || chunk_is_mmapped (mem2chunk (victim)) || &main_arena == arena_for_chunk (mem2chunk (victim))); return victim; } arena_get (ar_ptr, bytes); victim = _int_malloc (ar_ptr, bytes); /* Retry with another arena only if we were able to find a usable arena before. */ if (!victim && ar_ptr != NULL) { LIBC_PROBE (memory_malloc_retry, 1, bytes); ar_ptr = arena_get_retry (ar_ptr, bytes); victim = _int_malloc (ar_ptr, bytes); } if (ar_ptr != NULL) __libc_lock_unlock (ar_ptr->mutex); victim = tag_new_usable (victim); assert (!victim || chunk_is_mmapped (mem2chunk (victim)) || ar_ptr == arena_for_chunk (mem2chunk (victim))); return victim; } static void * _int_malloc (mstate av, size_t bytes) { INTERNAL_SIZE_T nb; /* normalized request size */ unsigned int idx; /* associated bin index */ mbinptr bin; /* associated bin */ mchunkptr victim; /* inspected/selected chunk */ INTERNAL_SIZE_T size; /* its size */ int victim_index; /* its bin index */ mchunkptr remainder; /* remainder from a split */ unsigned long remainder_size; /* its size */ unsigned int block; /* bit map traverser */ unsigned int bit; /* bit map traverser */ unsigned int map; /* current word of binmap */ mchunkptr fwd; /* misc temp for linking */ mchunkptr bck; /* misc temp for linking */ #if USE_TCACHE size_t tcache_unsorted_count; /* count of unsorted chunks processed */ #endif /* Convert request size to internal form by adding SIZE_SZ bytes overhead plus possibly more to obtain necessary alignment and/or to obtain a size of at least MINSIZE, the smallest allocatable size. Also, checked_request2size returns false for request sizes that are so large that they wrap around zero when padded and aligned. */ nb = checked_request2size (bytes); if (nb == 0) { __set_errno (ENOMEM); return NULL; } /* There are no usable arenas. Fall back to sysmalloc to get a chunk from mmap. */ if (__glibc_unlikely (av == NULL)) { void *p = sysmalloc (nb, av); if (p != NULL) alloc_perturb (p, bytes); return p; } /* If the size qualifies as a fastbin, first check corresponding bin. This code is safe to execute even if av is not yet initialized, so we can try it without checking, which saves some time on this fast path. */ #define REMOVE_FB(fb, victim, pp) \ do \ { \ victim = pp; \ if (victim == NULL) \ break; \ pp = REVEAL_PTR (victim->fd); \ if (__glibc_unlikely (pp != NULL && misaligned_chunk (pp))) \ malloc_printerr ("malloc(): unaligned fastbin chunk detected"); \ } \ while ((pp = catomic_compare_and_exchange_val_acq (fb, pp, victim)) \ != victim); \ if ((unsigned long) (nb) <= (unsigned long) (get_max_fast ())) { idx = fastbin_index (nb); mfastbinptr *fb = &fastbin (av, idx); mchunkptr pp; victim = *fb; if (victim != NULL) { if (__glibc_unlikely (misaligned_chunk (victim))) malloc_printerr ("malloc(): unaligned fastbin chunk detected 2"); if (SINGLE_THREAD_P) *fb = REVEAL_PTR (victim->fd); else REMOVE_FB (fb, pp, victim); if (__glibc_likely (victim != NULL)) { size_t victim_idx = fastbin_index (chunksize (victim)); if (__builtin_expect (victim_idx != idx, 0)) malloc_printerr ("malloc(): memory corruption (fast)"); check_remalloced_chunk (av, victim, nb); #if USE_TCACHE /* While we're here, if we see other chunks of the same size, stash them in the tcache. */ size_t tc_idx = csize2tidx (nb); if (tcache != NULL && tc_idx < mp_.tcache_bins) { mchunkptr tc_victim; /* While bin not empty and tcache not full, copy chunks. */ while (tcache->counts[tc_idx] < mp_.tcache_count && (tc_victim = *fb) != NULL) { if (__glibc_unlikely (misaligned_chunk (tc_victim))) malloc_printerr ("malloc(): unaligned fastbin chunk detected 3"); if (SINGLE_THREAD_P) *fb = REVEAL_PTR (tc_victim->fd); else { REMOVE_FB (fb, pp, tc_victim); if (__glibc_unlikely (tc_victim == NULL)) break; } tcache_put (tc_victim, tc_idx); } } #endif void *p = chunk2mem (victim); alloc_perturb (p, bytes); return p; } } } /* If a small request, check regular bin. Since these "smallbins" hold one size each, no searching within bins is necessary. (For a large request, we need to wait until unsorted chunks are processed to find best fit. But for small ones, fits are exact anyway, so we can check now, which is faster.) */ if (in_smallbin_range (nb)) { idx = smallbin_index (nb); bin = bin_at (av, idx); if ((victim = last (bin)) != bin) { bck = victim->bk; if (__glibc_unlikely (bck->fd != victim)) malloc_printerr ("malloc(): smallbin double linked list corrupted"); set_inuse_bit_at_offset (victim, nb); bin->bk = bck; bck->fd = bin; if (av != &main_arena) set_non_main_arena (victim); check_malloced_chunk (av, victim, nb); #if USE_TCACHE /* While we're here, if we see other chunks of the same size, stash them in the tcache. */ size_t tc_idx = csize2tidx (nb); if (tcache != NULL && tc_idx < mp_.tcache_bins) { mchunkptr tc_victim; /* While bin not empty and tcache not full, copy chunks over. */ while (tcache->counts[tc_idx] < mp_.tcache_count && (tc_victim = last (bin)) != bin) { if (tc_victim != 0) { bck = tc_victim->bk; set_inuse_bit_at_offset (tc_victim, nb); if (av != &main_arena) set_non_main_arena (tc_victim); bin->bk = bck; bck->fd = bin; tcache_put (tc_victim, tc_idx); } } } #endif void *p = chunk2mem (victim); alloc_perturb (p, bytes); return p; } } /* If this is a large request, consolidate fastbins before continuing. While it might look excessive to kill all fastbins before even seeing if there is space available, this avoids fragmentation problems normally associated with fastbins. Also, in practice, programs tend to have runs of either small or large requests, but less often mixtures, so consolidation is not invoked all that often in most programs. And the programs that it is called frequently in otherwise tend to fragment. */ else { idx = largebin_index (nb); if (atomic_load_relaxed (&av->have_fastchunks)) malloc_consolidate (av); } /* Process recently freed or remaindered chunks, taking one only if it is exact fit, or, if this a small request, the chunk is remainder from the most recent non-exact fit. Place other traversed chunks in bins. Note that this step is the only place in any routine where chunks are placed in bins. The outer loop here is needed because we might not realize until near the end of malloc that we should have consolidated, so must do so and retry. This happens at most once, and only when we would otherwise need to expand memory to service a "small" request. */ #if USE_TCACHE INTERNAL_SIZE_T tcache_nb = 0; size_t tc_idx = csize2tidx (nb); if (tcache != NULL && tc_idx < mp_.tcache_bins) tcache_nb = nb; int return_cached = 0; tcache_unsorted_count = 0; #endif for (;; ) { int iters = 0; while ((victim = unsorted_chunks (av)->bk) != unsorted_chunks (av)) { bck = victim->bk; size = chunksize (victim); mchunkptr next = chunk_at_offset (victim, size); if (__glibc_unlikely (size <= CHUNK_HDR_SZ) || __glibc_unlikely (size > av->system_mem)) malloc_printerr ("malloc(): invalid size (unsorted)"); if (__glibc_unlikely (chunksize_nomask (next) < CHUNK_HDR_SZ) || __glibc_unlikely (chunksize_nomask (next) > av->system_mem)) malloc_printerr ("malloc(): invalid next size (unsorted)"); if (__glibc_unlikely ((prev_size (next) & ~(SIZE_BITS)) != size)) malloc_printerr ("malloc(): mismatching next->prev_size (unsorted)"); if (__glibc_unlikely (bck->fd != victim) || __glibc_unlikely (victim->fd != unsorted_chunks (av))) malloc_printerr ("malloc(): unsorted double linked list corrupted"); if (__glibc_unlikely (prev_inuse (next))) malloc_printerr ("malloc(): invalid next->prev_inuse (unsorted)"); /* If a small request, try to use last remainder if it is the only chunk in unsorted bin. This helps promote locality for runs of consecutive small requests. This is the only exception to best-fit, and applies only when there is no exact fit for a small chunk. */ if (in_smallbin_range (nb) && bck == unsorted_chunks (av) && victim == av->last_remainder && (unsigned long) (size) > (unsigned long) (nb + MINSIZE)) { /* split and reattach remainder */ remainder_size = size - nb; remainder = chunk_at_offset (victim, nb); unsorted_chunks (av)->bk = unsorted_chunks (av)->fd = remainder; av->last_remainder = remainder; remainder->bk = remainder->fd = unsorted_chunks (av); if (!in_smallbin_range (remainder_size)) { remainder->fd_nextsize = NULL; remainder->bk_nextsize = NULL; } set_head (victim, nb | PREV_INUSE | (av != &main_arena ? NON_MAIN_ARENA : 0)); set_head (remainder, remainder_size | PREV_INUSE); set_foot (remainder, remainder_size); check_malloced_chunk (av, victim, nb); void *p = chunk2mem (victim); alloc_perturb (p, bytes); return p; } /* remove from unsorted list */ unsorted_chunks (av)->bk = bck; bck->fd = unsorted_chunks (av); /* Take now instead of binning if exact fit */ if (size == nb) { set_inuse_bit_at_offset (victim, size); if (av != &main_arena) set_non_main_arena (victim); #if USE_TCACHE /* Fill cache first, return to user only if cache fills. We may return one of these chunks later. */ if (tcache_nb > 0 && tcache->counts[tc_idx] < mp_.tcache_count) { tcache_put (victim, tc_idx); return_cached = 1; continue; } else { #endif check_malloced_chunk (av, victim, nb); void *p = chunk2mem (victim); alloc_perturb (p, bytes); return p; #if USE_TCACHE } #endif } /* place chunk in bin */ if (in_smallbin_range (size)) { victim_index = smallbin_index (size); bck = bin_at (av, victim_index); fwd = bck->fd; } else { victim_index = largebin_index (size); bck = bin_at (av, victim_index); fwd = bck->fd; /* maintain large bins in sorted order */ if (fwd != bck) { /* Or with inuse bit to speed comparisons */ size |= PREV_INUSE; /* if smaller than smallest, bypass loop below */ assert (chunk_main_arena (bck->bk)); if ((unsigned long) (size) < (unsigned long) chunksize_nomask (bck->bk)) { fwd = bck; bck = bck->bk; victim->fd_nextsize = fwd->fd; victim->bk_nextsize = fwd->fd->bk_nextsize; fwd->fd->bk_nextsize = victim->bk_nextsize->fd_nextsize = victim; } else { assert (chunk_main_arena (fwd)); while ((unsigned long) size < chunksize_nomask (fwd)) { fwd = fwd->fd_nextsize; assert (chunk_main_arena (fwd)); } if ((unsigned long) size == (unsigned long) chunksize_nomask (fwd)) /* Always insert in the second position. */ fwd = fwd->fd; else { victim->fd_nextsize = fwd; victim->bk_nextsize = fwd->bk_nextsize; if (__glibc_unlikely (fwd->bk_nextsize->fd_nextsize != fwd)) malloc_printerr ("malloc(): largebin double linked list corrupted (nextsize)"); fwd->bk_nextsize = victim; victim->bk_nextsize->fd_nextsize = victim; } bck = fwd->bk; if (bck->fd != fwd) malloc_printerr ("malloc(): largebin double linked list corrupted (bk)"); } } else victim->fd_nextsize = victim->bk_nextsize = victim; } mark_bin (av, victim_index); victim->bk = bck; victim->fd = fwd; fwd->bk = victim; bck->fd = victim; #if USE_TCACHE /* If we've processed as many chunks as we're allowed while filling the cache, return one of the cached ones. */ ++tcache_unsorted_count; if (return_cached && mp_.tcache_unsorted_limit > 0 && tcache_unsorted_count > mp_.tcache_unsorted_limit) { return tcache_get (tc_idx); } #endif #define MAX_ITERS 10000 if (++iters >= MAX_ITERS) break; } #if USE_TCACHE /* If all the small chunks we found ended up cached, return one now. */ if (return_cached) { return tcache_get (tc_idx); } #endif /* If a large request, scan through the chunks of current bin in sorted order to find smallest that fits. Use the skip list for this. */ if (!in_smallbin_range (nb)) { bin = bin_at (av, idx); /* skip scan if empty or largest chunk is too small */ if ((victim = first (bin)) != bin && (unsigned long) chunksize_nomask (victim) >= (unsigned long) (nb)) { victim = victim->bk_nextsize; while (((unsigned long) (size = chunksize (victim)) < (unsigned long) (nb))) victim = victim->bk_nextsize; /* Avoid removing the first entry for a size so that the skip list does not have to be rerouted. */ if (victim != last (bin) && chunksize_nomask (victim) == chunksize_nomask (victim->fd)) victim = victim->fd; remainder_size = size - nb; unlink_chunk (av, victim); /* Exhaust */ if (remainder_size < MINSIZE) { set_inuse_bit_at_offset (victim, size); if (av != &main_arena) set_non_main_arena (victim); } /* Split */ else { remainder = chunk_at_offset (victim, nb); /* We cannot assume the unsorted list is empty and therefore have to perform a complete insert here. */ bck = unsorted_chunks (av); fwd = bck->fd; if (__glibc_unlikely (fwd->bk != bck)) malloc_printerr ("malloc(): corrupted unsorted chunks"); remainder->bk = bck; remainder->fd = fwd; bck->fd = remainder; fwd->bk = remainder; if (!in_smallbin_range (remainder_size)) { remainder->fd_nextsize = NULL; remainder->bk_nextsize = NULL; } set_head (victim, nb | PREV_INUSE | (av != &main_arena ? NON_MAIN_ARENA : 0)); set_head (remainder, remainder_size | PREV_INUSE); set_foot (remainder, remainder_size); } check_malloced_chunk (av, victim, nb); void *p = chunk2mem (victim); alloc_perturb (p, bytes); return p; } } /* Search for a chunk by scanning bins, starting with next largest bin. This search is strictly by best-fit; i.e., the smallest (with ties going to approximately the least recently used) chunk that fits is selected. The bitmap avoids needing to check that most blocks are nonempty. The particular case of skipping all bins during warm-up phases when no chunks have been returned yet is faster than it might look. */ ++idx; bin = bin_at (av, idx); block = idx2block (idx); map = av->binmap[block]; bit = idx2bit (idx); for (;; ) { /* Skip rest of block if there are no more set bits in this block. */ if (bit > map || bit == 0) { do { if (++block >= BINMAPSIZE) /* out of bins */ goto use_top; } while ((map = av->binmap[block]) == 0); bin = bin_at (av, (block << BINMAPSHIFT)); bit = 1; } /* Advance to bin with set bit. There must be one. */ while ((bit & map) == 0) { bin = next_bin (bin); bit <<= 1; assert (bit != 0); } /* Inspect the bin. It is likely to be non-empty */ victim = last (bin); /* If a false alarm (empty bin), clear the bit. */ if (victim == bin) { av->binmap[block] = map &= ~bit; /* Write through */ bin = next_bin (bin); bit <<= 1; } else { size = chunksize (victim); /* We know the first chunk in this bin is big enough to use. */ assert ((unsigned long) (size) >= (unsigned long) (nb)); remainder_size = size - nb; /* unlink */ unlink_chunk (av, victim); /* Exhaust */ if (remainder_size < MINSIZE) { set_inuse_bit_at_offset (victim, size); if (av != &main_arena) set_non_main_arena (victim); } /* Split */ else { remainder = chunk_at_offset (victim, nb); /* We cannot assume the unsorted list is empty and therefore have to perform a complete insert here. */ bck = unsorted_chunks (av); fwd = bck->fd; if (__glibc_unlikely (fwd->bk != bck)) malloc_printerr ("malloc(): corrupted unsorted chunks 2"); remainder->bk = bck; remainder->fd = fwd; bck->fd = remainder; fwd->bk = remainder; /* advertise as last remainder */ if (in_smallbin_range (nb)) av->last_remainder = remainder; if (!in_smallbin_range (remainder_size)) { remainder->fd_nextsize = NULL; remainder->bk_nextsize = NULL; } set_head (victim, nb | PREV_INUSE | (av != &main_arena ? NON_MAIN_ARENA : 0)); set_head (remainder, remainder_size | PREV_INUSE); set_foot (remainder, remainder_size); } check_malloced_chunk (av, victim, nb); void *p = chunk2mem (victim); alloc_perturb (p, bytes); return p; } } use_top: /* If large enough, split off the chunk bordering the end of memory (held in av->top). Note that this is in accord with the best-fit search rule. In effect, av->top is treated as larger (and thus less well fitting) than any other available chunk since it can be extended to be as large as necessary (up to system limitations). We require that av->top always exists (i.e., has size >= MINSIZE) after initialization, so if it would otherwise be exhausted by current request, it is replenished. (The main reason for ensuring it exists is that we may need MINSIZE space to put in fenceposts in sysmalloc.) */ victim = av->top; size = chunksize (victim); if (__glibc_unlikely (size > av->system_mem)) malloc_printerr ("malloc(): corrupted top size"); if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE)) { remainder_size = size - nb; remainder = chunk_at_offset (victim, nb); av->top = remainder; set_head (victim, nb | PREV_INUSE | (av != &main_arena ? NON_MAIN_ARENA : 0)); set_head (remainder, remainder_size | PREV_INUSE); check_malloced_chunk (av, victim, nb); void *p = chunk2mem (victim); alloc_perturb (p, bytes); return p; } /* When we are using atomic ops to free fast chunks we can get here for all block sizes. */ else if (atomic_load_relaxed (&av->have_fastchunks)) { malloc_consolidate (av); /* restore original bin index */ if (in_smallbin_range (nb)) idx = smallbin_index (nb); else idx = largebin_index (nb); } /* Otherwise, relay to handle system-dependent cases */ else { void *p = sysmalloc (nb, av); if (p != NULL) alloc_perturb (p, bytes); return p; } } } static void _int_free (mstate av, mchunkptr p, int have_lock) { INTERNAL_SIZE_T size; /* its size */ mfastbinptr *fb; /* associated fastbin */ mchunkptr nextchunk; /* next contiguous chunk */ INTERNAL_SIZE_T nextsize; /* its size */ int nextinuse; /* true if nextchunk is used */ INTERNAL_SIZE_T prevsize; /* size of previous contiguous chunk */ mchunkptr bck; /* misc temp for linking */ mchunkptr fwd; /* misc temp for linking */ size = chunksize (p); /* Little security check which won't hurt performance: the allocator never wraps around at the end of the address space. Therefore we can exclude some size values which might appear here by accident or by "design" from some intruder. */ if (__builtin_expect ((uintptr_t) p > (uintptr_t) -size, 0) || __builtin_expect (misaligned_chunk (p), 0)) malloc_printerr ("free(): invalid pointer"); /* We know that each chunk is at least MINSIZE bytes in size or a multiple of MALLOC_ALIGNMENT. */ if (__glibc_unlikely (size < MINSIZE || !aligned_OK (size))) malloc_printerr ("free(): invalid size"); check_inuse_chunk(av, p); #if USE_TCACHE { size_t tc_idx = csize2tidx (size); if (tcache != NULL && tc_idx < mp_.tcache_bins) { /* Check to see if it's already in the tcache. */ tcache_entry *e = (tcache_entry *) chunk2mem (p); /* This test succeeds on double free. However, we don't 100% trust it (it also matches random payload data at a 1 in 2^<size_t> chance), so verify it's not an unlikely coincidence before aborting. */ if (__glibc_unlikely (e->key == tcache_key)) { tcache_entry *tmp; size_t cnt = 0; LIBC_PROBE (memory_tcache_double_free, 2, e, tc_idx); for (tmp = tcache->entries[tc_idx]; tmp; tmp = REVEAL_PTR (tmp->next), ++cnt) { if (cnt >= mp_.tcache_count) malloc_printerr ("free(): too many chunks detected in tcache"); if (__glibc_unlikely (!aligned_OK (tmp))) malloc_printerr ("free(): unaligned chunk detected in tcache 2"); if (tmp == e) malloc_printerr ("free(): double free detected in tcache 2"); /* If we get here, it was a coincidence. We've wasted a few cycles, but don't abort. */ } } if (tcache->counts[tc_idx] < mp_.tcache_count) { tcache_put (p, tc_idx); return; } } } #endif /* If eligible, place chunk on a fastbin so it can be found and used quickly in malloc. */ if ((unsigned long)(size) <= (unsigned long)(get_max_fast ()) #if TRIM_FASTBINS /* If TRIM_FASTBINS set, don't place chunks bordering top into fastbins */ && (chunk_at_offset(p, size) != av->top) #endif ) { if (__builtin_expect (chunksize_nomask (chunk_at_offset (p, size)) <= CHUNK_HDR_SZ, 0) || __builtin_expect (chunksize (chunk_at_offset (p, size)) >= av->system_mem, 0)) { bool fail = true; /* We might not have a lock at this point and concurrent modifications of system_mem might result in a false positive. Redo the test after getting the lock. */ if (!have_lock) { __libc_lock_lock (av->mutex); fail = (chunksize_nomask (chunk_at_offset (p, size)) <= CHUNK_HDR_SZ || chunksize (chunk_at_offset (p, size)) >= av->system_mem); __libc_lock_unlock (av->mutex); } if (fail) malloc_printerr ("free(): invalid next size (fast)"); } free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ); atomic_store_relaxed (&av->have_fastchunks, true); unsigned int idx = fastbin_index(size); fb = &fastbin (av, idx); /* Atomically link P to its fastbin: P->FD = *FB; *FB = P; */ mchunkptr old = *fb, old2; if (SINGLE_THREAD_P) { /* Check that the top of the bin is not the record we are going to add (i.e., double free). */ if (__builtin_expect (old == p, 0)) malloc_printerr ("double free or corruption (fasttop)"); p->fd = PROTECT_PTR (&p->fd, old); *fb = p; } else do { /* Check that the top of the bin is not the record we are going to add (i.e., double free). */ if (__builtin_expect (old == p, 0)) malloc_printerr ("double free or corruption (fasttop)"); old2 = old; p->fd = PROTECT_PTR (&p->fd, old); } while ((old = catomic_compare_and_exchange_val_rel (fb, p, old2)) != old2); /* Check that size of fastbin chunk at the top is the same as size of the chunk that we are adding. We can dereference OLD only if we have the lock, otherwise it might have already been allocated again. */ if (have_lock && old != NULL && __builtin_expect (fastbin_index (chunksize (old)) != idx, 0)) malloc_printerr ("invalid fastbin entry (free)"); } /* Consolidate other non-mmapped chunks as they arrive. */ else if (!chunk_is_mmapped(p)) { /* If we're single-threaded, don't lock the arena. */ if (SINGLE_THREAD_P) have_lock = true; if (!have_lock) __libc_lock_lock (av->mutex); nextchunk = chunk_at_offset(p, size); /* Lightweight tests: check whether the block is already the top block. */ if (__glibc_unlikely (p == av->top)) malloc_printerr ("double free or corruption (top)"); /* Or whether the next chunk is beyond the boundaries of the arena. */ if (__builtin_expect (contiguous (av) && (char *) nextchunk >= ((char *) av->top + chunksize(av->top)), 0)) malloc_printerr ("double free or corruption (out)"); /* Or whether the block is actually not marked used. */ if (__glibc_unlikely (!prev_inuse(nextchunk))) malloc_printerr ("double free or corruption (!prev)"); nextsize = chunksize(nextchunk); if (__builtin_expect (chunksize_nomask (nextchunk) <= CHUNK_HDR_SZ, 0) || __builtin_expect (nextsize >= av->system_mem, 0)) malloc_printerr ("free(): invalid next size (normal)"); free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ); /* consolidate backward */ if (!prev_inuse(p)) { prevsize = prev_size (p); size += prevsize; p = chunk_at_offset(p, -((long) prevsize)); if (__glibc_unlikely (chunksize(p) != prevsize)) malloc_printerr ("corrupted size vs. prev_size while consolidating"); unlink_chunk (av, p); } if (nextchunk != av->top) { /* get and clear inuse bit */ nextinuse = inuse_bit_at_offset(nextchunk, nextsize); /* consolidate forward */ if (!nextinuse) { unlink_chunk (av, nextchunk); size += nextsize; } else clear_inuse_bit_at_offset(nextchunk, 0); /* Place the chunk in unsorted chunk list. Chunks are not placed into regular bins until after they have been given one chance to be used in malloc. */ bck = unsorted_chunks(av); fwd = bck->fd; if (__glibc_unlikely (fwd->bk != bck)) malloc_printerr ("free(): corrupted unsorted chunks"); p->fd = fwd; p->bk = bck; if (!in_smallbin_range(size)) { p->fd_nextsize = NULL; p->bk_nextsize = NULL; } bck->fd = p; fwd->bk = p; set_head(p, size | PREV_INUSE); set_foot(p, size); check_free_chunk(av, p); } /* If the chunk borders the current high end of memory, consolidate into top */ else { size += nextsize; set_head(p, size | PREV_INUSE); av->top = p; check_chunk(av, p); } /* If freeing a large space, consolidate possibly-surrounding chunks. Then, if the total unused topmost memory exceeds trim threshold, ask malloc_trim to reduce top. Unless max_fast is 0, we don't know if there are fastbins bordering top, so we cannot tell for sure whether threshold has been reached unless fastbins are consolidated. But we don't want to consolidate on each free. As a compromise, consolidation is performed if FASTBIN_CONSOLIDATION_THRESHOLD is reached. */ if ((unsigned long)(size) >= FASTBIN_CONSOLIDATION_THRESHOLD) { if (atomic_load_relaxed (&av->have_fastchunks)) malloc_consolidate(av); if (av == &main_arena) { #ifndef MORECORE_CANNOT_TRIM if ((unsigned long)(chunksize(av->top)) >= (unsigned long)(mp_.trim_threshold)) systrim(mp_.top_pad, av); #endif } else { /* Always try heap_trim(), even if the top chunk is not large, because the corresponding heap might go away. */ heap_info *heap = heap_for_ptr(top(av)); assert(heap->ar_ptr == av); heap_trim(heap, mp_.top_pad); } } if (!have_lock) __libc_lock_unlock (av->mutex); } /* If the chunk was allocated via mmap, release via munmap(). */ else { munmap_chunk (p); } } static void malloc_consolidate(mstate av) { mfastbinptr* fb; /* current fastbin being consolidated */ mfastbinptr* maxfb; /* last fastbin (for loop control) */ mchunkptr p; /* current chunk being consolidated */ mchunkptr nextp; /* next chunk to consolidate */ mchunkptr unsorted_bin; /* bin header */ mchunkptr first_unsorted; /* chunk to link to */ /* These have same use as in free() */ mchunkptr nextchunk; INTERNAL_SIZE_T size; INTERNAL_SIZE_T nextsize; INTERNAL_SIZE_T prevsize; int nextinuse; atomic_store_relaxed (&av->have_fastchunks, false); unsorted_bin = unsorted_chunks(av); /* Remove each chunk from fast bin and consolidate it, placing it then in unsorted bin. Among other reasons for doing this, placing in unsorted bin avoids needing to calculate actual bins until malloc is sure that chunks aren't immediately going to be reused anyway. */ maxfb = &fastbin (av, NFASTBINS - 1); fb = &fastbin (av, 0); do { p = atomic_exchange_acquire (fb, NULL); if (p != 0) { do { { if (__glibc_unlikely (misaligned_chunk (p))) malloc_printerr ("malloc_consolidate(): " "unaligned fastbin chunk detected"); unsigned int idx = fastbin_index (chunksize (p)); if ((&fastbin (av, idx)) != fb) malloc_printerr ("malloc_consolidate(): invalid chunk size"); } check_inuse_chunk(av, p); nextp = REVEAL_PTR (p->fd); /* Slightly streamlined version of consolidation code in free() */ size = chunksize (p); nextchunk = chunk_at_offset(p, size); nextsize = chunksize(nextchunk); if (!prev_inuse(p)) { prevsize = prev_size (p); size += prevsize; p = chunk_at_offset(p, -((long) prevsize)); if (__glibc_unlikely (chunksize(p) != prevsize)) malloc_printerr ("corrupted size vs. prev_size in fastbins"); unlink_chunk (av, p); } if (nextchunk != av->top) { nextinuse = inuse_bit_at_offset(nextchunk, nextsize); if (!nextinuse) { size += nextsize; unlink_chunk (av, nextchunk); } else clear_inuse_bit_at_offset(nextchunk, 0); first_unsorted = unsorted_bin->fd; unsorted_bin->fd = p; first_unsorted->bk = p; if (!in_smallbin_range (size)) { p->fd_nextsize = NULL; p->bk_nextsize = NULL; } set_head(p, size | PREV_INUSE); p->bk = unsorted_bin; p->fd = first_unsorted; set_foot(p, size); } else { size += nextsize; set_head(p, size | PREV_INUSE); av->top = p; } } while ( (p = nextp) != 0); } } while (fb++ != maxfb); } static __always_inline int do_set_trim_threshold (size_t value) { LIBC_PROBE (memory_mallopt_trim_threshold, 3, value, mp_.trim_threshold, mp_.no_dyn_threshold); mp_.trim_threshold = value; mp_.no_dyn_threshold = 1; return 1; } static __always_inline int do_set_top_pad (size_t value) { LIBC_PROBE (memory_mallopt_top_pad, 3, value, mp_.top_pad, mp_.no_dyn_threshold); mp_.top_pad = value; mp_.no_dyn_threshold = 1; return 1; } static __always_inline int do_set_mmap_threshold (size_t value) { LIBC_PROBE (memory_mallopt_mmap_threshold, 3, value, mp_.mmap_threshold, mp_.no_dyn_threshold); mp_.mmap_threshold = value; mp_.no_dyn_threshold = 1; return 1; } static __always_inline int do_set_mmaps_max (int32_t value) { LIBC_PROBE (memory_mallopt_mmap_max, 3, value, mp_.n_mmaps_max, mp_.no_dyn_threshold); mp_.n_mmaps_max = value; mp_.no_dyn_threshold = 1; return 1; } static __always_inline int do_set_perturb_byte (int32_t value) { LIBC_PROBE (memory_mallopt_perturb, 2, value, perturb_byte); perturb_byte = value; return 1; } static __always_inline int do_set_arena_test (size_t value) { LIBC_PROBE (memory_mallopt_arena_test, 2, value, mp_.arena_test); mp_.arena_test = value; return 1; } static __always_inline int do_set_arena_max (size_t value) { LIBC_PROBE (memory_mallopt_arena_max, 2, value, mp_.arena_max); mp_.arena_max = value; return 1; } static __always_inline int do_set_tcache_max (size_t value) { if (value <= MAX_TCACHE_SIZE) { LIBC_PROBE (memory_tunable_tcache_max_bytes, 2, value, mp_.tcache_max_bytes); mp_.tcache_max_bytes = value; mp_.tcache_bins = csize2tidx (request2size(value)) + 1; return 1; } return 0; } static __always_inline int do_set_tcache_count (size_t value) { if (value <= MAX_TCACHE_COUNT) { LIBC_PROBE (memory_tunable_tcache_count, 2, value, mp_.tcache_count); mp_.tcache_count = value; return 1; } return 0; } static __always_inline int do_set_tcache_unsorted_limit (size_t value) { LIBC_PROBE (memory_tunable_tcache_unsorted_limit, 2, value, mp_.tcache_unsorted_limit); mp_.tcache_unsorted_limit = value; return 1; } static __always_inline int do_set_mxfast (size_t value) { if (value <= MAX_FAST_SIZE) { LIBC_PROBE (memory_mallopt_mxfast, 2, value, get_max_fast ()); set_max_fast (value); return 1; } return 0; } static __always_inline int do_set_hugetlb (size_t value) { if (value == 1) { enum malloc_thp_mode_t thp_mode = __malloc_thp_mode (); /* Only enable THP madvise usage if system does support it and has 'madvise' mode. Otherwise the madvise() call is wasteful. */ if (thp_mode == malloc_thp_mode_madvise) mp_.thp_pagesize = __malloc_default_thp_pagesize (); } else if (value >= 2) __malloc_hugepage_config (value == 2 ? 0 : value, &mp_.hp_pagesize, &mp_.hp_flags); return 0; } static void malloc_printerr (const char *str) { #if IS_IN (libc) __libc_message ("%s\n", str); #else __libc_fatal (str); #endif __builtin_unreachable (); }
Become a Patron
Sponsor on GitHub
Donate via PayPal
Source on GitHub
Mailing list
Installed libraries
Wiki
Report an issue
How it works
Contact the author
CE on Mastodon
About the author
Statistics
Changelog
Version tree