This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
Re: [PATCH 4/4] x86: Remove arch-specific low level lock implementation
- From: Adhemerval Zanella <adhemerval dot zanella at linaro dot org>
- To: libc-alpha at sourceware dot org
- Date: Tue, 9 Apr 2019 09:11:42 -0300
- Subject: Re: [PATCH 4/4] x86: Remove arch-specific low level lock implementation
- References: <20190222192703.18177-1-adhemerval.zanella@linaro.org> <20190222192703.18177-4-adhemerval.zanella@linaro.org>
Ping.
On 22/02/2019 16:27, Adhemerval Zanella wrote:
> This patch removes the arch-specific x86 assembly implementation for
> low level locking and consolidate both 64 bits and 32 bits in a
> single implementation.
>
> Different than other architectures, x86 lll_trylock, lll_lock, and
> lll_unlock implements a single-thread optimization to avoid atomic
> operation, using cmpxchgl instead. This patch implements by using
> the new single-thread.h definitions in a generic way, although using
> the previous semantic.
>
> The lll_cond_trylock, lll_cond_lock, and lll_timedlock just use
> atomic operations plus calls to lll_lock_wait*.
>
> For __lll_lock_wait_private and __lll_lock_wait the generic implemtation
> there is no indication that assembly implementation is required
> performance-wise.
>
> Checked on x86_64-linux-gnu and i686-linux-gnu.
>
> * sysdeps/nptl/lowlevellock.h (__lll_trylock): New macro.
> (lll_trylock): Call __lll_trylock.
> * sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S: Remove file.
> * sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c: Likewise.
> * sysdeps/unix/sysv/linux/i386/lowlevellock.S: Likewise.
> * sysdeps/unix/sysv/linux/i386/lowlevellock.h: Likewise.
> * sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S: Likewise.
> * sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c: Likewise.
> * sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Likewise.
> * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise.
> * sysdeps/unix/sysv/linux/x86/lowlevellock.h: New file.
> * sysdeps/unix/sysv/linux/x86_64/cancellation.S: Include
> lowlevellock-futex.h.
> ---
> sysdeps/nptl/lowlevellock.h | 4 +-
> .../unix/sysv/linux/i386/libc-lowlevellock.S | 19 -
> .../unix/sysv/linux/i386/lll_timedlock_wait.c | 1 -
> sysdeps/unix/sysv/linux/i386/lowlevellock.S | 368 ------------------
> sysdeps/unix/sysv/linux/i386/lowlevellock.h | 240 ------------
> sysdeps/unix/sysv/linux/x86/lowlevellock.h | 110 ++++++
> sysdeps/unix/sysv/linux/x86_64/cancellation.S | 2 +-
> .../sysv/linux/x86_64/libc-lowlevellock.S | 19 -
> .../sysv/linux/x86_64/lll_timedlock_wait.c | 1 -
> sysdeps/unix/sysv/linux/x86_64/lowlevellock.S | 348 -----------------
> sysdeps/unix/sysv/linux/x86_64/lowlevellock.h | 243 ------------
> 11 files changed, 114 insertions(+), 1241 deletions(-)
> delete mode 100644 sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S
> delete mode 100644 sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c
> delete mode 100644 sysdeps/unix/sysv/linux/i386/lowlevellock.S
> delete mode 100644 sysdeps/unix/sysv/linux/i386/lowlevellock.h
> create mode 100644 sysdeps/unix/sysv/linux/x86/lowlevellock.h
> delete mode 100644 sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S
> delete mode 100644 sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c
> delete mode 100644 sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
> delete mode 100644 sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
>
> diff --git a/sysdeps/nptl/lowlevellock.h b/sysdeps/nptl/lowlevellock.h
> index 6f017afdd5..e905829ee4 100644
> --- a/sysdeps/nptl/lowlevellock.h
> +++ b/sysdeps/nptl/lowlevellock.h
> @@ -63,8 +63,10 @@
> /* If LOCK is 0 (not acquired), set to 1 (acquired with no waiters) and return
> 0. Otherwise leave lock unchanged and return non-zero to indicate that the
> lock was not acquired. */
> +#define __lll_trylock(lock) \
> + __glibc_unlikely (atomic_compare_and_exchange_bool_acq ((lock), 1, 0))
> #define lll_trylock(lock) \
> - __glibc_unlikely (atomic_compare_and_exchange_bool_acq (&(lock), 1, 0))
> + __lll_trylock (&(lock))
>
> /* If LOCK is 0 (not acquired), set to 2 (acquired, possibly with waiters) and
> return 0. Otherwise leave lock unchanged and return non-zero to indicate
> diff --git a/sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S b/sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S
> deleted file mode 100644
> index 32025277f3..0000000000
> --- a/sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S
> +++ /dev/null
> @@ -1,19 +0,0 @@
> -/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
> - This file is part of the GNU C Library.
> - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
> -
> - The GNU C Library is free software; you can redistribute it and/or
> - modify it under the terms of the GNU Lesser General Public
> - License as published by the Free Software Foundation; either
> - version 2.1 of the License, or (at your option) any later version.
> -
> - The GNU C Library is distributed in the hope that it will be useful,
> - but WITHOUT ANY WARRANTY; without even the implied warranty of
> - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> - Lesser General Public License for more details.
> -
> - You should have received a copy of the GNU Lesser General Public
> - License along with the GNU C Library; if not, see
> - <http://www.gnu.org/licenses/>. */
> -
> -#include "lowlevellock.S"
> diff --git a/sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c b/sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c
> deleted file mode 100644
> index f6875b8f89..0000000000
> --- a/sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c
> +++ /dev/null
> @@ -1 +0,0 @@
> -/* __lll_timedlock_wait is in lowlevellock.S. */
> diff --git a/sysdeps/unix/sysv/linux/i386/lowlevellock.S b/sysdeps/unix/sysv/linux/i386/lowlevellock.S
> deleted file mode 100644
> index 83191a3899..0000000000
> --- a/sysdeps/unix/sysv/linux/i386/lowlevellock.S
> +++ /dev/null
> @@ -1,368 +0,0 @@
> -/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
> - This file is part of the GNU C Library.
> - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
> -
> - The GNU C Library is free software; you can redistribute it and/or
> - modify it under the terms of the GNU Lesser General Public
> - License as published by the Free Software Foundation; either
> - version 2.1 of the License, or (at your option) any later version.
> -
> - The GNU C Library is distributed in the hope that it will be useful,
> - but WITHOUT ANY WARRANTY; without even the implied warranty of
> - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> - Lesser General Public License for more details.
> -
> - You should have received a copy of the GNU Lesser General Public
> - License along with the GNU C Library; if not, see
> - <http://www.gnu.org/licenses/>. */
> -
> -#include <sysdep.h>
> -#include <pthread-errnos.h>
> -#include <kernel-features.h>
> -#include <lowlevellock.h>
> -
> -#include <stap-probe.h>
> -
> - .text
> -
> -#define LOAD_PRIVATE_FUTEX_WAIT(reg) \
> - movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
> -#define LOAD_PRIVATE_FUTEX_WAKE(reg) \
> - movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
> -#define LOAD_FUTEX_WAIT(reg) \
> - xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
> -#define LOAD_FUTEX_WAIT_ABS(reg) \
> - xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
> -#define LOAD_FUTEX_WAKE(reg) \
> - xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
> -
> - .globl __lll_lock_wait_private
> - .type __lll_lock_wait_private,@function
> - .hidden __lll_lock_wait_private
> - .align 16
> -__lll_lock_wait_private:
> - cfi_startproc
> - pushl %edx
> - cfi_adjust_cfa_offset(4)
> - pushl %ebx
> - cfi_adjust_cfa_offset(4)
> - pushl %esi
> - cfi_adjust_cfa_offset(4)
> - cfi_offset(%edx, -8)
> - cfi_offset(%ebx, -12)
> - cfi_offset(%esi, -16)
> -
> - movl $2, %edx
> - movl %ecx, %ebx
> - xorl %esi, %esi /* No timeout. */
> - LOAD_PRIVATE_FUTEX_WAIT (%ecx)
> -
> - cmpl %edx, %eax /* NB: %edx == 2 */
> - jne 2f
> -
> -1: LIBC_PROBE (lll_lock_wait_private, 1, %ebx)
> - movl $SYS_futex, %eax
> - ENTER_KERNEL
> -
> -2: movl %edx, %eax
> - xchgl %eax, (%ebx) /* NB: lock is implied */
> -
> - testl %eax, %eax
> - jnz 1b
> -
> - popl %esi
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%esi)
> - popl %ebx
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%ebx)
> - popl %edx
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%edx)
> - ret
> - cfi_endproc
> - .size __lll_lock_wait_private,.-__lll_lock_wait_private
> -
> -#if !IS_IN (libc)
> - .globl __lll_lock_wait
> - .type __lll_lock_wait,@function
> - .hidden __lll_lock_wait
> - .align 16
> -__lll_lock_wait:
> - cfi_startproc
> - pushl %edx
> - cfi_adjust_cfa_offset(4)
> - pushl %ebx
> - cfi_adjust_cfa_offset(4)
> - pushl %esi
> - cfi_adjust_cfa_offset(4)
> - cfi_offset(%edx, -8)
> - cfi_offset(%ebx, -12)
> - cfi_offset(%esi, -16)
> -
> - movl %edx, %ebx
> - movl $2, %edx
> - xorl %esi, %esi /* No timeout. */
> - LOAD_FUTEX_WAIT (%ecx)
> -
> - cmpl %edx, %eax /* NB: %edx == 2 */
> - jne 2f
> -
> -1: movl $SYS_futex, %eax
> - ENTER_KERNEL
> -
> -2: movl %edx, %eax
> - xchgl %eax, (%ebx) /* NB: lock is implied */
> -
> - testl %eax, %eax
> - jnz 1b
> -
> - popl %esi
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%esi)
> - popl %ebx
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%ebx)
> - popl %edx
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%edx)
> - ret
> - cfi_endproc
> - .size __lll_lock_wait,.-__lll_lock_wait
> -
> - /* %ecx: futex
> - %esi: flags
> - %edx: timeout
> - %eax: futex value
> - */
> - .globl __lll_timedlock_wait
> - .type __lll_timedlock_wait,@function
> - .hidden __lll_timedlock_wait
> - .align 16
> -__lll_timedlock_wait:
> - cfi_startproc
> - pushl %ebp
> - cfi_adjust_cfa_offset(4)
> - cfi_rel_offset(%ebp, 0)
> - pushl %ebx
> - cfi_adjust_cfa_offset(4)
> - cfi_rel_offset(%ebx, 0)
> -
> -# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
> -# ifdef PIC
> - LOAD_PIC_REG (bx)
> - cmpl $0, __have_futex_clock_realtime@GOTOFF(%ebx)
> -# else
> - cmpl $0, __have_futex_clock_realtime
> -# endif
> - je .Lreltmo
> -# endif
> -
> - cmpl $0, (%edx)
> - js 8f
> -
> - movl %ecx, %ebx
> - movl %esi, %ecx
> - movl %edx, %esi
> - movl $0xffffffff, %ebp
> - LOAD_FUTEX_WAIT_ABS (%ecx)
> -
> - movl $2, %edx
> - cmpl %edx, %eax
> - jne 2f
> -
> -1: movl $SYS_futex, %eax
> - movl $2, %edx
> - ENTER_KERNEL
> -
> -2: xchgl %edx, (%ebx) /* NB: lock is implied */
> -
> - testl %edx, %edx
> - jz 3f
> -
> - cmpl $-ETIMEDOUT, %eax
> - je 4f
> - cmpl $-EINVAL, %eax
> - jne 1b
> -4: movl %eax, %edx
> - negl %edx
> -
> -3: movl %edx, %eax
> -7: popl %ebx
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%ebx)
> - popl %ebp
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%ebp)
> - ret
> -
> -8: movl $ETIMEDOUT, %eax
> - jmp 7b
> -
> -# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
> -.Lreltmo:
> - /* Check for a valid timeout value. */
> - cmpl $1000000000, 4(%edx)
> - jae 3f
> -
> - pushl %esi
> - cfi_adjust_cfa_offset(4)
> - cfi_rel_offset(%esi, 0)
> - pushl %edi
> - cfi_adjust_cfa_offset(4)
> - cfi_rel_offset(%edi, 0)
> -
> - /* Stack frame for the timespec and timeval structs. */
> - subl $8, %esp
> - cfi_adjust_cfa_offset(8)
> -
> - movl %ecx, %ebp
> - movl %edx, %edi
> -
> - movl $2, %edx
> - xchgl %edx, (%ebp)
> -
> - test %edx, %edx
> - je 6f
> -
> -1:
> - /* Get current time. */
> - movl %esp, %ebx
> - xorl %ecx, %ecx
> - movl $__NR_gettimeofday, %eax
> - ENTER_KERNEL
> -
> - /* Compute relative timeout. */
> - movl 4(%esp), %eax
> - movl $1000, %edx
> - mul %edx /* Milli seconds to nano seconds. */
> - movl (%edi), %ecx
> - movl 4(%edi), %edx
> - subl (%esp), %ecx
> - subl %eax, %edx
> - jns 4f
> - addl $1000000000, %edx
> - subl $1, %ecx
> -4: testl %ecx, %ecx
> - js 2f /* Time is already up. */
> -
> - /* Store relative timeout. */
> - movl %ecx, (%esp)
> - movl %edx, 4(%esp)
> -
> - /* Futex call. */
> - movl %ebp, %ebx
> - movl $2, %edx
> - movl %esp, %esi
> - movl 16(%esp), %ecx
> - LOAD_FUTEX_WAIT (%ecx)
> - movl $SYS_futex, %eax
> - ENTER_KERNEL
> -
> - /* NB: %edx == 2 */
> - xchgl %edx, (%ebp)
> -
> - testl %edx, %edx
> - je 6f
> -
> - cmpl $-ETIMEDOUT, %eax
> - jne 1b
> -2: movl $ETIMEDOUT, %edx
> -
> -6: addl $8, %esp
> - cfi_adjust_cfa_offset(-8)
> - popl %edi
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%edi)
> - popl %esi
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%esi)
> -7: popl %ebx
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%ebx)
> - popl %ebp
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%ebp)
> - movl %edx, %eax
> - ret
> -
> -3: movl $EINVAL, %edx
> - jmp 7b
> -# endif
> - cfi_endproc
> - .size __lll_timedlock_wait,.-__lll_timedlock_wait
> -#endif
> -
> - .globl __lll_unlock_wake_private
> - .type __lll_unlock_wake_private,@function
> - .hidden __lll_unlock_wake_private
> - .align 16
> -__lll_unlock_wake_private:
> - cfi_startproc
> - pushl %ebx
> - cfi_adjust_cfa_offset(4)
> - pushl %ecx
> - cfi_adjust_cfa_offset(4)
> - pushl %edx
> - cfi_adjust_cfa_offset(4)
> - cfi_offset(%ebx, -8)
> - cfi_offset(%ecx, -12)
> - cfi_offset(%edx, -16)
> -
> - movl %eax, %ebx
> - movl $0, (%eax)
> - LOAD_PRIVATE_FUTEX_WAKE (%ecx)
> - movl $1, %edx /* Wake one thread. */
> - movl $SYS_futex, %eax
> - ENTER_KERNEL
> -
> - popl %edx
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%edx)
> - popl %ecx
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%ecx)
> - popl %ebx
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%ebx)
> - ret
> - cfi_endproc
> - .size __lll_unlock_wake_private,.-__lll_unlock_wake_private
> -
> -#if !IS_IN (libc)
> - .globl __lll_unlock_wake
> - .type __lll_unlock_wake,@function
> - .hidden __lll_unlock_wake
> - .align 16
> -__lll_unlock_wake:
> - cfi_startproc
> - pushl %ebx
> - cfi_adjust_cfa_offset(4)
> - pushl %ecx
> - cfi_adjust_cfa_offset(4)
> - pushl %edx
> - cfi_adjust_cfa_offset(4)
> - cfi_offset(%ebx, -8)
> - cfi_offset(%ecx, -12)
> - cfi_offset(%edx, -16)
> -
> - movl %eax, %ebx
> - movl $0, (%eax)
> - LOAD_FUTEX_WAKE (%ecx)
> - movl $1, %edx /* Wake one thread. */
> - movl $SYS_futex, %eax
> - ENTER_KERNEL
> -
> - popl %edx
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%edx)
> - popl %ecx
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%ecx)
> - popl %ebx
> - cfi_adjust_cfa_offset(-4)
> - cfi_restore(%ebx)
> - ret
> - cfi_endproc
> - .size __lll_unlock_wake,.-__lll_unlock_wake
> -#endif
> diff --git a/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/sysdeps/unix/sysv/linux/i386/lowlevellock.h
> deleted file mode 100644
> index 94dccc4ce7..0000000000
> --- a/sysdeps/unix/sysv/linux/i386/lowlevellock.h
> +++ /dev/null
> @@ -1,240 +0,0 @@
> -/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
> - This file is part of the GNU C Library.
> - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
> -
> - The GNU C Library is free software; you can redistribute it and/or
> - modify it under the terms of the GNU Lesser General Public
> - License as published by the Free Software Foundation; either
> - version 2.1 of the License, or (at your option) any later version.
> -
> - The GNU C Library is distributed in the hope that it will be useful,
> - but WITHOUT ANY WARRANTY; without even the implied warranty of
> - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> - Lesser General Public License for more details.
> -
> - You should have received a copy of the GNU Lesser General Public
> - License along with the GNU C Library; if not, see
> - <http://www.gnu.org/licenses/>. */
> -
> -#ifndef _LOWLEVELLOCK_H
> -#define _LOWLEVELLOCK_H 1
> -
> -#ifndef __ASSEMBLER__
> -# include <time.h>
> -# include <sys/param.h>
> -# include <bits/pthreadtypes.h>
> -# include <kernel-features.h>
> -/* <tcb-offsets.h> is generated from tcb-offsets.sym to define offsets
> - and sizes of types in <tls.h> as well as <pthread.h> which includes
> - <lowlevellock.h> via nptl/descr.h. Don't include <tcb-offsets.h>
> - when generating <tcb-offsets.h> to avoid circular dependency which
> - may lead to build hang on a many-core machine. */
> -# ifndef GEN_AS_CONST_HEADERS
> -# include <tcb-offsets.h>
> -# endif
> -
> -# ifndef LOCK_INSTR
> -# ifdef UP
> -# define LOCK_INSTR /* nothing */
> -# else
> -# define LOCK_INSTR "lock;"
> -# endif
> -# endif
> -#else
> -# ifndef LOCK
> -# ifdef UP
> -# define LOCK
> -# else
> -# define LOCK lock
> -# endif
> -# endif
> -#endif
> -
> -#include <lowlevellock-futex.h>
> -
> -/* XXX Remove when no assembler code uses futexes anymore. */
> -#define SYS_futex __NR_futex
> -
> -#ifndef __ASSEMBLER__
> -
> -/* Initializer for compatibility lock. */
> -#define LLL_LOCK_INITIALIZER (0)
> -#define LLL_LOCK_INITIALIZER_LOCKED (1)
> -#define LLL_LOCK_INITIALIZER_WAITERS (2)
> -
> -
> -/* NB: in the lll_trylock macro we simply return the value in %eax
> - after the cmpxchg instruction. In case the operation succeded this
> - value is zero. In case the operation failed, the cmpxchg instruction
> - has loaded the current value of the memory work which is guaranteed
> - to be nonzero. */
> -#if !IS_IN (libc) || defined UP
> -# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
> -#else
> -# define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \
> - "je 0f\n\t" \
> - "lock\n" \
> - "0:\tcmpxchgl %2, %1"
> -#endif
> -
> -#define lll_trylock(futex) \
> - ({ int ret; \
> - __asm __volatile (__lll_trylock_asm \
> - : "=a" (ret), "=m" (futex) \
> - : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \
> - "0" (LLL_LOCK_INITIALIZER), \
> - "i" (MULTIPLE_THREADS_OFFSET) \
> - : "memory"); \
> - ret; })
> -
> -
> -#define lll_cond_trylock(futex) \
> - ({ int ret; \
> - __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \
> - : "=a" (ret), "=m" (futex) \
> - : "r" (LLL_LOCK_INITIALIZER_WAITERS), \
> - "m" (futex), "0" (LLL_LOCK_INITIALIZER) \
> - : "memory"); \
> - ret; })
> -
> -#if !IS_IN (libc) || defined UP
> -# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t"
> -#else
> -# define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t" \
> - "je 0f\n\t" \
> - "lock\n" \
> - "0:\tcmpxchgl %1, %2\n\t"
> -#endif
> -
> -#define lll_lock(futex, private) \
> - (void) \
> - ({ int ignore1, ignore2; \
> - if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
> - __asm __volatile (__lll_lock_asm_start \
> - "jz 18f\n\t" \
> - "1:\tleal %2, %%ecx\n" \
> - "2:\tcall __lll_lock_wait_private\n" \
> - "18:" \
> - : "=a" (ignore1), "=c" (ignore2), "=m" (futex) \
> - : "0" (0), "1" (1), "m" (futex), \
> - "i" (MULTIPLE_THREADS_OFFSET) \
> - : "memory"); \
> - else \
> - { \
> - int ignore3; \
> - __asm __volatile (__lll_lock_asm_start \
> - "jz 18f\n\t" \
> - "1:\tleal %2, %%edx\n" \
> - "0:\tmovl %8, %%ecx\n" \
> - "2:\tcall __lll_lock_wait\n" \
> - "18:" \
> - : "=a" (ignore1), "=c" (ignore2), \
> - "=m" (futex), "=&d" (ignore3) \
> - : "1" (1), "m" (futex), \
> - "i" (MULTIPLE_THREADS_OFFSET), "0" (0), \
> - "g" ((int) (private)) \
> - : "memory"); \
> - } \
> - })
> -
> -
> -/* Special version of lll_lock which causes the unlock function to
> - always wakeup waiters. */
> -#define lll_cond_lock(futex, private) \
> - (void) \
> - ({ int ignore1, ignore2, ignore3; \
> - __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
> - "jz 18f\n\t" \
> - "1:\tleal %2, %%edx\n" \
> - "0:\tmovl %7, %%ecx\n" \
> - "2:\tcall __lll_lock_wait\n" \
> - "18:" \
> - : "=a" (ignore1), "=c" (ignore2), "=m" (futex), \
> - "=&d" (ignore3) \
> - : "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\
> - : "memory"); \
> - })
> -
> -
> -#define lll_timedlock(futex, timeout, private) \
> - ({ int result, ignore1, ignore2, ignore3; \
> - __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \
> - "jz 18f\n\t" \
> - "1:\tleal %3, %%ecx\n" \
> - "0:\tmovl %8, %%edx\n" \
> - "2:\tcall __lll_timedlock_wait\n" \
> - "18:" \
> - : "=a" (result), "=c" (ignore1), "=&d" (ignore2), \
> - "=m" (futex), "=S" (ignore3) \
> - : "0" (0), "1" (1), "m" (futex), "m" (timeout), \
> - "4" ((int) (private)) \
> - : "memory"); \
> - result; })
> -
> -extern int __lll_timedlock_elision (int *futex, short *adapt_count,
> - const struct timespec *timeout,
> - int private) attribute_hidden;
> -
> -#define lll_timedlock_elision(futex, adapt_count, timeout, private) \
> - __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)
> -
> -#if !IS_IN (libc) || defined UP
> -# define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t"
> -#else
> -# define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t" \
> - "je 0f\n\t" \
> - "lock\n" \
> - "0:\tsubl $1,%0\n\t"
> -#endif
> -
> -#define lll_unlock(futex, private) \
> - (void) \
> - ({ int ignore; \
> - if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
> - __asm __volatile (__lll_unlock_asm \
> - "je 18f\n\t" \
> - "1:\tleal %0, %%eax\n" \
> - "2:\tcall __lll_unlock_wake_private\n" \
> - "18:" \
> - : "=m" (futex), "=&a" (ignore) \
> - : "m" (futex), "i" (MULTIPLE_THREADS_OFFSET) \
> - : "memory"); \
> - else \
> - { \
> - int ignore2; \
> - __asm __volatile (__lll_unlock_asm \
> - "je 18f\n\t" \
> - "1:\tleal %0, %%eax\n" \
> - "0:\tmovl %5, %%ecx\n" \
> - "2:\tcall __lll_unlock_wake\n" \
> - "18:" \
> - : "=m" (futex), "=&a" (ignore), "=&c" (ignore2) \
> - : "i" (MULTIPLE_THREADS_OFFSET), "m" (futex), \
> - "g" ((int) (private)) \
> - : "memory"); \
> - } \
> - })
> -
> -
> -#define lll_islocked(futex) \
> - (futex != LLL_LOCK_INITIALIZER)
> -
> -extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
> - attribute_hidden;
> -
> -extern int __lll_unlock_elision(int *lock, int private)
> - attribute_hidden;
> -
> -extern int __lll_trylock_elision(int *lock, short *adapt_count)
> - attribute_hidden;
> -
> -#define lll_lock_elision(futex, adapt_count, private) \
> - __lll_lock_elision (&(futex), &(adapt_count), private)
> -#define lll_unlock_elision(futex, adapt_count, private) \
> - __lll_unlock_elision (&(futex), private)
> -#define lll_trylock_elision(futex, adapt_count) \
> - __lll_trylock_elision(&(futex), &(adapt_count))
> -
> -#endif /* !__ASSEMBLER__ */
> -
> -#endif /* lowlevellock.h */
> diff --git a/sysdeps/unix/sysv/linux/x86/lowlevellock.h b/sysdeps/unix/sysv/linux/x86/lowlevellock.h
> new file mode 100644
> index 0000000000..2652a00736
> --- /dev/null
> +++ b/sysdeps/unix/sysv/linux/x86/lowlevellock.h
> @@ -0,0 +1,110 @@
> +/* Low-level lock implementation, x86 version.
> + Copyright (C) 2019 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +#ifndef _X86_64_LOWLEVELLOCK_H
> +#define _X86_64_LOWLEVELLOCK_H 1
> +
> +#ifndef __ASSEMBLER__
> +#include <sysdeps/nptl/lowlevellock.h>
> +#include <single-thread.h>
> +
> +/* The lll_trylock, lll_lock, and lll_unlock implements a single-thread
> + optimization using the cmpxchgl instruction. It checks if the process
> + is single thread and avoid a more expensive atomic instruction. */
> +
> +/* The single-thread optimization only works for libc itself, we need
> + atomicity for libpthread in case of shared futexes. */
> +#if !IS_IN(libc)
> +# define is_single_thread 0
> +#else
> +# define is_single_thread SINGLE_THREAD_P
> +#endif
> +
> +/* In the __lllc_as we simply return the value in %eax after the cmpxchg
> + instruction. In case the operation succeded this value is zero. In case
> + the operation failed, the cmpxchg instruction has loaded the current value
> + of the memory work which is guaranteed to be nonzero. */
> +static inline int
> +__attribute__ ((always_inline))
> +__lll_cas_lock (int *futex)
> +{
> + int ret;
> + asm volatile ("cmpxchgl %2, %1"
> + : "=a" (ret), "=m" (*futex)
> + : "r" (1), "m" (*futex), "0" (0)
> + : "memory");
> + return ret;
> +}
> +
> +#undef lll_trylock
> +#define lll_trylock(lock) \
> + ({ \
> + int __ret; \
> + if (is_single_thread) \
> + __ret = __lll_cas_lock (&(lock)); \
> + else \
> + __ret = __lll_trylock (&(lock)); \
> + __ret; \
> + })
> +
> +#undef lll_lock
> +#define lll_lock(lock, private) \
> + ((void) \
> + ({ \
> + if (is_single_thread) \
> + __lll_cas_lock (&(lock)); \
> + else \
> + __lll_lock (&(lock), private); \
> + }))
> +
> +#undef lll_unlock
> +#define lll_unlock(lock, private) \
> + ((void) \
> + ({ \
> + if (is_single_thread) \
> + (lock)--; \
> + else \
> + __lll_unlock (&(lock), private); \
> + }))
> +
> +extern int __lll_timedlock_elision (int *futex, short *adapt_count,
> + const struct timespec *timeout,
> + int private) attribute_hidden;
> +
> +#define lll_timedlock_elision(futex, adapt_count, timeout, private) \
> + __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)
> +
> +extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
> + attribute_hidden;
> +
> +extern int __lll_unlock_elision (int *lock, int private)
> + attribute_hidden;
> +
> +extern int __lll_trylock_elision (int *lock, short *adapt_count)
> + attribute_hidden;
> +
> +#define lll_lock_elision(futex, adapt_count, private) \
> + __lll_lock_elision (&(futex), &(adapt_count), private)
> +#define lll_unlock_elision(futex, adapt_count, private) \
> + __lll_unlock_elision (&(futex), private)
> +#define lll_trylock_elision(futex, adapt_count) \
> + __lll_trylock_elision (&(futex), &(adapt_count))
> +
> +#endif /* !__ASSEMBLER__ */
> +
> +#endif /* lowlevellock.h */
> diff --git a/sysdeps/unix/sysv/linux/x86_64/cancellation.S b/sysdeps/unix/sysv/linux/x86_64/cancellation.S
> index 7d169d9aca..bb4910764a 100644
> --- a/sysdeps/unix/sysv/linux/x86_64/cancellation.S
> +++ b/sysdeps/unix/sysv/linux/x86_64/cancellation.S
> @@ -19,7 +19,7 @@
> #include <sysdep.h>
> #include <tcb-offsets.h>
> #include <kernel-features.h>
> -#include "lowlevellock.h"
> +#include <lowlevellock-futex.h>
>
> #define PTHREAD_UNWIND JUMPTARGET(__pthread_unwind)
> #if IS_IN (libpthread)
> diff --git a/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S b/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S
> deleted file mode 100644
> index 32025277f3..0000000000
> --- a/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S
> +++ /dev/null
> @@ -1,19 +0,0 @@
> -/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
> - This file is part of the GNU C Library.
> - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
> -
> - The GNU C Library is free software; you can redistribute it and/or
> - modify it under the terms of the GNU Lesser General Public
> - License as published by the Free Software Foundation; either
> - version 2.1 of the License, or (at your option) any later version.
> -
> - The GNU C Library is distributed in the hope that it will be useful,
> - but WITHOUT ANY WARRANTY; without even the implied warranty of
> - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> - Lesser General Public License for more details.
> -
> - You should have received a copy of the GNU Lesser General Public
> - License along with the GNU C Library; if not, see
> - <http://www.gnu.org/licenses/>. */
> -
> -#include "lowlevellock.S"
> diff --git a/sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c b/sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c
> deleted file mode 100644
> index f6875b8f89..0000000000
> --- a/sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c
> +++ /dev/null
> @@ -1 +0,0 @@
> -/* __lll_timedlock_wait is in lowlevellock.S. */
> diff --git a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
> deleted file mode 100644
> index 166dbcfd3b..0000000000
> --- a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
> +++ /dev/null
> @@ -1,348 +0,0 @@
> -/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
> - This file is part of the GNU C Library.
> - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
> -
> - The GNU C Library is free software; you can redistribute it and/or
> - modify it under the terms of the GNU Lesser General Public
> - License as published by the Free Software Foundation; either
> - version 2.1 of the License, or (at your option) any later version.
> -
> - The GNU C Library is distributed in the hope that it will be useful,
> - but WITHOUT ANY WARRANTY; without even the implied warranty of
> - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> - Lesser General Public License for more details.
> -
> - You should have received a copy of the GNU Lesser General Public
> - License along with the GNU C Library; if not, see
> - <http://www.gnu.org/licenses/>. */
> -
> -#include <sysdep.h>
> -#include <pthread-errnos.h>
> -#include <kernel-features.h>
> -#include <lowlevellock.h>
> -
> -#include <stap-probe.h>
> -
> - .text
> -
> -#define LOAD_PRIVATE_FUTEX_WAIT(reg) \
> - movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
> -#define LOAD_PRIVATE_FUTEX_WAKE(reg) \
> - movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
> -#define LOAD_FUTEX_WAIT(reg) \
> - xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
> -#define LOAD_FUTEX_WAIT_ABS(reg) \
> - xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
> -#define LOAD_FUTEX_WAKE(reg) \
> - xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
> -
> -
> - .globl __lll_lock_wait_private
> - .type __lll_lock_wait_private,@function
> - .hidden __lll_lock_wait_private
> - .align 16
> -__lll_lock_wait_private:
> - cfi_startproc
> - pushq %r10
> - cfi_adjust_cfa_offset(8)
> - pushq %rdx
> - cfi_adjust_cfa_offset(8)
> - cfi_offset(%r10, -16)
> - cfi_offset(%rdx, -24)
> - xorq %r10, %r10 /* No timeout. */
> - movl $2, %edx
> - LOAD_PRIVATE_FUTEX_WAIT (%esi)
> -
> - cmpl %edx, %eax /* NB: %edx == 2 */
> - jne 2f
> -
> -1: LIBC_PROBE (lll_lock_wait_private, 1, %rdi)
> - movl $SYS_futex, %eax
> - syscall
> -
> -2: movl %edx, %eax
> - xchgl %eax, (%rdi) /* NB: lock is implied */
> -
> - testl %eax, %eax
> - jnz 1b
> -
> - popq %rdx
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%rdx)
> - popq %r10
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%r10)
> - retq
> - cfi_endproc
> - .size __lll_lock_wait_private,.-__lll_lock_wait_private
> -
> -#if !IS_IN (libc)
> - .globl __lll_lock_wait
> - .type __lll_lock_wait,@function
> - .hidden __lll_lock_wait
> - .align 16
> -__lll_lock_wait:
> - cfi_startproc
> - pushq %r10
> - cfi_adjust_cfa_offset(8)
> - pushq %rdx
> - cfi_adjust_cfa_offset(8)
> - cfi_offset(%r10, -16)
> - cfi_offset(%rdx, -24)
> - xorq %r10, %r10 /* No timeout. */
> - movl $2, %edx
> - LOAD_FUTEX_WAIT (%esi)
> -
> - cmpl %edx, %eax /* NB: %edx == 2 */
> - jne 2f
> -
> -1: LIBC_PROBE (lll_lock_wait, 2, %rdi, %rsi)
> - movl $SYS_futex, %eax
> - syscall
> -
> -2: movl %edx, %eax
> - xchgl %eax, (%rdi) /* NB: lock is implied */
> -
> - testl %eax, %eax
> - jnz 1b
> -
> - popq %rdx
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%rdx)
> - popq %r10
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%r10)
> - retq
> - cfi_endproc
> - .size __lll_lock_wait,.-__lll_lock_wait
> -
> - /* %rdi: futex
> - %rsi: flags
> - %rdx: timeout
> - %eax: futex value
> - */
> - .globl __lll_timedlock_wait
> - .type __lll_timedlock_wait,@function
> - .hidden __lll_timedlock_wait
> - .align 16
> -__lll_timedlock_wait:
> - cfi_startproc
> -# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
> -# ifdef PIC
> - cmpl $0, __have_futex_clock_realtime(%rip)
> -# else
> - cmpl $0, __have_futex_clock_realtime
> -# endif
> - je .Lreltmo
> -# endif
> -
> - cmpq $0, (%rdx)
> - js 5f
> -
> - pushq %r9
> - cfi_adjust_cfa_offset(8)
> - cfi_rel_offset(%r9, 0)
> -
> - movq %rdx, %r10
> - movl $0xffffffff, %r9d
> - LOAD_FUTEX_WAIT_ABS (%esi)
> -
> - movl $2, %edx
> - cmpl %edx, %eax
> - jne 2f
> -
> -1: movl $SYS_futex, %eax
> - movl $2, %edx
> - syscall
> -
> -2: xchgl %edx, (%rdi) /* NB: lock is implied */
> -
> - testl %edx, %edx
> - jz 3f
> -
> - cmpl $-ETIMEDOUT, %eax
> - je 4f
> - cmpl $-EINVAL, %eax
> - jne 1b
> -4: movl %eax, %edx
> - negl %edx
> -
> -3: movl %edx, %eax
> - popq %r9
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%r9)
> - retq
> -
> -5: movl $ETIMEDOUT, %eax
> - retq
> -
> -# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
> -.Lreltmo:
> - /* Check for a valid timeout value. */
> - cmpq $1000000000, 8(%rdx)
> - jae 3f
> -
> - pushq %r8
> - cfi_adjust_cfa_offset(8)
> - pushq %r9
> - cfi_adjust_cfa_offset(8)
> - pushq %r12
> - cfi_adjust_cfa_offset(8)
> - pushq %r13
> - cfi_adjust_cfa_offset(8)
> - pushq %r14
> - cfi_adjust_cfa_offset(8)
> - cfi_offset(%r8, -16)
> - cfi_offset(%r9, -24)
> - cfi_offset(%r12, -32)
> - cfi_offset(%r13, -40)
> - cfi_offset(%r14, -48)
> - pushq %rsi
> - cfi_adjust_cfa_offset(8)
> -
> - /* Stack frame for the timespec and timeval structs. */
> - subq $24, %rsp
> - cfi_adjust_cfa_offset(24)
> -
> - movq %rdi, %r12
> - movq %rdx, %r13
> -
> - movl $2, %edx
> - xchgl %edx, (%r12)
> -
> - testl %edx, %edx
> - je 6f
> -
> -1:
> - /* Get current time. */
> - movq %rsp, %rdi
> - xorl %esi, %esi
> - /* This call works because we directly jump to a system call entry
> - which preserves all the registers. */
> - call JUMPTARGET(__gettimeofday)
> -
> - /* Compute relative timeout. */
> - movq 8(%rsp), %rax
> - movl $1000, %edi
> - mul %rdi /* Milli seconds to nano seconds. */
> - movq (%r13), %rdi
> - movq 8(%r13), %rsi
> - subq (%rsp), %rdi
> - subq %rax, %rsi
> - jns 4f
> - addq $1000000000, %rsi
> - decq %rdi
> -4: testq %rdi, %rdi
> - js 2f /* Time is already up. */
> -
> - /* Store relative timeout. */
> - movq %rdi, (%rsp)
> - movq %rsi, 8(%rsp)
> -
> - /* Futex call. */
> - movl $2, %edx
> - movl $1, %eax
> - movq %rsp, %r10
> - movl 24(%rsp), %esi
> - LOAD_FUTEX_WAIT (%esi)
> - movq %r12, %rdi
> - movl $SYS_futex, %eax
> - syscall
> -
> - /* NB: %edx == 2 */
> - xchgl %edx, (%r12)
> -
> - testl %edx, %edx
> - je 6f
> -
> - cmpl $-ETIMEDOUT, %eax
> - jne 1b
> -2: movl $ETIMEDOUT, %edx
> -
> -6: addq $32, %rsp
> - cfi_adjust_cfa_offset(-32)
> - popq %r14
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%r14)
> - popq %r13
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%r13)
> - popq %r12
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%r12)
> - popq %r9
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%r9)
> - popq %r8
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%r8)
> - movl %edx, %eax
> - retq
> -
> -3: movl $EINVAL, %eax
> - retq
> -# endif
> - cfi_endproc
> - .size __lll_timedlock_wait,.-__lll_timedlock_wait
> -#endif
> -
> -
> - .globl __lll_unlock_wake_private
> - .type __lll_unlock_wake_private,@function
> - .hidden __lll_unlock_wake_private
> - .align 16
> -__lll_unlock_wake_private:
> - cfi_startproc
> - pushq %rsi
> - cfi_adjust_cfa_offset(8)
> - pushq %rdx
> - cfi_adjust_cfa_offset(8)
> - cfi_offset(%rsi, -16)
> - cfi_offset(%rdx, -24)
> -
> - movl $0, (%rdi)
> - LOAD_PRIVATE_FUTEX_WAKE (%esi)
> - movl $1, %edx /* Wake one thread. */
> - movl $SYS_futex, %eax
> - syscall
> -
> - popq %rdx
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%rdx)
> - popq %rsi
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%rsi)
> - retq
> - cfi_endproc
> - .size __lll_unlock_wake_private,.-__lll_unlock_wake_private
> -
> -#if !IS_IN (libc)
> - .globl __lll_unlock_wake
> - .type __lll_unlock_wake,@function
> - .hidden __lll_unlock_wake
> - .align 16
> -__lll_unlock_wake:
> - cfi_startproc
> - pushq %rsi
> - cfi_adjust_cfa_offset(8)
> - pushq %rdx
> - cfi_adjust_cfa_offset(8)
> - cfi_offset(%rsi, -16)
> - cfi_offset(%rdx, -24)
> -
> - movl $0, (%rdi)
> - LOAD_FUTEX_WAKE (%esi)
> - movl $1, %edx /* Wake one thread. */
> - movl $SYS_futex, %eax
> - syscall
> -
> - popq %rdx
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%rdx)
> - popq %rsi
> - cfi_adjust_cfa_offset(-8)
> - cfi_restore(%rsi)
> - retq
> - cfi_endproc
> - .size __lll_unlock_wake,.-__lll_unlock_wake
> -#endif
> diff --git a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
> deleted file mode 100644
> index 8cbc1caa5b..0000000000
> --- a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
> +++ /dev/null
> @@ -1,243 +0,0 @@
> -/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
> - This file is part of the GNU C Library.
> - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
> -
> - The GNU C Library is free software; you can redistribute it and/or
> - modify it under the terms of the GNU Lesser General Public
> - License as published by the Free Software Foundation; either
> - version 2.1 of the License, or (at your option) any later version.
> -
> - The GNU C Library is distributed in the hope that it will be useful,
> - but WITHOUT ANY WARRANTY; without even the implied warranty of
> - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> - Lesser General Public License for more details.
> -
> - You should have received a copy of the GNU Lesser General Public
> - License along with the GNU C Library; if not, see
> - <http://www.gnu.org/licenses/>. */
> -
> -#ifndef _LOWLEVELLOCK_H
> -#define _LOWLEVELLOCK_H 1
> -
> -#ifndef __ASSEMBLER__
> -# include <time.h>
> -# include <sys/param.h>
> -# include <bits/pthreadtypes.h>
> -# include <kernel-features.h>
> -
> -# ifndef LOCK_INSTR
> -# ifdef UP
> -# define LOCK_INSTR /* nothing */
> -# else
> -# define LOCK_INSTR "lock;"
> -# endif
> -# endif
> -#else
> -# ifndef LOCK
> -# ifdef UP
> -# define LOCK
> -# else
> -# define LOCK lock
> -# endif
> -# endif
> -#endif
> -
> -#include <lowlevellock-futex.h>
> -
> -/* XXX Remove when no assembler code uses futexes anymore. */
> -#define SYS_futex __NR_futex
> -
> -#ifndef __ASSEMBLER__
> -
> -/* Initializer for lock. */
> -#define LLL_LOCK_INITIALIZER (0)
> -#define LLL_LOCK_INITIALIZER_LOCKED (1)
> -#define LLL_LOCK_INITIALIZER_WAITERS (2)
> -
> -
> -/* NB: in the lll_trylock macro we simply return the value in %eax
> - after the cmpxchg instruction. In case the operation succeded this
> - value is zero. In case the operation failed, the cmpxchg instruction
> - has loaded the current value of the memory work which is guaranteed
> - to be nonzero. */
> -#if !IS_IN (libc) || defined UP
> -# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
> -#else
> -# define __lll_trylock_asm "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \
> - "je 0f\n\t" \
> - "lock; cmpxchgl %2, %1\n\t" \
> - "jmp 1f\n\t" \
> - "0:\tcmpxchgl %2, %1\n\t" \
> - "1:"
> -#endif
> -
> -#define lll_trylock(futex) \
> - ({ int ret; \
> - __asm __volatile (__lll_trylock_asm \
> - : "=a" (ret), "=m" (futex) \
> - : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \
> - "0" (LLL_LOCK_INITIALIZER) \
> - : "memory"); \
> - ret; })
> -
> -#define lll_cond_trylock(futex) \
> - ({ int ret; \
> - __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \
> - : "=a" (ret), "=m" (futex) \
> - : "r" (LLL_LOCK_INITIALIZER_WAITERS), \
> - "m" (futex), "0" (LLL_LOCK_INITIALIZER) \
> - : "memory"); \
> - ret; })
> -
> -#if !IS_IN (libc) || defined UP
> -# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %4, %2\n\t" \
> - "jz 24f\n\t"
> -#else
> -# define __lll_lock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \
> - "je 0f\n\t" \
> - "lock; cmpxchgl %4, %2\n\t" \
> - "jnz 1f\n\t" \
> - "jmp 24f\n" \
> - "0:\tcmpxchgl %4, %2\n\t" \
> - "jz 24f\n\t"
> -#endif
> -
> -#define lll_lock(futex, private) \
> - (void) \
> - ({ int ignore1, ignore2, ignore3; \
> - if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
> - __asm __volatile (__lll_lock_asm_start \
> - "1:\tlea %2, %%" RDI_LP "\n" \
> - "2:\tsub $128, %%" RSP_LP "\n" \
> - ".cfi_adjust_cfa_offset 128\n" \
> - "3:\tcallq __lll_lock_wait_private\n" \
> - "4:\tadd $128, %%" RSP_LP "\n" \
> - ".cfi_adjust_cfa_offset -128\n" \
> - "24:" \
> - : "=S" (ignore1), "=&D" (ignore2), "=m" (futex), \
> - "=a" (ignore3) \
> - : "0" (1), "m" (futex), "3" (0) \
> - : "cx", "r11", "cc", "memory"); \
> - else \
> - __asm __volatile (__lll_lock_asm_start \
> - "1:\tlea %2, %%" RDI_LP "\n" \
> - "2:\tsub $128, %%" RSP_LP "\n" \
> - ".cfi_adjust_cfa_offset 128\n" \
> - "3:\tcallq __lll_lock_wait\n" \
> - "4:\tadd $128, %%" RSP_LP "\n" \
> - ".cfi_adjust_cfa_offset -128\n" \
> - "24:" \
> - : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \
> - "=a" (ignore3) \
> - : "1" (1), "m" (futex), "3" (0), "0" (private) \
> - : "cx", "r11", "cc", "memory"); \
> - }) \
> -
> -#define lll_cond_lock(futex, private) \
> - (void) \
> - ({ int ignore1, ignore2, ignore3; \
> - __asm __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t" \
> - "jz 24f\n" \
> - "1:\tlea %2, %%" RDI_LP "\n" \
> - "2:\tsub $128, %%" RSP_LP "\n" \
> - ".cfi_adjust_cfa_offset 128\n" \
> - "3:\tcallq __lll_lock_wait\n" \
> - "4:\tadd $128, %%" RSP_LP "\n" \
> - ".cfi_adjust_cfa_offset -128\n" \
> - "24:" \
> - : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \
> - "=a" (ignore3) \
> - : "1" (2), "m" (futex), "3" (0), "0" (private) \
> - : "cx", "r11", "cc", "memory"); \
> - })
> -
> -#define lll_timedlock(futex, timeout, private) \
> - ({ int result, ignore1, ignore2, ignore3; \
> - __asm __volatile (LOCK_INSTR "cmpxchgl %1, %4\n\t" \
> - "jz 24f\n" \
> - "1:\tlea %4, %%" RDI_LP "\n" \
> - "0:\tmov %8, %%" RDX_LP "\n" \
> - "2:\tsub $128, %%" RSP_LP "\n" \
> - ".cfi_adjust_cfa_offset 128\n" \
> - "3:\tcallq __lll_timedlock_wait\n" \
> - "4:\tadd $128, %%" RSP_LP "\n" \
> - ".cfi_adjust_cfa_offset -128\n" \
> - "24:" \
> - : "=a" (result), "=D" (ignore1), "=S" (ignore2), \
> - "=&d" (ignore3), "=m" (futex) \
> - : "0" (0), "1" (1), "m" (futex), "m" (timeout), \
> - "2" (private) \
> - : "memory", "cx", "cc", "r10", "r11"); \
> - result; })
> -
> -extern int __lll_timedlock_elision (int *futex, short *adapt_count,
> - const struct timespec *timeout,
> - int private) attribute_hidden;
> -
> -#define lll_timedlock_elision(futex, adapt_count, timeout, private) \
> - __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)
> -
> -#if !IS_IN (libc) || defined UP
> -# define __lll_unlock_asm_start LOCK_INSTR "decl %0\n\t" \
> - "je 24f\n\t"
> -#else
> -# define __lll_unlock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \
> - "je 0f\n\t" \
> - "lock; decl %0\n\t" \
> - "jne 1f\n\t" \
> - "jmp 24f\n\t" \
> - "0:\tdecl %0\n\t" \
> - "je 24f\n\t"
> -#endif
> -
> -#define lll_unlock(futex, private) \
> - (void) \
> - ({ int ignore; \
> - if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
> - __asm __volatile (__lll_unlock_asm_start \
> - "1:\tlea %0, %%" RDI_LP "\n" \
> - "2:\tsub $128, %%" RSP_LP "\n" \
> - ".cfi_adjust_cfa_offset 128\n" \
> - "3:\tcallq __lll_unlock_wake_private\n" \
> - "4:\tadd $128, %%" RSP_LP "\n" \
> - ".cfi_adjust_cfa_offset -128\n" \
> - "24:" \
> - : "=m" (futex), "=&D" (ignore) \
> - : "m" (futex) \
> - : "ax", "cx", "r11", "cc", "memory"); \
> - else \
> - __asm __volatile (__lll_unlock_asm_start \
> - "1:\tlea %0, %%" RDI_LP "\n" \
> - "2:\tsub $128, %%" RSP_LP "\n" \
> - ".cfi_adjust_cfa_offset 128\n" \
> - "3:\tcallq __lll_unlock_wake\n" \
> - "4:\tadd $128, %%" RSP_LP "\n" \
> - ".cfi_adjust_cfa_offset -128\n" \
> - "24:" \
> - : "=m" (futex), "=&D" (ignore) \
> - : "m" (futex), "S" (private) \
> - : "ax", "cx", "r11", "cc", "memory"); \
> - })
> -
> -#define lll_islocked(futex) \
> - (futex != LLL_LOCK_INITIALIZER)
> -
> -extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
> - attribute_hidden;
> -
> -extern int __lll_unlock_elision (int *lock, int private)
> - attribute_hidden;
> -
> -extern int __lll_trylock_elision (int *lock, short *adapt_count)
> - attribute_hidden;
> -
> -#define lll_lock_elision(futex, adapt_count, private) \
> - __lll_lock_elision (&(futex), &(adapt_count), private)
> -#define lll_unlock_elision(futex, adapt_count, private) \
> - __lll_unlock_elision (&(futex), private)
> -#define lll_trylock_elision(futex, adapt_count) \
> - __lll_trylock_elision (&(futex), &(adapt_count))
> -
> -#endif /* !__ASSEMBLER__ */
> -
> -#endif /* lowlevellock.h */
>