[patch v2] Correct range checking in mallopt/mxfast/tcache [BZ #25194]

[DJ Delorie <dj at redhat dot com>]

>From f5c9a8a75de6d1c114193411634e9abd00618a21 Mon Sep 17 00:00:00 2001
From: DJ Delorie <dj@redhat.com>
Date: Tue, 3 Dec 2019 17:44:36 -0500
Subject: Correct range checking in mallopt/mxfast/tcache [BZ #25194]

do_set_tcache_max, do_set_mxfast:
Fix two instances of comparing "size_t < 0"
Both cases have upper limit, so the "negative value" case
is already handled via overflow semantics.

do_set_tcache_max, do_set_tcache_count:
Fix return value on error.  Note: currently not used.

mallopt:
pass return value of helper functions to user.  Behavior should
only be actually changed for mxfast, where we restore the old
(pre-tunables) behavior.

diff --git a/malloc/malloc.c b/malloc/malloc.c
index 70cc35a473..a6f78cc6c1 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -5086,13 +5086,14 @@ do_set_arena_max (size_t value)
 static __always_inline int
 do_set_tcache_max (size_t value)
 {
-  if (value >= 0 && value <= MAX_TCACHE_SIZE)
+  if (value <= MAX_TCACHE_SIZE)
     {
       LIBC_PROBE (memory_tunable_tcache_max_bytes, 2, value, mp_.tcache_max_bytes);
       mp_.tcache_max_bytes = value;
       mp_.tcache_bins = csize2tidx (request2size(value)) + 1;
+      return 1;
     }
-  return 1;
+  return 0;
 }
 
 static __always_inline int
@@ -5102,8 +5103,9 @@ do_set_tcache_count (size_t value)
     {
       LIBC_PROBE (memory_tunable_tcache_count, 2, value, mp_.tcache_count);
       mp_.tcache_count = value;
+      return 1;
     }
-  return 1;
+  return 0;
 }
 
 static __always_inline int
@@ -5119,7 +5121,7 @@ static inline int
 __always_inline
 do_set_mxfast (size_t value)
 {
-  if (value >= 0 && value <= MAX_FAST_SIZE)
+  if (value <= MAX_FAST_SIZE)
     {
       LIBC_PROBE (memory_mallopt_mxfast, 2, value, get_max_fast ());
       set_max_fast (value);
@@ -5144,18 +5146,25 @@ __libc_mallopt (int param_number, int value)
      (see definition of set_max_fast).  */
   malloc_consolidate (av);
 
+  /* Many of these helper functions take a size_t.  We do not worry
+     about overflow here, because forcing a large size_t value to a
+     negative "int" might be the only way to pass a large size_t value
+     to mallopt, and because the helpers have sufficient range
+     checking already despite the conversions.  Many of these helpers
+     are also referenced in the tunables macros in arena.c.  */
+
   switch (param_number)
     {
     case M_MXFAST:
-      do_set_mxfast (value);
+      res = do_set_mxfast (value);
       break;
 
     case M_TRIM_THRESHOLD:
-      do_set_trim_threshold (value);
+      res = do_set_trim_threshold (value);
       break;
 
     case M_TOP_PAD:
-      do_set_top_pad (value);
+      res = do_set_top_pad (value);
       break;
 
     case M_MMAP_THRESHOLD:
@@ -5163,25 +5172,25 @@ __libc_mallopt (int param_number, int value)
       break;
 
     case M_MMAP_MAX:
-      do_set_mmaps_max (value);
+      res = do_set_mmaps_max (value);
       break;
 
     case M_CHECK_ACTION:
-      do_set_mallopt_check (value);
+      res = do_set_mallopt_check (value);
       break;
 
     case M_PERTURB:
-      do_set_perturb_byte (value);
+      res = do_set_perturb_byte (value);
       break;
 
     case M_ARENA_TEST:
       if (value > 0)
-	do_set_arena_test (value);
+	res = do_set_arena_test (value);
       break;
 
     case M_ARENA_MAX:
       if (value > 0)
-	do_set_arena_max (value);
+	res = do_set_arena_max (value);
       break;
     }
   __libc_lock_unlock (av->mutex);