Improve check against integer wraparound in hcreate_r [BZ #18240]

2016-01-22  Florian Weimer  <fweimer@redhat.com>

	[BZ #18240]
	* misc/hsearch_r.c (__hcreate_r): Protect against unsigned int
	wraparound.
	* misc/bug18240.c: New test.
	* misc/Makefile (tests): Add it.

diff --git a/misc/Makefile b/misc/Makefile
index b9f854e..d7bbc85 100644
--- a/misc/Makefile
+++ b/misc/Makefile
@@ -77,7 +77,7 @@  gpl2lgpl := error.c error.h
 
 tests := tst-dirname tst-tsearch tst-fdset tst-efgcvt tst-mntent tst-hsearch \
 	 tst-error1 tst-pselect tst-insremque tst-mntent2 bug-hsearch1 \
-	 tst-mntent-blank-corrupt tst-mntent-blank-passno
+	 tst-mntent-blank-corrupt tst-mntent-blank-passno bug18240
 ifeq ($(run-built-tests),yes)
 tests-special += $(objpfx)tst-error1-mem.out
 endif
diff --git a/misc/bug18240.c b/misc/bug18240.c
new file mode 100644
index 0000000..1223486
--- /dev/null
+++ b/misc/bug18240.c
@@ -0,0 +1,69 @@ 
+/* Test integer wraparound in hcreate.
+   Copyright (C) 2016 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <errno.h>
+#include <limits.h>
+#include <search.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+static void
+test_size (size_t size)
+{
+  int res = hcreate (size);
+  if (res == 0)
+    {
+      if (errno == ENOMEM)
+        return;
+      printf ("error: hcreate (%zu): %m\n", size);
+      exit (1);
+    }
+  char *keys[100];
+  for (int i = 0; i < 100; ++i)
+    {
+      if (asprintf (keys + i, "%d", i) < 0)
+        {
+          printf ("error: asprintf: %m\n");
+          exit (1);
+        }
+      ENTRY e = { keys[i], (char *) "value" };
+      if (hsearch (e, ENTER) == NULL)
+        {
+          printf ("error: hsearch (\"%s\"): %m\n", keys[i]);
+          exit (1);
+        }
+    }
+  hdestroy ();
+
+  for (int i = 0; i < 100; ++i)
+    free (keys[i]);
+}
+
+static int
+do_test (void)
+{
+  test_size (-1);
+  test_size (-3);
+  test_size (INT_MAX);
+  test_size (UINT_MAX);
+  return 0;
+}
+
+#define TEST_FUNCTION do_test ()
+#include "../test-skeleton.c"
diff --git a/misc/hsearch_r.c b/misc/hsearch_r.c
index f6f16ed..8955d85 100644
--- a/misc/hsearch_r.c
+++ b/misc/hsearch_r.c
@@ -71,7 +71,10 @@  __hcreate_r (size_t nel, struct hsearch_data *htab)
       return 0;
     }
 
-  if (nel >= SIZE_MAX / sizeof (_ENTRY))
+  /* This limit is sufficient to avoid unsigned wraparound below,
+     possibly after truncation to unsigned int.  (struct hsearch_data
+     is part of the public API and uses usigned ints.)  */
+  if (nel >= INT_MAX / sizeof (_ENTRY))
     {
       __set_errno (ENOMEM);
       return 0;