@@ -1,3 +1,26 @@
+2019-02-11 Kyeong Yoo <kyeong.yoo@alliedtelesis.co.nz>
+
+ * malloc/mtrace.c: New environment variable MALLOC_TRACE_LEVEL is
+ defined to enable backtrace output from mtrace(). If the value set
+ in MALLOC_TRACE_LEVEL is between 1 and 15 when mtrace() is called,
+ extra backtrace info is recorded to the mtrace output.
+ * malloc/mtrace.c (tr_backtrace): Print backtrace of the function
+ calls up to the specified call level.
+ * malloc/mtrace.c (tr_freehook, tr_mallochook,tr_reallochook,
+ tr_memalignhook): Record backtrace if needed.
+ * malloc/mtrace.c (mtrace): Parse MALLOC_TRACE_LEVEL environment
+ variable. Valid number is between 1 and 15.
+ * malloc/mtrace.c (muntrace): Reset backtrace level to 0.
+ * malloc/mtrace.pl: Adjust to print backtrace of function calls.
+ * malloc/mtrace.pl: Accept new command argument for "Maps" which makes
+ possible to resolve addresses from shared libraries cross-compilation.
+ * malloc/mtrace.pl: Add "--addr2line=CMD" command-line option to specify
+ alternative 'addr2line' command to use.
+ * malloc/mtrace.pl: Add "--call-offset=NUM" command-line option to be
+ useful to find more accurate source code linei by 'addr2line'.
+ * malloc/mtrace.pl: Add "--solib-path=PATH" command-line option to be
+ useful to specify a list directorys to search missing shared libraries.
+
2019-02-11 Paul A. Clarke <pc@us.ibm.com>
* sysdeps/powerpc/fpu/e_sqrt.c (__slow_ieee754_sqrtf):
@@ -14,6 +14,11 @@ Major new features:
* On Linux, the gettid function has been added.
+* mtrace() records backtraces to identify caller functions
+ if MALLOC_TRACE_LEVEL environment variable is set. mtrace Perl script
+ is updated accordingly to resolve and display backtraces. New command-line
+ options are also added to the script to be useful for cross-compilation.
+
Deprecated and removed features, and other changes affecting compatibility:
* The functions clock_gettime, clock_getres, clock_settime,
@@ -24,6 +24,7 @@
# include <mcheck.h>
# include <libc-lock.h>
#endif
+#include <execinfo.h>
#include <dlfcn.h>
#include <fcntl.h>
@@ -44,8 +45,12 @@
#define TRACE_BUFFER_SIZE 512
+#define MAX_BACKTRACE_LEVEL 15
+
static FILE *mallstream;
static const char mallenv[] = "MALLOC_TRACE";
+static const char mall_level_env[] = "MALLOC_TRACE_LEVEL";
+static int mall_trace_level = 0;
static char *malloc_trace_buffer;
__libc_lock_define_initialized (static, lock);
@@ -61,6 +66,9 @@ static void *(*tr_old_realloc_hook) (void *ptr, size_t size,
static void *(*tr_old_memalign_hook) (size_t __alignment, size_t __size,
const void *);
+static void *tr_mallochook (size_t size, const void *caller);
+static void *tr_reallochook (void *ptr, size_t size, const void *caller);
+
/* This function is called when the block being alloc'd, realloc'd, or
freed has an address matching the variable "mallwatch". In a debugger,
set "mallwatch" to the address of interest, then put a breakpoint on
@@ -108,6 +116,26 @@ tr_where (const void *caller, Dl_info *info)
}
}
+static void
+tr_backtrace (void)
+{
+ void *bt_addrs[MAX_BACKTRACE_LEVEL + 2];
+ size_t bt_size;
+
+ bt_size = __backtrace (bt_addrs, mall_trace_level + 2);
+
+ /* Print backtrace (skip the first two) */
+ if (bt_size > 2)
+ {
+ size_t i;
+
+ fprintf (mallstream, "# %p", bt_addrs[2]);
+ for (i = 3; i < bt_size; i++)
+ fprintf (mallstream, ",%p", bt_addrs[i]);
+ fprintf (mallstream, "\n");
+ }
+}
+
static Dl_info *
lock_and_info (const void *caller, Dl_info *mem)
{
@@ -127,6 +155,18 @@ tr_freehook (void *ptr, const void *caller)
if (ptr == NULL)
return;
+ /* Print backtrace */
+ if (mall_trace_level > 0)
+ {
+ __free_hook = tr_old_free_hook;
+ __malloc_hook = tr_old_malloc_hook;
+ __realloc_hook = tr_old_realloc_hook;
+ tr_backtrace ();
+ __free_hook = tr_freehook;
+ __malloc_hook = tr_mallochook;
+ __realloc_hook = tr_reallochook;
+ }
+
Dl_info mem;
Dl_info *info = lock_and_info (caller, &mem);
tr_where (caller, info);
@@ -160,6 +200,11 @@ tr_mallochook (size_t size, const void *caller)
hdr = (void *) (*tr_old_malloc_hook)(size, caller);
else
hdr = (void *) malloc (size);
+
+ /* Print backtrace */
+ if (mall_trace_level > 0)
+ tr_backtrace ();
+
__malloc_hook = tr_mallochook;
tr_where (caller, info);
@@ -192,6 +237,11 @@ tr_reallochook (void *ptr, size_t size, const void *caller)
hdr = (void *) (*tr_old_realloc_hook)(ptr, size, caller);
else
hdr = (void *) realloc (ptr, size);
+
+ /* Collect backtrace */
+ if (mall_trace_level > 0)
+ tr_backtrace ();
+
__free_hook = tr_freehook;
__malloc_hook = tr_mallochook;
__realloc_hook = tr_reallochook;
@@ -236,6 +286,11 @@ tr_memalignhook (size_t alignment, size_t size, const void *caller)
hdr = (void *) (*tr_old_memalign_hook)(alignment, size, caller);
else
hdr = (void *) memalign (alignment, size);
+
+ /* Collect backtrace */
+ if (mall_trace_level > 0)
+ tr_backtrace ();
+
__memalign_hook = tr_memalignhook;
__malloc_hook = tr_mallochook;
@@ -321,6 +376,15 @@ mtrace (void)
__dso_handle);
}
#endif
+
+ /* Check backtrace level */
+ const char *level_str = getenv (mall_level_env);
+ if (level_str != NULL)
+ {
+ int num = atoi (level_str);
+ if (0 < num && num <= MAX_BACKTRACE_LEVEL)
+ mall_trace_level = num;
+ }
}
else
free (mtb);
@@ -338,6 +402,7 @@ muntrace (void)
file. */
FILE *f = mallstream;
mallstream = NULL;
+ mall_trace_level = 0;
__free_hook = tr_old_free_hook;
__malloc_hook = tr_old_malloc_hook;
__realloc_hook = tr_old_realloc_hook;
@@ -24,49 +24,59 @@ $VERSION = "@VERSION@";
$PKGVERSION = "@PKGVERSION@";
$REPORT_BUGS_TO = '@REPORT_BUGS_TO@';
$progname = $0;
+$addr2line_cmd = "addr2line";
+$call_offset = 0;
+$solib_path = "";
+
+use Class::Struct;
+use File::Basename;
+use Getopt::Long;
+
+struct Map => {
+ start => '$', # start address
+ end => '$', # end address
+ path => '$', # library path
+};
sub usage {
- print "Usage: mtrace [OPTION]... [Binary] MtraceData\n";
- print " --help print this help, then exit\n";
- print " --version print version number, then exit\n";
+ print "Usage: mtrace [OPTION]... [Binary] MtraceData [Maps]\n";
+ print " --help print this help, then exit\n";
+ print " --version print version number, then exit\n";
+ print " --addr2line=CMD specify addr2line command for cross-compilation (default: $addr2line_cmd)\n";
+ print " --call-offset=NUM CALL instruction length to subtract from return address (default: $call_offset)\n";
+ print " --solib-path=PATH shared library search path for cross-compilation (default: $solib_path)\n";
print "\n";
print "For bug reporting instructions, please see:\n";
print "$REPORT_BUGS_TO.\n";
exit 0;
}
-# We expect two arguments:
+sub version {
+ print "mtrace $PKGVERSION$VERSION\n";
+ print "Copyright (C) 2019 Free Software Foundation, Inc.\n";
+ print "This is free software; see the source for copying conditions. There is NO\n";
+ print "warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n";
+ print "Written by Ulrich Drepper <drepper\@gnu.org>\n";
+ exit 0;
+}
+
+# We expect three arguments:
# #1: the complete path to the binary
# #2: the mtrace data filename
+# #3: the memory map of the process, /proc/<PID>/maps (optional)
# The usual options are also recognized.
-
-arglist: while (@ARGV) {
- if ($ARGV[0] eq "--v" || $ARGV[0] eq "--ve" || $ARGV[0] eq "--ver" ||
- $ARGV[0] eq "--vers" || $ARGV[0] eq "--versi" ||
- $ARGV[0] eq "--versio" || $ARGV[0] eq "--version") {
- print "mtrace $PKGVERSION$VERSION\n";
- print "Copyright (C) 2019 Free Software Foundation, Inc.\n";
- print "This is free software; see the source for copying conditions. There is NO\n";
- print "warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n";
- print "Written by Ulrich Drepper <drepper\@gnu.org>\n";
-
- exit 0;
- } elsif ($ARGV[0] eq "--h" || $ARGV[0] eq "--he" || $ARGV[0] eq "--hel" ||
- $ARGV[0] eq "--help") {
- &usage;
- } elsif ($ARGV[0] =~ /^-/) {
- print "$progname: unrecognized option `$ARGV[0]'\n";
- print "Try `$progname --help' for more information.\n";
- exit 1;
- } else {
- last arglist;
- }
-}
+GetOptions ("help" => \&usage,
+ "version" => \&version,
+ "addr2line=s" => \$addr2line_cmd, # string
+ "call-offset=i" => \$call_offset,
+ "solib-path=s" => \@solib_path, # array of string
+ ) or die "Try `$progname --help' for more information.\n";
+@solib_path = split(/:/, join(':', @solib_path));
if ($#ARGV == 0) {
$binary="";
$data=$ARGV[0];
-} elsif ($#ARGV == 1) {
+} elsif ($#ARGV == 1 or $#ARGV == 2) {
$binary=$ARGV[0];
$data=$ARGV[1];
@@ -84,10 +94,77 @@ if ($#ARGV == 0) {
}
close (LOCS);
}
+
+ # Parse maps file (if specified)
+ if (defined $ARGV[2]) {
+ open(MAPS, "<$ARGV[2]") || die "Cannot open maps file";
+ my $prog_base = basename($prog);
+ while (<MAPS>) {
+ chop;
+ my @cols = split (' ');
+ # Find executable memory segment for library (except the program itself)
+ if ($#cols == 5 and $cols[1] =~ /^..x.$/ and $cols[5] =~ /^\//) {
+ $lib = basename($cols[5]);
+ next if ($lib eq $prog_base);
+ foreach $dir (@solib_path) {
+ my $path = $dir . '/' . $lib;
+ if (-e $path and $cols[0] =~ /^(.*)-(.*)$/) {
+ my $m = Map->new();
+ $m->start(hex($1));
+ $m->end(hex($2));
+ $m->path($path);
+ push(@maps, $m);
+ next MAP;
+ }
+ }
+ printf ("Warning: cannot find library %s\n", $lib);
+ }
+ }
+ close (MAPS);
+ }
} else {
die "Wrong number of arguments, run $progname --help for help.";
}
+sub find_address_map {
+ my $addr = pop(@_);
+ foreach my $m (@maps) {
+ return $m if ($m->start <= $addr and $addr <= $m->end);
+ }
+ return undef;
+}
+
+sub print_backtrace {
+ foreach my $addr (split (',', pop(@_))) {
+ printf (" # %-16s %s\n", $addr, &addr2line(hex($addr)));
+ }
+}
+
+sub addr2line {
+ my $addr = pop(@_);
+ return $cache{$addr} if (exists $cache{$addr});
+
+ my $executable;
+ my $searchaddr;
+ my $m = find_address_map($addr);
+ if ($m) {
+ $executable = $m->path;
+ $searchaddr = sprintf ("%#x", $addr - $m->start);
+ } else {
+ $executable = $binary;
+ $searchaddr = sprintf ("%#x", $addr);
+ }
+ if ($executable ne "" && open (ADDR, "$addr2line_cmd -e $executable $searchaddr|")) {
+ my $line = <ADDR>;
+ chomp $line;
+ close (ADDR);
+ $cache{$addr} = $line;
+ return $cache{$addr};
+ }
+ $cache{$addr} = "unknown";
+ return $cache{$addr};
+}
+
sub location {
my $str = pop(@_);
return $str if ($str eq "");
@@ -95,7 +172,7 @@ sub location {
my $addr = $1;
my $fct = $2;
return $cache{$addr} if (exists $cache{$addr});
- if ($binary ne "" && open (ADDR, "addr2line -e $binary $addr|")) {
+ if ($binary ne "" && open (ADDR, "$addr2line_cmd -e $binary $addr|")) {
my $line = <ADDR>;
chomp $line;
close (ADDR);
@@ -116,7 +193,7 @@ sub location {
$searchaddr = $addr;
$prog = $binary;
}
- if ($binary ne "" && open (ADDR, "addr2line -e $prog $searchaddr|")) {
+ if ($binary ne "" && open (ADDR, "$addr2line_cmd -e $prog $searchaddr|")) {
my $line = <ADDR>;
chomp $line;
close (ADDR);
@@ -129,7 +206,7 @@ sub location {
} elsif ($str =~ /^.*[[](0x[^]]*)]$/) {
my $addr = $1;
return $cache{$addr} if (exists $cache{$addr});
- if ($binary ne "" && open (ADDR, "addr2line -e $binary $addr|")) {
+ if ($binary ne "" && open (ADDR, "$addr2line_cmd -e $binary $addr|")) {
my $line = <ADDR>;
chomp $line;
close (ADDR);
@@ -144,6 +221,7 @@ sub location {
}
$nr=0;
+$bt="";
open(DATA, "<$data") || die "Cannot open mtrace data file";
while (<DATA>) {
my @cols = split (' ');
@@ -167,9 +245,11 @@ while (<DATA>) {
printf ("+ %#0@XXX@x Alloc %d duplicate: %s %s\n",
hex($allocaddr), $nr, &location($addrwas{$allocaddr}),
$where);
+ &print_backtrace ($bt) if ($bt ne "");
} elsif ($allocaddr =~ /^0x/) {
$allocated{$allocaddr}=$howmuch;
$addrwas{$allocaddr}=$where;
+ $backtrace{$allocaddr}=$bt if ($bt ne "");
}
last SWITCH;
}
@@ -177,9 +257,11 @@ while (<DATA>) {
if (defined $allocated{$allocaddr}) {
undef $allocated{$allocaddr};
undef $addrwas{$allocaddr};
+ undef $backtrace{$allocaddr};
} else {
printf ("- %#0@XXX@x Free %d was never alloc'd %s\n",
hex($allocaddr), $nr, &location($where));
+ &print_backtrace ($bt) if ($bt ne "");
}
last SWITCH;
}
@@ -187,9 +269,11 @@ while (<DATA>) {
if (defined $allocated{$allocaddr}) {
undef $allocated{$allocaddr};
undef $addrwas{$allocaddr};
+ undef $backtrace{$allocaddr};
} else {
printf ("- %#0@XXX@x Realloc %d was never alloc'd %s\n",
hex($allocaddr), $nr, &location($where));
+ &print_backtrace ($bt) if ($bt ne "");
}
last SWITCH;
}
@@ -198,9 +282,11 @@ while (<DATA>) {
printf ("+ %#0@XXX@x Realloc %d duplicate: %#010x %s %s\n",
hex($allocaddr), $nr, $allocated{$allocaddr},
&location($addrwas{$allocaddr}), &location($where));
+ &print_backtrace ($bt) if ($bt ne "");
} else {
$allocated{$allocaddr}=$howmuch;
$addrwas{$allocaddr}=$where;
+ $backtrace{$allocaddr}=$bt if ($bt ne "");
}
last SWITCH;
}
@@ -213,6 +299,13 @@ while (<DATA>) {
last SWITCH;
}
}
+
+ # Save backtrace info (if any) for the next line
+ if ($cols[0] eq "#") {
+ $bt=$cols[1];
+ } else {
+ $bt="";
+ }
}
close (DATA);
@@ -229,6 +322,7 @@ if ($#addrs >= 0) {
}
printf ("%#0@XXX@x %#8x at %s\n", hex($addr), $allocated{$addr},
&location($addrwas{$addr}));
+ &print_backtrace ($backtrace{$addr}) if (defined $backtrace{$addr});
}
}
}
@@ -1694,7 +1694,7 @@ penalties for the program if the debugging mode is not enabled.
@c __cxa_atexit (once) dup @asulock @aculock @acsmem
@c free dup @ascuheap @acsmem
When the @code{mtrace} function is called it looks for an environment
-variable named @code{MALLOC_TRACE}. This variable is supposed to
+variable named @env{MALLOC_TRACE}. This variable is supposed to
contain a valid file name. The user must have write access. If the
file already exists it is truncated. If the environment variable is not
set or it does not name a valid file which can be opened for writing
@@ -1709,6 +1709,11 @@ functions are traced and protocolled into the file. There is now of
course a speed penalty for all calls to the traced functions so tracing
should not be enabled during normal use.
+To generate more detailed backtrace information of function calls to
+@code{malloc} etc, set an environment variable named @env{MALLOC_TRACE_LEVEL}
+with a number between 1 and 15. This indicates the maximum number of
+function pointers to generate in the backtrace information.
+
This function is a GNU extension and generally not available on other
systems. The prototype can be found in @file{mcheck.h}.
@end deftypefun
@@ -1818,7 +1823,7 @@ main (int argc, char *argv[])
@end example
I.e., the user can start the memory debugger any time s/he wants if the
-program was started with @code{MALLOC_TRACE} set in the environment.
+program was started with @env{MALLOC_TRACE} set in the environment.
The output will of course not show the allocations which happened before
the first signal but if there is a memory leak this will show up
nevertheless.
@@ -1830,13 +1835,13 @@ If you take a look at the output it will look similar to this:
@example
= Start
-@ [0x8048209] - 0x8064cc8
-@ [0x8048209] - 0x8064ce0
-@ [0x8048209] - 0x8064cf8
-@ [0x80481eb] + 0x8064c48 0x14
-@ [0x80481eb] + 0x8064c60 0x14
-@ [0x80481eb] + 0x8064c78 0x14
-@ [0x80481eb] + 0x8064c90 0x14
+@@ [0x8048209] - 0x8064cc8
+@@ [0x8048209] - 0x8064ce0
+@@ [0x8048209] - 0x8064cf8
+@@ [0x80481eb] + 0x8064c48 0x14
+@@ [0x80481eb] + 0x8064c60 0x14
+@@ [0x80481eb] + 0x8064c78 0x14
+@@ [0x80481eb] + 0x8064c90 0x14
= End
@end example
@@ -1915,6 +1920,72 @@ from line 33 in the source file @file{/home/drepper/tst-mtrace.c} four
times without freeing this memory before the program terminates.
Whether this is a real problem remains to be investigated.
+Sometimes the output from @code{mtrace} is pointing to an incorrect
+line of the source code. It may be caused by the compiler optimization
+but possibly it's due to the fact that the address recorded by
+@code{mtrace} is the return address to the caller. To adjust this
+interpretation, use command-line option @option{--call-offset=@var{NUM}}.
+
+@example
+drepper$ mtrace --call-offset=5 tst errlog
+@end example
+
+@node Interpreting the traces with backtrace
+@subsubsection Interpreting the traces with backtrace
+
+If backtrace information is generated by setting an environment variable
+named @env{MALLOC_TRACE_LEVEL} (for example,
+@code{export MALLOC_TRACE_LEVEL=3}), the output will look similar to this:
+
+@example
+= Start
+# 0x4009b4,0x400866,0x4008f3
+@@ [0x4009b4] - 0x184e280
+# 0x400879,0x4008f3,0x400923
+@@ [0x400879] + 0x184e280 0x14
+= End
+@end example
+
+A list of pointers are printed with @samp{#} mark before each memory
+allocation/deallocation, which indicates the return addresses from the
+corresponding stack frame.
+
+@example
+drepper$ mtrace tst errlog
+- 0x0184e280 Free 19 was never alloc'd /home/drepper/tst.c:51
+ # 0x4009b4 /home/drepper/tst.c:51
+ # 0x400866 /home/drepper/tst.c:13
+ # 0x4008f3 /home/drepper/tst.c:26
+
+Memory not freed:
+-----------------
+ Address Size Caller
+0x0184e280 0x14 at /home/drepper/tst.c:13
+ # 0x400879 /home/drepper/tst.c:13
+ # 0x4008f3 /home/drepper/tst.c:26
+ # 0x400923 /home/drepper/tst.c:36
+@end example
+
+When this mtrace output is processed, it shows better picture of where
+the memory allocation/deallocation was called.
+
+@node Use mtrace for cross-compilation
+@subsubsection Use mtrace for cross-compilation
+
+For cross-compilation, @code{mtrace} can generate meaningful output
+with additional command-line options @option{--solib-path=@var{path}} and
+@option{--addr2line=@var{cmd}} along with @var{maps} file copied from
+the target device (@file{/proc/<PID>/maps}).
+
+For example, if you have cross-compiled libraries exist in @code{~/staging}
+and compiled for powerpc, you can run command like this:
+
+@example
+drepper$ mtrace --solib-path=~/staging/lib:~/staging/usr/lib \
+ --addr2line=powerpc64-hardfloat-linux-gnu-addr2line \
+ tst errlog maps
+@end example
+
@node Replacing malloc
@subsection Replacing @code{malloc}