stackcollapse-perf.pl (13234B)
1 #!/usr/bin/perl -w 2 # 3 # stackcollapse-perf.pl collapse perf samples into single lines. 4 # 5 # Parses a list of multiline stacks generated by "perf script", and 6 # outputs a semicolon separated stack followed by a space and a count. 7 # If memory addresses (+0xd) are present, they are stripped, and resulting 8 # identical stacks are colased with their counts summed. 9 # 10 # USAGE: ./stackcollapse-perf.pl [options] infile > outfile 11 # 12 # Run "./stackcollapse-perf.pl -h" to list options. 13 # 14 # Example input: 15 # 16 # swapper 0 [000] 158665.570607: cpu-clock: 17 # ffffffff8103ce3b native_safe_halt ([kernel.kallsyms]) 18 # ffffffff8101c6a3 default_idle ([kernel.kallsyms]) 19 # ffffffff81013236 cpu_idle ([kernel.kallsyms]) 20 # ffffffff815bf03e rest_init ([kernel.kallsyms]) 21 # ffffffff81aebbfe start_kernel ([kernel.kallsyms].init.text) 22 # [...] 23 # 24 # Example output: 25 # 26 # swapper;start_kernel;rest_init;cpu_idle;default_idle;native_safe_halt 1 27 # 28 # Input may be created and processed using: 29 # 30 # perf record -a -g -F 997 sleep 60 31 # perf script | ./stackcollapse-perf.pl > out.stacks-folded 32 # 33 # The output of "perf script" should include stack traces. If these are missing 34 # for you, try manually selecting the perf script output; eg: 35 # 36 # perf script -f comm,pid,tid,cpu,time,event,ip,sym,dso,trace | ... 37 # 38 # This is also required for the --pid or --tid options, so that the output has 39 # both the PID and TID. 40 # 41 # Copyright 2012 Joyent, Inc. All rights reserved. 42 # Copyright 2012 Brendan Gregg. All rights reserved. 43 # 44 # CDDL HEADER START 45 # 46 # The contents of this file are subject to the terms of the 47 # Common Development and Distribution License (the "License"). 48 # You may not use this file except in compliance with the License. 49 # 50 # You can obtain a copy of the license at docs/cddl1.txt or 51 # http://opensource.org/licenses/CDDL-1.0. 52 # See the License for the specific language governing permissions 53 # and limitations under the License. 54 # 55 # When distributing Covered Code, include this CDDL HEADER in each 56 # file and include the License file at docs/cddl1.txt. 57 # If applicable, add the following below this CDDL HEADER, with the 58 # fields enclosed by brackets "[]" replaced with your own identifying 59 # information: Portions Copyright [yyyy] [name of copyright owner] 60 # 61 # CDDL HEADER END 62 # 63 # 02-Mar-2012 Brendan Gregg Created this. 64 # 02-Jul-2014 " " Added process name to stacks. 65 66 use strict; 67 use Getopt::Long; 68 69 my %collapsed; 70 71 sub remember_stack { 72 my ($stack, $count) = @_; 73 $collapsed{$stack} += $count; 74 } 75 my $annotate_kernel = 0; # put an annotation on kernel function 76 my $annotate_jit = 0; # put an annotation on jit symbols 77 my $annotate_all = 0; # enale all annotations 78 my $include_pname = 1; # include process names in stacks 79 my $include_pid = 0; # include process ID with process name 80 my $include_tid = 0; # include process & thread ID with process name 81 my $include_addrs = 0; # include raw address where a symbol can't be found 82 my $tidy_java = 1; # condense Java signatures 83 my $tidy_generic = 1; # clean up function names a little 84 my $target_pname; # target process name from perf invocation 85 my $event_filter = ""; # event type filter, defaults to first encountered event 86 my $event_defaulted = 0; # whether we defaulted to an event (none provided) 87 my $event_warning = 0; # if we printed a warning for the event 88 89 my $show_inline = 0; 90 my $show_context = 0; 91 92 my $srcline_in_input = 0; # if there are extra lines with source location (perf script -F+srcline) 93 GetOptions('inline' => \$show_inline, 94 'context' => \$show_context, 95 'srcline' => \$srcline_in_input, 96 'pid' => \$include_pid, 97 'kernel' => \$annotate_kernel, 98 'jit' => \$annotate_jit, 99 'all' => \$annotate_all, 100 'tid' => \$include_tid, 101 'addrs' => \$include_addrs, 102 'event-filter=s' => \$event_filter) 103 or die <<USAGE_END; 104 USAGE: $0 [options] infile > outfile\n 105 --pid # include PID with process names [1] 106 --tid # include TID and PID with process names [1] 107 --inline # un-inline using addr2line 108 --all # all annotations (--kernel --jit) 109 --kernel # annotate kernel functions with a _[k] 110 --jit # annotate jit functions with a _[j] 111 --context # adds source context to --inline 112 --srcline # parses output of 'perf script -F+srcline' and adds source context 113 --addrs # include raw addresses where symbols can't be found 114 --event-filter=EVENT # event name filter\n 115 [1] perf script must emit both PID and TIDs for these to work; eg, Linux < 4.1: 116 perf script -f comm,pid,tid,cpu,time,event,ip,sym,dso,trace 117 for Linux >= 4.1: 118 perf script -F comm,pid,tid,cpu,time,event,ip,sym,dso,trace 119 If you save this output add --header on Linux >= 3.14 to include perf info. 120 USAGE_END 121 122 if ($annotate_all) { 123 $annotate_kernel = $annotate_jit = 1; 124 } 125 126 my %inlineCache; 127 128 my %nmCache; 129 130 sub inlineCacheAdd { 131 my ($pc, $mod, $result) = @_; 132 if (defined($inlineCache{$pc})) { 133 $inlineCache{$pc}{$mod} = $result; 134 } else { 135 $inlineCache{$pc} = {$mod => $result}; 136 } 137 } 138 139 # for the --inline option 140 sub inline { 141 my ($pc, $rawfunc, $mod) = @_; 142 143 return $inlineCache{$pc}{$mod} if defined($inlineCache{$pc}{$mod}); 144 145 # capture addr2line output 146 my $a2l_output = `addr2line -a $pc -e $mod -i -f -s -C`; 147 148 # remove first line 149 $a2l_output =~ s/^(.*\n){1}//; 150 151 if ($a2l_output =~ /\?\?\n\?\?:0/) { 152 # if addr2line fails and rawfunc is func+offset, then fall back to it 153 if ($rawfunc =~ /^(.+)\+0x([0-9a-f]+)$/) { 154 my $func = $1; 155 my $addr = hex $2; 156 157 $nmCache{$mod}=`nm $mod` unless defined $nmCache{$mod}; 158 159 if ($nmCache{$mod} =~ /^([0-9a-f]+) . \Q$func\E$/m) { 160 my $base = hex $1; 161 my $newPc = sprintf "0x%x", $base+$addr; 162 my $result = inline($newPc, '', $mod); 163 inlineCacheAdd($pc, $mod, $result); 164 return $result; 165 } 166 } 167 } 168 169 my @fullfunc; 170 my $one_item = ""; 171 for (split /^/, $a2l_output) { 172 chomp $_; 173 174 # remove discriminator info if exists 175 $_ =~ s/ \(discriminator \S+\)//; 176 177 if ($one_item eq "") { 178 $one_item = $_; 179 } else { 180 if ($show_context == 1) { 181 unshift @fullfunc, $one_item . ":$_"; 182 } else { 183 unshift @fullfunc, $one_item; 184 } 185 $one_item = ""; 186 } 187 } 188 189 my $result = join ";" , @fullfunc; 190 191 inlineCacheAdd($pc, $mod, $result); 192 193 return $result; 194 } 195 196 my @stack; 197 my $pname; 198 my $m_pid; 199 my $m_tid; 200 my $m_period; 201 202 # 203 # Main loop 204 # 205 while (defined($_ = <>)) { 206 207 # find the name of the process launched by perf, by stepping backwards 208 # over the args to find the first non-option (no dash): 209 if (/^# cmdline/) { 210 my @args = split ' ', $_; 211 foreach my $arg (reverse @args) { 212 if ($arg !~ /^-/) { 213 $target_pname = $arg; 214 $target_pname =~ s:.*/::; # strip pathname 215 last; 216 } 217 } 218 } 219 220 # skip remaining comments 221 next if m/^#/; 222 chomp; 223 224 # end of stack. save cached data. 225 if (m/^$/) { 226 # ignore filtered samples 227 next if not $pname; 228 229 if ($include_pname) { 230 if (defined $pname) { 231 unshift @stack, $pname; 232 } else { 233 unshift @stack, ""; 234 } 235 } 236 remember_stack(join(";", @stack), $m_period) if @stack; 237 undef @stack; 238 undef $pname; 239 next; 240 } 241 242 # 243 # event record start 244 # 245 if (/^(\S.+?)\s+(\d+)\/*(\d+)*\s+/) { 246 # default "perf script" output has TID but not PID 247 # eg, "java 25607 4794564.109216: 1 cycles:" 248 # eg, "java 12688 [002] 6544038.708352: 235 cpu-clock:" 249 # eg, "V8 WorkerThread 25607 4794564.109216: 104345 cycles:" 250 # eg, "java 24636/25607 [000] 4794564.109216: 1 cycles:" 251 # eg, "java 12688/12764 6544038.708352: 10309278 cpu-clock:" 252 # eg, "V8 WorkerThread 24636/25607 [000] 94564.109216: 100 cycles:" 253 # other combinations possible 254 my ($comm, $pid, $tid, $period) = ($1, $2, $3, ""); 255 if (not $tid) { 256 $tid = $pid; 257 $pid = "?"; 258 } 259 260 if (/:\s*(\d+)*\s+(\S+):\s*$/) { 261 $period = $1; 262 my $event = $2; 263 264 if ($event_filter eq "") { 265 # By default only show events of the first encountered 266 # event type. Merging together different types, such as 267 # instructions and cycles, produces misleading results. 268 $event_filter = $event; 269 $event_defaulted = 1; 270 } elsif ($event ne $event_filter) { 271 if ($event_defaulted and $event_warning == 0) { 272 # only print this warning if necessary: 273 # when we defaulted and there was 274 # multiple event types. 275 print STDERR "Filtering for events of type: $event\n"; 276 $event_warning = 1; 277 } 278 next; 279 } 280 } 281 282 if (not $period) { 283 $period = 1 284 } 285 ($m_pid, $m_tid, $m_period) = ($pid, $tid, $period); 286 287 if ($include_tid) { 288 $pname = "$comm-$m_pid/$m_tid"; 289 } elsif ($include_pid) { 290 $pname = "$comm-$m_pid"; 291 } else { 292 $pname = "$comm"; 293 } 294 $pname =~ tr/ /_/; 295 296 # 297 # stack line 298 # 299 } elsif (/^\s*(\w+)\s*(.+) \((.*)\)/) { 300 # ignore filtered samples 301 next if not $pname; 302 303 my ($pc, $rawfunc, $mod) = ($1, $2, $3); 304 305 if ($show_inline == 1 && $mod !~ m/(perf-\d+.map|kernel\.|\[[^\]]+\])/) { 306 my $inlineRes = inline($pc, $rawfunc, $mod); 307 # - empty result this happens e.g., when $mod does not exist or is a path to a compressed kernel module 308 # if this happens, the user will see error message from addr2line written to stderr 309 # - if addr2line results in "??" , then it's much more sane to fall back than produce a '??' in graph 310 if($inlineRes ne "" and $inlineRes ne "??" and $inlineRes ne "??:??:0" ) { 311 unshift @stack, $inlineRes; 312 next; 313 } 314 } 315 316 # Linux 4.8 included symbol offsets in perf script output by default, eg: 317 # 7fffb84c9afc cpu_startup_entry+0x800047c022ec ([kernel.kallsyms]) 318 # strip these off: 319 $rawfunc =~ s/\+0x[\da-f]+$//; 320 321 next if $rawfunc =~ /^\(/; # skip process names 322 323 my $is_unknown=0; 324 my @inline; 325 for (split /\->/, $rawfunc) { 326 my $func = $_; 327 328 if ($func eq "[unknown]") { 329 if ($mod ne "[unknown]") { # use module name instead, if known 330 $func = $mod; 331 $func =~ s/.*\///; 332 } else { 333 $func = "unknown"; 334 $is_unknown=1; 335 } 336 337 if ($include_addrs) { 338 $func = "\[$func \<$pc\>\]"; 339 } else { 340 $func = "\[$func\]"; 341 } 342 } 343 344 if ($tidy_generic) { 345 $func =~ s/;/:/g; 346 if ($func !~ m/\.\(.*\)\./) { 347 # This doesn't look like a Go method name (such as 348 # "net/http.(*Client).Do"), so everything after the first open 349 # paren (that is not part of an "(anonymous namespace)") is 350 # just noise. 351 $func =~ s/\((?!anonymous namespace\)).*//; 352 } 353 # now tidy this horrible thing: 354 # 13a80b608e0a RegExp:[&<>\"\'] (/tmp/perf-7539.map) 355 $func =~ tr/"\'//d; 356 # fall through to $tidy_java 357 } 358 359 if ($tidy_java and $pname =~ m/^java/) { 360 # along with $tidy_generic, converts the following: 361 # Lorg/mozilla/javascript/ContextFactory;.call(Lorg/mozilla/javascript/ContextAction;)Ljava/lang/Object; 362 # Lorg/mozilla/javascript/ContextFactory;.call(Lorg/mozilla/javascript/C 363 # Lorg/mozilla/javascript/MemberBox;.<init>(Ljava/lang/reflect/Method;)V 364 # into: 365 # org/mozilla/javascript/ContextFactory:.call 366 # org/mozilla/javascript/ContextFactory:.call 367 # org/mozilla/javascript/MemberBox:.init 368 $func =~ s/^L// if $func =~ m:/:; 369 } 370 371 # 372 # Annotations 373 # 374 # detect inlined from the @inline array 375 # detect kernel from the module name; eg, frames to parse include: 376 # ffffffff8103ce3b native_safe_halt ([kernel.kallsyms]) 377 # 8c3453 tcp_sendmsg (/lib/modules/4.3.0-rc1-virtual/build/vmlinux) 378 # 7d8 ipv4_conntrack_local+0x7f8f80b8 ([nf_conntrack_ipv4]) 379 # detect jit from the module name; eg: 380 # 7f722d142778 Ljava/io/PrintStream;::print (/tmp/perf-19982.map) 381 if (scalar(@inline) > 0) { 382 $func .= "_[i]" unless $func =~ m/\_\[i\]/; # inlined 383 } elsif ($annotate_kernel == 1 && $mod =~ m/(^\[|vmlinux$)/ && $mod !~ /unknown/) { 384 $func .= "_[k]"; # kernel 385 } elsif ($annotate_jit == 1 && $mod =~ m:/tmp/perf-\d+\.map:) { 386 $func .= "_[j]" unless $func =~ m/\_\[j\]/; # jitted 387 } 388 389 # 390 # Source lines 391 # 392 # 393 # Sample outputs: 394 # | a.out 35081 252436.005167: 667783 cycles: 395 # | 408ebb some_method_name+0x8b (/full/path/to/a.out) 396 # | uniform_int_dist.h:300 397 # | 4069f5 main+0x935 (/full/path/to/a.out) 398 # | file.cpp:137 399 # | 7f6d2148eb25 __libc_start_main+0xd5 (/lib64/libc-2.33.so) 400 # | libc-2.33.so[27b25] 401 # 402 # | a.out 35081 252435.738165: 306459 cycles: 403 # | 7f6d213c2750 [unknown] (/usr/lib64/libkmod.so.2.3.6) 404 # | libkmod.so.2.3.6[6750] 405 # 406 # | a.out 35081 252435.738373: 315813 cycles: 407 # | 7f6d215ca51b __strlen_avx2+0x4b (/lib64/libc-2.33.so) 408 # | libc-2.33.so[16351b] 409 # | 7ffc71ee9580 [unknown] ([unknown]) 410 # | 411 # 412 # | a.out 35081 252435.718940: 247984 cycles: 413 # | ffffffff814f9302 up_write+0x32 ([kernel.kallsyms]) 414 # | [kernel.kallsyms][ffffffff814f9302] 415 if($srcline_in_input and not $is_unknown){ 416 $_ = <>; 417 chomp; 418 s/\[.*?\]//g; 419 s/^\s*//g; 420 s/\s*$//g; 421 $func.=':'.$_ unless $_ eq ""; 422 } 423 424 push @inline, $func; 425 } 426 427 unshift @stack, @inline; 428 } else { 429 warn "Unrecognized line: $_"; 430 } 431 } 432 433 foreach my $k (sort { $a cmp $b } keys %collapsed) { 434 print "$k $collapsed{$k}\n"; 435 }