flamegraph

Fork of flamegraph without j***s*****
git clone git://git.laack.co/flamegraph.git
Log | Files | Refs | README

stackcollapse-perf-sched.awk (6527B)


      1 #!/usr/bin/awk -f
      2 
      3 #
      4 # This program generates collapsed off-cpu stacks fit for use by flamegraph.pl
      5 # from scheduler data collected via perf_events.
      6 #
      7 # Outputs the cumulative time off cpu in us for each distinct stack observed.
      8 #
      9 # Some awk variables further control behavior:
     10 #
     11 #   record_tid          If truthy, causes all stack traces to include the
     12 #                       command and LWP id.
     13 #
     14 #   record_wake_stack   If truthy, stacks include the frames from the wakeup
     15 #                       event in addition to the sleep event.
     16 #                       See http://www.brendangregg.com/FlameGraphs/offcpuflamegraphs.html#Wakeup
     17 #
     18 #   recurse             If truthy, attempt to recursively identify and
     19 #                       visualize the full wakeup stack chain.
     20 #                       See http://www.brendangregg.com/FlameGraphs/offcpuflamegraphs.html#ChainGraph
     21 #
     22 #                       Note that this is only an approximation, as only the
     23 #                       last sleep event is recorded (e.g. if a thread slept
     24 #                       multiple times before waking another thread, only the
     25 #                       last sleep event is used). Implies record_wake_stack=1
     26 #
     27 # To set any of these variables from the command line, run via:
     28 #
     29 #    stackcollapse-perf-sched.awk -v recurse=1
     30 #
     31 # == Important warning ==
     32 #
     33 # WARNING: tracing all scheduler events is very high overhead in perf. Even
     34 # more alarmingly, there appear to be bugs in perf that prevent it from reliably
     35 # getting consistent traces (even with large trace buffers), causing it to
     36 # produce empty perf.data files with error messages of the form:
     37 #
     38 #   0x952790 [0x736d]: failed to process type: 3410
     39 #
     40 # This failure is not determinisitic, so re-executing perf record will
     41 # eventually succeed.
     42 #
     43 # == Usage ==
     44 #
     45 # First, record data via perf_events:
     46 #
     47 # sudo perf record -g -e 'sched:sched_switch' \
     48 #       -e 'sched:sched_stat_sleep' -e 'sched:sched_stat_blocked' \
     49 #       -p <pid> -o perf.data  -- sleep 1
     50 #
     51 # Then post process with this script:
     52 #
     53 # sudo perf script -f time,comm,pid,tid,event,ip,sym,dso,trace -i perf.data | \
     54 #       stackcollapse-perf-sched.awk -v recurse=1 | \
     55 #       flamegraph.pl --color=io --countname=us >out.svg
     56 #
     57 
     58 #
     59 # CDDL HEADER START
     60 #
     61 # The contents of this file are subject to the terms of the
     62 # Common Development and Distribution License (the "License").
     63 # You may not use this file except in compliance with the License.
     64 #
     65 # You can obtain a copy of the license at docs/cddl1.txt or
     66 # http://opensource.org/licenses/CDDL-1.0.
     67 # See the License for the specific language governing permissions
     68 # and limitations under the License.
     69 #
     70 # When distributing Covered Code, include this CDDL HEADER in each
     71 # file and include the License file at docs/cddl1.txt.
     72 # If applicable, add the following below this CDDL HEADER, with the
     73 # fields enclosed by brackets "[]" replaced with your own identifying
     74 # information: Portions Copyright [yyyy] [name of copyright owner]
     75 #
     76 # CDDL HEADER END
     77 #
     78 
     79 #
     80 # Copyright (c) 2015 by MemSQL. All rights reserved.
     81 #
     82 
     83 #
     84 # Match a perf captured variable, returning just the contents. For example, for
     85 # the following line, get_perf_captured_variable("pid") would return "27235":
     86 #
     87 #     swapper     0 [006] 708189.626415: sched:sched_stat_sleep: comm=memsqld pid=27235 delay=100078421 [ns
     88 #
     89 function get_perf_captured_variable(variable)
     90 {
     91 	match($0, variable "=[^[:space:]]+")
     92 	return substr($0, RSTART + length(variable) + 1,
     93                       RLENGTH - length(variable) - 1)
     94 }
     95 
     96 #
     97 # The timestamp is the first field that ends in a colon, e.g.:
     98 #
     99 #     swapper     0 [006] 708189.626415: sched:sched_stat_sleep: comm=memsqld pid=27235 delay=100078421 [ns
    100 #
    101 # or
    102 #
    103 #     swapper     0/0     708189.626415: sched:sched_stat_sleep: comm=memsqld pid=27235 delay=100078421 [ns]
    104 #
    105 function get_perf_timestamp()
    106 {
    107 	match($0, " [^ :]+:")
    108 	return substr($0, RSTART + 1, RLENGTH - 2)
    109 }
    110 
    111 !/^#/ && /sched:sched_switch/ {
    112 	switchcommand = get_perf_captured_variable("comm")
    113 
    114 	switchpid = get_perf_captured_variable("prev_pid")
    115 
    116 	switchtime=get_perf_timestamp()
    117 
    118 	switchstack=""
    119 }
    120 
    121 #
    122 # Strip the function name from a stack entry
    123 #
    124 # Stack entry is expected to be formatted like:
    125 #           c60849 MyClass::Foo(unsigned long) (/home/areece/a.out)
    126 #
    127 function get_function_name()
    128 {
    129 	# We start from 2 since we don't need the hex offset.
    130 	# We stop at NF - 1 since we don't need the library path.
    131 	funcname = $2
    132 	for (i = 3; i <= NF - 1; i++) {
    133 		funcname = funcname " " $i
    134 	}
    135 	return funcname
    136 }
    137 
    138 (switchpid != 0 && /^\s/) {
    139 	if (switchstack == "")  {
    140 		switchstack = get_function_name()
    141 	} else {
    142 		switchstack = get_function_name() ";" switchstack
    143 	}
    144 }
    145 
    146 (switchpid != 0 && /^$/) {
    147 	switch_stacks[switchpid] = switchstack
    148 	delete last_switch_stacks[switchpid]
    149 	switch_time[switchpid] = switchtime
    150 
    151 	switchpid=0
    152 	switchcommand=""
    153 	switchstack=""
    154 }
    155 
    156 !/^#/ && (/sched:sched_stat_sleep/ || /sched:sched_stat_blocked/) {
    157 	wakecommand=$1
    158 	wakepid=$2
    159 
    160 	waketime=get_perf_timestamp()
    161 
    162 	stat_next_command = get_perf_captured_variable("comm")
    163 
    164 	stat_next_pid = get_perf_captured_variable("pid")
    165 
    166 	stat_delay_ns = int(get_perf_captured_variable("delay"))
    167 
    168 	wakestack=""
    169 }
    170 
    171 (stat_next_pid != 0 && /^\s/) {
    172 	if (wakestack == "") {
    173 		wakestack = get_function_name()
    174 	} else {
    175 		# We build the wakestack in reverse order.
    176 		wakestack = wakestack ";" get_function_name()
    177 	}
    178 }
    179 
    180 (stat_next_pid != 0 && /^$/) {
    181 	#
    182 	# For some reason, perf appears to output duplicate
    183 	# sched:sched_stat_sleep and sched:sched_stat_blocked events. We only
    184 	# handle the first event.
    185 	#
    186 	if (stat_next_pid in switch_stacks) {
    187 		last_wake_time[stat_next_pid] = waketime
    188 
    189 		stack = switch_stacks[stat_next_pid]
    190 		if (recurse || record_wake_stack) {
    191 			stack = stack ";" wakestack
    192 			if (record_tid) {
    193 				stack = stack ";" wakecommand "-" wakepid
    194 			} else {
    195 				stack = stack ";" wakecommand
    196 			}
    197 		}
    198 
    199 		if (recurse) {
    200 			if (last_wake_time[wakepid] > last_switch_time[stat_next_pid]) {
    201 				stack = stack ";-;" last_switch_stacks[wakepid]
    202 			}
    203 			last_switch_stacks[stat_next_pid] = stack
    204 		}
    205 
    206 		delete switch_stacks[stat_next_pid]
    207 
    208 		if (record_tid) {
    209 			stack_times[stat_next_command "-" stat_next_pid ";" stack] += stat_delay_ns
    210 		} else {
    211 			stack_times[stat_next_command ";" stack] += stat_delay_ns
    212 		}
    213 	}
    214 
    215 	wakecommand=""
    216 	wakepid=0
    217 	stat_next_pid=0
    218 	stat_next_command=""
    219 	stat_delay_ms=0
    220 }
    221 
    222 END {
    223 	for (stack in stack_times) {
    224 		if (int(stack_times[stack] / 1000) > 0) {
    225 			print stack, int(stack_times[stack] / 1000)
    226 		}
    227 	}
    228 }