stackcollapse-perf-sched.awk (6527B)
1 #!/usr/bin/awk -f 2 3 # 4 # This program generates collapsed off-cpu stacks fit for use by flamegraph.pl 5 # from scheduler data collected via perf_events. 6 # 7 # Outputs the cumulative time off cpu in us for each distinct stack observed. 8 # 9 # Some awk variables further control behavior: 10 # 11 # record_tid If truthy, causes all stack traces to include the 12 # command and LWP id. 13 # 14 # record_wake_stack If truthy, stacks include the frames from the wakeup 15 # event in addition to the sleep event. 16 # See http://www.brendangregg.com/FlameGraphs/offcpuflamegraphs.html#Wakeup 17 # 18 # recurse If truthy, attempt to recursively identify and 19 # visualize the full wakeup stack chain. 20 # See http://www.brendangregg.com/FlameGraphs/offcpuflamegraphs.html#ChainGraph 21 # 22 # Note that this is only an approximation, as only the 23 # last sleep event is recorded (e.g. if a thread slept 24 # multiple times before waking another thread, only the 25 # last sleep event is used). Implies record_wake_stack=1 26 # 27 # To set any of these variables from the command line, run via: 28 # 29 # stackcollapse-perf-sched.awk -v recurse=1 30 # 31 # == Important warning == 32 # 33 # WARNING: tracing all scheduler events is very high overhead in perf. Even 34 # more alarmingly, there appear to be bugs in perf that prevent it from reliably 35 # getting consistent traces (even with large trace buffers), causing it to 36 # produce empty perf.data files with error messages of the form: 37 # 38 # 0x952790 [0x736d]: failed to process type: 3410 39 # 40 # This failure is not determinisitic, so re-executing perf record will 41 # eventually succeed. 42 # 43 # == Usage == 44 # 45 # First, record data via perf_events: 46 # 47 # sudo perf record -g -e 'sched:sched_switch' \ 48 # -e 'sched:sched_stat_sleep' -e 'sched:sched_stat_blocked' \ 49 # -p <pid> -o perf.data -- sleep 1 50 # 51 # Then post process with this script: 52 # 53 # sudo perf script -f time,comm,pid,tid,event,ip,sym,dso,trace -i perf.data | \ 54 # stackcollapse-perf-sched.awk -v recurse=1 | \ 55 # flamegraph.pl --color=io --countname=us >out.svg 56 # 57 58 # 59 # CDDL HEADER START 60 # 61 # The contents of this file are subject to the terms of the 62 # Common Development and Distribution License (the "License"). 63 # You may not use this file except in compliance with the License. 64 # 65 # You can obtain a copy of the license at docs/cddl1.txt or 66 # http://opensource.org/licenses/CDDL-1.0. 67 # See the License for the specific language governing permissions 68 # and limitations under the License. 69 # 70 # When distributing Covered Code, include this CDDL HEADER in each 71 # file and include the License file at docs/cddl1.txt. 72 # If applicable, add the following below this CDDL HEADER, with the 73 # fields enclosed by brackets "[]" replaced with your own identifying 74 # information: Portions Copyright [yyyy] [name of copyright owner] 75 # 76 # CDDL HEADER END 77 # 78 79 # 80 # Copyright (c) 2015 by MemSQL. All rights reserved. 81 # 82 83 # 84 # Match a perf captured variable, returning just the contents. For example, for 85 # the following line, get_perf_captured_variable("pid") would return "27235": 86 # 87 # swapper 0 [006] 708189.626415: sched:sched_stat_sleep: comm=memsqld pid=27235 delay=100078421 [ns 88 # 89 function get_perf_captured_variable(variable) 90 { 91 match($0, variable "=[^[:space:]]+") 92 return substr($0, RSTART + length(variable) + 1, 93 RLENGTH - length(variable) - 1) 94 } 95 96 # 97 # The timestamp is the first field that ends in a colon, e.g.: 98 # 99 # swapper 0 [006] 708189.626415: sched:sched_stat_sleep: comm=memsqld pid=27235 delay=100078421 [ns 100 # 101 # or 102 # 103 # swapper 0/0 708189.626415: sched:sched_stat_sleep: comm=memsqld pid=27235 delay=100078421 [ns] 104 # 105 function get_perf_timestamp() 106 { 107 match($0, " [^ :]+:") 108 return substr($0, RSTART + 1, RLENGTH - 2) 109 } 110 111 !/^#/ && /sched:sched_switch/ { 112 switchcommand = get_perf_captured_variable("comm") 113 114 switchpid = get_perf_captured_variable("prev_pid") 115 116 switchtime=get_perf_timestamp() 117 118 switchstack="" 119 } 120 121 # 122 # Strip the function name from a stack entry 123 # 124 # Stack entry is expected to be formatted like: 125 # c60849 MyClass::Foo(unsigned long) (/home/areece/a.out) 126 # 127 function get_function_name() 128 { 129 # We start from 2 since we don't need the hex offset. 130 # We stop at NF - 1 since we don't need the library path. 131 funcname = $2 132 for (i = 3; i <= NF - 1; i++) { 133 funcname = funcname " " $i 134 } 135 return funcname 136 } 137 138 (switchpid != 0 && /^\s/) { 139 if (switchstack == "") { 140 switchstack = get_function_name() 141 } else { 142 switchstack = get_function_name() ";" switchstack 143 } 144 } 145 146 (switchpid != 0 && /^$/) { 147 switch_stacks[switchpid] = switchstack 148 delete last_switch_stacks[switchpid] 149 switch_time[switchpid] = switchtime 150 151 switchpid=0 152 switchcommand="" 153 switchstack="" 154 } 155 156 !/^#/ && (/sched:sched_stat_sleep/ || /sched:sched_stat_blocked/) { 157 wakecommand=$1 158 wakepid=$2 159 160 waketime=get_perf_timestamp() 161 162 stat_next_command = get_perf_captured_variable("comm") 163 164 stat_next_pid = get_perf_captured_variable("pid") 165 166 stat_delay_ns = int(get_perf_captured_variable("delay")) 167 168 wakestack="" 169 } 170 171 (stat_next_pid != 0 && /^\s/) { 172 if (wakestack == "") { 173 wakestack = get_function_name() 174 } else { 175 # We build the wakestack in reverse order. 176 wakestack = wakestack ";" get_function_name() 177 } 178 } 179 180 (stat_next_pid != 0 && /^$/) { 181 # 182 # For some reason, perf appears to output duplicate 183 # sched:sched_stat_sleep and sched:sched_stat_blocked events. We only 184 # handle the first event. 185 # 186 if (stat_next_pid in switch_stacks) { 187 last_wake_time[stat_next_pid] = waketime 188 189 stack = switch_stacks[stat_next_pid] 190 if (recurse || record_wake_stack) { 191 stack = stack ";" wakestack 192 if (record_tid) { 193 stack = stack ";" wakecommand "-" wakepid 194 } else { 195 stack = stack ";" wakecommand 196 } 197 } 198 199 if (recurse) { 200 if (last_wake_time[wakepid] > last_switch_time[stat_next_pid]) { 201 stack = stack ";-;" last_switch_stacks[wakepid] 202 } 203 last_switch_stacks[stat_next_pid] = stack 204 } 205 206 delete switch_stacks[stat_next_pid] 207 208 if (record_tid) { 209 stack_times[stat_next_command "-" stat_next_pid ";" stack] += stat_delay_ns 210 } else { 211 stack_times[stat_next_command ";" stack] += stat_delay_ns 212 } 213 } 214 215 wakecommand="" 216 wakepid=0 217 stat_next_pid=0 218 stat_next_command="" 219 stat_delay_ms=0 220 } 221 222 END { 223 for (stack in stack_times) { 224 if (int(stack_times[stack] / 1000) > 0) { 225 print stack, int(stack_times[stack] / 1000) 226 } 227 } 228 }