Vous avez reçu un message "Your GitLab account has been locked ..." ? Pas d'inquiétude : lisez cet article https://docs.gricad-pages.univ-grenoble-alpes.fr/help/unlock/

Commit d5b38ae3 authored by aumgn's avatar aumgn
Browse files

Remove results processing tools

parent 77b70537
#! /usr/bin/python
#
# migrate_stacks
#
# USAGE: migrate_stacks.py [-h] [-p PID | -u | -k] [-U | -K] [-f] [duration]
#
# Based on bcc / offcputime (created by Brendan Gregg)
# https://github.com/iovisor/bcc/blob/master/tool/offcputime.py
#
from __future__ import print_function
from bcc import BPF
from sys import stderr
from time import sleep, strftime
import argparse
import errno
import signal
# arg validation
def positive_int(val):
try:
ival = int(val)
except ValueError:
raise argparse.ArgumentTypeError("must be an integer")
if ival < 0:
raise argparse.ArgumentTypeError("must be positive")
return ival
def positive_nonzero_int(val):
ival = positive_int(val)
if ival == 0:
raise argparse.ArgumentTypeError("must be nonzero")
return ival
# arguments
examples = """examples:
./migrate_stacks.py # trace off-CPU stack time until Ctrl-C
./migrate_stacks.py 5 # trace for 5 seconds only
./migrate_stacks.py -f 5 # 5 seconds, and output in folded format
./migrate_stacks.py -p 185 # only trace threads for PID 185
./migrate_stacks.py -t 188 # only trace thread 188
./migrate_stacks.py -u # only trace user threads (no kernel)
./migrate_stacks.py -k # only trace kernel threads (no user)
./migrate_stacks.py -U # only show user space stacks (no kernel)
./migrate_stacks.py -K # only show kernel space stacks (no user)
"""
parser = argparse.ArgumentParser(
description="Summarize off-CPU time by stack trace",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=examples)
thread_group = parser.add_mutually_exclusive_group()
# Note: this script provides --pid and --tid flags but their arguments are
# referred to internally using kernel nomenclature: TGID and PID.
thread_group.add_argument("-p", "--pid", metavar="PID", dest="tgid",
help="trace this PID only", type=positive_int)
thread_group.add_argument("-t", "--tid", metavar="TID", dest="pid",
help="trace this TID only", type=positive_int)
thread_group.add_argument("-u", "--user-threads-only", action="store_true",
help="user threads only (no kernel threads)")
thread_group.add_argument("-k", "--kernel-threads-only", action="store_true",
help="kernel threads only (no user threads)")
stack_group = parser.add_mutually_exclusive_group()
stack_group.add_argument("-U", "--user-stacks-only", action="store_true",
help="show stacks from user space only (no kernel space stacks)")
stack_group.add_argument("-K", "--kernel-stacks-only", action="store_true",
help="show stacks from kernel space only (no user space stacks)")
parser.add_argument("-d", "--delimited", action="store_true",
help="insert delimiter between kernel/user stacks")
parser.add_argument("-f", "--folded", action="store_true",
help="output folded format")
parser.add_argument("--stack-storage-size", default=1024,
type=positive_nonzero_int,
help="the number of unique stack traces that can be stored and "
"displayed (default 1024)")
parser.add_argument("duration", nargs="?", default=99999999,
type=positive_nonzero_int,
help="duration of trace, in seconds")
args = parser.parse_args()
if args.pid and args.tgid:
parser.error("specify only one of -p and -t")
folded = args.folded
duration = int(args.duration)
# signal handler
def signal_ignore(signal, frame):
print()
# define BPF program
bpf_text = """
#include <uapi/linux/ptrace.h>
#include <linux/sched.h>
struct key_t {
u32 pid;
u32 tgid;
int user_stack_id;
int kernel_stack_id;
char name[TASK_COMM_LEN];
};
BPF_HASH(counts, struct key_t);
BPF_STACK_TRACE(stack_traces, STACK_STORAGE_SIZE)
int on_set_task_cpu(struct pt_regs *ctx, struct task_struct *p, int new_cpu) {
u32 pid = p->pid;
u32 tgid = p->tgid;
if (!(THREAD_FILTER) && task_cpu(p) == new_cpu) {
return 0;
}
u64 zero = 0, *val;
struct key_t key = {};
key.pid = pid;
key.tgid = tgid;
key.user_stack_id = USER_STACK_GET;
key.kernel_stack_id = KERNEL_STACK_GET;
bpf_get_current_comm(&key.name, sizeof(key.name));
val = counts.lookup_or_init(&key, &zero);
(*val) ++;
return 0;
}
"""
# set thread filter
thread_context = ""
if args.tgid is not None:
thread_context = "PID %d" % args.tgid
thread_filter = 'tgid == %d' % args.tgid
elif args.pid is not None:
thread_context = "TID %d" % args.pid
thread_filter = 'pid == %d' % args.pid
elif args.user_threads_only:
thread_context = "user threads"
thread_filter = '!(p->flags & PF_KTHREAD)'
elif args.kernel_threads_only:
thread_context = "kernel threads"
thread_filter = 'p->flags & PF_KTHREAD'
else:
thread_context = "all threads"
thread_filter = '1'
bpf_text = bpf_text.replace('THREAD_FILTER', thread_filter)
# set stack storage size
bpf_text = bpf_text.replace('STACK_STORAGE_SIZE', str(args.stack_storage_size))
# handle stack args
kernel_stack_get = "stack_traces.get_stackid(ctx, BPF_F_REUSE_STACKID)"
user_stack_get = \
"stack_traces.get_stackid(ctx, BPF_F_REUSE_STACKID | BPF_F_USER_STACK)"
stack_context = ""
if args.user_stacks_only:
stack_context = "user"
kernel_stack_get = "-1"
elif args.kernel_stacks_only:
stack_context = "kernel"
user_stack_get = "-1"
else:
stack_context = "user + kernel"
bpf_text = bpf_text.replace('USER_STACK_GET', user_stack_get)
bpf_text = bpf_text.replace('KERNEL_STACK_GET', kernel_stack_get)
need_delimiter = args.delimited and not (args.kernel_stacks_only or
args.user_stacks_only)
# check for an edge case; the code below will handle this case correctly
# but ultimately nothing will be displayed
if args.kernel_threads_only and args.user_stacks_only:
print("ERROR: Displaying user stacks for kernel threads " +
"doesn't make sense.", file=stderr)
exit(1)
# initialize BPF
b = BPF(text=bpf_text)
b.attach_kprobe(event="set_task_cpu", fn_name="on_set_task_cpu")
matched = b.num_open_kprobes()
if matched == 0:
print("error: 0 functions traced. Exiting.", file=stderr)
exit(1)
# header
if not folded:
print("Tracing off-CPU time (us) of %s by %s stack" %
(thread_context, stack_context), end="")
if duration < 99999999:
print(" for %d secs." % duration)
else:
print("... Hit Ctrl-C to end.")
try:
sleep(duration)
except KeyboardInterrupt:
# as cleanup can take many seconds, trap Ctrl-C:
signal.signal(signal.SIGINT, signal_ignore)
if not folded:
print()
missing_stacks = 0
has_enomem = False
counts = b.get_table("counts")
stack_traces = b.get_table("stack_traces")
for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
# handle get_stackid erorrs
if (not args.user_stacks_only and k.kernel_stack_id < 0) or \
(not args.kernel_stacks_only and k.user_stack_id < 0 and
k.user_stack_id != -errno.EFAULT):
missing_stacks += 1
# check for an ENOMEM error
if k.kernel_stack_id == -errno.ENOMEM or \
k.user_stack_id == -errno.ENOMEM:
has_enomem = True
continue
# user stacks will be symbolized by tgid, not pid, to avoid the overhead
# of one symbol resolver per thread
user_stack = [] if k.user_stack_id < 0 else \
stack_traces.walk(k.user_stack_id)
kernel_stack = [] if k.kernel_stack_id < 0 else \
stack_traces.walk(k.kernel_stack_id)
if folded:
# print folded stack output
user_stack = list(user_stack)
kernel_stack = list(kernel_stack)
line = [k.name.decode()] + \
[b.sym(addr, k.tgid) for addr in reversed(user_stack)] + \
(need_delimiter and ["-"] or []) + \
[b.ksym(addr) for addr in reversed(kernel_stack)]
print("%s %d" % (";".join(line), v.value))
else:
# print default multi-line stack output
for addr in kernel_stack:
print(" %s" % b.ksym(addr))
if need_delimiter:
print(" --")
for addr in user_stack:
print(" %s" % b.sym(addr, k.tgid))
print(" %-16s %s (%d)" % ("-", k.name.decode(), k.pid))
print(" %d\n" % v.value)
if missing_stacks > 0:
enomem_str = "" if not has_enomem else \
" Consider increasing --stack-storage-size."
print("WARNING: %d stack traces could not be displayed.%s" %
(missing_stacks, enomem_str),
file=stderr)
#! /usr/bin/python
import sys
import schedstatparser as parser
import schedstatnames as names
cpus = parser.parse(sys.argv[1])
for cpu in cpus.values():
for domain in cpu.domains.values():
mask = ''.join([ hex(m)[2:].zfill(8) for m in domain.cpusmask ])
for i, stat in enumerate(domain.stats):
if names.table[i]:
print("{},{},{},{},{}".format(
cpu.index,
domain.index,
mask,
names.table[i],
stat
))
#! /usr/bin/python
import sys
import schedstatparser as parser
cpus1 = parser.parse(sys.argv[1])
cpus2 = parser.parse(sys.argv[2])
print("version {}".format(parser.SCHEDSTAT_VERSION))
print("timestamp 0")
for cpu1 in cpus1.values():
cpu2 = cpus2[cpu1.index]
print("cpu{} {}".format(
cpu1.index,
' '.join([ str(stat2 - stat1) for (stat1, stat2) in zip(cpu1.stats, cpu2.stats) ])
))
for domain1 in cpu1.domains.values():
domain2 = cpu2.domains[domain1.index]
print("domain{} {} {}".format(
domain1.index,
','.join([ hex(m)[2:].zfill(8) for m in domain1.cpusmask ]),
' '.join([ str(stat2 - stat1) for (stat1, stat2) in zip(domain1.stats, domain2.stats) ])
))
# References :
# - http://elixir.free-electrons.com/linux/v4.9.9/source/include/linux/sched.h#L1110
# - http://elixir.free-electrons.com/linux/v4.9.9/source/Documentation/scheduler/sched-stats.txt
# - http://elixir.free-electrons.com/linux/v4.9.9/source/kernel/sched/stats.c
table = [
### load_balance() stats when idle (CPU_IDLE)
# # of times in this domain load_balance() was called when the
# cpu was idle
"idle_lb_count",
# # of times in this domain load_balance() checked but found
# the load did not require balancing when the cpu was idle
"idle_lb_balanced",
# # of times in this domain load_balance() tried to move one or
# more tasks and failed, when the cpu was idle
"idle_lb_failed",
# sum of imbalances discovered (if any) with each call to
# load_balance() in this domain when the cpu was idle
"idle_lb_imbalance",
# # of times in this domain pull_task() was called when the cpu
# was idle
"idle_lb_gained",
# # of times in this domain pull_task() was called even though
# the target task was cache-hot when idle
"idle_lb_hot_gained",
# # of times in this domain load_balance() was called but did
# not find a busier queue while the cpu was idle
"idle_lb_nobusyq",
# # of times in this domain a busier queue was found while the
# cpu was idle but no busier group was found
"idle_lb_nobusyg",
### load_balance() stats when busy (CPU_NOT_IDLE)
# # of times in this domain load_balance() was called when the
# cpu was busy
"busy_lb_count",
# # of times in this domain load_balance() checked but found the
# load did not require balancing when busy
"busy_lb_balanced",
# # of times in this domain load_balance() tried to move one or
# more tasks and failed, when the cpu was busy
"busy_lb_failed",
# sum of imbalances discovered (if any) with each call to
# load_balance() in this domain when the cpu was busy
"busy_lb_imbalance",
# # of times in this domain pull_task() was called when busy
"busy_lb_gained",
# # of times in this domain pull_task() was called even though the
# target task was cache-hot when busy
"busy_lb_hot_gained",
# # of times in this domain load_balance() was called but did not
# find a busier queue while the cpu was busy
"busy_lb_nobusyq",
# # of times in this domain a busier queue was found while the cpu
# was busy but no busier group was found
"busy_lb_nobusyg",
### load_balance() stats when newly idle (CPU_NEWLY_IDLE)
# # of times in this domain load_balance() was called when the
# cpu was just becoming idle
"newly_idle_lb_count",
# # of times in this domain load_balance() checked but found the
# load did not require balancing when the cpu was just becoming idle
"newly_idle_lb_balanced",
# # of times in this domain load_balance() tried to move one or more
# tasks and failed, when the cpu was just becoming idle
# was just becoming idle but no busier group was found
"newly_idle_lb_failed",
# sum of imbalances discovered (if any) with each call to
"newly_idle_lb_imbalance",
# load_balance() in this domain when the cpu was just becoming idle
"newly_idle_lb_gained",
# # of times in this domain pull_task() was called when newly idle
"newly_idle_lb_hot_gained",
# # of times in this domain pull_task() was called even though the
# target task was cache-hot when just becoming idle
"newly_idle_lb_nobusyq",
# # of times in this domain load_balance() was called but did not
# find a busier queue while the cpu was just becoming idle
"newly_idle_lb_nobusyg",
# # of times in this domain a busier queue was found while the cpu
### Active load balancing
# # of times active_load_balance() was called
"alb_count",
# # of times active_load_balance() tried to move a task and failed
"alb_failed",
# # of times active_load_balance() successfully moved a task
"alb_pushed",
### Unused
"sbe_count",
"sbe_balanced",
"sbe_pushed",
"sbf_count",
"sbf_balanced",
"sbf_pushed",
### try_to_wake_up() stats
# # of times in this domain try_to_wake_up() awoke a task that
# last ran on a different cpu in this domain
"ttwu_wake_remote",
# # of times in this domain try_to_wake_up() moved a task to the
# waking cpu because it was cache-cold on its own cpu anyway
"ttwu_move_affine",
# # of times in this domain try_to_wake_up() started passive balancing
"ttwu_move_balance",
]
import re
import sys
SCHEDSTAT_VERSION = 15
version_re = re.compile('^version ([0-9]+)$')
def check_version(match):
version = int(match.group(1))
if SCHEDSTAT_VERSION != version:
print("Unknown version {}, expected {}".format(
version,
SCHEDSTAT_VERSION))
exit(1)
empty_re = re.compile('^\s*$')
class Cpu:
RE = re.compile('^cpu([0-9]+) ([\-0-9 ]+)$')
def __init__(self, match):
self.index = int(match.group(1))
self.stats = [ int(stat) for stat in match.group(2).split(" ") ]
self.domains = {}
class Domain:
RE = re.compile('^domain([0-9]+) ([\-0-9a-f,]+) ([\-0-9 ]+)$')
def __init__(self, match):
self.index = int(match.group(1))
self.cpusmask = [ int(part, 16) for part in match.group(2).split(",") ]
self.stats = [ int(stat) for stat in match.group(3).split(" ") ]
def parse(filename):
cpus = {}
with open(filename) as f:
cpu = None
domain = None
for line in f:
version_match = version_re.match(line)
if version_match:
check_version(version_match)
continue
empty_match = empty_re.match(line)
if empty_match:
continue
cpu_match = Cpu.RE.match(line)
if cpu_match:
cpu = Cpu(cpu_match)
cpus[cpu.index] = cpu
continue
domain_match = Domain.RE.match(line)
if domain_match:
domain = Domain(domain_match)
cpu.domains[domain.index] = domain
continue
return cpus
library('tidyverse')
library('ggthemr')
light_theme <- ggthemr('flat', set_theme = FALSE)
dark_theme <- ggthemr('flat dark', set_theme = FALSE)
args = commandArgs(trailingOnly=TRUE)
if (length(args) != 1) {
stop("Exactly one argument required (csv input)", call.=FALSE)
}
migrations <- read_csv(args[1], col_names = TRUE) %>%
filter(Trigger %in% c('NLB', 'SLB', 'ILB', 'WA', 'WLB')) %>%
group_by(Orig, Dest) %>%
summarize(Count = n()) %>%
ungroup()
cpus_x_breaks <- seq(0, 63)
cpus_y_breaks <- seq(0, 63)
png('migrations_matrix.png', width = 940, height = 900)
ggplot(migrations, aes(x = Dest, y = Orig)) +
geom_tile(aes(fill = Count)) +
scale_x_continuous(breaks = cpus_x_breaks, position = 'top', expand = c(0, 0)) +
scale_y_continuous(breaks = cpus_y_breaks, trans = "reverse", expand = c(0, 0)) +
scale_fill_distiller(palette = "Spectral") +
xlab("Destination CPU") +
ylab("Origin CPU") +
theme(
line = element_blank(),
plot.margin = unit(c(0.2, 0.2, 0.2, 0.2), 'cm'),
legend.text = element_blank()
)
# -*- coding: utf-8 -*-
import os
import sys
import getopt
sys.path.append(os.environ['PERF_EXEC_PATH'] + \
'/scripts/python/Perf-Trace-Util/lib/Perf/Trace')
from perf_trace_context import *
from Core import *
import csv
TRIGGER_SLB = 0x0001
TRIGGER_NLB = 0x0002
TRIGGER_ILB = 0x0004
TRIGGER_WLB = 0x0008
TRIGGER_WAFF = 0x0010
TRIGGER_FLB = 0x0020
TRIGGER_ELB = 0x0040
TRIGGER_RT = 0x0080
TRIGGER_DL = 0x0100
TRIGGER_NB = 0x0200
TRIGGER_CA = 0x0400
TRIGGER_HP = 0x0800
TRIGGER_OTH = 0x1000
TRIGGER_EXIT = 0x8000
TRIGGER_SYMBOLS = {
TRIGGER_SLB : 'SLB',
TRIGGER_NLB : 'NLB',
TRIGGER_ILB : 'ILB',
TRIGGER_WLB : 'WLB',
TRIGGER_WAFF : 'WAFF',
TRIGGER_FLB : 'FLB',
TRIGGER_ELB : 'ELB',
TRIGGER_RT : 'RT',
TRIGGER_DL : 'DL',
TRIGGER_NB : 'NB',
TRIGGER_CA : 'CA',
TRIGGER_HP : 'HP',
TRIGGER_OTH : 'OTH',
TRIGGER_EXIT : 'EXIT'
}
data = None
class GlobalData:
def __init__(self):
if len(sys.argv) != 5:
print("Invalid number of arguments expected 4, got {}".format(len(sys.argv) - 1))
print("perf script ... <comm> <exec> <migrations-output> <rq-sizes-output>")
exit(15)
self.comm = sys.argv[1]
self.exec_comm = sys.argv[2]
self.last_secs = 0
self.rq_start = [None] * 64
self.rq_sizes = [0] * 64
try:
self.migrations_file = open(sys.argv[3], 'w')
self.rq_sizes_file = open(sys.argv[4], 'w')
except:
exit(10)