分类 Linux 下的文章

strace_system call tracing

作者: matsudaira
时间: 2023-10-24
分类: Linux,运维日寄
评论

你可以使用strace -c cmd /-p pid来追踪某个进程或者指令的系统调用，他们的打印结果略有不同

[root@master ~]# strace -c pwd
/root
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 21.71    0.000104          11         9           mmap
 16.91    0.000081          16         5           close
 10.86    0.000052          17         3         3 access
 10.02    0.000048          12         4           mprotect
  8.98    0.000043          14         3           open
  8.14    0.000039          19         2           munmap
  6.68    0.000032           8         4           fstat
  6.05    0.000029           7         4           brk
  4.38    0.000021          21         1           write
  2.71    0.000013          13         1           execve
  2.09    0.000010          10         1           arch_prctl
  0.84    0.000004           4         1           getcwd
  0.63    0.000003           3         1           read
------ ----------- ----------- --------- --------- ----------------
100.00    0.000479                    39         3 total

当使用-p 的时候，打印出来的文本量可能会有点超出你的想象..

[root@master ~]# strace -p 1
strace: Process 1 attached
epoll_wait(4, [{EPOLLIN, {u32=1532991024, u64=93829388539440}}], 33, -1) = 1
clock_gettime(CLOCK_BOOTTIME, {tv_sec=319337, tv_nsec=129368879}) = 0
recvmsg(18, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="WATCHDOG=1", iov_len=4096}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_SOCKET, cmsg_type=SCM_CREDENTIALS, cmsg_data={pid=365, uid=0, gid=0}}], msg_controllen=32, msg_flags=MSG_CMSG_CLOEXEC}, MSG_DONTWAIT|MSG_CMSG_CLOEXEC) = 10
open("/proc/365/cgroup", O_RDONLY|O_CLOEXEC) = 16
fstat(16, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f32446e7000
read(16, "11:memory:/system.slice/systemd-"..., 1024) = 370
close(16)                               = 0
munmap(0x7f32446e7000, 4096)            = 0
timerfd_settime(3, TFD_TIMER_ABSTIME, {it_interval={tv_sec=0, tv_nsec=0}, it_value={tv_sec=319494, tv_nsec=691126000}}, NULL) = 0
epoll_wait(4, [{EPOLLIN, {u32=1532730112, u64=93829388278528}}], 33, -1) = 1
clock_gettime(CLOCK_BOOTTIME, {tv_sec=319337, tv_nsec=131285975}) = 0
recvmsg(38, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="l\4\1\1 \0\0\0@\r\0\0\211\0\0\0\1\1o\0\25\0\0\0", iov_len=24}], msg_iovlen=1, msg_controllen=0, msg_flags=MSG_CMSG_CLOEXEC}, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_CMSG_CLOEXEC) = 24
recvmsg(38, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="/org/freedesktop/DBus\0\0\0\2\1s\0\24\0\0\0"..., iov_len=168}], msg_iovlen=1, msg_controllen=0, msg_flags=MSG_CMSG_CLOEXEC}, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_CMSG_CLOEXEC) = 168
epoll_wait(4, [{EPOLLIN, {u32=1532730112, u64=93829388278528}}], 33, -1) = 1
clock_gettime(CLOCK_BOOTTIME, {tv_sec=319337, tv_nsec=131777119}) = 0
recvmsg(38, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="l\1\0\1@\1\0\0~/\0\0\305\0\0\0\1\1o\0\31\0\0\0", iov_len=24}], msg_iovlen=1, msg_controllen=0, msg_flags=MSG_CMSG_CLOEXEC}, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_CMSG_CLOEXEC) = 24
recvmsg(38, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="/org/freedesktop/systemd1\0\0\0\0\0\0\0"..., iov_len=512}], msg_iovlen=1, msg_controllen=0, msg_flags=MSG_CMSG_CLOEXEC}, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_CMSG_CLOEXEC) = 512
sendmsg(38, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="l\1\0\1\t\0\0\0\0E\1\0\207\0\0\0\1\1o\0\25\0\0\0/org/fre"..., iov_len=152}, {iov_base="\4\0\0\0:1.1\0", iov_len=9}], msg_iovlen=2, msg_controllen=0, msg_flags=0}, MSG_DONTWAIT|MSG_NOSIGNAL) = 161
recvmsg(38, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base="l\2\1\1\4\0\0\0\311\237\0\0=\0\0\0\6\1s\0\4\0\0\0", iov_len=24}], msg_iovlen=1, msg_controllen=0, msg_flags=MSG_CMSG_CLOEXEC}, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_CMSG_CLOEXEC) = 24
recvmsg(38, {msg_name=NULL, msg_namelen=0, msg_iov=[{iov_base=":1.0\0\0\0\0\5\1u\0\0E\1\0\10\1g\0\1u\0\0\7\1s\0\24\0\0\0"..., iov_len=60}], msg_iovlen=1, msg_controllen=0, msg_flags=MSG_CMSG_CLOEXEC}, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_CMSG_CLOEXEC) = 60
getuid()                                = 0
stat("/run/systemd", {st_mode=S_IFDIR|0755, st_size=400, ...}) = 0
mkdir("/run/systemd/system", 0755)      = -1 EEXIST (File exists)
stat("/run/systemd/system", {st_mode=S_IFDIR|0755, st_size=1200, ...}) = 0
umask(077)                              = 000
open("/run/systemd/system/.#session-811.scopeHaHZNz", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0600) = 16
umask(000)                              = 077
fcntl(16, F_GETFL)                      = 0x8002 (flags O_RDWR|O_LARGEFILE)
umask(0777)                             = 000
fchmod(16, 0644)                        = 0
umask(000)                              = 0777
fstat(16, {st_mode=S_IFREG|0644, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f32446e7000
write(16, "# Transient stub\n", 17)     = 17
rename("/run/systemd/system/.#session-811.scopeHaHZNz", "/run/systemd/system/session-811.scope") = 0
close(16)                               = 0
munmap(0x7f32446e7000, 4096)            = 0
stat("/run/systemd/system", {st_mode=S_IFDIR|0755, st_size=1220, ...}) = 0
mkdir("/run/systemd/system/session-811.scope.d", 0755) = 0
umask(077)                              = 000
open("/run/systemd/system/session-811.scope.d/.#50-Slice.confd5TEip", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0600) = 16
umask(000)                              = 077
fcntl(16, F_GETFL)                      = 0x8002 (flags O_RDWR|O_LARGEFILE)
umask(0777)                             = 000
fchmod(16, 0644)                        = 0
umask(000)                              = 0777
fstat(16, {st_mode=S_IFREG|0644, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f32446e7000
write(16, "[Scope]\nSlice=user-0.slice\n", 27) = 27
rename("/run/systemd/system/session-811.scope.d/.#50-Slice.confd5TEip", "/run/systemd/system/session-811.scope.d/50-Slice.conf") = 0
close(16)                               = 0

如果你想保存打印结果，你可以使用 strace -o outputfilename pwd这种方式，他会输出一个打印日志在当前目录下
如果你想知道某个系统调用在某个命令当中实时的调用情况，你可以这样这（以open这个syscall和ls这个命令为例子）

[root@master ~]# strace -t -e open ls
10:26:01 open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
10:26:01 open("/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
10:26:01 open("/lib64/libcap.so.2", O_RDONLY|O_CLOEXEC) = 3
10:26:01 open("/lib64/libacl.so.1", O_RDONLY|O_CLOEXEC) = 3
10:26:01 open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
10:26:01 open("/lib64/libpcre.so.1", O_RDONLY|O_CLOEXEC) = 3
10:26:01 open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
10:26:01 open("/lib64/libattr.so.1", O_RDONLY|O_CLOEXEC) = 3
10:26:01 open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
10:26:01 open("/proc/filesystems", O_RDONLY) = 3
10:26:01 open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
flannel.yaml  jen-dep.yaml  jen-pv.yaml  jen-role.yaml  jen-svc.yaml  output
10:26:01 +++ exited with 0 +++

如果你想追踪基于某个文件运行起来的进程的子进程，那么你可以使用strace -f filename

僵尸进程问题排查

作者: matsudaira
时间: 2023-10-24
分类: Linux,运维日寄
评论

一个进程结束后，会返回一个返回值，父进程会用wait()来获取这个返回值，等获取到之后，系统会回收进程的pcd并删除对应的进程，但如果父进程一直不获取，那么这个进程会处于僵尸状态

处理思路：

僵尸进程是已经结束的进程，无法用kill杀死，首先得去看看他的父进程出了什么事情，为什么不获取

ps -p pid -o stat | tail -n 1

如果父进程停止了（T），那么恢复父进程就行
kill -SIGCONT pid

如果父进程正常，用strace -p pid去追踪他的行为，看看是不是发生了死锁之类的

如果是孤儿进程（父进程先一步退出了），那么这种进程一般会被初始进程接管，这种情况下要是还是出了问题，那看看是不是初始进程出事了，方法参上

注：高版本的初始进程不再会被kill发送的停止信号影响，也不能被strace

prometheus-relabel

作者: matsudaira
时间: 2023-10-24
分类: Linux
评论

https://blog.csdn.net/lovely_nn/article/details/123043447

- job_name: kubelet/metrics
  honor_labels: true
  honor_timestamps: true
  scrape_interval: 15s
  scrape_timeout: 10s
  metrics_path: /metrics
  scheme: https
  kubernetes_sd_configs:
  - role: node
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    insecure_skip_verify: false
  relabel_configs:
  - separator: ;
    regex: __meta_kubernetes_node_label_(.+)
    replacement: $1
    action: labelmap
  - separator: ;
    regex: (.*)
    target_label: __address__
    replacement: kubernetes.default.svc:443
    action: replace
  - source_labels: [__meta_kubernetes_node_name]
    separator: ;
    regex: (.+)
    target_label: __metrics_path__
    replacement: /api/v1/nodes/${1}/proxy/metrics
    action: replace
  - separator: ;
    regex: (.*)
    target_label: job
    replacement: kubelet
    action: replace

磁盘占满但是找不到大文件

作者: matsudaira
时间: 2023-10-24
分类: Linux,运维日寄
评论

某个磁盘满了，但是进去找不到文件，排除掉权限、隐藏文件、硬件问题等之后，那还有种可能

lsof -n |grep deleted

kill 杀掉对应进程

出现问题的原因是文件虽然被删除，但是进程没有释放文件的句柄，导致内核不能正常释放对应的存储空间

容器内存告警但实际无异常

作者: matsudaira
时间: 2023-10-24
分类: Linux,运维日寄
评论

监控报容器的内存报警，但是进容器查看发现实际没有使用那么多的内存，再看监控数据，确实是超了预警值触发了告警
看配置的指标，用的是container_memory_working_set_bytes和kube_pod_container_resource_limits_memory_bytes ，kube_pod_container_resource_limits_memory_bytes 是当前容器内存限制大小，container_memory_working_set_bytes通过查看cadvisor可知是memory.usage-total_inactive_file，也就是cgroup的memory.usage_in_bytes

进容器的/sys/fs/cgroup/memory目录下

memory.usage_in_bytes 已使用的内存量(包含cache和buffer)(字节)，相当于linux的used_meme
memory.limit_in_bytes 限制的内存总量(字节)，相当于linux的total_mem
memory.failcnt 申请内存失败次数计数
memory.stat 内存相关状态

memory.usage_in_bytes的计算方式包含rss+cache+buffer，total_active_file和total_inactive_file都包含在cache，cache是不会主动回收的，只有当容器被销毁或者系统内存不足才回去回收，所以才会出现上面的情况

参考博客：https://blog.csdn.net/weixin_39961559/article/details/80496419
https://blog.csdn.net/weixin_39961559/article/details/86432283
https://www.cnblogs.com/276815076/p/5478966.html

strace_system call tracing

僵尸进程问题排查

prometheus-relabel

磁盘占满但是找不到大文件

容器内存告警但实际无异常

最新文章

最近回复

分类

归档

其它