codekingpro/portable-devtools
114k
1# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Linux platform implementation."""
6
7from __future__ import division
8
9import base64
10import collections
11import errno
12import functools
13import glob
14import os
15import re
16import socket
17import struct
18import sys
19import warnings
20from collections import defaultdict
21from collections import namedtuple
22
23from . import _common
24from . import _psposix
25from . import _psutil_linux as cext
26from . import _psutil_posix as cext_posix
27from ._common import NIC_DUPLEX_FULL
28from ._common import NIC_DUPLEX_HALF
29from ._common import NIC_DUPLEX_UNKNOWN
30from ._common import AccessDenied
31from ._common import NoSuchProcess
32from ._common import ZombieProcess
33from ._common import bcat
34from ._common import cat
35from ._common import debug
36from ._common import decode
37from ._common import get_procfs_path
38from ._common import isfile_strict
39from ._common import memoize
40from ._common import memoize_when_activated
41from ._common import open_binary
42from ._common import open_text
43from ._common import parse_environ_block
44from ._common import path_exists_strict
45from ._common import supports_ipv6
46from ._common import usage_percent
47from ._compat import PY3
48from ._compat import FileNotFoundError
49from ._compat import PermissionError
50from ._compat import ProcessLookupError
51from ._compat import b
52from ._compat import basestring
53
54
55if PY3:
56 import enum
57else:
58 enum = None
59
60
61# fmt: off
62__extra__all__ = [
63 #
64 'PROCFS_PATH',
65 # io prio constants
66 "IOPRIO_CLASS_NONE", "IOPRIO_CLASS_RT", "IOPRIO_CLASS_BE",
67 "IOPRIO_CLASS_IDLE",
68 # connection status constants
69 "CONN_ESTABLISHED", "CONN_SYN_SENT", "CONN_SYN_RECV", "CONN_FIN_WAIT1",
70 "CONN_FIN_WAIT2", "CONN_TIME_WAIT", "CONN_CLOSE", "CONN_CLOSE_WAIT",
71 "CONN_LAST_ACK", "CONN_LISTEN", "CONN_CLOSING",
72]
73# fmt: on
74
75
76# =====================================================================
77# --- globals
78# =====================================================================
79
80
81POWER_SUPPLY_PATH = "/sys/class/power_supply"
82HAS_PROC_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid())
83HAS_PROC_SMAPS_ROLLUP = os.path.exists('/proc/%s/smaps_rollup' % os.getpid())
84HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_ioprio_get")
85HAS_CPU_AFFINITY = hasattr(cext, "proc_cpu_affinity_get")
86
87# Number of clock ticks per second
88CLOCK_TICKS = os.sysconf("SC_CLK_TCK")
89PAGESIZE = cext_posix.getpagesize()
90BOOT_TIME = None # set later
91LITTLE_ENDIAN = sys.byteorder == 'little'
92
93# "man iostat" states that sectors are equivalent with blocks and have
94# a size of 512 bytes. Despite this value can be queried at runtime
95# via /sys/block/{DISK}/queue/hw_sector_size and results may vary
96# between 1k, 2k, or 4k... 512 appears to be a magic constant used
97# throughout Linux source code:
98# * https://stackoverflow.com/a/38136179/376587
99# * https://lists.gt.net/linux/kernel/2241060
100# * https://github.com/giampaolo/psutil/issues/1305
101# * https://github.com/torvalds/linux/blob/
102# 4f671fe2f9523a1ea206f63fe60a7c7b3a56d5c7/include/linux/bio.h#L99
103# * https://lkml.org/lkml/2015/8/17/234
104DISK_SECTOR_SIZE = 512
105
106if enum is None:
107 AF_LINK = socket.AF_PACKET
108else:
109 AddressFamily = enum.IntEnum(
110 'AddressFamily', {'AF_LINK': int(socket.AF_PACKET)}
111 )
112 AF_LINK = AddressFamily.AF_LINK
113
114# ioprio_* constants http://linux.die.net/man/2/ioprio_get
115if enum is None:
116 IOPRIO_CLASS_NONE = 0
117 IOPRIO_CLASS_RT = 1
118 IOPRIO_CLASS_BE = 2
119 IOPRIO_CLASS_IDLE = 3
120else:
121
122 class IOPriority(enum.IntEnum):
123 IOPRIO_CLASS_NONE = 0
124 IOPRIO_CLASS_RT = 1
125 IOPRIO_CLASS_BE = 2
126 IOPRIO_CLASS_IDLE = 3
127
128 globals().update(IOPriority.__members__)
129
130# See:
131# https://github.com/torvalds/linux/blame/master/fs/proc/array.c
132# ...and (TASK_* constants):
133# https://github.com/torvalds/linux/blob/master/include/linux/sched.h
134PROC_STATUSES = {
135 "R": _common.STATUS_RUNNING,
136 "S": _common.STATUS_SLEEPING,
137 "D": _common.STATUS_DISK_SLEEP,
138 "T": _common.STATUS_STOPPED,
139 "t": _common.STATUS_TRACING_STOP,
140 "Z": _common.STATUS_ZOMBIE,
141 "X": _common.STATUS_DEAD,
142 "x": _common.STATUS_DEAD,
143 "K": _common.STATUS_WAKE_KILL,
144 "W": _common.STATUS_WAKING,
145 "I": _common.STATUS_IDLE,
146 "P": _common.STATUS_PARKED,
147}
148
149# https://github.com/torvalds/linux/blob/master/include/net/tcp_states.h
150TCP_STATUSES = {
151 "01": _common.CONN_ESTABLISHED,
152 "02": _common.CONN_SYN_SENT,
153 "03": _common.CONN_SYN_RECV,
154 "04": _common.CONN_FIN_WAIT1,
155 "05": _common.CONN_FIN_WAIT2,
156 "06": _common.CONN_TIME_WAIT,
157 "07": _common.CONN_CLOSE,
158 "08": _common.CONN_CLOSE_WAIT,
159 "09": _common.CONN_LAST_ACK,
160 "0A": _common.CONN_LISTEN,
161 "0B": _common.CONN_CLOSING,
162}
163
164
165# =====================================================================
166# --- named tuples
167# =====================================================================
168
169
170# fmt: off
171# psutil.virtual_memory()
172svmem = namedtuple(
173 'svmem', ['total', 'available', 'percent', 'used', 'free',
174 'active', 'inactive', 'buffers', 'cached', 'shared', 'slab'])
175# psutil.disk_io_counters()
176sdiskio = namedtuple(
177 'sdiskio', ['read_count', 'write_count',
178 'read_bytes', 'write_bytes',
179 'read_time', 'write_time',
180 'read_merged_count', 'write_merged_count',
181 'busy_time'])
182# psutil.Process().open_files()
183popenfile = namedtuple(
184 'popenfile', ['path', 'fd', 'position', 'mode', 'flags'])
185# psutil.Process().memory_info()
186pmem = namedtuple('pmem', 'rss vms shared text lib data dirty')
187# psutil.Process().memory_full_info()
188pfullmem = namedtuple('pfullmem', pmem._fields + ('uss', 'pss', 'swap'))
189# psutil.Process().memory_maps(grouped=True)
190pmmap_grouped = namedtuple(
191 'pmmap_grouped',
192 ['path', 'rss', 'size', 'pss', 'shared_clean', 'shared_dirty',
193 'private_clean', 'private_dirty', 'referenced', 'anonymous', 'swap'])
194# psutil.Process().memory_maps(grouped=False)
195pmmap_ext = namedtuple(
196 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields))
197# psutil.Process.io_counters()
198pio = namedtuple('pio', ['read_count', 'write_count',
199 'read_bytes', 'write_bytes',
200 'read_chars', 'write_chars'])
201# psutil.Process.cpu_times()
202pcputimes = namedtuple('pcputimes',
203 ['user', 'system', 'children_user', 'children_system',
204 'iowait'])
205# fmt: on
206
207
208# =====================================================================
209# --- utils
210# =====================================================================
211
212
213def readlink(path):
214 """Wrapper around os.readlink()."""
215 assert isinstance(path, basestring), path
216 path = os.readlink(path)
217 # readlink() might return paths containing null bytes ('\x00')
218 # resulting in "TypeError: must be encoded string without NULL
219 # bytes, not str" errors when the string is passed to other
220 # fs-related functions (os.*, open(), ...).
221 # Apparently everything after '\x00' is garbage (we can have
222 # ' (deleted)', 'new' and possibly others), see:
223 # https://github.com/giampaolo/psutil/issues/717
224 path = path.split('\x00')[0]
225 # Certain paths have ' (deleted)' appended. Usually this is
226 # bogus as the file actually exists. Even if it doesn't we
227 # don't care.
228 if path.endswith(' (deleted)') and not path_exists_strict(path):
229 path = path[:-10]
230 return path
231
232
233def file_flags_to_mode(flags):
234 """Convert file's open() flags into a readable string.
235 Used by Process.open_files().
236 """
237 modes_map = {os.O_RDONLY: 'r', os.O_WRONLY: 'w', os.O_RDWR: 'w+'}
238 mode = modes_map[flags & (os.O_RDONLY | os.O_WRONLY | os.O_RDWR)]
239 if flags & os.O_APPEND:
240 mode = mode.replace('w', 'a', 1)
241 mode = mode.replace('w+', 'r+')
242 # possible values: r, w, a, r+, a+
243 return mode
244
245
246def is_storage_device(name):
247 """Return True if the given name refers to a root device (e.g.
248 "sda", "nvme0n1") as opposed to a logical partition (e.g. "sda1",
249 "nvme0n1p1"). If name is a virtual device (e.g. "loop1", "ram")
250 return True.
251 """
252 # Re-adapted from iostat source code, see:
253 # https://github.com/sysstat/sysstat/blob/
254 # 97912938cd476645b267280069e83b1c8dc0e1c7/common.c#L208
255 # Some devices may have a slash in their name (e.g. cciss/c0d0...).
256 name = name.replace('/', '!')
257 including_virtual = True
258 if including_virtual:
259 path = "/sys/block/%s" % name
260 else:
261 path = "/sys/block/%s/device" % name
262 return os.access(path, os.F_OK)
263
264
265@memoize
266def set_scputimes_ntuple(procfs_path):
267 """Set a namedtuple of variable fields depending on the CPU times
268 available on this Linux kernel version which may be:
269 (user, nice, system, idle, iowait, irq, softirq, [steal, [guest,
270 [guest_nice]]])
271 Used by cpu_times() function.
272 """
273 global scputimes
274 with open_binary('%s/stat' % procfs_path) as f:
275 values = f.readline().split()[1:]
276 fields = ['user', 'nice', 'system', 'idle', 'iowait', 'irq', 'softirq']
277 vlen = len(values)
278 if vlen >= 8:
279 # Linux >= 2.6.11
280 fields.append('steal')
281 if vlen >= 9:
282 # Linux >= 2.6.24
283 fields.append('guest')
284 if vlen >= 10:
285 # Linux >= 3.2.0
286 fields.append('guest_nice')
287 scputimes = namedtuple('scputimes', fields)
288
289
290try:
291 set_scputimes_ntuple("/proc")
292except Exception as err: # noqa: BLE001
293 # Don't want to crash at import time.
294 debug("ignoring exception on import: %r" % err)
295 scputimes = namedtuple('scputimes', 'user system idle')(0.0, 0.0, 0.0)
296
297
298# =====================================================================
299# --- prlimit
300# =====================================================================
301
302# Backport of resource.prlimit() for Python 2. Originally this was done
303# in C, but CentOS-6 which we use to create manylinux wheels is too old
304# and does not support prlimit() syscall. As such the resulting wheel
305# would not include prlimit(), even when installed on newer systems.
306# This is the only part of psutil using ctypes.
307
308prlimit = None
309try:
310 from resource import prlimit # python >= 3.4
311except ImportError:
312 import ctypes
313
314 libc = ctypes.CDLL(None, use_errno=True)
315
316 if hasattr(libc, "prlimit"):
317
318 def prlimit(pid, resource_, limits=None):
319 class StructRlimit(ctypes.Structure):
320 _fields_ = [
321 ('rlim_cur', ctypes.c_longlong),
322 ('rlim_max', ctypes.c_longlong),
323 ]
324
325 current = StructRlimit()
326 if limits is None:
327 # get
328 ret = libc.prlimit(pid, resource_, None, ctypes.byref(current))
329 else:
330 # set
331 new = StructRlimit()
332 new.rlim_cur = limits[0]
333 new.rlim_max = limits[1]
334 ret = libc.prlimit(
335 pid, resource_, ctypes.byref(new), ctypes.byref(current)
336 )
337
338 if ret != 0:
339 errno_ = ctypes.get_errno()
340 raise OSError(errno_, os.strerror(errno_))
341 return (current.rlim_cur, current.rlim_max)
342
343
344if prlimit is not None:
345 __extra__all__.extend(
346 [x for x in dir(cext) if x.startswith('RLIM') and x.isupper()]
347 )
348
349
350# =====================================================================
351# --- system memory
352# =====================================================================
353
354
355def calculate_avail_vmem(mems):
356 """Fallback for kernels < 3.14 where /proc/meminfo does not provide
357 "MemAvailable", see:
358 https://blog.famzah.net/2014/09/24/.
359
360 This code reimplements the algorithm outlined here:
361 https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/
362 commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773
363
364 We use this function also when "MemAvailable" returns 0 (possibly a
365 kernel bug, see: https://github.com/giampaolo/psutil/issues/1915).
366 In that case this routine matches "free" CLI tool result ("available"
367 column).
368
369 XXX: on recent kernels this calculation may differ by ~1.5% compared
370 to "MemAvailable:", as it's calculated slightly differently.
371 It is still way more realistic than doing (free + cached) though.
372 See:
373 * https://gitlab.com/procps-ng/procps/issues/42
374 * https://github.com/famzah/linux-memavailable-procfs/issues/2
375 """
376 # Note about "fallback" value. According to:
377 # https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/
378 # commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773
379 # ...long ago "available" memory was calculated as (free + cached),
380 # We use fallback when one of these is missing from /proc/meminfo:
381 # "Active(file)": introduced in 2.6.28 / Dec 2008
382 # "Inactive(file)": introduced in 2.6.28 / Dec 2008
383 # "SReclaimable": introduced in 2.6.19 / Nov 2006
384 # /proc/zoneinfo: introduced in 2.6.13 / Aug 2005
385 free = mems[b'MemFree:']
386 fallback = free + mems.get(b"Cached:", 0)
387 try:
388 lru_active_file = mems[b'Active(file):']
389 lru_inactive_file = mems[b'Inactive(file):']
390 slab_reclaimable = mems[b'SReclaimable:']
391 except KeyError as err:
392 debug(
393 "%s is missing from /proc/meminfo; using an approximation for "
394 "calculating available memory"
395 % err.args[0]
396 )
397 return fallback
398 try:
399 f = open_binary('%s/zoneinfo' % get_procfs_path())
400 except IOError:
401 return fallback # kernel 2.6.13
402
403 watermark_low = 0
404 with f:
405 for line in f:
406 line = line.strip()
407 if line.startswith(b'low'):
408 watermark_low += int(line.split()[1])
409 watermark_low *= PAGESIZE
410
411 avail = free - watermark_low
412 pagecache = lru_active_file + lru_inactive_file
413 pagecache -= min(pagecache / 2, watermark_low)
414 avail += pagecache
415 avail += slab_reclaimable - min(slab_reclaimable / 2.0, watermark_low)
416 return int(avail)
417
418
419def virtual_memory():
420 """Report virtual memory stats.
421 This implementation mimicks procps-ng-3.3.12, aka "free" CLI tool:
422 https://gitlab.com/procps-ng/procps/blob/
423 24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L778-791
424 The returned values are supposed to match both "free" and "vmstat -s"
425 CLI tools.
426 """
427 missing_fields = []
428 mems = {}
429 with open_binary('%s/meminfo' % get_procfs_path()) as f:
430 for line in f:
431 fields = line.split()
432 mems[fields[0]] = int(fields[1]) * 1024
433
434 # /proc doc states that the available fields in /proc/meminfo vary
435 # by architecture and compile options, but these 3 values are also
436 # returned by sysinfo(2); as such we assume they are always there.
437 total = mems[b'MemTotal:']
438 free = mems[b'MemFree:']
439 try:
440 buffers = mems[b'Buffers:']
441 except KeyError:
442 # https://github.com/giampaolo/psutil/issues/1010
443 buffers = 0
444 missing_fields.append('buffers')
445 try:
446 cached = mems[b"Cached:"]
447 except KeyError:
448 cached = 0
449 missing_fields.append('cached')
450 else:
451 # "free" cmdline utility sums reclaimable to cached.
452 # Older versions of procps used to add slab memory instead.
453 # This got changed in:
454 # https://gitlab.com/procps-ng/procps/commit/
455 # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e
456 cached += mems.get(b"SReclaimable:", 0) # since kernel 2.6.19
457
458 try:
459 shared = mems[b'Shmem:'] # since kernel 2.6.32
460 except KeyError:
461 try:
462 shared = mems[b'MemShared:'] # kernels 2.4
463 except KeyError:
464 shared = 0
465 missing_fields.append('shared')
466
467 try:
468 active = mems[b"Active:"]
469 except KeyError:
470 active = 0
471 missing_fields.append('active')
472
473 try:
474 inactive = mems[b"Inactive:"]
475 except KeyError:
476 try:
477 inactive = (
478 mems[b"Inact_dirty:"]
479 + mems[b"Inact_clean:"]
480 + mems[b"Inact_laundry:"]
481 )
482 except KeyError:
483 inactive = 0
484 missing_fields.append('inactive')
485
486 try:
487 slab = mems[b"Slab:"]
488 except KeyError:
489 slab = 0
490
491 used = total - free - cached - buffers
492 if used < 0:
493 # May be symptomatic of running within a LCX container where such
494 # values will be dramatically distorted over those of the host.
495 used = total - free
496
497 # - starting from 4.4.0 we match free's "available" column.
498 # Before 4.4.0 we calculated it as (free + buffers + cached)
499 # which matched htop.
500 # - free and htop available memory differs as per:
501 # http://askubuntu.com/a/369589
502 # http://unix.stackexchange.com/a/65852/168884
503 # - MemAvailable has been introduced in kernel 3.14
504 try:
505 avail = mems[b'MemAvailable:']
506 except KeyError:
507 avail = calculate_avail_vmem(mems)
508 else:
509 if avail == 0:
510 # Yes, it can happen (probably a kernel bug):
511 # https://github.com/giampaolo/psutil/issues/1915
512 # In this case "free" CLI tool makes an estimate. We do the same,
513 # and it matches "free" CLI tool.
514 avail = calculate_avail_vmem(mems)
515
516 if avail < 0:
517 avail = 0
518 missing_fields.append('available')
519 elif avail > total:
520 # If avail is greater than total or our calculation overflows,
521 # that's symptomatic of running within a LCX container where such
522 # values will be dramatically distorted over those of the host.
523 # https://gitlab.com/procps-ng/procps/blob/
524 # 24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L764
525 avail = free
526
527 percent = usage_percent((total - avail), total, round_=1)
528
529 # Warn about missing metrics which are set to 0.
530 if missing_fields:
531 msg = "%s memory stats couldn't be determined and %s set to 0" % (
532 ", ".join(missing_fields),
533 "was" if len(missing_fields) == 1 else "were",
534 )
535 warnings.warn(msg, RuntimeWarning, stacklevel=2)
536
537 return svmem(
538 total,
539 avail,
540 percent,
541 used,
542 free,
543 active,
544 inactive,
545 buffers,
546 cached,
547 shared,
548 slab,
549 )
550
551
552def swap_memory():
553 """Return swap memory metrics."""
554 mems = {}
555 with open_binary('%s/meminfo' % get_procfs_path()) as f:
556 for line in f:
557 fields = line.split()
558 mems[fields[0]] = int(fields[1]) * 1024
559 # We prefer /proc/meminfo over sysinfo() syscall so that
560 # psutil.PROCFS_PATH can be used in order to allow retrieval
561 # for linux containers, see:
562 # https://github.com/giampaolo/psutil/issues/1015
563 try:
564 total = mems[b'SwapTotal:']
565 free = mems[b'SwapFree:']
566 except KeyError:
567 _, _, _, _, total, free, unit_multiplier = cext.linux_sysinfo()
568 total *= unit_multiplier
569 free *= unit_multiplier
570
571 used = total - free
572 percent = usage_percent(used, total, round_=1)
573 # get pgin/pgouts
574 try:
575 f = open_binary("%s/vmstat" % get_procfs_path())
576 except IOError as err:
577 # see https://github.com/giampaolo/psutil/issues/722
578 msg = (
579 "'sin' and 'sout' swap memory stats couldn't "
580 + "be determined and were set to 0 (%s)" % str(err)
581 )
582 warnings.warn(msg, RuntimeWarning, stacklevel=2)
583 sin = sout = 0
584 else:
585 with f:
586 sin = sout = None
587 for line in f:
588 # values are expressed in 4 kilo bytes, we want
589 # bytes instead
590 if line.startswith(b'pswpin'):
591 sin = int(line.split(b' ')[1]) * 4 * 1024
592 elif line.startswith(b'pswpout'):
593 sout = int(line.split(b' ')[1]) * 4 * 1024
594 if sin is not None and sout is not None:
595 break
596 else:
597 # we might get here when dealing with exotic Linux
598 # flavors, see:
599 # https://github.com/giampaolo/psutil/issues/313
600 msg = "'sin' and 'sout' swap memory stats couldn't "
601 msg += "be determined and were set to 0"
602 warnings.warn(msg, RuntimeWarning, stacklevel=2)
603 sin = sout = 0
604 return _common.sswap(total, used, free, percent, sin, sout)
605
606
607# =====================================================================
608# --- CPU
609# =====================================================================
610
611
612def cpu_times():
613 """Return a named tuple representing the following system-wide
614 CPU times:
615 (user, nice, system, idle, iowait, irq, softirq [steal, [guest,
616 [guest_nice]]])
617 Last 3 fields may not be available on all Linux kernel versions.
618 """
619 procfs_path = get_procfs_path()
620 set_scputimes_ntuple(procfs_path)
621 with open_binary('%s/stat' % procfs_path) as f:
622 values = f.readline().split()
623 fields = values[1 : len(scputimes._fields) + 1]
624 fields = [float(x) / CLOCK_TICKS for x in fields]
625 return scputimes(*fields)
626
627
628def per_cpu_times():
629 """Return a list of namedtuple representing the CPU times
630 for every CPU available on the system.
631 """
632 procfs_path = get_procfs_path()
633 set_scputimes_ntuple(procfs_path)
634 cpus = []
635 with open_binary('%s/stat' % procfs_path) as f:
636 # get rid of the first line which refers to system wide CPU stats
637 f.readline()
638 for line in f:
639 if line.startswith(b'cpu'):
640 values = line.split()
641 fields = values[1 : len(scputimes._fields) + 1]
642 fields = [float(x) / CLOCK_TICKS for x in fields]
643 entry = scputimes(*fields)
644 cpus.append(entry)
645 return cpus
646
647
648def cpu_count_logical():
649 """Return the number of logical CPUs in the system."""
650 try:
651 return os.sysconf("SC_NPROCESSORS_ONLN")
652 except ValueError:
653 # as a second fallback we try to parse /proc/cpuinfo
654 num = 0
655 with open_binary('%s/cpuinfo' % get_procfs_path()) as f:
656 for line in f:
657 if line.lower().startswith(b'processor'):
658 num += 1
659
660 # unknown format (e.g. amrel/sparc architectures), see:
661 # https://github.com/giampaolo/psutil/issues/200
662 # try to parse /proc/stat as a last resort
663 if num == 0:
664 search = re.compile(r'cpu\d')
665 with open_text('%s/stat' % get_procfs_path()) as f:
666 for line in f:
667 line = line.split(' ')[0]
668 if search.match(line):
669 num += 1
670
671 if num == 0:
672 # mimic os.cpu_count()
673 return None
674 return num
675
676
677def cpu_count_cores():
678 """Return the number of CPU cores in the system."""
679 # Method #1
680 ls = set()
681 # These 2 files are the same but */core_cpus_list is newer while
682 # */thread_siblings_list is deprecated and may disappear in the future.
683 # https://www.kernel.org/doc/Documentation/admin-guide/cputopology.rst
684 # https://github.com/giampaolo/psutil/pull/1727#issuecomment-707624964
685 # https://lkml.org/lkml/2019/2/26/41
686 p1 = "/sys/devices/system/cpu/cpu[0-9]*/topology/core_cpus_list"
687 p2 = "/sys/devices/system/cpu/cpu[0-9]*/topology/thread_siblings_list"
688 for path in glob.glob(p1) or glob.glob(p2):
689 with open_binary(path) as f:
690 ls.add(f.read().strip())
691 result = len(ls)
692 if result != 0:
693 return result
694
695 # Method #2
696 mapping = {}
697 current_info = {}
698 with open_binary('%s/cpuinfo' % get_procfs_path()) as f:
699 for line in f:
700 line = line.strip().lower()
701 if not line:
702 # new section
703 try:
704 mapping[current_info[b'physical id']] = current_info[
705 b'cpu cores'
706 ]
707 except KeyError:
708 pass
709 current_info = {}
710 else:
711 # ongoing section
712 if line.startswith((b'physical id', b'cpu cores')):
713 key, value = line.split(b'\t:', 1)
714 current_info[key] = int(value)
715
716 result = sum(mapping.values())
717 return result or None # mimic os.cpu_count()
718
719
720def cpu_stats():
721 """Return various CPU stats as a named tuple."""
722 with open_binary('%s/stat' % get_procfs_path()) as f:
723 ctx_switches = None
724 interrupts = None
725 soft_interrupts = None
726 for line in f:
727 if line.startswith(b'ctxt'):
728 ctx_switches = int(line.split()[1])
729 elif line.startswith(b'intr'):
730 interrupts = int(line.split()[1])
731 elif line.startswith(b'softirq'):
732 soft_interrupts = int(line.split()[1])
733 if (
734 ctx_switches is not None
735 and soft_interrupts is not None
736 and interrupts is not None
737 ):
738 break
739 syscalls = 0
740 return _common.scpustats(
741 ctx_switches, interrupts, soft_interrupts, syscalls
742 )
743
744
745def _cpu_get_cpuinfo_freq():
746 """Return current CPU frequency from cpuinfo if available."""
747 ret = []
748 with open_binary('%s/cpuinfo' % get_procfs_path()) as f:
749 for line in f:
750 if line.lower().startswith(b'cpu mhz'):
751 ret.append(float(line.split(b':', 1)[1]))
752 return ret
753
754
755if os.path.exists("/sys/devices/system/cpu/cpufreq/policy0") or os.path.exists(
756 "/sys/devices/system/cpu/cpu0/cpufreq"
757):
758
759 def cpu_freq():
760 """Return frequency metrics for all CPUs.
761 Contrarily to other OSes, Linux updates these values in
762 real-time.
763 """
764 cpuinfo_freqs = _cpu_get_cpuinfo_freq()
765 paths = glob.glob(
766 "/sys/devices/system/cpu/cpufreq/policy[0-9]*"
767 ) or glob.glob("/sys/devices/system/cpu/cpu[0-9]*/cpufreq")
768 paths.sort(key=lambda x: int(re.search(r"[0-9]+", x).group()))
769 ret = []
770 pjoin = os.path.join
771 for i, path in enumerate(paths):
772 if len(paths) == len(cpuinfo_freqs):
773 # take cached value from cpuinfo if available, see:
774 # https://github.com/giampaolo/psutil/issues/1851
775 curr = cpuinfo_freqs[i] * 1000
776 else:
777 curr = bcat(pjoin(path, "scaling_cur_freq"), fallback=None)
778 if curr is None:
779 # Likely an old RedHat, see:
780 # https://github.com/giampaolo/psutil/issues/1071
781 curr = bcat(pjoin(path, "cpuinfo_cur_freq"), fallback=None)
782 if curr is None:
783 msg = "can't find current frequency file"
784 raise NotImplementedError(msg)
785 curr = int(curr) / 1000
786 max_ = int(bcat(pjoin(path, "scaling_max_freq"))) / 1000
787 min_ = int(bcat(pjoin(path, "scaling_min_freq"))) / 1000
788 ret.append(_common.scpufreq(curr, min_, max_))
789 return ret
790
791else:
792
793 def cpu_freq():
794 """Alternate implementation using /proc/cpuinfo.
795 min and max frequencies are not available and are set to None.
796 """
797 return [_common.scpufreq(x, 0.0, 0.0) for x in _cpu_get_cpuinfo_freq()]
798
799
800# =====================================================================
801# --- network
802# =====================================================================
803
804
805net_if_addrs = cext_posix.net_if_addrs
806
807
808class _Ipv6UnsupportedError(Exception):
809 pass
810
811
812class Connections:
813 """A wrapper on top of /proc/net/* files, retrieving per-process
814 and system-wide open connections (TCP, UDP, UNIX) similarly to
815 "netstat -an".
816
817 Note: in case of UNIX sockets we're only able to determine the
818 local endpoint/path, not the one it's connected to.
819 According to [1] it would be possible but not easily.
820
821 [1] http://serverfault.com/a/417946
822 """
823
824 def __init__(self):
825 # The string represents the basename of the corresponding
826 # /proc/net/{proto_name} file.
827 tcp4 = ("tcp", socket.AF_INET, socket.SOCK_STREAM)
828 tcp6 = ("tcp6", socket.AF_INET6, socket.SOCK_STREAM)
829 udp4 = ("udp", socket.AF_INET, socket.SOCK_DGRAM)
830 udp6 = ("udp6", socket.AF_INET6, socket.SOCK_DGRAM)
831 unix = ("unix", socket.AF_UNIX, None)
832 self.tmap = {
833 "all": (tcp4, tcp6, udp4, udp6, unix),
834 "tcp": (tcp4, tcp6),
835 "tcp4": (tcp4,),
836 "tcp6": (tcp6,),
837 "udp": (udp4, udp6),
838 "udp4": (udp4,),
839 "udp6": (udp6,),
840 "unix": (unix,),
841 "inet": (tcp4, tcp6, udp4, udp6),
842 "inet4": (tcp4, udp4),
843 "inet6": (tcp6, udp6),
844 }
845 self._procfs_path = None
846
847 def get_proc_inodes(self, pid):
848 inodes = defaultdict(list)
849 for fd in os.listdir("%s/%s/fd" % (self._procfs_path, pid)):
850 try:
851 inode = readlink("%s/%s/fd/%s" % (self._procfs_path, pid, fd))
852 except (FileNotFoundError, ProcessLookupError):
853 # ENOENT == file which is gone in the meantime;
854 # os.stat('/proc/%s' % self.pid) will be done later
855 # to force NSP (if it's the case)
856 continue
857 except OSError as err:
858 if err.errno == errno.EINVAL:
859 # not a link
860 continue
861 if err.errno == errno.ENAMETOOLONG:
862 # file name too long
863 debug(err)
864 continue
865 raise
866 else:
867 if inode.startswith('socket:['):
868 # the process is using a socket
869 inode = inode[8:][:-1]
870 inodes[inode].append((pid, int(fd)))
871 return inodes
872
873 def get_all_inodes(self):
874 inodes = {}
875 for pid in pids():
876 try:
877 inodes.update(self.get_proc_inodes(pid))
878 except (FileNotFoundError, ProcessLookupError, PermissionError):
879 # os.listdir() is gonna raise a lot of access denied
880 # exceptions in case of unprivileged user; that's fine
881 # as we'll just end up returning a connection with PID
882 # and fd set to None anyway.
883 # Both netstat -an and lsof does the same so it's
884 # unlikely we can do any better.
885 # ENOENT just means a PID disappeared on us.
886 continue
887 return inodes
888
889 @staticmethod
890 def decode_address(addr, family):
891 """Accept an "ip:port" address as displayed in /proc/net/*
892 and convert it into a human readable form, like:
893
894 "0500000A:0016" -> ("10.0.0.5", 22)
895 "0000000000000000FFFF00000100007F:9E49" -> ("::ffff:127.0.0.1", 40521)
896
897 The IP address portion is a little or big endian four-byte
898 hexadecimal number; that is, the least significant byte is listed
899 first, so we need to reverse the order of the bytes to convert it
900 to an IP address.
901 The port is represented as a two-byte hexadecimal number.
902
903 Reference:
904 http://linuxdevcenter.com/pub/a/linux/2000/11/16/LinuxAdmin.html
905 """
906 ip, port = addr.split(':')
907 port = int(port, 16)
908 # this usually refers to a local socket in listen mode with
909 # no end-points connected
910 if not port:
911 return ()
912 if PY3:
913 ip = ip.encode('ascii')
914 if family == socket.AF_INET:
915 # see: https://github.com/giampaolo/psutil/issues/201
916 if LITTLE_ENDIAN:
917 ip = socket.inet_ntop(family, base64.b16decode(ip)[::-1])
918 else:
919 ip = socket.inet_ntop(family, base64.b16decode(ip))
920 else: # IPv6
921 ip = base64.b16decode(ip)
922 try:
923 # see: https://github.com/giampaolo/psutil/issues/201
924 if LITTLE_ENDIAN:
925 ip = socket.inet_ntop(
926 socket.AF_INET6,
927 struct.pack('>4I', *struct.unpack('<4I', ip)),
928 )
929 else:
930 ip = socket.inet_ntop(
931 socket.AF_INET6,
932 struct.pack('<4I', *struct.unpack('<4I', ip)),
933 )
934 except ValueError:
935 # see: https://github.com/giampaolo/psutil/issues/623
936 if not supports_ipv6():
937 raise _Ipv6UnsupportedError
938 else:
939 raise
940 return _common.addr(ip, port)
941
942 @staticmethod
943 def process_inet(file, family, type_, inodes, filter_pid=None):
944 """Parse /proc/net/tcp* and /proc/net/udp* files."""
945 if file.endswith('6') and not os.path.exists(file):
946 # IPv6 not supported
947 return
948 with open_text(file) as f:
949 f.readline() # skip the first line
950 for lineno, line in enumerate(f, 1):
951 try:
952 _, laddr, raddr, status, _, _, _, _, _, inode = (
953 line.split()[:10]
954 )
955 except ValueError:
956 raise RuntimeError(
957 "error while parsing %s; malformed line %s %r"
958 % (file, lineno, line)
959 )
960 if inode in inodes:
961 # # We assume inet sockets are unique, so we error
962 # # out if there are multiple references to the
963 # # same inode. We won't do this for UNIX sockets.
964 # if len(inodes[inode]) > 1 and family != socket.AF_UNIX:
965 # raise ValueError("ambiguous inode with multiple "
966 # "PIDs references")
967 pid, fd = inodes[inode][0]
968 else:
969 pid, fd = None, -1
970 if filter_pid is not None and filter_pid != pid:
971 continue
972 else:
973 if type_ == socket.SOCK_STREAM:
974 status = TCP_STATUSES[status]
975 else:
976 status = _common.CONN_NONE
977 try:
978 laddr = Connections.decode_address(laddr, family)
979 raddr = Connections.decode_address(raddr, family)
980 except _Ipv6UnsupportedError:
981 continue
982 yield (fd, family, type_, laddr, raddr, status, pid)
983
984 @staticmethod
985 def process_unix(file, family, inodes, filter_pid=None):
986 """Parse /proc/net/unix files."""
987 with open_text(file) as f:
988 f.readline() # skip the first line
989 for line in f:
990 tokens = line.split()
991 try:
992 _, _, _, _, type_, _, inode = tokens[0:7]
993 except ValueError:
994 if ' ' not in line:
995 # see: https://github.com/giampaolo/psutil/issues/766
996 continue
997 raise RuntimeError(
998 "error while parsing %s; malformed line %r"
999 % (file, line)
1000 )
1001 if inode in inodes: # noqa
1002 # With UNIX sockets we can have a single inode
1003 # referencing many file descriptors.
1004 pairs = inodes[inode]
1005 else:
1006 pairs = [(None, -1)]
1007 for pid, fd in pairs:
1008 if filter_pid is not None and filter_pid != pid:
1009 continue
1010 else:
1011 path = tokens[-1] if len(tokens) == 8 else ''
1012 type_ = _common.socktype_to_enum(int(type_))
1013 # XXX: determining the remote endpoint of a
1014 # UNIX socket on Linux is not possible, see:
1015 # https://serverfault.com/questions/252723/
1016 raddr = ""
1017 status = _common.CONN_NONE
1018 yield (fd, family, type_, path, raddr, status, pid)
1019
1020 def retrieve(self, kind, pid=None):
1021 if kind not in self.tmap:
1022 raise ValueError(
1023 "invalid %r kind argument; choose between %s"
1024 % (kind, ', '.join([repr(x) for x in self.tmap]))
1025 )
1026 self._procfs_path = get_procfs_path()
1027 if pid is not None:
1028 inodes = self.get_proc_inodes(pid)
1029 if not inodes:
1030 # no connections for this process
1031 return []
1032 else:
1033 inodes = self.get_all_inodes()
1034 ret = set()
1035 for proto_name, family, type_ in self.tmap[kind]:
1036 path = "%s/net/%s" % (self._procfs_path, proto_name)
1037 if family in (socket.AF_INET, socket.AF_INET6):
1038 ls = self.process_inet(
1039 path, family, type_, inodes, filter_pid=pid
1040 )
1041 else:
1042 ls = self.process_unix(path, family, inodes, filter_pid=pid)
1043 for fd, family, type_, laddr, raddr, status, bound_pid in ls:
1044 if pid:
1045 conn = _common.pconn(
1046 fd, family, type_, laddr, raddr, status
1047 )
1048 else:
1049 conn = _common.sconn(
1050 fd, family, type_, laddr, raddr, status, bound_pid
1051 )
1052 ret.add(conn)
1053 return list(ret)
1054
1055
1056_connections = Connections()
1057
1058
1059def net_connections(kind='inet'):
1060 """Return system-wide open connections."""
1061 return _connections.retrieve(kind)
1062
1063
1064def net_io_counters():
1065 """Return network I/O statistics for every network interface
1066 installed on the system as a dict of raw tuples.
1067 """
1068 with open_text("%s/net/dev" % get_procfs_path()) as f:
1069 lines = f.readlines()
1070 retdict = {}
1071 for line in lines[2:]:
1072 colon = line.rfind(':')
1073 assert colon > 0, repr(line)
1074 name = line[:colon].strip()
1075 fields = line[colon + 1 :].strip().split()
1076
1077 # in
1078 (
1079 bytes_recv,
1080 packets_recv,
1081 errin,
1082 dropin,
1083 fifoin, # unused
1084 framein, # unused
1085 compressedin, # unused
1086 multicastin, # unused
1087 # out
1088 bytes_sent,
1089 packets_sent,
1090 errout,
1091 dropout,
1092 fifoout, # unused
1093 collisionsout, # unused
1094 carrierout, # unused
1095 compressedout,
1096 ) = map(int, fields)
1097
1098 retdict[name] = (
1099 bytes_sent,
1100 bytes_recv,
1101 packets_sent,
1102 packets_recv,
1103 errin,
1104 errout,
1105 dropin,
1106 dropout,
1107 )
1108 return retdict
1109
1110
1111def net_if_stats():
1112 """Get NIC stats (isup, duplex, speed, mtu)."""
1113 duplex_map = {
1114 cext.DUPLEX_FULL: NIC_DUPLEX_FULL,
1115 cext.DUPLEX_HALF: NIC_DUPLEX_HALF,
1116 cext.DUPLEX_UNKNOWN: NIC_DUPLEX_UNKNOWN,
1117 }
1118 names = net_io_counters().keys()
1119 ret = {}
1120 for name in names:
1121 try:
1122 mtu = cext_posix.net_if_mtu(name)
1123 flags = cext_posix.net_if_flags(name)
1124 duplex, speed = cext.net_if_duplex_speed(name)
1125 except OSError as err:
1126 # https://github.com/giampaolo/psutil/issues/1279
1127 if err.errno != errno.ENODEV:
1128 raise
1129 else:
1130 debug(err)
1131 else:
1132 output_flags = ','.join(flags)
1133 isup = 'running' in flags
1134 ret[name] = _common.snicstats(
1135 isup, duplex_map[duplex], speed, mtu, output_flags
1136 )
1137 return ret
1138
1139
1140# =====================================================================
1141# --- disks
1142# =====================================================================
1143
1144
1145disk_usage = _psposix.disk_usage
1146
1147
1148def disk_io_counters(perdisk=False):
1149 """Return disk I/O statistics for every disk installed on the
1150 system as a dict of raw tuples.
1151 """
1152
1153 def read_procfs():
1154 # OK, this is a bit confusing. The format of /proc/diskstats can
1155 # have 3 variations.
1156 # On Linux 2.4 each line has always 15 fields, e.g.:
1157 # "3 0 8 hda 8 8 8 8 8 8 8 8 8 8 8"
1158 # On Linux 2.6+ each line *usually* has 14 fields, and the disk
1159 # name is in another position, like this:
1160 # "3 0 hda 8 8 8 8 8 8 8 8 8 8 8"
1161 # ...unless (Linux 2.6) the line refers to a partition instead
1162 # of a disk, in which case the line has less fields (7):
1163 # "3 1 hda1 8 8 8 8"
1164 # 4.18+ has 4 fields added:
1165 # "3 0 hda 8 8 8 8 8 8 8 8 8 8 8 0 0 0 0"
1166 # 5.5 has 2 more fields.
1167 # See:
1168 # https://www.kernel.org/doc/Documentation/iostats.txt
1169 # https://www.kernel.org/doc/Documentation/ABI/testing/procfs-diskstats
1170 with open_text("%s/diskstats" % get_procfs_path()) as f:
1171 lines = f.readlines()
1172 for line in lines:
1173 fields = line.split()
1174 flen = len(fields)
1175 # fmt: off
1176 if flen == 15:
1177 # Linux 2.4
1178 name = fields[3]
1179 reads = int(fields[2])
1180 (reads_merged, rbytes, rtime, writes, writes_merged,
1181 wbytes, wtime, _, busy_time, _) = map(int, fields[4:14])
1182 elif flen == 14 or flen >= 18:
1183 # Linux 2.6+, line referring to a disk
1184 name = fields[2]
1185 (reads, reads_merged, rbytes, rtime, writes, writes_merged,
1186 wbytes, wtime, _, busy_time, _) = map(int, fields[3:14])
1187 elif flen == 7:
1188 # Linux 2.6+, line referring to a partition
1189 name = fields[2]
1190 reads, rbytes, writes, wbytes = map(int, fields[3:])
1191 rtime = wtime = reads_merged = writes_merged = busy_time = 0
1192 else:
1193 raise ValueError("not sure how to interpret line %r" % line)
1194 yield (name, reads, writes, rbytes, wbytes, rtime, wtime,
1195 reads_merged, writes_merged, busy_time)
1196 # fmt: on
1197
1198 def read_sysfs():
1199 for block in os.listdir('/sys/block'):
1200 for root, _, files in os.walk(os.path.join('/sys/block', block)):