diff options
Diffstat (limited to 'contrib')
-rwxr-xr-x | contrib/ps_mem.py | 477 |
1 files changed, 477 insertions, 0 deletions
diff --git a/contrib/ps_mem.py b/contrib/ps_mem.py new file mode 100755 index 000000000..1b0dbd411 --- /dev/null +++ b/contrib/ps_mem.py | |||
@@ -0,0 +1,477 @@ | |||
1 | #!/usr/bin/env python | ||
2 | |||
3 | # Try to determine how much RAM is currently being used per program. | ||
4 | # Note per _program_, not per process. So for example this script | ||
5 | # will report RAM used by all httpd process together. In detail it reports: | ||
6 | # sum(private RAM for program processes) + sum(Shared RAM for program processes) | ||
7 | # The shared RAM is problematic to calculate, and this script automatically | ||
8 | # selects the most accurate method available for your kernel. | ||
9 | |||
10 | # Licence: LGPLv2 | ||
11 | # Author: P@draigBrady.com | ||
12 | # Source: http://www.pixelbeat.org/scripts/ps_mem.py | ||
13 | |||
14 | # V1.0 06 Jul 2005 Initial release | ||
15 | # V1.1 11 Aug 2006 root permission required for accuracy | ||
16 | # V1.2 08 Nov 2006 Add total to output | ||
17 | # Use KiB,MiB,... for units rather than K,M,... | ||
18 | # V1.3 22 Nov 2006 Ignore shared col from /proc/$pid/statm for | ||
19 | # 2.6 kernels up to and including 2.6.9. | ||
20 | # There it represented the total file backed extent | ||
21 | # V1.4 23 Nov 2006 Remove total from output as it's meaningless | ||
22 | # (the shared values overlap with other programs). | ||
23 | # Display the shared column. This extra info is | ||
24 | # useful, especially as it overlaps between programs. | ||
25 | # V1.5 26 Mar 2007 Remove redundant recursion from human() | ||
26 | # V1.6 05 Jun 2007 Also report number of processes with a given name. | ||
27 | # Patch from riccardo.murri@gmail.com | ||
28 | # V1.7 20 Sep 2007 Use PSS from /proc/$pid/smaps if available, which | ||
29 | # fixes some over-estimation and allows totalling. | ||
30 | # Enumerate the PIDs directly rather than using ps, | ||
31 | # which fixes the possible race between reading | ||
32 | # RSS with ps, and shared memory with this program. | ||
33 | # Also we can show non truncated command names. | ||
34 | # V1.8 28 Sep 2007 More accurate matching for stats in /proc/$pid/smaps | ||
35 | # as otherwise could match libraries causing a crash. | ||
36 | # Patch from patrice.bouchand.fedora@gmail.com | ||
37 | # V1.9 20 Feb 2008 Fix invalid values reported when PSS is available. | ||
38 | # Reported by Andrey Borzenkov <arvidjaar@mail.ru> | ||
39 | # V3.1 10 May 2013 | ||
40 | # http://github.com/pixelb/scripts/commits/master/scripts/ps_mem.py | ||
41 | |||
42 | # Notes: | ||
43 | # | ||
44 | # All interpreted programs where the interpreter is started | ||
45 | # by the shell or with env, will be merged to the interpreter | ||
46 | # (as that's what's given to exec). For e.g. all python programs | ||
47 | # starting with "#!/usr/bin/env python" will be grouped under python. | ||
48 | # You can change this by using the full command line but that will | ||
49 | # have the undesirable affect of splitting up programs started with | ||
50 | # differing parameters (for e.g. mingetty tty[1-6]). | ||
51 | # | ||
52 | # For 2.6 kernels up to and including 2.6.13 and later 2.4 redhat kernels | ||
53 | # (rmap vm without smaps) it can not be accurately determined how many pages | ||
54 | # are shared between processes in general or within a program in our case: | ||
55 | # http://lkml.org/lkml/2005/7/6/250 | ||
56 | # A warning is printed if overestimation is possible. | ||
57 | # In addition for 2.6 kernels up to 2.6.9 inclusive, the shared | ||
58 | # value in /proc/$pid/statm is the total file-backed extent of a process. | ||
59 | # We ignore that, introducing more overestimation, again printing a warning. | ||
60 | # Since kernel 2.6.23-rc8-mm1 PSS is available in smaps, which allows | ||
61 | # us to calculate a more accurate value for the total RAM used by programs. | ||
62 | # | ||
63 | # Programs that use CLONE_VM without CLONE_THREAD are discounted by assuming | ||
64 | # they're the only programs that have the same /proc/$PID/smaps file for | ||
65 | # each instance. This will fail if there are multiple real instances of a | ||
66 | # program that then use CLONE_VM without CLONE_THREAD, or if a clone changes | ||
67 | # its memory map while we're checksumming each /proc/$PID/smaps. | ||
68 | # | ||
69 | # I don't take account of memory allocated for a program | ||
70 | # by other programs. For e.g. memory used in the X server for | ||
71 | # a program could be determined, but is not. | ||
72 | # | ||
73 | # FreeBSD is supported if linprocfs is mounted at /compat/linux/proc/ | ||
74 | # FreeBSD 8.0 supports up to a level of Linux 2.6.16 | ||
75 | |||
76 | # TODO/FIXME: The script currently requires root permission to gather | ||
77 | # memory usage details about all the processes. This restriction | ||
78 | # has to be relaxed --- when running without root only the user's | ||
79 | # processes details should be displayed | ||
80 | |||
81 | import getopt | ||
82 | import time | ||
83 | import errno | ||
84 | import os | ||
85 | import sys | ||
86 | |||
87 | try: | ||
88 | # md5 module is deprecated on python 2.6 | ||
89 | # so try the newer hashlib first | ||
90 | import hashlib | ||
91 | md5_new = hashlib.md5 | ||
92 | except ImportError: | ||
93 | import md5 | ||
94 | md5_new = md5.new | ||
95 | |||
96 | |||
97 | # The following exits cleanly on Ctrl-C or EPIPE | ||
98 | # while treating other exceptions as before. | ||
99 | def std_exceptions(etype, value, tb): | ||
100 | sys.excepthook = sys.__excepthook__ | ||
101 | if issubclass(etype, KeyboardInterrupt): | ||
102 | pass | ||
103 | elif issubclass(etype, IOError) and value.errno == errno.EPIPE: | ||
104 | pass | ||
105 | else: | ||
106 | sys.__excepthook__(etype, value, tb) | ||
107 | sys.excepthook = std_exceptions | ||
108 | |||
109 | # | ||
110 | # Define some global variables | ||
111 | # | ||
112 | |||
113 | PAGESIZE = os.sysconf("SC_PAGE_SIZE") / 1024 #KiB | ||
114 | our_pid = os.getpid() | ||
115 | |||
116 | have_pss = 0 | ||
117 | |||
118 | class Proc: | ||
119 | def __init__(self): | ||
120 | uname = os.uname() | ||
121 | if uname[0] == "FreeBSD": | ||
122 | self.proc = '/compat/linux/proc' | ||
123 | else: | ||
124 | self.proc = '/proc' | ||
125 | |||
126 | def path(self, *args): | ||
127 | return os.path.join(self.proc, *(str(a) for a in args)) | ||
128 | |||
129 | def open(self, *args): | ||
130 | try: | ||
131 | return open(self.path(*args)) | ||
132 | except (IOError, OSError): | ||
133 | val = sys.exc_info()[1] | ||
134 | if (val.errno == errno.ENOENT or # kernel thread or process gone | ||
135 | val.errno == errno.EPERM): | ||
136 | raise LookupError | ||
137 | |||
138 | proc = Proc() | ||
139 | |||
140 | |||
141 | # | ||
142 | # Functions | ||
143 | # | ||
144 | |||
145 | def parse_options(): | ||
146 | try: | ||
147 | long_options = ['split-args', 'help'] | ||
148 | opts, args = getopt.getopt(sys.argv[1:], "shp:w:", long_options) | ||
149 | except getopt.GetoptError: | ||
150 | sys.stderr.write(help()) | ||
151 | sys.exit(3) | ||
152 | |||
153 | # ps_mem.py options | ||
154 | split_args = False | ||
155 | pids_to_show = None | ||
156 | watch = None | ||
157 | |||
158 | for o, a in opts: | ||
159 | if o in ('-s', '--split-args'): | ||
160 | split_args = True | ||
161 | if o in ('-h', '--help'): | ||
162 | sys.stdout.write(help()) | ||
163 | sys.exit(0) | ||
164 | if o in ('-p',): | ||
165 | try: | ||
166 | pids_to_show = [int(x) for x in a.split(',')] | ||
167 | except: | ||
168 | sys.stderr.write(help()) | ||
169 | sys.exit(3) | ||
170 | if o in ('-w',): | ||
171 | try: | ||
172 | watch = int(a) | ||
173 | except: | ||
174 | sys.stderr.write(help()) | ||
175 | sys.exit(3) | ||
176 | |||
177 | return (split_args, pids_to_show, watch) | ||
178 | |||
179 | def help(): | ||
180 | help_msg = 'ps_mem.py - Show process memory usage\n'\ | ||
181 | '\n'\ | ||
182 | '-h Show this help\n'\ | ||
183 | '-w <N> Measure and show process memory every N seconds\n'\ | ||
184 | '-p <pid>[,pid2,...pidN] Only show memory usage PIDs in the specified list\n' | ||
185 | |||
186 | return help_msg | ||
187 | |||
188 | #(major,minor,release) | ||
189 | def kernel_ver(): | ||
190 | kv = proc.open('sys/kernel/osrelease').readline().split(".")[:3] | ||
191 | last = len(kv) | ||
192 | if last == 2: | ||
193 | kv.append('0') | ||
194 | last -= 1 | ||
195 | for char in "-_": | ||
196 | kv[last] = kv[last].split(char)[0] | ||
197 | try: | ||
198 | int(kv[last]) | ||
199 | except: | ||
200 | kv[last] = 0 | ||
201 | return (int(kv[0]), int(kv[1]), int(kv[2])) | ||
202 | |||
203 | |||
204 | #return Private,Shared | ||
205 | #Note shared is always a subset of rss (trs is not always) | ||
206 | def getMemStats(pid): | ||
207 | global have_pss | ||
208 | mem_id = pid #unique | ||
209 | Private_lines = [] | ||
210 | Shared_lines = [] | ||
211 | Pss_lines = [] | ||
212 | Rss = (int(proc.open(pid, 'statm').readline().split()[1]) | ||
213 | * PAGESIZE) | ||
214 | if os.path.exists(proc.path(pid, 'smaps')): #stat | ||
215 | digester = md5_new() | ||
216 | for line in proc.open(pid, 'smaps').readlines(): #open | ||
217 | # Note we checksum smaps as maps is usually but | ||
218 | # not always different for separate processes. | ||
219 | digester.update(line.encode('latin1')) | ||
220 | if line.startswith("Shared"): | ||
221 | Shared_lines.append(line) | ||
222 | elif line.startswith("Private"): | ||
223 | Private_lines.append(line) | ||
224 | elif line.startswith("Pss"): | ||
225 | have_pss = 1 | ||
226 | Pss_lines.append(line) | ||
227 | mem_id = digester.hexdigest() | ||
228 | Shared = sum([int(line.split()[1]) for line in Shared_lines]) | ||
229 | Private = sum([int(line.split()[1]) for line in Private_lines]) | ||
230 | #Note Shared + Private = Rss above | ||
231 | #The Rss in smaps includes video card mem etc. | ||
232 | if have_pss: | ||
233 | pss_adjust = 0.5 # add 0.5KiB as this avg error due to trunctation | ||
234 | Pss = sum([float(line.split()[1])+pss_adjust for line in Pss_lines]) | ||
235 | Shared = Pss - Private | ||
236 | elif (2,6,1) <= kernel_ver() <= (2,6,9): | ||
237 | Shared = 0 #lots of overestimation, but what can we do? | ||
238 | Private = Rss | ||
239 | else: | ||
240 | Shared = int(proc.open(pid, 'statm').readline().split()[2]) | ||
241 | Shared *= PAGESIZE | ||
242 | Private = Rss - Shared | ||
243 | return (Private, Shared, mem_id) | ||
244 | |||
245 | |||
246 | def getCmdName(pid, split_args): | ||
247 | cmdline = proc.open(pid, 'cmdline').read().split("\0") | ||
248 | if cmdline[-1] == '' and len(cmdline) > 1: | ||
249 | cmdline = cmdline[:-1] | ||
250 | |||
251 | path = proc.path(pid, 'exe') | ||
252 | try: | ||
253 | path = os.readlink(path) | ||
254 | # Some symlink targets were seen to contain NULs on RHEL 5 at least | ||
255 | # https://github.com/pixelb/scripts/pull/10, so take string up to NUL | ||
256 | path = path.split('\0')[0] | ||
257 | except OSError: | ||
258 | val = sys.exc_info()[1] | ||
259 | if (val.errno == errno.ENOENT or # either kernel thread or process gone | ||
260 | val.errno == errno.EPERM): | ||
261 | raise LookupError | ||
262 | |||
263 | if split_args: | ||
264 | return " ".join(cmdline) | ||
265 | if path.endswith(" (deleted)"): | ||
266 | path = path[:-10] | ||
267 | if os.path.exists(path): | ||
268 | path += " [updated]" | ||
269 | else: | ||
270 | #The path could be have prelink stuff so try cmdline | ||
271 | #which might have the full path present. This helped for: | ||
272 | #/usr/libexec/notification-area-applet.#prelink#.fX7LCT (deleted) | ||
273 | if os.path.exists(cmdline[0]): | ||
274 | path = cmdline[0] + " [updated]" | ||
275 | else: | ||
276 | path += " [deleted]" | ||
277 | exe = os.path.basename(path) | ||
278 | cmd = proc.open(pid, 'status').readline()[6:-1] | ||
279 | if exe.startswith(cmd): | ||
280 | cmd = exe #show non truncated version | ||
281 | #Note because we show the non truncated name | ||
282 | #one can have separated programs as follows: | ||
283 | #584.0 KiB + 1.0 MiB = 1.6 MiB mozilla-thunder (exe -> bash) | ||
284 | # 56.0 MiB + 22.2 MiB = 78.2 MiB mozilla-thunderbird-bin | ||
285 | return cmd | ||
286 | |||
287 | |||
288 | #The following matches "du -h" output | ||
289 | #see also human.py | ||
290 | def human(num, power="Ki"): | ||
291 | powers = ["Ki", "Mi", "Gi", "Ti"] | ||
292 | while num >= 1000: #4 digits | ||
293 | num /= 1024.0 | ||
294 | power = powers[powers.index(power)+1] | ||
295 | return "%.1f %s" % (num, power) | ||
296 | |||
297 | |||
298 | def cmd_with_count(cmd, count): | ||
299 | if count > 1: | ||
300 | return "%s (%u)" % (cmd, count) | ||
301 | else: | ||
302 | return cmd | ||
303 | |||
304 | #Warn of possible inaccuracies | ||
305 | #2 = accurate & can total | ||
306 | #1 = accurate only considering each process in isolation | ||
307 | #0 = some shared mem not reported | ||
308 | #-1= all shared mem not reported | ||
309 | def shared_val_accuracy(): | ||
310 | """http://wiki.apache.org/spamassassin/TopSharedMemoryBug""" | ||
311 | kv = kernel_ver() | ||
312 | if kv[:2] == (2,4): | ||
313 | if proc.open('meminfo').read().find("Inact_") == -1: | ||
314 | return 1 | ||
315 | return 0 | ||
316 | elif kv[:2] == (2,6): | ||
317 | pid = os.getpid() | ||
318 | if os.path.exists(proc.path(pid, 'smaps')): | ||
319 | if proc.open(pid, 'smaps').read().find("Pss:")!=-1: | ||
320 | return 2 | ||
321 | else: | ||
322 | return 1 | ||
323 | if (2,6,1) <= kv <= (2,6,9): | ||
324 | return -1 | ||
325 | return 0 | ||
326 | elif kv[0] > 2: | ||
327 | return 2 | ||
328 | else: | ||
329 | return 1 | ||
330 | |||
331 | def show_shared_val_accuracy( possible_inacc ): | ||
332 | if possible_inacc == -1: | ||
333 | sys.stderr.write( | ||
334 | "Warning: Shared memory is not reported by this system.\n" | ||
335 | ) | ||
336 | sys.stderr.write( | ||
337 | "Values reported will be too large, and totals are not reported\n" | ||
338 | ) | ||
339 | elif possible_inacc == 0: | ||
340 | sys.stderr.write( | ||
341 | "Warning: Shared memory is not reported accurately by this system.\n" | ||
342 | ) | ||
343 | sys.stderr.write( | ||
344 | "Values reported could be too large, and totals are not reported\n" | ||
345 | ) | ||
346 | elif possible_inacc == 1: | ||
347 | sys.stderr.write( | ||
348 | "Warning: Shared memory is slightly over-estimated by this system\n" | ||
349 | "for each program, so totals are not reported.\n" | ||
350 | ) | ||
351 | sys.stderr.close() | ||
352 | |||
353 | def get_memory_usage( pids_to_show, split_args, include_self=False, only_self=False ): | ||
354 | cmds = {} | ||
355 | shareds = {} | ||
356 | mem_ids = {} | ||
357 | count = {} | ||
358 | for pid in os.listdir(proc.path('')): | ||
359 | if not pid.isdigit(): | ||
360 | continue | ||
361 | pid = int(pid) | ||
362 | |||
363 | # Some filters | ||
364 | if only_self and pid != our_pid: | ||
365 | continue | ||
366 | if pid == our_pid and not include_self: | ||
367 | continue | ||
368 | if pids_to_show is not None and pid not in pids_to_show: | ||
369 | continue | ||
370 | |||
371 | try: | ||
372 | cmd = getCmdName(pid, split_args) | ||
373 | except LookupError: | ||
374 | #permission denied or | ||
375 | #kernel threads don't have exe links or | ||
376 | #process gone | ||
377 | continue | ||
378 | |||
379 | try: | ||
380 | private, shared, mem_id = getMemStats(pid) | ||
381 | except RuntimeError: | ||
382 | continue #process gone | ||
383 | if shareds.get(cmd): | ||
384 | if have_pss: #add shared portion of PSS together | ||
385 | shareds[cmd] += shared | ||
386 | elif shareds[cmd] < shared: #just take largest shared val | ||
387 | shareds[cmd] = shared | ||
388 | else: | ||
389 | shareds[cmd] = shared | ||
390 | cmds[cmd] = cmds.setdefault(cmd, 0) + private | ||
391 | if cmd in count: | ||
392 | count[cmd] += 1 | ||
393 | else: | ||
394 | count[cmd] = 1 | ||
395 | mem_ids.setdefault(cmd, {}).update({mem_id:None}) | ||
396 | |||
397 | #Add shared mem for each program | ||
398 | total = 0 | ||
399 | for cmd in cmds: | ||
400 | cmd_count = count[cmd] | ||
401 | if len(mem_ids[cmd]) == 1 and cmd_count > 1: | ||
402 | # Assume this program is using CLONE_VM without CLONE_THREAD | ||
403 | # so only account for one of the processes | ||
404 | cmds[cmd] /= cmd_count | ||
405 | if have_pss: | ||
406 | shareds[cmd] /= cmd_count | ||
407 | cmds[cmd] = cmds[cmd] + shareds[cmd] | ||
408 | total += cmds[cmd] #valid if PSS available | ||
409 | |||
410 | sorted_cmds = sorted(cmds.items(), key=lambda x:x[1]) | ||
411 | sorted_cmds = [x for x in sorted_cmds if x[1]] | ||
412 | |||
413 | return sorted_cmds, shareds, count, total | ||
414 | |||
415 | def print_header(): | ||
416 | sys.stdout.write(" Private + Shared = RAM used\tProgram \n\n") | ||
417 | |||
418 | def print_memory_usage(sorted_cmds, shareds, count, total): | ||
419 | for cmd in sorted_cmds: | ||
420 | sys.stdout.write("%8sB + %8sB = %8sB\t%s\n" % | ||
421 | (human(cmd[1]-shareds[cmd[0]]), | ||
422 | human(shareds[cmd[0]]), human(cmd[1]), | ||
423 | cmd_with_count(cmd[0], count[cmd[0]]))) | ||
424 | if have_pss: | ||
425 | sys.stdout.write("%s\n%s%8sB\n%s\n" % | ||
426 | ("-" * 33, " " * 24, human(total), "=" * 33)) | ||
427 | |||
428 | def verify_environment(): | ||
429 | if os.geteuid() != 0: | ||
430 | sys.stderr.write("Sorry, root permission required.\n") | ||
431 | if __name__ == '__main__': | ||
432 | sys.stderr.close() | ||
433 | sys.exit(1) | ||
434 | |||
435 | try: | ||
436 | kv = kernel_ver() | ||
437 | except (IOError, OSError): | ||
438 | val = sys.exc_info()[1] | ||
439 | if val.errno == errno.ENOENT: | ||
440 | sys.stderr.write( | ||
441 | "Couldn't access " + proc.path('') + "\n" | ||
442 | "Only GNU/Linux and FreeBSD (with linprocfs) are supported\n") | ||
443 | sys.exit(2) | ||
444 | else: | ||
445 | raise | ||
446 | |||
447 | if __name__ == '__main__': | ||
448 | verify_environment() | ||
449 | split_args, pids_to_show, watch = parse_options() | ||
450 | |||
451 | print_header() | ||
452 | |||
453 | if watch is not None: | ||
454 | try: | ||
455 | sorted_cmds = True | ||
456 | while sorted_cmds: | ||
457 | sorted_cmds, shareds, count, total = get_memory_usage( pids_to_show, split_args ) | ||
458 | print_memory_usage(sorted_cmds, shareds, count, total) | ||
459 | time.sleep(watch) | ||
460 | else: | ||
461 | sys.stdout.write('Process does not exist anymore.\n') | ||
462 | except KeyboardInterrupt: | ||
463 | pass | ||
464 | else: | ||
465 | # This is the default behavior | ||
466 | sorted_cmds, shareds, count, total = get_memory_usage( pids_to_show, split_args ) | ||
467 | print_memory_usage(sorted_cmds, shareds, count, total) | ||
468 | |||
469 | |||
470 | # We must close explicitly, so that any EPIPE exception | ||
471 | # is handled by our excepthook, rather than the default | ||
472 | # one which is reenabled after this script finishes. | ||
473 | sys.stdout.close() | ||
474 | |||
475 | vm_accuracy = shared_val_accuracy() | ||
476 | show_shared_val_accuracy( vm_accuracy ) | ||
477 | |||