4 * @remark Copyright 2002 OProfile authors
5 * @remark Read the file COPYING
7 * @author John Levon <levon@movementarian.org>
9 * This is the core of the buffer management. Each
10 * CPU buffer is processed and entered into the
11 * global event buffer. Such processing is necessary
12 * in several circumstances, mentioned below.
14 * The processing does the job of converting the
15 * transitory EIP value into a persistent dentry/offset
16 * value that the profiler can record at its leisure.
18 * See fs/dcookies.c for a description of the dentry/offset
23 #include <linux/workqueue.h>
24 #include <linux/notifier.h>
25 #include <linux/dcookies.h>
26 #include <linux/profile.h>
27 #include <linux/module.h>
30 #include "oprofile_stats.h"
31 #include "event_buffer.h"
32 #include "cpu_buffer.h"
33 #include "buffer_sync.h"
35 #define DEFAULT_EXPIRE (HZ / 4)
37 static void wq_sync_buffers(void *);
38 static DECLARE_WORK(sync_wq, wq_sync_buffers, NULL);
40 static struct timer_list sync_timer;
41 static void timer_ping(unsigned long data);
42 static void sync_cpu_buffers(void);
45 /* We must make sure to process every entry in the CPU buffers
46 * before a task got the PF_EXITING flag, otherwise we will hold
47 * references to a possibly freed task_struct. We are safe with
48 * samples past the PF_EXITING point in do_exit(), because we
49 * explicitly check for that in cpu_buffer.c
51 static int exit_task_notify(struct notifier_block * self, unsigned long val, void * data)
57 /* There are two cases of tasks modifying task->mm->mmap list we
58 * must concern ourselves with. First, when a task is about to
59 * exit (exit_mmap()), we should process the buffer to deal with
60 * any samples in the CPU buffer, before we lose the ->mmap information
61 * we need. It is vital to get this case correct, otherwise we can
62 * end up trying to access a freed task_struct.
64 static int mm_notify(struct notifier_block * self, unsigned long val, void * data)
71 /* Second, a task may unmap (part of) an executable mmap,
72 * so we want to process samples before that happens too. This is merely
73 * a QOI issue not a correctness one.
75 static int munmap_notify(struct notifier_block * self, unsigned long val, void * data)
77 /* Note that we cannot sync the buffers directly, because we might end up
78 * taking the the mmap_sem that we hold now inside of event_buffer_read()
79 * on a page fault, whilst holding buffer_sem - deadlock.
81 * This would mean a threaded reader of the event buffer, but we should
84 * Delaying the work in a context that doesn't hold the mmap_sem means
85 * that we won't lose samples from other mappings that current() may
86 * have. Note that either way, we lose any pending samples for what is
89 schedule_work(&sync_wq);
94 /* We need to be told about new modules so we don't attribute to a previously
95 * loaded module, or drop the samples on the floor.
97 static int module_load_notify(struct notifier_block * self, unsigned long val, void * data)
100 if (val != MODULE_STATE_COMING)
105 add_event_entry(ESCAPE_CODE);
106 add_event_entry(MODULE_LOADED_CODE);
113 static struct notifier_block exit_task_nb = {
114 .notifier_call = exit_task_notify,
117 static struct notifier_block exec_unmap_nb = {
118 .notifier_call = munmap_notify,
121 static struct notifier_block exit_mmap_nb = {
122 .notifier_call = mm_notify,
125 static struct notifier_block module_load_nb = {
126 .notifier_call = module_load_notify,
130 static void end_sync_timer(void)
132 del_timer_sync(&sync_timer);
133 /* timer might have queued work, make sure it's completed. */
134 flush_scheduled_work();
142 init_timer(&sync_timer);
143 sync_timer.function = timer_ping;
144 sync_timer.expires = jiffies + DEFAULT_EXPIRE;
145 add_timer(&sync_timer);
147 err = profile_event_register(EXIT_TASK, &exit_task_nb);
150 err = profile_event_register(EXIT_MMAP, &exit_mmap_nb);
153 err = profile_event_register(EXEC_UNMAP, &exec_unmap_nb);
156 err = register_module_notifier(&module_load_nb);
163 profile_event_unregister(EXEC_UNMAP, &exec_unmap_nb);
165 profile_event_unregister(EXIT_MMAP, &exit_mmap_nb);
167 profile_event_unregister(EXIT_TASK, &exit_task_nb);
176 unregister_module_notifier(&module_load_nb);
177 profile_event_unregister(EXIT_TASK, &exit_task_nb);
178 profile_event_unregister(EXIT_MMAP, &exit_mmap_nb);
179 profile_event_unregister(EXEC_UNMAP, &exec_unmap_nb);
184 /* Optimisation. We can manage without taking the dcookie sem
185 * because we cannot reach this code without at least one
186 * dcookie user still being registered (namely, the reader
187 * of the event buffer). */
188 static inline unsigned long fast_get_dcookie(struct dentry * dentry,
189 struct vfsmount * vfsmnt)
191 unsigned long cookie;
193 if (dentry->d_cookie)
194 return (unsigned long)dentry;
195 get_dcookie(dentry, vfsmnt, &cookie);
200 /* Look up the dcookie for the task's first VM_EXECUTABLE mapping,
201 * which corresponds loosely to "application name". This is
202 * not strictly necessary but allows oprofile to associate
203 * shared-library samples with particular applications
205 static unsigned long get_exec_dcookie(struct mm_struct * mm)
207 unsigned long cookie = 0;
208 struct vm_area_struct * vma;
213 for (vma = mm->mmap; vma; vma = vma->vm_next) {
216 if (!(vma->vm_flags & VM_EXECUTABLE))
218 cookie = fast_get_dcookie(vma->vm_file->f_dentry,
219 vma->vm_file->f_vfsmnt);
228 /* Convert the EIP value of a sample into a persistent dentry/offset
229 * pair that can then be added to the global event buffer. We make
230 * sure to do this lookup before a mm->mmap modification happens so
231 * we don't lose track.
233 static unsigned long lookup_dcookie(struct mm_struct * mm, unsigned long addr, off_t * offset)
235 unsigned long cookie = 0;
236 struct vm_area_struct * vma;
238 for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
243 if (addr < vma->vm_start || addr >= vma->vm_end)
246 cookie = fast_get_dcookie(vma->vm_file->f_dentry,
247 vma->vm_file->f_vfsmnt);
248 *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
256 static unsigned long last_cookie = ~0UL;
258 static void add_cpu_switch(int i)
260 add_event_entry(ESCAPE_CODE);
261 add_event_entry(CPU_SWITCH_CODE);
266 static void add_kernel_ctx_switch(unsigned int in_kernel)
268 add_event_entry(ESCAPE_CODE);
270 add_event_entry(KERNEL_ENTER_SWITCH_CODE);
272 add_event_entry(KERNEL_EXIT_SWITCH_CODE);
276 add_user_ctx_switch(struct task_struct const * task, unsigned long cookie)
278 add_event_entry(ESCAPE_CODE);
279 add_event_entry(CTX_SWITCH_CODE);
280 add_event_entry(task->pid);
281 add_event_entry(cookie);
282 /* Another code for daemon back-compat */
283 add_event_entry(ESCAPE_CODE);
284 add_event_entry(CTX_TGID_CODE);
285 add_event_entry(task->tgid);
289 static void add_cookie_switch(unsigned long cookie)
291 add_event_entry(ESCAPE_CODE);
292 add_event_entry(COOKIE_SWITCH_CODE);
293 add_event_entry(cookie);
297 static void add_sample_entry(unsigned long offset, unsigned long event)
299 add_event_entry(offset);
300 add_event_entry(event);
304 static void add_us_sample(struct mm_struct * mm, struct op_sample * s)
306 unsigned long cookie;
309 cookie = lookup_dcookie(mm, s->eip, &offset);
312 atomic_inc(&oprofile_stats.sample_lost_no_mapping);
316 if (cookie != last_cookie) {
317 add_cookie_switch(cookie);
318 last_cookie = cookie;
321 add_sample_entry(offset, s->event);
325 /* Add a sample to the global event buffer. If possible the
326 * sample is converted into a persistent dentry/offset pair
327 * for later lookup from userspace.
329 static void add_sample(struct mm_struct * mm, struct op_sample * s, int in_kernel)
332 add_sample_entry(s->eip, s->event);
334 add_us_sample(mm, s);
336 atomic_inc(&oprofile_stats.sample_lost_no_mm);
341 static void release_mm(struct mm_struct * mm)
344 up_read(&mm->mmap_sem);
348 /* Take the task's mmap_sem to protect ourselves from
349 * races when we do lookup_dcookie().
351 static struct mm_struct * take_tasks_mm(struct task_struct * task)
353 struct mm_struct * mm;
355 /* Subtle. We don't need to keep a reference to this task's mm,
356 * because, for the mm to be freed on another CPU, that would have
357 * to go through the task exit notifier, which ends up sleeping
358 * on the buffer_sem we hold, so we end up with mutual exclusion
366 /* needed to walk the task's VMAs */
367 down_read(&mm->mmap_sem);
374 static inline int is_ctx_switch(unsigned long val)
380 /* "acquire" as many cpu buffer slots as we can */
381 static unsigned long get_slots(struct oprofile_cpu_buffer * b)
383 unsigned long head = b->head_pos;
384 unsigned long tail = b->tail_pos;
387 * Subtle. This resets the persistent last_task
388 * and in_kernel values used for switching notes.
389 * BUT, there is a small window between reading
390 * head_pos, and this call, that means samples
391 * can appear at the new head position, but not
392 * be prefixed with the notes for switching
393 * kernel mode or a task switch. This small hole
394 * can lead to mis-attribution or samples where
395 * we don't know if it's in the kernel or not,
396 * at the start of an event buffer.
403 return head + (b->buffer_size - tail);
407 static void increment_tail(struct oprofile_cpu_buffer * b)
409 unsigned long new_tail = b->tail_pos + 1;
413 if (new_tail < (b->buffer_size))
414 b->tail_pos = new_tail;
420 /* Sync one of the CPU's buffers into the global event buffer.
421 * Here we need to go through each batch of samples punctuated
422 * by context switch notes, taking the task's mmap_sem and doing
423 * lookup in task->mm->mmap to convert EIP into dcookie/offset
426 static void sync_buffer(struct oprofile_cpu_buffer * cpu_buf)
428 struct mm_struct *mm = NULL;
429 struct task_struct * new;
430 unsigned long cookie = 0;
434 /* Remember, only we can modify tail_pos */
436 unsigned long const available = get_slots(cpu_buf);
438 for (i=0; i < available; ++i) {
439 struct op_sample * s = &cpu_buf->buffer[cpu_buf->tail_pos];
441 if (is_ctx_switch(s->eip)) {
443 /* kernel/userspace switch */
444 in_kernel = s->event;
445 add_kernel_ctx_switch(s->event);
447 struct mm_struct * oldmm = mm;
449 /* userspace context switch */
450 new = (struct task_struct *)s->event;
453 mm = take_tasks_mm(new);
455 cookie = get_exec_dcookie(mm);
456 add_user_ctx_switch(new, cookie);
459 add_sample(mm, s, in_kernel);
462 increment_tail(cpu_buf);
468 /* Process each CPU's local buffer into the global
471 static void sync_cpu_buffers(void)
477 for (i = 0; i < NR_CPUS; ++i) {
478 struct oprofile_cpu_buffer * cpu_buf;
480 if (!cpu_possible(i))
483 cpu_buf = &cpu_buffer[i];
486 sync_buffer(cpu_buf);
491 mod_timer(&sync_timer, jiffies + DEFAULT_EXPIRE);
495 static void wq_sync_buffers(void * data)
501 /* It is possible that we could have no munmap() or
502 * other events for a period of time. This will lead
503 * the CPU buffers to overflow and lose samples and
504 * context switches. We try to reduce the problem
505 * by timing out when nothing happens for a while.
507 static void timer_ping(unsigned long data)
509 schedule_work(&sync_wq);
510 /* timer is re-added by the scheduled task */