4 * @remark Copyright 2002 OProfile authors
5 * @remark Read the file COPYING
7 * @author John Levon <levon@movementarian.org>
9 * This is the core of the buffer management. Each
10 * CPU buffer is processed and entered into the
11 * global event buffer. Such processing is necessary
12 * in several circumstances, mentioned below.
14 * The processing does the job of converting the
15 * transitory EIP value into a persistent dentry/offset
16 * value that the profiler can record at its leisure.
18 * See fs/dcookies.c for a description of the dentry/offset
23 #include <linux/workqueue.h>
24 #include <linux/notifier.h>
25 #include <linux/dcookies.h>
26 #include <linux/profile.h>
27 #include <linux/module.h>
30 #include "oprofile_stats.h"
31 #include "event_buffer.h"
32 #include "cpu_buffer.h"
33 #include "buffer_sync.h"
35 static LIST_HEAD(dying_tasks);
36 static LIST_HEAD(dead_tasks);
37 cpumask_t marked_cpus = CPU_MASK_NONE;
38 static spinlock_t task_mortuary = SPIN_LOCK_UNLOCKED;
39 void process_task_mortuary(void);
42 /* Take ownership of the task struct and place it on the
43 * list for processing. Only after two full buffer syncs
44 * does the task eventually get freed, because by then
45 * we are sure we will not reference it again.
47 static int task_free_notify(struct notifier_block * self, unsigned long val, void * data)
49 struct task_struct * task = (struct task_struct *)data;
50 spin_lock(&task_mortuary);
51 list_add(&task->tasks, &dying_tasks);
52 spin_unlock(&task_mortuary);
57 /* The task is on its way out. A sync of the buffer means we can catch
58 * any remaining samples for this task.
60 static int task_exit_notify(struct notifier_block * self, unsigned long val, void * data)
62 /* To avoid latency problems, we only process the current CPU,
63 * hoping that most samples for the task are on this CPU
65 sync_buffer(smp_processor_id());
70 /* The task is about to try a do_munmap(). We peek at what it's going to
71 * do, and if it's an executable region, process the samples first, so
72 * we don't lose any. This does not have to be exact, it's a QoI issue
75 static int munmap_notify(struct notifier_block * self, unsigned long val, void * data)
77 unsigned long addr = (unsigned long)data;
78 struct mm_struct * mm = current->mm;
79 struct vm_area_struct * mpnt;
81 down_read(&mm->mmap_sem);
83 mpnt = find_vma(mm, addr);
84 if (mpnt && mpnt->vm_file && (mpnt->vm_flags & VM_EXEC)) {
85 up_read(&mm->mmap_sem);
86 /* To avoid latency problems, we only process the current CPU,
87 * hoping that most samples for the task are on this CPU
89 sync_buffer(smp_processor_id());
93 up_read(&mm->mmap_sem);
98 /* We need to be told about new modules so we don't attribute to a previously
99 * loaded module, or drop the samples on the floor.
101 static int module_load_notify(struct notifier_block * self, unsigned long val, void * data)
103 #ifdef CONFIG_MODULES
104 if (val != MODULE_STATE_COMING)
107 /* FIXME: should we process all CPU buffers ? */
109 add_event_entry(ESCAPE_CODE);
110 add_event_entry(MODULE_LOADED_CODE);
117 static struct notifier_block task_free_nb = {
118 .notifier_call = task_free_notify,
121 static struct notifier_block task_exit_nb = {
122 .notifier_call = task_exit_notify,
125 static struct notifier_block munmap_nb = {
126 .notifier_call = munmap_notify,
129 static struct notifier_block module_load_nb = {
130 .notifier_call = module_load_notify,
134 static void end_sync(void)
137 /* make sure we don't leak task structs */
138 process_task_mortuary();
139 process_task_mortuary();
149 err = task_handoff_register(&task_free_nb);
152 err = profile_event_register(PROFILE_TASK_EXIT, &task_exit_nb);
155 err = profile_event_register(PROFILE_MUNMAP, &munmap_nb);
158 err = register_module_notifier(&module_load_nb);
165 profile_event_unregister(PROFILE_MUNMAP, &munmap_nb);
167 profile_event_unregister(PROFILE_TASK_EXIT, &task_exit_nb);
169 task_handoff_unregister(&task_free_nb);
178 unregister_module_notifier(&module_load_nb);
179 profile_event_unregister(PROFILE_MUNMAP, &munmap_nb);
180 profile_event_unregister(PROFILE_TASK_EXIT, &task_exit_nb);
181 task_handoff_unregister(&task_free_nb);
186 /* Optimisation. We can manage without taking the dcookie sem
187 * because we cannot reach this code without at least one
188 * dcookie user still being registered (namely, the reader
189 * of the event buffer). */
190 static inline unsigned long fast_get_dcookie(struct dentry * dentry,
191 struct vfsmount * vfsmnt)
193 unsigned long cookie;
195 if (dentry->d_cookie)
196 return (unsigned long)dentry;
197 get_dcookie(dentry, vfsmnt, &cookie);
202 /* Look up the dcookie for the task's first VM_EXECUTABLE mapping,
203 * which corresponds loosely to "application name". This is
204 * not strictly necessary but allows oprofile to associate
205 * shared-library samples with particular applications
207 static unsigned long get_exec_dcookie(struct mm_struct * mm)
209 unsigned long cookie = 0;
210 struct vm_area_struct * vma;
215 for (vma = mm->mmap; vma; vma = vma->vm_next) {
218 if (!(vma->vm_flags & VM_EXECUTABLE))
220 cookie = fast_get_dcookie(vma->vm_file->f_dentry,
221 vma->vm_file->f_vfsmnt);
230 /* Convert the EIP value of a sample into a persistent dentry/offset
231 * pair that can then be added to the global event buffer. We make
232 * sure to do this lookup before a mm->mmap modification happens so
233 * we don't lose track.
235 static unsigned long lookup_dcookie(struct mm_struct * mm, unsigned long addr, off_t * offset)
237 unsigned long cookie = 0;
238 struct vm_area_struct * vma;
240 for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
245 if (addr < vma->vm_start || addr >= vma->vm_end)
248 cookie = fast_get_dcookie(vma->vm_file->f_dentry,
249 vma->vm_file->f_vfsmnt);
250 *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
258 static unsigned long last_cookie = ~0UL;
260 static void add_cpu_switch(int i)
262 add_event_entry(ESCAPE_CODE);
263 add_event_entry(CPU_SWITCH_CODE);
268 static void add_kernel_ctx_switch(unsigned int in_kernel)
270 add_event_entry(ESCAPE_CODE);
272 add_event_entry(KERNEL_ENTER_SWITCH_CODE);
274 add_event_entry(KERNEL_EXIT_SWITCH_CODE);
278 add_user_ctx_switch(struct task_struct const * task, unsigned long cookie)
280 add_event_entry(ESCAPE_CODE);
281 add_event_entry(CTX_SWITCH_CODE);
282 add_event_entry(task->pid);
283 add_event_entry(cookie);
284 /* Another code for daemon back-compat */
285 add_event_entry(ESCAPE_CODE);
286 add_event_entry(CTX_TGID_CODE);
287 add_event_entry(task->tgid);
291 static void add_cookie_switch(unsigned long cookie)
293 add_event_entry(ESCAPE_CODE);
294 add_event_entry(COOKIE_SWITCH_CODE);
295 add_event_entry(cookie);
299 static void add_sample_entry(unsigned long offset, unsigned long event)
301 add_event_entry(offset);
302 add_event_entry(event);
306 static void add_us_sample(struct mm_struct * mm, struct op_sample * s)
308 unsigned long cookie;
311 cookie = lookup_dcookie(mm, s->eip, &offset);
314 atomic_inc(&oprofile_stats.sample_lost_no_mapping);
318 if (cookie != last_cookie) {
319 add_cookie_switch(cookie);
320 last_cookie = cookie;
323 add_sample_entry(offset, s->event);
327 /* Add a sample to the global event buffer. If possible the
328 * sample is converted into a persistent dentry/offset pair
329 * for later lookup from userspace.
331 static void add_sample(struct mm_struct * mm, struct op_sample * s, int in_kernel)
334 add_sample_entry(s->eip, s->event);
336 add_us_sample(mm, s);
338 atomic_inc(&oprofile_stats.sample_lost_no_mm);
343 static void release_mm(struct mm_struct * mm)
347 up_read(&mm->mmap_sem);
352 static struct mm_struct * take_tasks_mm(struct task_struct * task)
354 struct mm_struct * mm = get_task_mm(task);
356 down_read(&mm->mmap_sem);
361 static inline int is_ctx_switch(unsigned long val)
367 /* "acquire" as many cpu buffer slots as we can */
368 static unsigned long get_slots(struct oprofile_cpu_buffer * b)
370 unsigned long head = b->head_pos;
371 unsigned long tail = b->tail_pos;
374 * Subtle. This resets the persistent last_task
375 * and in_kernel values used for switching notes.
376 * BUT, there is a small window between reading
377 * head_pos, and this call, that means samples
378 * can appear at the new head position, but not
379 * be prefixed with the notes for switching
380 * kernel mode or a task switch. This small hole
381 * can lead to mis-attribution or samples where
382 * we don't know if it's in the kernel or not,
383 * at the start of an event buffer.
390 return head + (b->buffer_size - tail);
394 static void increment_tail(struct oprofile_cpu_buffer * b)
396 unsigned long new_tail = b->tail_pos + 1;
400 if (new_tail < (b->buffer_size))
401 b->tail_pos = new_tail;
407 /* Move tasks along towards death. Any tasks on dead_tasks
408 * will definitely have no remaining references in any
409 * CPU buffers at this point, because we use two lists,
410 * and to have reached the list, it must have gone through
411 * one full sync already.
413 void process_task_mortuary(void)
415 struct list_head * pos;
416 struct list_head * pos2;
417 struct task_struct * task;
419 spin_lock(&task_mortuary);
421 list_for_each_safe(pos, pos2, &dead_tasks) {
422 task = list_entry(pos, struct task_struct, tasks);
423 list_del(&task->tasks);
427 list_for_each_safe(pos, pos2, &dying_tasks) {
428 task = list_entry(pos, struct task_struct, tasks);
429 list_del(&task->tasks);
430 list_add_tail(&task->tasks, &dead_tasks);
433 spin_unlock(&task_mortuary);
437 static void mark_done(int cpu)
441 cpu_set(cpu, marked_cpus);
443 for_each_online_cpu(i) {
444 if (!cpu_isset(i, marked_cpus))
448 /* All CPUs have been processed at least once,
449 * we can process the mortuary once
451 process_task_mortuary();
453 cpus_clear(marked_cpus);
457 /* Sync one of the CPU's buffers into the global event buffer.
458 * Here we need to go through each batch of samples punctuated
459 * by context switch notes, taking the task's mmap_sem and doing
460 * lookup in task->mm->mmap to convert EIP into dcookie/offset
463 void sync_buffer(int cpu)
465 struct oprofile_cpu_buffer * cpu_buf = &cpu_buffer[cpu];
466 struct mm_struct *mm = NULL;
467 struct task_struct * new;
468 unsigned long cookie = 0;
471 unsigned long available;
477 /* Remember, only we can modify tail_pos */
479 available = get_slots(cpu_buf);
481 for (i=0; i < available; ++i) {
482 struct op_sample * s = &cpu_buf->buffer[cpu_buf->tail_pos];
484 if (is_ctx_switch(s->eip)) {
486 /* kernel/userspace switch */
487 in_kernel = s->event;
488 add_kernel_ctx_switch(s->event);
490 struct mm_struct * oldmm = mm;
492 /* userspace context switch */
493 new = (struct task_struct *)s->event;
496 mm = take_tasks_mm(new);
498 cookie = get_exec_dcookie(mm);
499 add_user_ctx_switch(new, cookie);
502 add_sample(mm, s, in_kernel);
505 increment_tail(cpu_buf);