From: Marc Fiuczynski <mef@cs.princeton.edu>
Date: Wed, 8 Sep 2004 19:41:04 +0000 (+0000)
Subject: ckrm-E15 memory controller
X-Git-Tag: ckrm_E15-mem-controller~1
X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=041b7df03818bb535b04201c39a6b290bf2f36f0;p=linux-2.6.git
ckrm-E15 memory controller
---
diff --git a/fs/exec.c b/fs/exec.c
index 61fba8f37..ba44671ed 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -47,6 +47,7 @@
#include <linux/syscalls.h>
#include <linux/rmap.h>
#include <linux/ckrm.h>
+#include <linux/ckrm_mem.h>
#include <asm/uaccess.h>
#include <asm/pgalloc.h>
@@ -547,6 +548,18 @@ static int exec_mmap(struct mm_struct *mm)
tsk->active_mm = mm;
activate_mm(active_mm, mm);
task_unlock(tsk);
+#ifdef CONFIG_CKRM_RES_MEM
+ if (old_mm) {
+ spin_lock(&old_mm->peertask_lock);
+ list_del(&tsk->mm_peers);
+ ckrm_mem_evaluate_mm(old_mm);
+ spin_unlock(&old_mm->peertask_lock);
+ }
+ spin_lock(&mm->peertask_lock);
+ list_add_tail(&tsk->mm_peers, &mm->tasklist);
+ ckrm_mem_evaluate_mm(mm);
+ spin_unlock(&mm->peertask_lock);
+#endif
if (old_mm) {
if (active_mm != old_mm) BUG();
mmput(old_mm);
diff --git a/include/linux/ckrm_rc.h b/include/linux/ckrm_rc.h
index b46cfd9f3..f4031671e 100644
--- a/include/linux/ckrm_rc.h
+++ b/include/linux/ckrm_rc.h
@@ -223,7 +223,17 @@ typedef struct ckrm_core_class {
* OTHER
******************************************************************************/
-#define ckrm_get_res_class(rescls,resid,type) ((type*)((rescls)->res_class[resid]))
+#define ckrm_get_res_class(rescls, resid, type) \
+ ((type*) (((resid != -1) && ((rescls) != NULL) \
+ && ((rescls) != (void *)-1)) ? \
+ ((struct ckrm_core_class *)(rescls))->res_class[resid] : NULL))
+
+#define ckrm_set_res_class(rescls, resid, value) \
+do { \
+ if ((resid != -1) && ((rescls) != NULL)) { \
+ ((struct ckrm_core_class *)(rescls))->res_class[resid] = value; \
+ } \
+} while (0)
extern int ckrm_register_res_ctlr(struct ckrm_classtype *, ckrm_res_ctlr_t *);
extern int ckrm_unregister_res_ctlr(ckrm_res_ctlr_t *);
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 8f8a8a3a3..a2bd104b7 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -228,6 +228,9 @@ struct page {
void *virtual; /* Kernel virtual address (NULL if
not kmapped, ie. highmem) */
#endif /* WANT_PAGE_VIRTUAL */
+#ifdef CONFIG_CKRM_RES_MEM
+ void *memclass;
+#endif // CONFIG_CKRM_RES_MEM
};
/*
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 47762ca69..5edb739b4 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -1,9 +1,11 @@
+#include <linux/ckrm_mem_inline.h>
static inline void
add_page_to_active_list(struct zone *zone, struct page *page)
{
list_add(&page->lru, &zone->active_list);
zone->nr_active++;
+ ckrm_mem_inc_active(page);
}
static inline void
@@ -11,6 +13,7 @@ add_page_to_inactive_list(struct zone *zone, struct page *page)
{
list_add(&page->lru, &zone->inactive_list);
zone->nr_inactive++;
+ ckrm_mem_inc_inactive(page);
}
static inline void
@@ -18,6 +21,7 @@ del_page_from_active_list(struct zone *zone, struct page *page)
{
list_del(&page->lru);
zone->nr_active--;
+ ckrm_mem_dec_active(page);
}
static inline void
@@ -25,6 +29,7 @@ del_page_from_inactive_list(struct zone *zone, struct page *page)
{
list_del(&page->lru);
zone->nr_inactive--;
+ ckrm_mem_dec_inactive(page);
}
static inline void
@@ -34,7 +39,9 @@ del_page_from_lru(struct zone *zone, struct page *page)
if (PageActive(page)) {
ClearPageActive(page);
zone->nr_active--;
+ ckrm_mem_dec_active(page);
} else {
zone->nr_inactive--;
+ ckrm_mem_dec_inactive(page);
}
}
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 0a3efe462..a5a1d8ed9 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -235,6 +235,11 @@ struct mm_struct {
struct kioctx *ioctx_list;
struct kioctx default_kioctx;
+#ifdef CONFIG_CKRM_RES_MEM
+ struct ckrm_mem_res *memclass;
+ struct list_head tasklist; /* list of all tasks sharing this address space */
+ spinlock_t peertask_lock; /* protect above tasklist */
+#endif
};
extern int mmlist_nr;
@@ -522,12 +527,11 @@ struct task_struct {
// .. Hubertus should change to CONFIG_CKRM_TYPE_TASKCLASS
struct ckrm_task_class *taskclass;
struct list_head taskclass_link;
-#ifdef CONFIG_CKRM_CPU_SCHEDULE
- struct ckrm_cpu_class *cpu_class;
-#endif
#endif // CONFIG_CKRM_TYPE_TASKCLASS
+#ifdef CONFIG_CKRM_RES_MEM
+ struct list_head mm_peers; // list of tasks using same mm_struct
+#endif // CONFIG_CKRM_RES_MEM
#endif // CONFIG_CKRM
-
struct task_delay_info delays;
};
@@ -974,7 +978,8 @@ static inline struct mm_struct * get_task_mm(struct task_struct * task)
return mm;
}
-
+
+
/* set thread flags in other task's structures
* - see asm/thread_info.h for TIF_xxxx flags available
*/
diff --git a/init/Kconfig b/init/Kconfig
index 579d847e1..c6888dbea 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -168,23 +168,25 @@ config CKRM_RES_NUMTASKS
Say N if unsure, Y to use the feature.
-config CKRM_CPU_SCHEDULE
- bool "CKRM CPU scheduler"
- depends on CKRM_TYPE_TASKCLASS
- default m
+config CKRM_RES_MEM
+ bool "Class based physical memory controller"
+ default y
+ depends on CKRM
help
- Use CKRM CPU scheduler instead of Linux Scheduler
-
- Say N if unsure, Y to use the feature.
+ Provide the basic support for collecting physical memory usage information
+ among classes. Say Y if you want to know the memory usage of each class.
-config CKRM_CPU_MONITOR
- tristate "CKRM CPU Resoure Monitor"
- depends on CKRM_CPU_SCHEDULE
- default m
+config CKRM_MEM_LRUORDER_CHANGE
+ bool "Change the LRU ordering of scanned pages"
+ default n
+ depends on CKRM_RES_MEM
help
- Monitor CPU Resource Usage of the classes
-
- Say N if unsure, Y to use the feature.
+ While trying to free pages, by default(n), scanned pages are left were they
+ are found if they belong to relatively under-used class. In this case the
+ LRU ordering of the memory subsystemis left intact. If this option is chosen,
+ then the scanned pages are moved to the tail of the list(active or inactive).
+ Changing this to yes reduces the checking overhead but violates the approximate
+ LRU order that is maintained by the paging subsystem.
config CKRM_TYPE_SOCKETCLASS
bool "Class Manager for socket groups"
diff --git a/init/main.c b/init/main.c
index 6cc592bec..7080522cd 100644
--- a/init/main.c
+++ b/init/main.c
@@ -43,12 +43,11 @@
#include <linux/efi.h>
#include <linux/unistd.h>
#include <linux/rmap.h>
+
#include <asm/io.h>
#include <asm/bugs.h>
#include <linux/ckrm.h>
-int __init init_ckrm_sched_res(void);
-
/*
* This is one of the first .c files built. Error out early
@@ -416,6 +415,7 @@ asmlinkage void __init start_kernel(void)
* printk() and can access its per-cpu storage.
*/
smp_prepare_boot_cpu();
+
/*
* Set up the scheduler prior starting any interrupts (such as the
* timer interrupt). Full topology setup happens at smp_init()
@@ -629,8 +629,8 @@ static int init(void * unused)
* firmware files.
*/
populate_rootfs();
+
do_basic_setup();
- init_ckrm_sched_res();
/*
* check if there is an early userspace init. If yes, let it do all
diff --git a/kernel/Makefile b/kernel/Makefile
index bbba09802..107df8eb1 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -9,7 +9,6 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o profile.o \
rcupdate.o intermodule.o extable.o params.o posix-timers.o \
kthread.o ckrm/
-obj-$(CONFIG_ZTRACE) += swap_hook.o
obj-$(CONFIG_FUTEX) += futex.o
obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o
obj-$(CONFIG_SMP) += cpu.o
@@ -22,12 +21,8 @@ obj-$(CONFIG_COMPAT) += compat.o
obj-$(CONFIG_IKCONFIG) += configs.o
obj-$(CONFIG_IKCONFIG_PROC) += configs.o
obj-$(CONFIG_STOP_MACHINE) += stop_machine.o
-obj-$(CONFIG_CKRM_CPU_SCHEDULE) += ckrm_classqueue.o
-obj-$(CONFIG_CKRM_CPU_SCHEDULE) += ckrm_sched.o
obj-$(CONFIG_AUDIT) += audit.o
obj-$(CONFIG_AUDITSYSCALL) += auditsc.o
-obj-$(CONFIG_KGDB) += kgdbstub.o
-
ifneq ($(CONFIG_IA64),y)
# According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
diff --git a/kernel/ckrm/Makefile b/kernel/ckrm/Makefile
index 3da88775d..0ff956a92 100644
--- a/kernel/ckrm/Makefile
+++ b/kernel/ckrm/Makefile
@@ -7,7 +7,6 @@ ifeq ($(CONFIG_CKRM),y)
endif
obj-$(CONFIG_CKRM_TYPE_TASKCLASS) += ckrm_tc.o
obj-$(CONFIG_CKRM_RES_NUMTASKS) += ckrm_tasks.o
+ obj-$(CONFIG_CKRM_RES_MEM) += ckrm_mem.o
obj-$(CONFIG_CKRM_TYPE_SOCKETCLASS) += ckrm_sockc.o
obj-$(CONFIG_CKRM_RES_LISTENAQ) += ckrm_listenaq.o
- obj-$(CONFIG_CKRM_CPU_SCHEDULE) += ckrm_cpu_class.o
- obj-$(CONFIG_CKRM_CPU_MONITOR) += ckrm_cpu_monitor.o
diff --git a/kernel/ckrm/rbce/rbcemod.c b/kernel/ckrm/rbce/rbcemod.c
index f61d0879c..fffa9ad1f 100644
--- a/kernel/ckrm/rbce/rbcemod.c
+++ b/kernel/ckrm/rbce/rbcemod.c
@@ -497,7 +497,6 @@ rbce_class_deletecb(const char *classname, void *classobj, int classtype)
}
}
}
- put_class(cls);
if ((cls = find_class_name(classname)) != NULL) {
printk(KERN_ERR
"rbce ERROR: class %s exists in rbce after "
diff --git a/kernel/exit.c b/kernel/exit.c
index f53583e2b..333f8003e 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -25,6 +25,7 @@
#include <linux/mempolicy.h>
#include <linux/ckrm.h>
#include <linux/ckrm_tsk.h>
+#include <linux/ckrm_mem.h>
#include <asm/uaccess.h>
#include <asm/unistd.h>
@@ -513,6 +514,12 @@ static inline void __exit_mm(struct task_struct * tsk)
task_lock(tsk);
tsk->mm = NULL;
up_read(&mm->mmap_sem);
+#ifdef CONFIG_CKRM_RES_MEM
+ spin_lock(&mm->peertask_lock);
+ list_del_init(&tsk->mm_peers);
+ ckrm_mem_evaluate_mm(mm);
+ spin_unlock(&mm->peertask_lock);
+#endif
enter_lazy_tlb(mm, current);
task_unlock(tsk);
mmput(mm);
diff --git a/kernel/fork.c b/kernel/fork.c
index 4af488db0..deb264307 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -38,6 +38,7 @@
#include <linux/rmap.h>
#include <linux/ckrm.h>
#include <linux/ckrm_tsk.h>
+#include <linux/ckrm_mem_inline.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
@@ -265,6 +266,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig)
ckrm_cb_newtask(tsk);
/* One for us, one for whoever does the "release_task()" (usually parent) */
atomic_set(&tsk->usage,2);
+#ifdef CONFIG_CKRM_RES_MEM
+ INIT_LIST_HEAD(&tsk->mm_peers);
+#endif
return tsk;
}
@@ -417,6 +421,10 @@ static struct mm_struct * mm_init(struct mm_struct * mm)
mm->ioctx_list = NULL;
mm->default_kioctx = (struct kioctx)INIT_KIOCTX(mm->default_kioctx, *mm);
mm->free_area_cache = TASK_UNMAPPED_BASE;
+#ifdef CONFIG_CKRM_RES_MEM
+ INIT_LIST_HEAD(&mm->tasklist);
+ mm->peertask_lock = SPIN_LOCK_UNLOCKED;
+#endif
if (likely(!mm_alloc_pgd(mm))) {
mm->def_flags = 0;
@@ -437,6 +445,10 @@ struct mm_struct * mm_alloc(void)
if (mm) {
memset(mm, 0, sizeof(*mm));
mm = mm_init(mm);
+#ifdef CONFIG_CKRM_RES_MEM
+ mm->memclass = GET_MEM_CLASS(current);
+ mem_class_get(mm->memclass);
+#endif
}
return mm;
}
@@ -451,6 +463,13 @@ void fastcall __mmdrop(struct mm_struct *mm)
BUG_ON(mm == &init_mm);
mm_free_pgd(mm);
destroy_context(mm);
+#ifdef CONFIG_CKRM_RES_MEM
+ /* class can be null and mm's tasklist can be empty here */
+ if (mm->memclass) {
+ mem_class_put(mm->memclass);
+ mm->memclass = NULL;
+ }
+#endif
free_mm(mm);
}
@@ -578,6 +597,7 @@ static int copy_mm(unsigned long clone_flags, struct task_struct * tsk)
good_mm:
tsk->mm = mm;
tsk->active_mm = mm;
+ ckrm_init_mm_to_task(mm, tsk);
return 0;
free_pt:
diff --git a/kernel/sched.c b/kernel/sched.c
index 63e8835e8..b5d3eb51d 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -17,6 +17,7 @@
* 2003-09-03 Interactivity tuning by Con Kolivas.
* 2004-04-02 Scheduler domains code by Nick Piggin
*/
+
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
@@ -155,6 +156,9 @@
#define LOW_CREDIT(p) \
((p)->interactive_credit < -CREDIT_LIMIT)
+#define TASK_PREEMPTS_CURR(p, rq) \
+ ((p)->prio < (rq)->curr->prio)
+
/*
* BASE_TIMESLICE scales user-nice values [ -20 ... 19 ]
* to time slice values.
@@ -180,24 +184,16 @@ static unsigned int task_timeslice(task_t *p)
/*
* These are the runqueue data structures:
*/
-typedef struct runqueue runqueue_t;
-#ifdef CONFIG_CKRM_CPU_SCHEDULE
-#include <linux/ckrm_classqueue.h>
-#endif
+#define BITMAP_SIZE ((((MAX_PRIO+1+7)/8)+sizeof(long)-1)/sizeof(long))
-#ifdef CONFIG_CKRM_CPU_SCHEDULE
+typedef struct runqueue runqueue_t;
-/**
- * if belong to different class, compare class priority
- * otherwise compare task priority
- */
-#define TASK_PREEMPTS_CURR(p, rq) \
- (((p)->cpu_class != (rq)->curr->cpu_class) && ((rq)->curr != (rq)->idle))? class_preempts_curr((p),(rq)->curr) : ((p)->prio < (rq)->curr->prio)
-#else
-#define TASK_PREEMPTS_CURR(p, rq) \
- ((p)->prio < (rq)->curr->prio)
-#endif
+struct prio_array {
+ unsigned int nr_active;
+ unsigned long bitmap[BITMAP_SIZE];
+ struct list_head queue[MAX_PRIO];
+};
/*
* This is the main, per-CPU runqueue data structure.
@@ -214,7 +210,7 @@ struct runqueue {
* remote CPUs use both these fields when doing load calculation.
*/
unsigned long nr_running;
-#if defined(CONFIG_SMP)
+#ifdef CONFIG_SMP
unsigned long cpu_load;
#endif
unsigned long long nr_switches;
@@ -222,12 +218,7 @@ struct runqueue {
unsigned long long timestamp_last_tick;
task_t *curr, *idle;
struct mm_struct *prev_mm;
-#ifdef CONFIG_CKRM_CPU_SCHEDULE
- unsigned long ckrm_cpu_load;
- struct classqueue_struct classqueue;
-#else
- prio_array_t *active, *expired, arrays[2];
-#endif
+ prio_array_t *active, *expired, arrays[2];
int best_expired_prio;
atomic_t nr_iowait;
@@ -262,160 +253,12 @@ static DEFINE_PER_CPU(struct runqueue, runqueues);
# define task_running(rq, p) ((rq)->curr == (p))
#endif
-#ifdef CONFIG_CKRM_CPU_SCHEDULE
-#include <linux/ckrm_sched.h>
-spinlock_t cvt_lock = SPIN_LOCK_UNLOCKED;
-rwlock_t class_list_lock = RW_LOCK_UNLOCKED;
-LIST_HEAD(active_cpu_classes); // list of active cpu classes; anchor
-struct ckrm_cpu_class default_cpu_class_obj;
-
-/*
- * the minimum CVT allowed is the base_cvt
- * otherwise, it will starve others
- */
-CVT_t get_min_cvt(int cpu)
-{
- cq_node_t *node;
- struct ckrm_local_runqueue * lrq;
- CVT_t min_cvt;
-
- node = classqueue_get_head(bpt_queue(cpu));
- lrq = (node) ? class_list_entry(node) : NULL;
-
- if (lrq)
- min_cvt = lrq->local_cvt;
- else
- min_cvt = 0;
-
- return min_cvt;
-}
-
-/*
- * update the classueue base for all the runqueues
- * TODO: we can only update half of the min_base to solve the movebackward issue
- */
-static inline void check_update_class_base(int this_cpu) {
- unsigned long min_base = 0xFFFFFFFF;
- cq_node_t *node;
- int i;
-
- if (! cpu_online(this_cpu)) return;
-
- /*
- * find the min_base across all the processors
- */
- for_each_online_cpu(i) {
- /*
- * I should change it to directly use bpt->base
- */
- node = classqueue_get_head(bpt_queue(i));
- if (node && node->prio < min_base) {
- min_base = node->prio;
- }
- }
- if (min_base != 0xFFFFFFFF)
- classqueue_update_base(bpt_queue(this_cpu),min_base);
-}
-
-static inline void ckrm_rebalance_tick(int j,int this_cpu)
-{
-#ifdef CONFIG_CKRM_CPU_SCHEDULE
- read_lock(&class_list_lock);
- if (!(j % CVT_UPDATE_TICK))
- update_global_cvts(this_cpu);
-
-#define CKRM_BASE_UPDATE_RATE 400
- if (! (jiffies % CKRM_BASE_UPDATE_RATE))
- check_update_class_base(this_cpu);
-
- read_unlock(&class_list_lock);
-#endif
-}
-
-static inline struct ckrm_local_runqueue *rq_get_next_class(struct runqueue *rq)
-{
- cq_node_t *node = classqueue_get_head(&rq->classqueue);
- return ((node) ? class_list_entry(node) : NULL);
-}
-
-static inline struct task_struct * rq_get_next_task(struct runqueue* rq)
-{
- prio_array_t *array;
- struct task_struct *next;
- struct ckrm_local_runqueue *queue;
- int cpu = smp_processor_id();
-
- next = rq->idle;
- retry_next_class:
- if ((queue = rq_get_next_class(rq))) {
- array = queue->active;
- //check switch active/expired queue
- if (unlikely(!queue->active->nr_active)) {
- prio_array_t *array;
-
- array = queue->active;
- queue->active = queue->expired;
- queue->expired = array;
- queue->expired_timestamp = 0;
-
- if (queue->active->nr_active)
- set_top_priority(queue,
- find_first_bit(queue->active->bitmap, MAX_PRIO));
- else {
- classqueue_dequeue(queue->classqueue,
- &queue->classqueue_linkobj);
- cpu_demand_event(get_rq_local_stat(queue,cpu),CPU_DEMAND_DEQUEUE,0);
- }
-
- goto retry_next_class;
- }
- BUG_ON(!queue->active->nr_active);
- next = task_list_entry(array->queue[queue->top_priority].next);
- }
- return next;
-}
-
-static inline void rq_load_inc(runqueue_t *rq, struct task_struct *p) { rq->ckrm_cpu_load += cpu_class_weight(p->cpu_class); }
-static inline void rq_load_dec(runqueue_t *rq, struct task_struct *p) { rq->ckrm_cpu_load -= cpu_class_weight(p->cpu_class); }
-
-#else /*CONFIG_CKRM_CPU_SCHEDULE*/
-
-static inline struct task_struct * rq_get_next_task(struct runqueue* rq)
-{
- prio_array_t *array;
- struct list_head *queue;
- int idx;
-
- array = rq->active;
- if (unlikely(!array->nr_active)) {
- /*
- * Switch the active and expired arrays.
- */
- rq->active = rq->expired;
- rq->expired = array;
- array = rq->active;
- rq->expired_timestamp = 0;
- }
-
- idx = sched_find_first_bit(array->bitmap);
- queue = array->queue + idx;
- return list_entry(queue->next, task_t, run_list);
-}
-
-static inline void class_enqueue_task(struct task_struct* p, prio_array_t *array) { }
-static inline void class_dequeue_task(struct task_struct* p, prio_array_t *array) { }
-static inline void init_cpu_classes(void) { }
-static inline void rq_load_inc(runqueue_t *rq, struct task_struct *p) { }
-static inline void rq_load_dec(runqueue_t *rq, struct task_struct *p) { }
-#endif /* CONFIG_CKRM_CPU_SCHEDULE */
-
-
/*
* task_rq_lock - lock the runqueue a given task resides on and disable
* interrupts. Note the ordering: we can safely lookup the task_rq without
* explicitly disabling preemption.
*/
-runqueue_t *task_rq_lock(task_t *p, unsigned long *flags)
+static runqueue_t *task_rq_lock(task_t *p, unsigned long *flags)
{
struct runqueue *rq;
@@ -430,7 +273,7 @@ repeat_lock_task:
return rq;
}
-void task_rq_unlock(runqueue_t *rq, unsigned long *flags)
+static inline void task_rq_unlock(runqueue_t *rq, unsigned long *flags)
{
spin_unlock_irqrestore(&rq->lock, *flags);
}
@@ -457,23 +300,20 @@ static inline void rq_unlock(runqueue_t *rq)
/*
* Adding/removing a task to/from a priority array:
*/
-void dequeue_task(struct task_struct *p, prio_array_t *array)
+static void dequeue_task(struct task_struct *p, prio_array_t *array)
{
- BUG_ON(! array);
array->nr_active--;
list_del(&p->run_list);
if (list_empty(array->queue + p->prio))
__clear_bit(p->prio, array->bitmap);
- class_dequeue_task(p,array);
}
-void enqueue_task(struct task_struct *p, prio_array_t *array)
+static void enqueue_task(struct task_struct *p, prio_array_t *array)
{
list_add_tail(&p->run_list, array->queue + p->prio);
__set_bit(p->prio, array->bitmap);
array->nr_active++;
p->array = array;
- class_enqueue_task(p,array);
}
/*
@@ -487,7 +327,6 @@ static inline void enqueue_task_head(struct task_struct *p, prio_array_t *array)
__set_bit(p->prio, array->bitmap);
array->nr_active++;
p->array = array;
- class_enqueue_task(p,array);
}
/*
@@ -526,9 +365,8 @@ static int effective_prio(task_t *p)
*/
static inline void __activate_task(task_t *p, runqueue_t *rq)
{
- enqueue_task(p, rq_active(p,rq));
+ enqueue_task(p, rq->active);
rq->nr_running++;
- rq_load_inc(rq,p);
}
/*
@@ -536,9 +374,8 @@ static inline void __activate_task(task_t *p, runqueue_t *rq)
*/
static inline void __activate_idle_task(task_t *p, runqueue_t *rq)
{
- enqueue_task_head(p, rq_active(p,rq));
+ enqueue_task_head(p, rq->active);
rq->nr_running++;
- rq_load_inc(rq,p);
}
static void recalc_task_prio(task_t *p, unsigned long long now)
@@ -670,7 +507,6 @@ static void activate_task(task_t *p, runqueue_t *rq, int local)
static void deactivate_task(struct task_struct *p, runqueue_t *rq)
{
rq->nr_running--;
- rq_load_dec(rq,p);
if (p->state == TASK_UNINTERRUPTIBLE)
rq->nr_uninterruptible++;
dequeue_task(p, p->array);
@@ -1120,7 +956,6 @@ void fastcall wake_up_forked_process(task_t * p)
p->array = current->array;
p->array->nr_active++;
rq->nr_running++;
- rq_load_inc(rq,p);
}
task_rq_unlock(rq, &flags);
}
@@ -1443,7 +1278,6 @@ lock_again:
p->array = current->array;
p->array->nr_active++;
rq->nr_running++;
- rq_load_inc(rq,p);
}
} else {
/* Not the local CPU - must adjust timestamp */
@@ -1548,13 +1382,9 @@ void pull_task(runqueue_t *src_rq, prio_array_t *src_array, task_t *p,
{
dequeue_task(p, src_array);
src_rq->nr_running--;
- rq_load_dec(src_rq,p);
-
set_task_cpu(p, this_cpu);
this_rq->nr_running++;
- rq_load_inc(this_rq,p);
enqueue_task(p, this_array);
-
p->timestamp = (p->timestamp - src_rq->timestamp_last_tick)
+ this_rq->timestamp_last_tick;
/*
@@ -1593,194 +1423,6 @@ int can_migrate_task(task_t *p, runqueue_t *rq, int this_cpu,
return 1;
}
-#ifdef CONFIG_CKRM_CPU_SCHEDULE
-
-struct ckrm_cpu_class *find_unbalanced_class(int busiest_cpu, int this_cpu, unsigned long *cls_imbalance)
-{
- struct ckrm_cpu_class *most_unbalanced_class = NULL;
- struct ckrm_cpu_class *clsptr;
- int max_unbalance = 0;
-
- list_for_each_entry(clsptr,&active_cpu_classes,links) {
- struct ckrm_local_runqueue *this_lrq = get_ckrm_local_runqueue(clsptr,this_cpu);
- struct ckrm_local_runqueue *busiest_lrq = get_ckrm_local_runqueue(clsptr,busiest_cpu);
- int unbalance_degree;
-
- unbalance_degree = (local_queue_nr_running(busiest_lrq) - local_queue_nr_running(this_lrq)) * cpu_class_weight(clsptr);
- if (unbalance_degree >= *cls_imbalance)
- continue; // already looked at this class
-
- if (unbalance_degree > max_unbalance) {
- max_unbalance = unbalance_degree;
- most_unbalanced_class = clsptr;
- }
- }
- *cls_imbalance = max_unbalance;
- return most_unbalanced_class;
-}
-
-
-/*
- * find_busiest_queue - find the busiest runqueue among the cpus in cpumask.
- */
-static int find_busiest_cpu(runqueue_t *this_rq, int this_cpu, int idle,
- int *imbalance)
-{
- int cpu_load, load, max_load, i, busiest_cpu;
- runqueue_t *busiest, *rq_src;
-
-
- /*Hubertus ... the concept of nr_running is replace with cpu_load */
- cpu_load = this_rq->ckrm_cpu_load;
-
- busiest = NULL;
- busiest_cpu = -1;
-
- max_load = -1;
- for_each_online_cpu(i) {
- rq_src = cpu_rq(i);
- load = rq_src->ckrm_cpu_load;
-
- if ((load > max_load) && (rq_src != this_rq)) {
- busiest = rq_src;
- busiest_cpu = i;
- max_load = load;
- }
- }
-
- if (likely(!busiest))
- goto out;
-
- *imbalance = max_load - cpu_load;
-
- /* It needs an at least ~25% imbalance to trigger balancing. */
- if (!idle && ((*imbalance)*4 < max_load)) {
- busiest = NULL;
- goto out;
- }
-
- double_lock_balance(this_rq, busiest);
- /*
- * Make sure nothing changed since we checked the
- * runqueue length.
- */
- if (busiest->ckrm_cpu_load <= cpu_load) {
- spin_unlock(&busiest->lock);
- busiest = NULL;
- }
-out:
- return (busiest ? busiest_cpu : -1);
-}
-
-static int load_balance(int this_cpu, runqueue_t *this_rq,
- struct sched_domain *sd, enum idle_type idle)
-{
- int imbalance, idx;
- int busiest_cpu;
- runqueue_t *busiest;
- prio_array_t *array;
- struct list_head *head, *curr;
- task_t *tmp;
- struct ckrm_local_runqueue * busiest_local_queue;
- struct ckrm_cpu_class *clsptr;
- int weight;
- unsigned long cls_imbalance; // so we can retry other classes
-
- // need to update global CVT based on local accumulated CVTs
- read_lock(&class_list_lock);
- busiest_cpu = find_busiest_cpu(this_rq, this_cpu, idle, &imbalance);
- if (busiest_cpu == -1)
- goto out;
-
- busiest = cpu_rq(busiest_cpu);
-
- /*
- * We only want to steal a number of tasks equal to 1/2 the imbalance,
- * otherwise we'll just shift the imbalance to the new queue:
- */
- imbalance /= 2;
-
- /* now find class on that runqueue with largest inbalance */
- cls_imbalance = 0xFFFFFFFF;
-
- retry_other_class:
- clsptr = find_unbalanced_class(busiest_cpu, this_cpu, &cls_imbalance);
- if (!clsptr)
- goto out_unlock;
-
- busiest_local_queue = get_ckrm_local_runqueue(clsptr,busiest_cpu);
- weight = cpu_class_weight(clsptr);
-
- /*
- * We first consider expired tasks. Those will likely not be
- * executed in the near future, and they are most likely to
- * be cache-cold, thus switching CPUs has the least effect
- * on them.
- */
- if (busiest_local_queue->expired->nr_active)
- array = busiest_local_queue->expired;
- else
- array = busiest_local_queue->active;
-
- new_array:
- /* Start searching at priority 0: */
- idx = 0;
- skip_bitmap:
- if (!idx)
- idx = sched_find_first_bit(array->bitmap);
- else
- idx = find_next_bit(array->bitmap, MAX_PRIO, idx);
- if (idx >= MAX_PRIO) {
- if (array == busiest_local_queue->expired && busiest_local_queue->active->nr_active) {
- array = busiest_local_queue->active;
- goto new_array;
- }
- goto retry_other_class;
- }
-
- head = array->queue + idx;
- curr = head->prev;
- skip_queue:
- tmp = list_entry(curr, task_t, run_list);
-
- curr = curr->prev;
-
- if (!can_migrate_task(tmp, busiest, this_cpu, sd,idle)) {
- if (curr != head)
- goto skip_queue;
- idx++;
- goto skip_bitmap;
- }
- pull_task(busiest, array, tmp, this_rq, rq_active(tmp,this_rq),this_cpu);
- /*
- * tmp BUG FIX: hzheng
- * load balancing can make the busiest local queue empty
- * thus it should be removed from bpt
- */
- if (! local_queue_nr_running(busiest_local_queue)) {
- classqueue_dequeue(busiest_local_queue->classqueue,&busiest_local_queue->classqueue_linkobj);
- cpu_demand_event(get_rq_local_stat(busiest_local_queue,busiest_cpu),CPU_DEMAND_DEQUEUE,0);
- }
-
- imbalance -= weight;
- if (!idle && (imbalance>0)) {
- if (curr != head)
- goto skip_queue;
- idx++;
- goto skip_bitmap;
- }
- out_unlock:
- spin_unlock(&busiest->lock);
- out:
- read_unlock(&class_list_lock);
- return 0;
-}
-
-
-static inline void idle_balance(int this_cpu, runqueue_t *this_rq)
-{
-}
-#else /* CONFIG_CKRM_CPU_SCHEDULE */
/*
* move_tasks tries to move up to max_nr_move tasks from busiest to this_rq,
* as part of a balancing operation within "domain". Returns the number of
@@ -2230,7 +1872,6 @@ next_group:
group = group->next;
} while (group != sd->groups);
}
-#endif /* CONFIG_CKRM_CPU_SCHEDULE*/
/*
* rebalance_tick will get called every timer tick, on every CPU.
@@ -2251,8 +1892,6 @@ static void rebalance_tick(int this_cpu, runqueue_t *this_rq,
unsigned long j = jiffies + CPU_OFFSET(this_cpu);
struct sched_domain *sd;
- ckrm_rebalance_tick(j,this_cpu);
-
/* Update our load */
old_load = this_rq->cpu_load;
this_load = this_rq->nr_running * SCHED_LOAD_SCALE;
@@ -2285,15 +1924,13 @@ static void rebalance_tick(int this_cpu, runqueue_t *this_rq,
}
}
}
-#else /* SMP*/
+#else
/*
* on UP we do not need to balance between CPUs:
*/
static inline void rebalance_tick(int cpu, runqueue_t *rq, enum idle_type idle)
{
- ckrm_rebalance_tick(jiffies,cpu);
}
-
static inline void idle_balance(int cpu, runqueue_t *rq)
{
}
@@ -2314,7 +1951,7 @@ static inline int wake_priority_sleeper(runqueue_t *rq)
return 0;
}
-DEFINE_PER_CPU(struct kernel_stat, kstat) = { { 0 } };
+DEFINE_PER_CPU(struct kernel_stat, kstat);
EXPORT_PER_CPU_SYMBOL(kstat);
@@ -2328,19 +1965,11 @@ EXPORT_PER_CPU_SYMBOL(kstat);
* increasing number of running tasks. We also ignore the interactivity
* if a better static_prio task has expired:
*/
-
-#ifndef CONFIG_CKRM_CPU_SCHEDULE
#define EXPIRED_STARVING(rq) \
((STARVATION_LIMIT && ((rq)->expired_timestamp && \
(jiffies - (rq)->expired_timestamp >= \
STARVATION_LIMIT * ((rq)->nr_running) + 1))) || \
((rq)->curr->static_prio > (rq)->best_expired_prio))
-#else
-#define EXPIRED_STARVING(rq) \
- (STARVATION_LIMIT && ((rq)->expired_timestamp && \
- (jiffies - (rq)->expired_timestamp >= \
- STARVATION_LIMIT * (local_queue_nr_running(rq)) + 1)))
-#endif
/*
* This function gets called by the timer code, with HZ frequency.
@@ -2387,7 +2016,7 @@ void scheduler_tick(int user_ticks, int sys_ticks)
cpustat->system += sys_ticks;
/* Task might have expired already, but not scheduled off yet */
- if (p->array != rq_active(p,rq)) {
+ if (p->array != rq->active) {
set_tsk_need_resched(p);
goto out;
}
@@ -2410,16 +2039,12 @@ void scheduler_tick(int user_ticks, int sys_ticks)
set_tsk_need_resched(p);
/* put it at the end of the queue: */
- dequeue_task(p, rq_active(p,rq));
- enqueue_task(p, rq_active(p,rq));
+ dequeue_task(p, rq->active);
+ enqueue_task(p, rq->active);
}
goto out_unlock;
}
if (!--p->time_slice) {
-#ifdef CONFIG_CKRM_CPU_SCHEDULE
- /* Hubertus ... we can abstract this out */
- struct ckrm_local_runqueue* rq = get_task_class_queue(p);
-#endif
dequeue_task(p, rq->active);
set_tsk_need_resched(p);
p->prio = effective_prio(p);
@@ -2430,8 +2055,8 @@ void scheduler_tick(int user_ticks, int sys_ticks)
rq->expired_timestamp = jiffies;
if (!TASK_INTERACTIVE(p) || EXPIRED_STARVING(rq)) {
enqueue_task(p, rq->expired);
- if (p->static_prio < this_rq()->best_expired_prio)
- this_rq()->best_expired_prio = p->static_prio;
+ if (p->static_prio < rq->best_expired_prio)
+ rq->best_expired_prio = p->static_prio;
} else
enqueue_task(p, rq->active);
} else {
@@ -2454,12 +2079,12 @@ void scheduler_tick(int user_ticks, int sys_ticks)
if (TASK_INTERACTIVE(p) && !((task_timeslice(p) -
p->time_slice) % TIMESLICE_GRANULARITY(p)) &&
(p->time_slice >= TIMESLICE_GRANULARITY(p)) &&
- (p->array == rq_active(p,rq))) {
+ (p->array == rq->active)) {
- dequeue_task(p, rq_active(p,rq));
+ dequeue_task(p, rq->active);
set_tsk_need_resched(p);
p->prio = effective_prio(p);
- enqueue_task(p, rq_active(p,rq));
+ enqueue_task(p, rq->active);
}
}
out_unlock:
@@ -2562,9 +2187,10 @@ asmlinkage void __sched schedule(void)
task_t *prev, *next;
runqueue_t *rq;
prio_array_t *array;
+ struct list_head *queue;
unsigned long long now;
unsigned long run_time;
- int cpu;
+ int cpu, idx;
/*
* Test if we are atomic. Since do_exit() needs to call into
@@ -2625,9 +2251,21 @@ need_resched:
}
}
- next = rq_get_next_task(rq);
- if (next == rq->idle)
- goto switch_tasks;
+ array = rq->active;
+ if (unlikely(!array->nr_active)) {
+ /*
+ * Switch the active and expired arrays.
+ */
+ rq->active = rq->expired;
+ rq->expired = array;
+ array = rq->active;
+ rq->expired_timestamp = 0;
+ rq->best_expired_prio = MAX_PRIO;
+ }
+
+ idx = sched_find_first_bit(array->bitmap);
+ queue = array->queue + idx;
+ next = list_entry(queue->next, task_t, run_list);
if (dependent_sleeper(cpu, rq, next)) {
next = rq->idle;
@@ -2651,14 +2289,6 @@ switch_tasks:
clear_tsk_need_resched(prev);
RCU_qsctr(task_cpu(prev))++;
-#ifdef CONFIG_CKRM_CPU_SCHEDULE
- if (prev != rq->idle) {
- unsigned long long run = now - prev->timestamp;
- cpu_demand_event(get_task_local_stat(prev),CPU_DEMAND_DESCHEDULE,run);
- update_local_cvt(prev, run);
- }
-#endif
-
prev->sleep_avg -= run_time;
if ((long)prev->sleep_avg <= 0) {
prev->sleep_avg = 0;
@@ -3364,7 +2994,7 @@ asmlinkage long sys_sched_yield(void)
{
runqueue_t *rq = this_rq_lock();
prio_array_t *array = current->array;
- prio_array_t *target = rq_expired(current,rq);
+ prio_array_t *target = rq->expired;
/*
* We implement yielding by moving the task into the expired
@@ -3374,7 +3004,7 @@ asmlinkage long sys_sched_yield(void)
* array.)
*/
if (unlikely(rt_task(current)))
- target = rq_active(current,rq);
+ target = rq->active;
dequeue_task(current, array);
enqueue_task(current, target);
@@ -3799,9 +3429,7 @@ static int migration_thread(void * data)
}
if (rq->active_balance) {
-#ifndef CONFIG_CKRM_CPU_SCHEDULE
active_load_balance(rq, cpu);
-#endif
rq->active_balance = 0;
}
@@ -4277,10 +3905,7 @@ int in_sched_functions(unsigned long addr)
void __init sched_init(void)
{
runqueue_t *rq;
- int i;
-#ifndef CONFIG_CKRM_CPU_SCHEDULE
- int j, k;
-#endif
+ int i, j, k;
#ifdef CONFIG_SMP
/* Set up an initial dummy domain for early boot */
@@ -4300,21 +3925,13 @@ void __init sched_init(void)
sched_group_init.cpu_power = SCHED_LOAD_SCALE;
#endif
- init_cpu_classes();
-
for (i = 0; i < NR_CPUS; i++) {
-#ifndef CONFIG_CKRM_CPU_SCHEDULE
prio_array_t *array;
-#endif
+
rq = cpu_rq(i);
spin_lock_init(&rq->lock);
-
-#ifndef CONFIG_CKRM_CPU_SCHEDULE
rq->active = rq->arrays;
rq->expired = rq->arrays + 1;
-#else
- rq->ckrm_cpu_load = 0;
-#endif
rq->best_expired_prio = MAX_PRIO;
#ifdef CONFIG_SMP
@@ -4327,7 +3944,6 @@ void __init sched_init(void)
#endif
atomic_set(&rq->nr_iowait, 0);
-#ifndef CONFIG_CKRM_CPU_SCHEDULE
for (j = 0; j < 2; j++) {
array = rq->arrays + j;
for (k = 0; k < MAX_PRIO; k++) {
@@ -4337,9 +3953,7 @@ void __init sched_init(void)
// delimiter for bitsearch
__set_bit(MAX_PRIO, array->bitmap);
}
-#endif
}
-
/*
* We have to do a little magic to get the first
* thread right in SMP mode.
@@ -4348,10 +3962,6 @@ void __init sched_init(void)
rq->curr = current;
rq->idle = current;
set_task_cpu(current, smp_processor_id());
-#ifdef CONFIG_CKRM_CPU_SCHEDULE
- current->cpu_class = default_cpu_class;
- current->array = NULL;
-#endif
wake_up_forked_process(current);
/*
@@ -4437,13 +4047,3 @@ int task_running_sys(struct task_struct *p)
EXPORT_SYMBOL(task_running_sys);
#endif
-#ifdef CONFIG_CKRM_CPU_SCHEDULE
-/**
- * return the classqueue object of a certain processor
- * Note: not supposed to be used in performance sensitive functions
- */
-struct classqueue_struct * get_cpu_classqueue(int cpu)
-{
- return (& (cpu_rq(cpu)->classqueue) );
-}
-#endif
diff --git a/mm/fremap.c b/mm/fremap.c
index eb056db90..5cbe6779a 100644
--- a/mm/fremap.c
+++ b/mm/fremap.c
@@ -14,6 +14,7 @@
#include <linux/swapops.h>
#include <linux/rmap.h>
#include <linux/module.h>
+#include <linux/ckrm_mem_inline.h>
#include <asm/mmu_context.h>
#include <asm/cacheflush.h>
@@ -83,6 +84,7 @@ int install_page(struct mm_struct *mm, struct vm_area_struct *vma,
mm->rss++;
flush_icache_page(vma, page);
set_pte(pte, mk_pte(page, prot));
+ ckrm_mem_evaluate_page_byadd(page, mm);
page_add_file_rmap(page);
pte_val = *pte;
pte_unmap(pte);
diff --git a/mm/memory.c b/mm/memory.c
index ddf7049ff..75d75a38b 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -55,6 +55,7 @@
#include <linux/swapops.h>
#include <linux/elf.h>
+#include <linux/ckrm_mem_inline.h>
#ifndef CONFIG_DISCONTIGMEM
/* use the per-pgdat data instead for discontigmem - mbligh */
@@ -1534,6 +1535,7 @@ retry:
if (write_access)
entry = maybe_mkwrite(pte_mkdirty(entry), vma);
set_pte(page_table, entry);
+ ckrm_mem_evaluate_page_byadd(new_page, mm);
if (anon) {
lru_cache_add_active(new_page);
page_add_anon_rmap(new_page, vma, address);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 3dbbeb2f3..ea8ec23e8 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -31,6 +31,7 @@
#include <linux/topology.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
+#include <linux/ckrm_mem_inline.h>
#include <asm/tlbflush.h>
@@ -267,6 +268,7 @@ free_pages_bulk(struct zone *zone, int count,
page = list_entry(list->prev, struct page, lru);
/* have to delete it as __free_pages_bulk list manipulates */
list_del(&page->lru);
+ ckrm_clear_page_class(page);
__free_pages_bulk(page, base, zone, area, mask, order);
ret++;
}
@@ -598,6 +600,10 @@ __alloc_pages(unsigned int gfp_mask, unsigned int order,
might_sleep_if(wait);
+ if (!ckrm_class_limit_ok((GET_MEM_CLASS(current)))) {
+ return NULL;
+ }
+
zones = zonelist->zones; /* the list of zones suitable for gfp_mask */
if (zones[0] == NULL) /* no zones in the zonelist */
return NULL;
@@ -727,6 +733,7 @@ nopage:
return NULL;
got_pg:
kernel_map_pages(page, 1 << order, 1);
+ ckrm_set_pages_class(page, 1 << order, GET_MEM_CLASS(current));
return page;
}
diff --git a/mm/rmap.c b/mm/rmap.c
index 1f3c84f8c..59c1fc120 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -33,6 +33,7 @@
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/rmap.h>
+#include <linux/ckrm_mem_inline.h>
#include <asm/tlbflush.h>
@@ -360,6 +361,7 @@ void page_add_anon_rmap(struct page *page,
*/
page_map_lock(page);
if (!page->mapcount) {
+ ckrm_mem_evaluate_page_byadd(page, vma->vm_mm);
BUG_ON(PageAnon(page));
BUG_ON(page->mapping);
SetPageAnon(page);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index e5f0b0919..0908b4e15 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -38,6 +38,7 @@
#include <asm/div64.h>
#include <linux/swapops.h>
+#include <linux/ckrm_mem.h>
/*
* From 0 .. 100. Higher means more swappy.
@@ -329,6 +330,9 @@ struct scan_control {
/* This context's GFP mask */
unsigned int gfp_mask;
+ /* Flag used by CKRM */
+ unsigned int ckrm_flags;
+
int may_writepage;
};
@@ -539,19 +543,23 @@ static void shrink_cache(struct zone *zone, struct scan_control *sc)
{
LIST_HEAD(page_list);
struct pagevec pvec;
- int max_scan = sc->nr_to_scan;
+ int max_scan = sc->nr_to_scan, nr_pass;
+ unsigned int ckrm_flags = sc->ckrm_flags, bit_flag;
pagevec_init(&pvec, 1);
lru_add_drain();
spin_lock_irq(&zone->lru_lock);
+redo:
+ ckrm_get_reclaim_bits(&ckrm_flags, &bit_flag);
+ nr_pass = zone->nr_inactive;
while (max_scan > 0) {
struct page *page;
int nr_taken = 0;
int nr_scan = 0;
int nr_freed;
- while (nr_scan++ < SWAP_CLUSTER_MAX &&
+ while (nr_pass-- && nr_scan++ < SWAP_CLUSTER_MAX &&
!list_empty(&zone->inactive_list)) {
page = lru_to_page(&zone->inactive_list);
@@ -569,15 +577,25 @@ static void shrink_cache(struct zone *zone, struct scan_control *sc)
SetPageLRU(page);
list_add(&page->lru, &zone->inactive_list);
continue;
+ } else if (bit_flag && !ckrm_kick_page(page, bit_flag)) {
+ __put_page(page);
+ SetPageLRU(page);
+#ifdef CONFIG_CKRM_MEM_LRUORDER_CHANGE
+ list_add_tail(&page->lru, &zone->inactive_list);
+#else
+ list_add(&page->lru, &zone->inactive_list);
+#endif
+ continue;
}
list_add(&page->lru, &page_list);
+ ckrm_mem_dec_inactive(page);
nr_taken++;
}
zone->nr_inactive -= nr_taken;
zone->pages_scanned += nr_taken;
spin_unlock_irq(&zone->lru_lock);
- if (nr_taken == 0)
+ if ((bit_flag == 0) && (nr_taken == 0))
goto done;
max_scan -= nr_scan;
@@ -609,6 +627,9 @@ static void shrink_cache(struct zone *zone, struct scan_control *sc)
spin_lock_irq(&zone->lru_lock);
}
}
+ if (ckrm_flags && (nr_pass <= 0)) {
+ goto redo;
+ }
}
spin_unlock_irq(&zone->lru_lock);
done:
@@ -648,10 +669,15 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc)
long mapped_ratio;
long distress;
long swap_tendency;
+ unsigned int ckrm_flags = sc->ckrm_flags, bit_flag;
+ int nr_pass;
lru_add_drain();
pgmoved = 0;
spin_lock_irq(&zone->lru_lock);
+redo:
+ ckrm_get_reclaim_bits(&ckrm_flags, &bit_flag);
+ nr_pass = zone->nr_active;
while (pgscanned < nr_pages && !list_empty(&zone->active_list)) {
page = lru_to_page(&zone->active_list);
prefetchw_prev_lru_page(page, &zone->active_list, flags);
@@ -668,11 +694,24 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc)
__put_page(page);
SetPageLRU(page);
list_add(&page->lru, &zone->active_list);
+ pgscanned++;
+ } else if (bit_flag && !ckrm_kick_page(page, bit_flag)) {
+ __put_page(page);
+ SetPageLRU(page);
+#ifdef CONFIG_CKRM_MEM_LRUORDER_CHANGE
+ list_add_tail(&page->lru, &zone->active_list);
+#else
+ list_add(&page->lru, &zone->active_list);
+#endif
} else {
list_add(&page->lru, &l_hold);
+ ckrm_mem_dec_active(page);
pgmoved++;
- }
pgscanned++;
+ }
+ if (ckrm_flags && !--nr_pass) {
+ goto redo;
+ }
}
zone->nr_active -= pgmoved;
spin_unlock_irq(&zone->lru_lock);
@@ -746,6 +785,7 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc)
if (!TestClearPageActive(page))
BUG();
list_move(&page->lru, &zone->inactive_list);
+ ckrm_mem_inc_inactive(page);
pgmoved++;
if (!pagevec_add(&pvec, page)) {
zone->nr_inactive += pgmoved;
@@ -774,6 +814,7 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc)
BUG();
BUG_ON(!PageActive(page));
list_move(&page->lru, &zone->active_list);
+ ckrm_mem_inc_active(page);
pgmoved++;
if (!pagevec_add(&pvec, page)) {
zone->nr_active += pgmoved;
@@ -826,6 +867,7 @@ shrink_zone(struct zone *zone, struct scan_control *sc)
scan_active = (unsigned long)tmp;
}
+ sc->ckrm_flags = ckrm_setup_reclamation();
atomic_add(scan_active + 1, &zone->nr_scan_active);
count = atomic_read(&zone->nr_scan_active);
if (count >= SWAP_CLUSTER_MAX) {
@@ -841,8 +883,101 @@ shrink_zone(struct zone *zone, struct scan_control *sc)
sc->nr_to_scan = count;
shrink_cache(zone, sc);
}
+ ckrm_teardown_reclamation();
}
+#ifdef CONFIG_CKRM_RES_MEM
+// This function needs to be given more thought.
+static void
+ckrm_shrink_class(ckrm_mem_res_t *cls)
+{
+ struct scan_control sc;
+ struct zone *zone;
+ int zindex = 0, active_credit = 0, inactive_credit = 0;
+
+ if (ckrm_test_set_shrink(cls)) { // set the SHRINK bit atomically
+ // if it is already set somebody is working on it. so... leave
+ return;
+ }
+ sc.nr_mapped = read_page_state(nr_mapped);
+ sc.nr_scanned = 0;
+ sc.ckrm_flags = ckrm_get_reclaim_flags(cls);
+ sc.nr_reclaimed = 0;
+ sc.priority = 0; // always very high priority
+
+ for_each_zone(zone) {
+ int zone_total, zone_limit, active_limit, inactive_limit;
+ int active_over, inactive_over, count;
+ u64 temp;
+
+ zone->temp_priority = zone->prev_priority;
+ zone->prev_priority = sc.priority;
+
+ zone_total = zone->nr_active + zone->nr_inactive + zone->free_pages;
+
+ temp = (u64) cls->pg_limit * zone_total;
+ do_div(temp, ckrm_tot_lru_pages);
+ zone_limit = (int) temp;
+ active_limit = (6 * zone_limit) / 10; // 2/3rd in active list
+ inactive_limit = (3 * zone_limit) / 10; // 1/3rd in inactive list
+
+ active_over = cls->nr_active[zindex] - active_limit + active_credit;
+ inactive_over = active_over +
+ (cls->nr_inactive[zindex] - inactive_limit) + inactive_credit;
+
+ if (active_over > 0) {
+ atomic_add(active_over + 1, &zone->nr_scan_active);
+ count = atomic_read(&zone->nr_scan_active);
+ if (count >= SWAP_CLUSTER_MAX) {
+ atomic_set(&zone->nr_scan_active, 0);
+ sc.nr_to_scan = count;
+ refill_inactive_zone(zone, &sc);
+ }
+ active_credit = 0;
+ } else {
+ active_credit = active_over;
+ }
+
+ if (inactive_over > 0) {
+ atomic_add(inactive_over, &zone->nr_scan_inactive);
+ count = atomic_read(&zone->nr_scan_inactive);
+ if (count >= SWAP_CLUSTER_MAX) {
+ atomic_set(&zone->nr_scan_inactive, 0);
+ sc.nr_to_scan = count;
+ shrink_cache(zone, &sc);
+ }
+ inactive_credit = 0;
+ } else {
+ inactive_credit = inactive_over;
+ }
+ zone->prev_priority = zone->temp_priority;
+ zindex++;
+ }
+ ckrm_clear_shrink(cls);
+}
+
+static void
+ckrm_shrink_classes(void)
+{
+ ckrm_mem_res_t *cls;
+
+ spin_lock(&ckrm_mem_lock);
+ while (!ckrm_shrink_list_empty()) {
+ cls = list_entry(ckrm_shrink_list.next, ckrm_mem_res_t,
+ shrink_list);
+ spin_unlock(&ckrm_mem_lock);
+ ckrm_shrink_class(cls);
+ spin_lock(&ckrm_mem_lock);
+ list_del(&cls->shrink_list);
+ cls->flags &= ~MEM_NEAR_LIMIT;
+ }
+ spin_unlock(&ckrm_mem_lock);
+}
+
+#else
+#define ckrm_shrink_classes() do { } while(0)
+#endif
+
/*
* This is the direct reclaim path, for page-allocating processes. We only
* try to reclaim pages from zones which will satisfy the caller's allocation
@@ -1137,6 +1272,9 @@ int kswapd(void *p)
schedule();
finish_wait(&pgdat->kswapd_wait, &wait);
+ if (!ckrm_shrink_list_empty())
+ ckrm_shrink_classes();
+ else
balance_pgdat(pgdat, 0);
}
}
@@ -1146,7 +1284,7 @@ int kswapd(void *p)
*/
void wakeup_kswapd(struct zone *zone)
{
- if (zone->free_pages > zone->pages_low)
+ if ((zone->free_pages > zone->pages_low) && ckrm_shrink_list_empty())
return;
if (!waitqueue_active(&zone->zone_pgdat->kswapd_wait))
return;