From 041b7df03818bb535b04201c39a6b290bf2f36f0 Mon Sep 17 00:00:00 2001 From: Marc Fiuczynski Date: Wed, 8 Sep 2004 19:41:04 +0000 Subject: [PATCH] ckrm-E15 memory controller --- fs/exec.c | 13 + include/linux/ckrm_rc.h | 12 +- include/linux/mm.h | 3 + include/linux/mm_inline.h | 7 + include/linux/sched.h | 15 +- init/Kconfig | 30 +-- init/main.c | 6 +- kernel/Makefile | 5 - kernel/ckrm/Makefile | 3 +- kernel/ckrm/rbce/rbcemod.c | 1 - kernel/exit.c | 7 + kernel/fork.c | 20 ++ kernel/sched.c | 500 ++++--------------------------------- mm/fremap.c | 2 + mm/memory.c | 2 + mm/page_alloc.c | 7 + mm/rmap.c | 2 + mm/vmscan.c | 148 ++++++++++- 18 files changed, 297 insertions(+), 486 deletions(-) diff --git a/fs/exec.c b/fs/exec.c index 61fba8f37..ba44671ed 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -47,6 +47,7 @@ #include #include #include +#include #include #include @@ -547,6 +548,18 @@ static int exec_mmap(struct mm_struct *mm) tsk->active_mm = mm; activate_mm(active_mm, mm); task_unlock(tsk); +#ifdef CONFIG_CKRM_RES_MEM + if (old_mm) { + spin_lock(&old_mm->peertask_lock); + list_del(&tsk->mm_peers); + ckrm_mem_evaluate_mm(old_mm); + spin_unlock(&old_mm->peertask_lock); + } + spin_lock(&mm->peertask_lock); + list_add_tail(&tsk->mm_peers, &mm->tasklist); + ckrm_mem_evaluate_mm(mm); + spin_unlock(&mm->peertask_lock); +#endif if (old_mm) { if (active_mm != old_mm) BUG(); mmput(old_mm); diff --git a/include/linux/ckrm_rc.h b/include/linux/ckrm_rc.h index b46cfd9f3..f4031671e 100644 --- a/include/linux/ckrm_rc.h +++ b/include/linux/ckrm_rc.h @@ -223,7 +223,17 @@ typedef struct ckrm_core_class { * OTHER ******************************************************************************/ -#define ckrm_get_res_class(rescls,resid,type) ((type*)((rescls)->res_class[resid])) +#define ckrm_get_res_class(rescls, resid, type) \ + ((type*) (((resid != -1) && ((rescls) != NULL) \ + && ((rescls) != (void *)-1)) ? \ + ((struct ckrm_core_class *)(rescls))->res_class[resid] : NULL)) + +#define ckrm_set_res_class(rescls, resid, value) \ +do { \ + if ((resid != -1) && ((rescls) != NULL)) { \ + ((struct ckrm_core_class *)(rescls))->res_class[resid] = value; \ + } \ +} while (0) extern int ckrm_register_res_ctlr(struct ckrm_classtype *, ckrm_res_ctlr_t *); extern int ckrm_unregister_res_ctlr(ckrm_res_ctlr_t *); diff --git a/include/linux/mm.h b/include/linux/mm.h index 8f8a8a3a3..a2bd104b7 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -228,6 +228,9 @@ struct page { void *virtual; /* Kernel virtual address (NULL if not kmapped, ie. highmem) */ #endif /* WANT_PAGE_VIRTUAL */ +#ifdef CONFIG_CKRM_RES_MEM + void *memclass; +#endif // CONFIG_CKRM_RES_MEM }; /* diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 47762ca69..5edb739b4 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -1,9 +1,11 @@ +#include static inline void add_page_to_active_list(struct zone *zone, struct page *page) { list_add(&page->lru, &zone->active_list); zone->nr_active++; + ckrm_mem_inc_active(page); } static inline void @@ -11,6 +13,7 @@ add_page_to_inactive_list(struct zone *zone, struct page *page) { list_add(&page->lru, &zone->inactive_list); zone->nr_inactive++; + ckrm_mem_inc_inactive(page); } static inline void @@ -18,6 +21,7 @@ del_page_from_active_list(struct zone *zone, struct page *page) { list_del(&page->lru); zone->nr_active--; + ckrm_mem_dec_active(page); } static inline void @@ -25,6 +29,7 @@ del_page_from_inactive_list(struct zone *zone, struct page *page) { list_del(&page->lru); zone->nr_inactive--; + ckrm_mem_dec_inactive(page); } static inline void @@ -34,7 +39,9 @@ del_page_from_lru(struct zone *zone, struct page *page) if (PageActive(page)) { ClearPageActive(page); zone->nr_active--; + ckrm_mem_dec_active(page); } else { zone->nr_inactive--; + ckrm_mem_dec_inactive(page); } } diff --git a/include/linux/sched.h b/include/linux/sched.h index 0a3efe462..a5a1d8ed9 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -235,6 +235,11 @@ struct mm_struct { struct kioctx *ioctx_list; struct kioctx default_kioctx; +#ifdef CONFIG_CKRM_RES_MEM + struct ckrm_mem_res *memclass; + struct list_head tasklist; /* list of all tasks sharing this address space */ + spinlock_t peertask_lock; /* protect above tasklist */ +#endif }; extern int mmlist_nr; @@ -522,12 +527,11 @@ struct task_struct { // .. Hubertus should change to CONFIG_CKRM_TYPE_TASKCLASS struct ckrm_task_class *taskclass; struct list_head taskclass_link; -#ifdef CONFIG_CKRM_CPU_SCHEDULE - struct ckrm_cpu_class *cpu_class; -#endif #endif // CONFIG_CKRM_TYPE_TASKCLASS +#ifdef CONFIG_CKRM_RES_MEM + struct list_head mm_peers; // list of tasks using same mm_struct +#endif // CONFIG_CKRM_RES_MEM #endif // CONFIG_CKRM - struct task_delay_info delays; }; @@ -974,7 +978,8 @@ static inline struct mm_struct * get_task_mm(struct task_struct * task) return mm; } - + + /* set thread flags in other task's structures * - see asm/thread_info.h for TIF_xxxx flags available */ diff --git a/init/Kconfig b/init/Kconfig index 579d847e1..c6888dbea 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -168,23 +168,25 @@ config CKRM_RES_NUMTASKS Say N if unsure, Y to use the feature. -config CKRM_CPU_SCHEDULE - bool "CKRM CPU scheduler" - depends on CKRM_TYPE_TASKCLASS - default m +config CKRM_RES_MEM + bool "Class based physical memory controller" + default y + depends on CKRM help - Use CKRM CPU scheduler instead of Linux Scheduler - - Say N if unsure, Y to use the feature. + Provide the basic support for collecting physical memory usage information + among classes. Say Y if you want to know the memory usage of each class. -config CKRM_CPU_MONITOR - tristate "CKRM CPU Resoure Monitor" - depends on CKRM_CPU_SCHEDULE - default m +config CKRM_MEM_LRUORDER_CHANGE + bool "Change the LRU ordering of scanned pages" + default n + depends on CKRM_RES_MEM help - Monitor CPU Resource Usage of the classes - - Say N if unsure, Y to use the feature. + While trying to free pages, by default(n), scanned pages are left were they + are found if they belong to relatively under-used class. In this case the + LRU ordering of the memory subsystemis left intact. If this option is chosen, + then the scanned pages are moved to the tail of the list(active or inactive). + Changing this to yes reduces the checking overhead but violates the approximate + LRU order that is maintained by the paging subsystem. config CKRM_TYPE_SOCKETCLASS bool "Class Manager for socket groups" diff --git a/init/main.c b/init/main.c index 6cc592bec..7080522cd 100644 --- a/init/main.c +++ b/init/main.c @@ -43,12 +43,11 @@ #include #include #include + #include #include #include -int __init init_ckrm_sched_res(void); - /* * This is one of the first .c files built. Error out early @@ -416,6 +415,7 @@ asmlinkage void __init start_kernel(void) * printk() and can access its per-cpu storage. */ smp_prepare_boot_cpu(); + /* * Set up the scheduler prior starting any interrupts (such as the * timer interrupt). Full topology setup happens at smp_init() @@ -629,8 +629,8 @@ static int init(void * unused) * firmware files. */ populate_rootfs(); + do_basic_setup(); - init_ckrm_sched_res(); /* * check if there is an early userspace init. If yes, let it do all diff --git a/kernel/Makefile b/kernel/Makefile index bbba09802..107df8eb1 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -9,7 +9,6 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o profile.o \ rcupdate.o intermodule.o extable.o params.o posix-timers.o \ kthread.o ckrm/ -obj-$(CONFIG_ZTRACE) += swap_hook.o obj-$(CONFIG_FUTEX) += futex.o obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o obj-$(CONFIG_SMP) += cpu.o @@ -22,12 +21,8 @@ obj-$(CONFIG_COMPAT) += compat.o obj-$(CONFIG_IKCONFIG) += configs.o obj-$(CONFIG_IKCONFIG_PROC) += configs.o obj-$(CONFIG_STOP_MACHINE) += stop_machine.o -obj-$(CONFIG_CKRM_CPU_SCHEDULE) += ckrm_classqueue.o -obj-$(CONFIG_CKRM_CPU_SCHEDULE) += ckrm_sched.o obj-$(CONFIG_AUDIT) += audit.o obj-$(CONFIG_AUDITSYSCALL) += auditsc.o -obj-$(CONFIG_KGDB) += kgdbstub.o - ifneq ($(CONFIG_IA64),y) # According to Alan Modra , the -fno-omit-frame-pointer is diff --git a/kernel/ckrm/Makefile b/kernel/ckrm/Makefile index 3da88775d..0ff956a92 100644 --- a/kernel/ckrm/Makefile +++ b/kernel/ckrm/Makefile @@ -7,7 +7,6 @@ ifeq ($(CONFIG_CKRM),y) endif obj-$(CONFIG_CKRM_TYPE_TASKCLASS) += ckrm_tc.o obj-$(CONFIG_CKRM_RES_NUMTASKS) += ckrm_tasks.o + obj-$(CONFIG_CKRM_RES_MEM) += ckrm_mem.o obj-$(CONFIG_CKRM_TYPE_SOCKETCLASS) += ckrm_sockc.o obj-$(CONFIG_CKRM_RES_LISTENAQ) += ckrm_listenaq.o - obj-$(CONFIG_CKRM_CPU_SCHEDULE) += ckrm_cpu_class.o - obj-$(CONFIG_CKRM_CPU_MONITOR) += ckrm_cpu_monitor.o diff --git a/kernel/ckrm/rbce/rbcemod.c b/kernel/ckrm/rbce/rbcemod.c index f61d0879c..fffa9ad1f 100644 --- a/kernel/ckrm/rbce/rbcemod.c +++ b/kernel/ckrm/rbce/rbcemod.c @@ -497,7 +497,6 @@ rbce_class_deletecb(const char *classname, void *classobj, int classtype) } } } - put_class(cls); if ((cls = find_class_name(classname)) != NULL) { printk(KERN_ERR "rbce ERROR: class %s exists in rbce after " diff --git a/kernel/exit.c b/kernel/exit.c index f53583e2b..333f8003e 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -513,6 +514,12 @@ static inline void __exit_mm(struct task_struct * tsk) task_lock(tsk); tsk->mm = NULL; up_read(&mm->mmap_sem); +#ifdef CONFIG_CKRM_RES_MEM + spin_lock(&mm->peertask_lock); + list_del_init(&tsk->mm_peers); + ckrm_mem_evaluate_mm(mm); + spin_unlock(&mm->peertask_lock); +#endif enter_lazy_tlb(mm, current); task_unlock(tsk); mmput(mm); diff --git a/kernel/fork.c b/kernel/fork.c index 4af488db0..deb264307 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -265,6 +266,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) ckrm_cb_newtask(tsk); /* One for us, one for whoever does the "release_task()" (usually parent) */ atomic_set(&tsk->usage,2); +#ifdef CONFIG_CKRM_RES_MEM + INIT_LIST_HEAD(&tsk->mm_peers); +#endif return tsk; } @@ -417,6 +421,10 @@ static struct mm_struct * mm_init(struct mm_struct * mm) mm->ioctx_list = NULL; mm->default_kioctx = (struct kioctx)INIT_KIOCTX(mm->default_kioctx, *mm); mm->free_area_cache = TASK_UNMAPPED_BASE; +#ifdef CONFIG_CKRM_RES_MEM + INIT_LIST_HEAD(&mm->tasklist); + mm->peertask_lock = SPIN_LOCK_UNLOCKED; +#endif if (likely(!mm_alloc_pgd(mm))) { mm->def_flags = 0; @@ -437,6 +445,10 @@ struct mm_struct * mm_alloc(void) if (mm) { memset(mm, 0, sizeof(*mm)); mm = mm_init(mm); +#ifdef CONFIG_CKRM_RES_MEM + mm->memclass = GET_MEM_CLASS(current); + mem_class_get(mm->memclass); +#endif } return mm; } @@ -451,6 +463,13 @@ void fastcall __mmdrop(struct mm_struct *mm) BUG_ON(mm == &init_mm); mm_free_pgd(mm); destroy_context(mm); +#ifdef CONFIG_CKRM_RES_MEM + /* class can be null and mm's tasklist can be empty here */ + if (mm->memclass) { + mem_class_put(mm->memclass); + mm->memclass = NULL; + } +#endif free_mm(mm); } @@ -578,6 +597,7 @@ static int copy_mm(unsigned long clone_flags, struct task_struct * tsk) good_mm: tsk->mm = mm; tsk->active_mm = mm; + ckrm_init_mm_to_task(mm, tsk); return 0; free_pt: diff --git a/kernel/sched.c b/kernel/sched.c index 63e8835e8..b5d3eb51d 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -17,6 +17,7 @@ * 2003-09-03 Interactivity tuning by Con Kolivas. * 2004-04-02 Scheduler domains code by Nick Piggin */ + #include #include #include @@ -155,6 +156,9 @@ #define LOW_CREDIT(p) \ ((p)->interactive_credit < -CREDIT_LIMIT) +#define TASK_PREEMPTS_CURR(p, rq) \ + ((p)->prio < (rq)->curr->prio) + /* * BASE_TIMESLICE scales user-nice values [ -20 ... 19 ] * to time slice values. @@ -180,24 +184,16 @@ static unsigned int task_timeslice(task_t *p) /* * These are the runqueue data structures: */ -typedef struct runqueue runqueue_t; -#ifdef CONFIG_CKRM_CPU_SCHEDULE -#include -#endif +#define BITMAP_SIZE ((((MAX_PRIO+1+7)/8)+sizeof(long)-1)/sizeof(long)) -#ifdef CONFIG_CKRM_CPU_SCHEDULE +typedef struct runqueue runqueue_t; -/** - * if belong to different class, compare class priority - * otherwise compare task priority - */ -#define TASK_PREEMPTS_CURR(p, rq) \ - (((p)->cpu_class != (rq)->curr->cpu_class) && ((rq)->curr != (rq)->idle))? class_preempts_curr((p),(rq)->curr) : ((p)->prio < (rq)->curr->prio) -#else -#define TASK_PREEMPTS_CURR(p, rq) \ - ((p)->prio < (rq)->curr->prio) -#endif +struct prio_array { + unsigned int nr_active; + unsigned long bitmap[BITMAP_SIZE]; + struct list_head queue[MAX_PRIO]; +}; /* * This is the main, per-CPU runqueue data structure. @@ -214,7 +210,7 @@ struct runqueue { * remote CPUs use both these fields when doing load calculation. */ unsigned long nr_running; -#if defined(CONFIG_SMP) +#ifdef CONFIG_SMP unsigned long cpu_load; #endif unsigned long long nr_switches; @@ -222,12 +218,7 @@ struct runqueue { unsigned long long timestamp_last_tick; task_t *curr, *idle; struct mm_struct *prev_mm; -#ifdef CONFIG_CKRM_CPU_SCHEDULE - unsigned long ckrm_cpu_load; - struct classqueue_struct classqueue; -#else - prio_array_t *active, *expired, arrays[2]; -#endif + prio_array_t *active, *expired, arrays[2]; int best_expired_prio; atomic_t nr_iowait; @@ -262,160 +253,12 @@ static DEFINE_PER_CPU(struct runqueue, runqueues); # define task_running(rq, p) ((rq)->curr == (p)) #endif -#ifdef CONFIG_CKRM_CPU_SCHEDULE -#include -spinlock_t cvt_lock = SPIN_LOCK_UNLOCKED; -rwlock_t class_list_lock = RW_LOCK_UNLOCKED; -LIST_HEAD(active_cpu_classes); // list of active cpu classes; anchor -struct ckrm_cpu_class default_cpu_class_obj; - -/* - * the minimum CVT allowed is the base_cvt - * otherwise, it will starve others - */ -CVT_t get_min_cvt(int cpu) -{ - cq_node_t *node; - struct ckrm_local_runqueue * lrq; - CVT_t min_cvt; - - node = classqueue_get_head(bpt_queue(cpu)); - lrq = (node) ? class_list_entry(node) : NULL; - - if (lrq) - min_cvt = lrq->local_cvt; - else - min_cvt = 0; - - return min_cvt; -} - -/* - * update the classueue base for all the runqueues - * TODO: we can only update half of the min_base to solve the movebackward issue - */ -static inline void check_update_class_base(int this_cpu) { - unsigned long min_base = 0xFFFFFFFF; - cq_node_t *node; - int i; - - if (! cpu_online(this_cpu)) return; - - /* - * find the min_base across all the processors - */ - for_each_online_cpu(i) { - /* - * I should change it to directly use bpt->base - */ - node = classqueue_get_head(bpt_queue(i)); - if (node && node->prio < min_base) { - min_base = node->prio; - } - } - if (min_base != 0xFFFFFFFF) - classqueue_update_base(bpt_queue(this_cpu),min_base); -} - -static inline void ckrm_rebalance_tick(int j,int this_cpu) -{ -#ifdef CONFIG_CKRM_CPU_SCHEDULE - read_lock(&class_list_lock); - if (!(j % CVT_UPDATE_TICK)) - update_global_cvts(this_cpu); - -#define CKRM_BASE_UPDATE_RATE 400 - if (! (jiffies % CKRM_BASE_UPDATE_RATE)) - check_update_class_base(this_cpu); - - read_unlock(&class_list_lock); -#endif -} - -static inline struct ckrm_local_runqueue *rq_get_next_class(struct runqueue *rq) -{ - cq_node_t *node = classqueue_get_head(&rq->classqueue); - return ((node) ? class_list_entry(node) : NULL); -} - -static inline struct task_struct * rq_get_next_task(struct runqueue* rq) -{ - prio_array_t *array; - struct task_struct *next; - struct ckrm_local_runqueue *queue; - int cpu = smp_processor_id(); - - next = rq->idle; - retry_next_class: - if ((queue = rq_get_next_class(rq))) { - array = queue->active; - //check switch active/expired queue - if (unlikely(!queue->active->nr_active)) { - prio_array_t *array; - - array = queue->active; - queue->active = queue->expired; - queue->expired = array; - queue->expired_timestamp = 0; - - if (queue->active->nr_active) - set_top_priority(queue, - find_first_bit(queue->active->bitmap, MAX_PRIO)); - else { - classqueue_dequeue(queue->classqueue, - &queue->classqueue_linkobj); - cpu_demand_event(get_rq_local_stat(queue,cpu),CPU_DEMAND_DEQUEUE,0); - } - - goto retry_next_class; - } - BUG_ON(!queue->active->nr_active); - next = task_list_entry(array->queue[queue->top_priority].next); - } - return next; -} - -static inline void rq_load_inc(runqueue_t *rq, struct task_struct *p) { rq->ckrm_cpu_load += cpu_class_weight(p->cpu_class); } -static inline void rq_load_dec(runqueue_t *rq, struct task_struct *p) { rq->ckrm_cpu_load -= cpu_class_weight(p->cpu_class); } - -#else /*CONFIG_CKRM_CPU_SCHEDULE*/ - -static inline struct task_struct * rq_get_next_task(struct runqueue* rq) -{ - prio_array_t *array; - struct list_head *queue; - int idx; - - array = rq->active; - if (unlikely(!array->nr_active)) { - /* - * Switch the active and expired arrays. - */ - rq->active = rq->expired; - rq->expired = array; - array = rq->active; - rq->expired_timestamp = 0; - } - - idx = sched_find_first_bit(array->bitmap); - queue = array->queue + idx; - return list_entry(queue->next, task_t, run_list); -} - -static inline void class_enqueue_task(struct task_struct* p, prio_array_t *array) { } -static inline void class_dequeue_task(struct task_struct* p, prio_array_t *array) { } -static inline void init_cpu_classes(void) { } -static inline void rq_load_inc(runqueue_t *rq, struct task_struct *p) { } -static inline void rq_load_dec(runqueue_t *rq, struct task_struct *p) { } -#endif /* CONFIG_CKRM_CPU_SCHEDULE */ - - /* * task_rq_lock - lock the runqueue a given task resides on and disable * interrupts. Note the ordering: we can safely lookup the task_rq without * explicitly disabling preemption. */ -runqueue_t *task_rq_lock(task_t *p, unsigned long *flags) +static runqueue_t *task_rq_lock(task_t *p, unsigned long *flags) { struct runqueue *rq; @@ -430,7 +273,7 @@ repeat_lock_task: return rq; } -void task_rq_unlock(runqueue_t *rq, unsigned long *flags) +static inline void task_rq_unlock(runqueue_t *rq, unsigned long *flags) { spin_unlock_irqrestore(&rq->lock, *flags); } @@ -457,23 +300,20 @@ static inline void rq_unlock(runqueue_t *rq) /* * Adding/removing a task to/from a priority array: */ -void dequeue_task(struct task_struct *p, prio_array_t *array) +static void dequeue_task(struct task_struct *p, prio_array_t *array) { - BUG_ON(! array); array->nr_active--; list_del(&p->run_list); if (list_empty(array->queue + p->prio)) __clear_bit(p->prio, array->bitmap); - class_dequeue_task(p,array); } -void enqueue_task(struct task_struct *p, prio_array_t *array) +static void enqueue_task(struct task_struct *p, prio_array_t *array) { list_add_tail(&p->run_list, array->queue + p->prio); __set_bit(p->prio, array->bitmap); array->nr_active++; p->array = array; - class_enqueue_task(p,array); } /* @@ -487,7 +327,6 @@ static inline void enqueue_task_head(struct task_struct *p, prio_array_t *array) __set_bit(p->prio, array->bitmap); array->nr_active++; p->array = array; - class_enqueue_task(p,array); } /* @@ -526,9 +365,8 @@ static int effective_prio(task_t *p) */ static inline void __activate_task(task_t *p, runqueue_t *rq) { - enqueue_task(p, rq_active(p,rq)); + enqueue_task(p, rq->active); rq->nr_running++; - rq_load_inc(rq,p); } /* @@ -536,9 +374,8 @@ static inline void __activate_task(task_t *p, runqueue_t *rq) */ static inline void __activate_idle_task(task_t *p, runqueue_t *rq) { - enqueue_task_head(p, rq_active(p,rq)); + enqueue_task_head(p, rq->active); rq->nr_running++; - rq_load_inc(rq,p); } static void recalc_task_prio(task_t *p, unsigned long long now) @@ -670,7 +507,6 @@ static void activate_task(task_t *p, runqueue_t *rq, int local) static void deactivate_task(struct task_struct *p, runqueue_t *rq) { rq->nr_running--; - rq_load_dec(rq,p); if (p->state == TASK_UNINTERRUPTIBLE) rq->nr_uninterruptible++; dequeue_task(p, p->array); @@ -1120,7 +956,6 @@ void fastcall wake_up_forked_process(task_t * p) p->array = current->array; p->array->nr_active++; rq->nr_running++; - rq_load_inc(rq,p); } task_rq_unlock(rq, &flags); } @@ -1443,7 +1278,6 @@ lock_again: p->array = current->array; p->array->nr_active++; rq->nr_running++; - rq_load_inc(rq,p); } } else { /* Not the local CPU - must adjust timestamp */ @@ -1548,13 +1382,9 @@ void pull_task(runqueue_t *src_rq, prio_array_t *src_array, task_t *p, { dequeue_task(p, src_array); src_rq->nr_running--; - rq_load_dec(src_rq,p); - set_task_cpu(p, this_cpu); this_rq->nr_running++; - rq_load_inc(this_rq,p); enqueue_task(p, this_array); - p->timestamp = (p->timestamp - src_rq->timestamp_last_tick) + this_rq->timestamp_last_tick; /* @@ -1593,194 +1423,6 @@ int can_migrate_task(task_t *p, runqueue_t *rq, int this_cpu, return 1; } -#ifdef CONFIG_CKRM_CPU_SCHEDULE - -struct ckrm_cpu_class *find_unbalanced_class(int busiest_cpu, int this_cpu, unsigned long *cls_imbalance) -{ - struct ckrm_cpu_class *most_unbalanced_class = NULL; - struct ckrm_cpu_class *clsptr; - int max_unbalance = 0; - - list_for_each_entry(clsptr,&active_cpu_classes,links) { - struct ckrm_local_runqueue *this_lrq = get_ckrm_local_runqueue(clsptr,this_cpu); - struct ckrm_local_runqueue *busiest_lrq = get_ckrm_local_runqueue(clsptr,busiest_cpu); - int unbalance_degree; - - unbalance_degree = (local_queue_nr_running(busiest_lrq) - local_queue_nr_running(this_lrq)) * cpu_class_weight(clsptr); - if (unbalance_degree >= *cls_imbalance) - continue; // already looked at this class - - if (unbalance_degree > max_unbalance) { - max_unbalance = unbalance_degree; - most_unbalanced_class = clsptr; - } - } - *cls_imbalance = max_unbalance; - return most_unbalanced_class; -} - - -/* - * find_busiest_queue - find the busiest runqueue among the cpus in cpumask. - */ -static int find_busiest_cpu(runqueue_t *this_rq, int this_cpu, int idle, - int *imbalance) -{ - int cpu_load, load, max_load, i, busiest_cpu; - runqueue_t *busiest, *rq_src; - - - /*Hubertus ... the concept of nr_running is replace with cpu_load */ - cpu_load = this_rq->ckrm_cpu_load; - - busiest = NULL; - busiest_cpu = -1; - - max_load = -1; - for_each_online_cpu(i) { - rq_src = cpu_rq(i); - load = rq_src->ckrm_cpu_load; - - if ((load > max_load) && (rq_src != this_rq)) { - busiest = rq_src; - busiest_cpu = i; - max_load = load; - } - } - - if (likely(!busiest)) - goto out; - - *imbalance = max_load - cpu_load; - - /* It needs an at least ~25% imbalance to trigger balancing. */ - if (!idle && ((*imbalance)*4 < max_load)) { - busiest = NULL; - goto out; - } - - double_lock_balance(this_rq, busiest); - /* - * Make sure nothing changed since we checked the - * runqueue length. - */ - if (busiest->ckrm_cpu_load <= cpu_load) { - spin_unlock(&busiest->lock); - busiest = NULL; - } -out: - return (busiest ? busiest_cpu : -1); -} - -static int load_balance(int this_cpu, runqueue_t *this_rq, - struct sched_domain *sd, enum idle_type idle) -{ - int imbalance, idx; - int busiest_cpu; - runqueue_t *busiest; - prio_array_t *array; - struct list_head *head, *curr; - task_t *tmp; - struct ckrm_local_runqueue * busiest_local_queue; - struct ckrm_cpu_class *clsptr; - int weight; - unsigned long cls_imbalance; // so we can retry other classes - - // need to update global CVT based on local accumulated CVTs - read_lock(&class_list_lock); - busiest_cpu = find_busiest_cpu(this_rq, this_cpu, idle, &imbalance); - if (busiest_cpu == -1) - goto out; - - busiest = cpu_rq(busiest_cpu); - - /* - * We only want to steal a number of tasks equal to 1/2 the imbalance, - * otherwise we'll just shift the imbalance to the new queue: - */ - imbalance /= 2; - - /* now find class on that runqueue with largest inbalance */ - cls_imbalance = 0xFFFFFFFF; - - retry_other_class: - clsptr = find_unbalanced_class(busiest_cpu, this_cpu, &cls_imbalance); - if (!clsptr) - goto out_unlock; - - busiest_local_queue = get_ckrm_local_runqueue(clsptr,busiest_cpu); - weight = cpu_class_weight(clsptr); - - /* - * We first consider expired tasks. Those will likely not be - * executed in the near future, and they are most likely to - * be cache-cold, thus switching CPUs has the least effect - * on them. - */ - if (busiest_local_queue->expired->nr_active) - array = busiest_local_queue->expired; - else - array = busiest_local_queue->active; - - new_array: - /* Start searching at priority 0: */ - idx = 0; - skip_bitmap: - if (!idx) - idx = sched_find_first_bit(array->bitmap); - else - idx = find_next_bit(array->bitmap, MAX_PRIO, idx); - if (idx >= MAX_PRIO) { - if (array == busiest_local_queue->expired && busiest_local_queue->active->nr_active) { - array = busiest_local_queue->active; - goto new_array; - } - goto retry_other_class; - } - - head = array->queue + idx; - curr = head->prev; - skip_queue: - tmp = list_entry(curr, task_t, run_list); - - curr = curr->prev; - - if (!can_migrate_task(tmp, busiest, this_cpu, sd,idle)) { - if (curr != head) - goto skip_queue; - idx++; - goto skip_bitmap; - } - pull_task(busiest, array, tmp, this_rq, rq_active(tmp,this_rq),this_cpu); - /* - * tmp BUG FIX: hzheng - * load balancing can make the busiest local queue empty - * thus it should be removed from bpt - */ - if (! local_queue_nr_running(busiest_local_queue)) { - classqueue_dequeue(busiest_local_queue->classqueue,&busiest_local_queue->classqueue_linkobj); - cpu_demand_event(get_rq_local_stat(busiest_local_queue,busiest_cpu),CPU_DEMAND_DEQUEUE,0); - } - - imbalance -= weight; - if (!idle && (imbalance>0)) { - if (curr != head) - goto skip_queue; - idx++; - goto skip_bitmap; - } - out_unlock: - spin_unlock(&busiest->lock); - out: - read_unlock(&class_list_lock); - return 0; -} - - -static inline void idle_balance(int this_cpu, runqueue_t *this_rq) -{ -} -#else /* CONFIG_CKRM_CPU_SCHEDULE */ /* * move_tasks tries to move up to max_nr_move tasks from busiest to this_rq, * as part of a balancing operation within "domain". Returns the number of @@ -2230,7 +1872,6 @@ next_group: group = group->next; } while (group != sd->groups); } -#endif /* CONFIG_CKRM_CPU_SCHEDULE*/ /* * rebalance_tick will get called every timer tick, on every CPU. @@ -2251,8 +1892,6 @@ static void rebalance_tick(int this_cpu, runqueue_t *this_rq, unsigned long j = jiffies + CPU_OFFSET(this_cpu); struct sched_domain *sd; - ckrm_rebalance_tick(j,this_cpu); - /* Update our load */ old_load = this_rq->cpu_load; this_load = this_rq->nr_running * SCHED_LOAD_SCALE; @@ -2285,15 +1924,13 @@ static void rebalance_tick(int this_cpu, runqueue_t *this_rq, } } } -#else /* SMP*/ +#else /* * on UP we do not need to balance between CPUs: */ static inline void rebalance_tick(int cpu, runqueue_t *rq, enum idle_type idle) { - ckrm_rebalance_tick(jiffies,cpu); } - static inline void idle_balance(int cpu, runqueue_t *rq) { } @@ -2314,7 +1951,7 @@ static inline int wake_priority_sleeper(runqueue_t *rq) return 0; } -DEFINE_PER_CPU(struct kernel_stat, kstat) = { { 0 } }; +DEFINE_PER_CPU(struct kernel_stat, kstat); EXPORT_PER_CPU_SYMBOL(kstat); @@ -2328,19 +1965,11 @@ EXPORT_PER_CPU_SYMBOL(kstat); * increasing number of running tasks. We also ignore the interactivity * if a better static_prio task has expired: */ - -#ifndef CONFIG_CKRM_CPU_SCHEDULE #define EXPIRED_STARVING(rq) \ ((STARVATION_LIMIT && ((rq)->expired_timestamp && \ (jiffies - (rq)->expired_timestamp >= \ STARVATION_LIMIT * ((rq)->nr_running) + 1))) || \ ((rq)->curr->static_prio > (rq)->best_expired_prio)) -#else -#define EXPIRED_STARVING(rq) \ - (STARVATION_LIMIT && ((rq)->expired_timestamp && \ - (jiffies - (rq)->expired_timestamp >= \ - STARVATION_LIMIT * (local_queue_nr_running(rq)) + 1))) -#endif /* * This function gets called by the timer code, with HZ frequency. @@ -2387,7 +2016,7 @@ void scheduler_tick(int user_ticks, int sys_ticks) cpustat->system += sys_ticks; /* Task might have expired already, but not scheduled off yet */ - if (p->array != rq_active(p,rq)) { + if (p->array != rq->active) { set_tsk_need_resched(p); goto out; } @@ -2410,16 +2039,12 @@ void scheduler_tick(int user_ticks, int sys_ticks) set_tsk_need_resched(p); /* put it at the end of the queue: */ - dequeue_task(p, rq_active(p,rq)); - enqueue_task(p, rq_active(p,rq)); + dequeue_task(p, rq->active); + enqueue_task(p, rq->active); } goto out_unlock; } if (!--p->time_slice) { -#ifdef CONFIG_CKRM_CPU_SCHEDULE - /* Hubertus ... we can abstract this out */ - struct ckrm_local_runqueue* rq = get_task_class_queue(p); -#endif dequeue_task(p, rq->active); set_tsk_need_resched(p); p->prio = effective_prio(p); @@ -2430,8 +2055,8 @@ void scheduler_tick(int user_ticks, int sys_ticks) rq->expired_timestamp = jiffies; if (!TASK_INTERACTIVE(p) || EXPIRED_STARVING(rq)) { enqueue_task(p, rq->expired); - if (p->static_prio < this_rq()->best_expired_prio) - this_rq()->best_expired_prio = p->static_prio; + if (p->static_prio < rq->best_expired_prio) + rq->best_expired_prio = p->static_prio; } else enqueue_task(p, rq->active); } else { @@ -2454,12 +2079,12 @@ void scheduler_tick(int user_ticks, int sys_ticks) if (TASK_INTERACTIVE(p) && !((task_timeslice(p) - p->time_slice) % TIMESLICE_GRANULARITY(p)) && (p->time_slice >= TIMESLICE_GRANULARITY(p)) && - (p->array == rq_active(p,rq))) { + (p->array == rq->active)) { - dequeue_task(p, rq_active(p,rq)); + dequeue_task(p, rq->active); set_tsk_need_resched(p); p->prio = effective_prio(p); - enqueue_task(p, rq_active(p,rq)); + enqueue_task(p, rq->active); } } out_unlock: @@ -2562,9 +2187,10 @@ asmlinkage void __sched schedule(void) task_t *prev, *next; runqueue_t *rq; prio_array_t *array; + struct list_head *queue; unsigned long long now; unsigned long run_time; - int cpu; + int cpu, idx; /* * Test if we are atomic. Since do_exit() needs to call into @@ -2625,9 +2251,21 @@ need_resched: } } - next = rq_get_next_task(rq); - if (next == rq->idle) - goto switch_tasks; + array = rq->active; + if (unlikely(!array->nr_active)) { + /* + * Switch the active and expired arrays. + */ + rq->active = rq->expired; + rq->expired = array; + array = rq->active; + rq->expired_timestamp = 0; + rq->best_expired_prio = MAX_PRIO; + } + + idx = sched_find_first_bit(array->bitmap); + queue = array->queue + idx; + next = list_entry(queue->next, task_t, run_list); if (dependent_sleeper(cpu, rq, next)) { next = rq->idle; @@ -2651,14 +2289,6 @@ switch_tasks: clear_tsk_need_resched(prev); RCU_qsctr(task_cpu(prev))++; -#ifdef CONFIG_CKRM_CPU_SCHEDULE - if (prev != rq->idle) { - unsigned long long run = now - prev->timestamp; - cpu_demand_event(get_task_local_stat(prev),CPU_DEMAND_DESCHEDULE,run); - update_local_cvt(prev, run); - } -#endif - prev->sleep_avg -= run_time; if ((long)prev->sleep_avg <= 0) { prev->sleep_avg = 0; @@ -3364,7 +2994,7 @@ asmlinkage long sys_sched_yield(void) { runqueue_t *rq = this_rq_lock(); prio_array_t *array = current->array; - prio_array_t *target = rq_expired(current,rq); + prio_array_t *target = rq->expired; /* * We implement yielding by moving the task into the expired @@ -3374,7 +3004,7 @@ asmlinkage long sys_sched_yield(void) * array.) */ if (unlikely(rt_task(current))) - target = rq_active(current,rq); + target = rq->active; dequeue_task(current, array); enqueue_task(current, target); @@ -3799,9 +3429,7 @@ static int migration_thread(void * data) } if (rq->active_balance) { -#ifndef CONFIG_CKRM_CPU_SCHEDULE active_load_balance(rq, cpu); -#endif rq->active_balance = 0; } @@ -4277,10 +3905,7 @@ int in_sched_functions(unsigned long addr) void __init sched_init(void) { runqueue_t *rq; - int i; -#ifndef CONFIG_CKRM_CPU_SCHEDULE - int j, k; -#endif + int i, j, k; #ifdef CONFIG_SMP /* Set up an initial dummy domain for early boot */ @@ -4300,21 +3925,13 @@ void __init sched_init(void) sched_group_init.cpu_power = SCHED_LOAD_SCALE; #endif - init_cpu_classes(); - for (i = 0; i < NR_CPUS; i++) { -#ifndef CONFIG_CKRM_CPU_SCHEDULE prio_array_t *array; -#endif + rq = cpu_rq(i); spin_lock_init(&rq->lock); - -#ifndef CONFIG_CKRM_CPU_SCHEDULE rq->active = rq->arrays; rq->expired = rq->arrays + 1; -#else - rq->ckrm_cpu_load = 0; -#endif rq->best_expired_prio = MAX_PRIO; #ifdef CONFIG_SMP @@ -4327,7 +3944,6 @@ void __init sched_init(void) #endif atomic_set(&rq->nr_iowait, 0); -#ifndef CONFIG_CKRM_CPU_SCHEDULE for (j = 0; j < 2; j++) { array = rq->arrays + j; for (k = 0; k < MAX_PRIO; k++) { @@ -4337,9 +3953,7 @@ void __init sched_init(void) // delimiter for bitsearch __set_bit(MAX_PRIO, array->bitmap); } -#endif } - /* * We have to do a little magic to get the first * thread right in SMP mode. @@ -4348,10 +3962,6 @@ void __init sched_init(void) rq->curr = current; rq->idle = current; set_task_cpu(current, smp_processor_id()); -#ifdef CONFIG_CKRM_CPU_SCHEDULE - current->cpu_class = default_cpu_class; - current->array = NULL; -#endif wake_up_forked_process(current); /* @@ -4437,13 +4047,3 @@ int task_running_sys(struct task_struct *p) EXPORT_SYMBOL(task_running_sys); #endif -#ifdef CONFIG_CKRM_CPU_SCHEDULE -/** - * return the classqueue object of a certain processor - * Note: not supposed to be used in performance sensitive functions - */ -struct classqueue_struct * get_cpu_classqueue(int cpu) -{ - return (& (cpu_rq(cpu)->classqueue) ); -} -#endif diff --git a/mm/fremap.c b/mm/fremap.c index eb056db90..5cbe6779a 100644 --- a/mm/fremap.c +++ b/mm/fremap.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -83,6 +84,7 @@ int install_page(struct mm_struct *mm, struct vm_area_struct *vma, mm->rss++; flush_icache_page(vma, page); set_pte(pte, mk_pte(page, prot)); + ckrm_mem_evaluate_page_byadd(page, mm); page_add_file_rmap(page); pte_val = *pte; pte_unmap(pte); diff --git a/mm/memory.c b/mm/memory.c index ddf7049ff..75d75a38b 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -55,6 +55,7 @@ #include #include +#include #ifndef CONFIG_DISCONTIGMEM /* use the per-pgdat data instead for discontigmem - mbligh */ @@ -1534,6 +1535,7 @@ retry: if (write_access) entry = maybe_mkwrite(pte_mkdirty(entry), vma); set_pte(page_table, entry); + ckrm_mem_evaluate_page_byadd(new_page, mm); if (anon) { lru_cache_add_active(new_page); page_add_anon_rmap(new_page, vma, address); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 3dbbeb2f3..ea8ec23e8 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -31,6 +31,7 @@ #include #include #include +#include #include @@ -267,6 +268,7 @@ free_pages_bulk(struct zone *zone, int count, page = list_entry(list->prev, struct page, lru); /* have to delete it as __free_pages_bulk list manipulates */ list_del(&page->lru); + ckrm_clear_page_class(page); __free_pages_bulk(page, base, zone, area, mask, order); ret++; } @@ -598,6 +600,10 @@ __alloc_pages(unsigned int gfp_mask, unsigned int order, might_sleep_if(wait); + if (!ckrm_class_limit_ok((GET_MEM_CLASS(current)))) { + return NULL; + } + zones = zonelist->zones; /* the list of zones suitable for gfp_mask */ if (zones[0] == NULL) /* no zones in the zonelist */ return NULL; @@ -727,6 +733,7 @@ nopage: return NULL; got_pg: kernel_map_pages(page, 1 << order, 1); + ckrm_set_pages_class(page, 1 << order, GET_MEM_CLASS(current)); return page; } diff --git a/mm/rmap.c b/mm/rmap.c index 1f3c84f8c..59c1fc120 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -33,6 +33,7 @@ #include #include #include +#include #include @@ -360,6 +361,7 @@ void page_add_anon_rmap(struct page *page, */ page_map_lock(page); if (!page->mapcount) { + ckrm_mem_evaluate_page_byadd(page, vma->vm_mm); BUG_ON(PageAnon(page)); BUG_ON(page->mapping); SetPageAnon(page); diff --git a/mm/vmscan.c b/mm/vmscan.c index e5f0b0919..0908b4e15 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -38,6 +38,7 @@ #include #include +#include /* * From 0 .. 100. Higher means more swappy. @@ -329,6 +330,9 @@ struct scan_control { /* This context's GFP mask */ unsigned int gfp_mask; + /* Flag used by CKRM */ + unsigned int ckrm_flags; + int may_writepage; }; @@ -539,19 +543,23 @@ static void shrink_cache(struct zone *zone, struct scan_control *sc) { LIST_HEAD(page_list); struct pagevec pvec; - int max_scan = sc->nr_to_scan; + int max_scan = sc->nr_to_scan, nr_pass; + unsigned int ckrm_flags = sc->ckrm_flags, bit_flag; pagevec_init(&pvec, 1); lru_add_drain(); spin_lock_irq(&zone->lru_lock); +redo: + ckrm_get_reclaim_bits(&ckrm_flags, &bit_flag); + nr_pass = zone->nr_inactive; while (max_scan > 0) { struct page *page; int nr_taken = 0; int nr_scan = 0; int nr_freed; - while (nr_scan++ < SWAP_CLUSTER_MAX && + while (nr_pass-- && nr_scan++ < SWAP_CLUSTER_MAX && !list_empty(&zone->inactive_list)) { page = lru_to_page(&zone->inactive_list); @@ -569,15 +577,25 @@ static void shrink_cache(struct zone *zone, struct scan_control *sc) SetPageLRU(page); list_add(&page->lru, &zone->inactive_list); continue; + } else if (bit_flag && !ckrm_kick_page(page, bit_flag)) { + __put_page(page); + SetPageLRU(page); +#ifdef CONFIG_CKRM_MEM_LRUORDER_CHANGE + list_add_tail(&page->lru, &zone->inactive_list); +#else + list_add(&page->lru, &zone->inactive_list); +#endif + continue; } list_add(&page->lru, &page_list); + ckrm_mem_dec_inactive(page); nr_taken++; } zone->nr_inactive -= nr_taken; zone->pages_scanned += nr_taken; spin_unlock_irq(&zone->lru_lock); - if (nr_taken == 0) + if ((bit_flag == 0) && (nr_taken == 0)) goto done; max_scan -= nr_scan; @@ -609,6 +627,9 @@ static void shrink_cache(struct zone *zone, struct scan_control *sc) spin_lock_irq(&zone->lru_lock); } } + if (ckrm_flags && (nr_pass <= 0)) { + goto redo; + } } spin_unlock_irq(&zone->lru_lock); done: @@ -648,10 +669,15 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc) long mapped_ratio; long distress; long swap_tendency; + unsigned int ckrm_flags = sc->ckrm_flags, bit_flag; + int nr_pass; lru_add_drain(); pgmoved = 0; spin_lock_irq(&zone->lru_lock); +redo: + ckrm_get_reclaim_bits(&ckrm_flags, &bit_flag); + nr_pass = zone->nr_active; while (pgscanned < nr_pages && !list_empty(&zone->active_list)) { page = lru_to_page(&zone->active_list); prefetchw_prev_lru_page(page, &zone->active_list, flags); @@ -668,11 +694,24 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc) __put_page(page); SetPageLRU(page); list_add(&page->lru, &zone->active_list); + pgscanned++; + } else if (bit_flag && !ckrm_kick_page(page, bit_flag)) { + __put_page(page); + SetPageLRU(page); +#ifdef CONFIG_CKRM_MEM_LRUORDER_CHANGE + list_add_tail(&page->lru, &zone->active_list); +#else + list_add(&page->lru, &zone->active_list); +#endif } else { list_add(&page->lru, &l_hold); + ckrm_mem_dec_active(page); pgmoved++; - } pgscanned++; + } + if (ckrm_flags && !--nr_pass) { + goto redo; + } } zone->nr_active -= pgmoved; spin_unlock_irq(&zone->lru_lock); @@ -746,6 +785,7 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc) if (!TestClearPageActive(page)) BUG(); list_move(&page->lru, &zone->inactive_list); + ckrm_mem_inc_inactive(page); pgmoved++; if (!pagevec_add(&pvec, page)) { zone->nr_inactive += pgmoved; @@ -774,6 +814,7 @@ refill_inactive_zone(struct zone *zone, struct scan_control *sc) BUG(); BUG_ON(!PageActive(page)); list_move(&page->lru, &zone->active_list); + ckrm_mem_inc_active(page); pgmoved++; if (!pagevec_add(&pvec, page)) { zone->nr_active += pgmoved; @@ -826,6 +867,7 @@ shrink_zone(struct zone *zone, struct scan_control *sc) scan_active = (unsigned long)tmp; } + sc->ckrm_flags = ckrm_setup_reclamation(); atomic_add(scan_active + 1, &zone->nr_scan_active); count = atomic_read(&zone->nr_scan_active); if (count >= SWAP_CLUSTER_MAX) { @@ -841,8 +883,101 @@ shrink_zone(struct zone *zone, struct scan_control *sc) sc->nr_to_scan = count; shrink_cache(zone, sc); } + ckrm_teardown_reclamation(); } +#ifdef CONFIG_CKRM_RES_MEM +// This function needs to be given more thought. +static void +ckrm_shrink_class(ckrm_mem_res_t *cls) +{ + struct scan_control sc; + struct zone *zone; + int zindex = 0, active_credit = 0, inactive_credit = 0; + + if (ckrm_test_set_shrink(cls)) { // set the SHRINK bit atomically + // if it is already set somebody is working on it. so... leave + return; + } + sc.nr_mapped = read_page_state(nr_mapped); + sc.nr_scanned = 0; + sc.ckrm_flags = ckrm_get_reclaim_flags(cls); + sc.nr_reclaimed = 0; + sc.priority = 0; // always very high priority + + for_each_zone(zone) { + int zone_total, zone_limit, active_limit, inactive_limit; + int active_over, inactive_over, count; + u64 temp; + + zone->temp_priority = zone->prev_priority; + zone->prev_priority = sc.priority; + + zone_total = zone->nr_active + zone->nr_inactive + zone->free_pages; + + temp = (u64) cls->pg_limit * zone_total; + do_div(temp, ckrm_tot_lru_pages); + zone_limit = (int) temp; + active_limit = (6 * zone_limit) / 10; // 2/3rd in active list + inactive_limit = (3 * zone_limit) / 10; // 1/3rd in inactive list + + active_over = cls->nr_active[zindex] - active_limit + active_credit; + inactive_over = active_over + + (cls->nr_inactive[zindex] - inactive_limit) + inactive_credit; + + if (active_over > 0) { + atomic_add(active_over + 1, &zone->nr_scan_active); + count = atomic_read(&zone->nr_scan_active); + if (count >= SWAP_CLUSTER_MAX) { + atomic_set(&zone->nr_scan_active, 0); + sc.nr_to_scan = count; + refill_inactive_zone(zone, &sc); + } + active_credit = 0; + } else { + active_credit = active_over; + } + + if (inactive_over > 0) { + atomic_add(inactive_over, &zone->nr_scan_inactive); + count = atomic_read(&zone->nr_scan_inactive); + if (count >= SWAP_CLUSTER_MAX) { + atomic_set(&zone->nr_scan_inactive, 0); + sc.nr_to_scan = count; + shrink_cache(zone, &sc); + } + inactive_credit = 0; + } else { + inactive_credit = inactive_over; + } + zone->prev_priority = zone->temp_priority; + zindex++; + } + ckrm_clear_shrink(cls); +} + +static void +ckrm_shrink_classes(void) +{ + ckrm_mem_res_t *cls; + + spin_lock(&ckrm_mem_lock); + while (!ckrm_shrink_list_empty()) { + cls = list_entry(ckrm_shrink_list.next, ckrm_mem_res_t, + shrink_list); + spin_unlock(&ckrm_mem_lock); + ckrm_shrink_class(cls); + spin_lock(&ckrm_mem_lock); + list_del(&cls->shrink_list); + cls->flags &= ~MEM_NEAR_LIMIT; + } + spin_unlock(&ckrm_mem_lock); +} + +#else +#define ckrm_shrink_classes() do { } while(0) +#endif + /* * This is the direct reclaim path, for page-allocating processes. We only * try to reclaim pages from zones which will satisfy the caller's allocation @@ -1137,6 +1272,9 @@ int kswapd(void *p) schedule(); finish_wait(&pgdat->kswapd_wait, &wait); + if (!ckrm_shrink_list_empty()) + ckrm_shrink_classes(); + else balance_pgdat(pgdat, 0); } } @@ -1146,7 +1284,7 @@ int kswapd(void *p) */ void wakeup_kswapd(struct zone *zone) { - if (zone->free_pages > zone->pages_low) + if ((zone->free_pages > zone->pages_low) && ckrm_shrink_list_empty()) return; if (!waitqueue_active(&zone->zone_pgdat->kswapd_wait)) return; -- 2.47.0