* Aug 28, 2003
* Created.
* July 07, 2004
- * clean up, add comments
+ * clean up, add comments
+ *
+ *
+ * Overview:
+ * ---------
+ *
+ * Please read Documentation/ckrm/cpu_sched for a general overview of
+ * how the O(1) CKRM scheduler.
+ *
+ * ckrm_classqueue.h provides the definition to maintain the
+ * per cpu class runqueue.
*
*/
#include <linux/list.h>
-#define CLASSQUEUE_SIZE 128
+#define CLASSQUEUE_SIZE_SHIFT 7
+#define CLASSQUEUE_SIZE ( 1 << CLASSQUEUE_SIZE_SHIFT )
#define CQ_BITMAP_SIZE ((((CLASSQUEUE_SIZE+1+7)/8)+sizeof(long)-1)/sizeof(long))
/**
* struct cq_prio_array: duplicates prio_array defined in sched.c
- *
- * I duplicate this data structure to make ckrm_classqueue implementation more modular
*/
struct cq_prio_array {
int nr_active;
* @base: base priority
* @base_offset: index in array for the base
*
- * classqueue can be thought of as runqueue of classes (instead of runqueue of tasks)
- * as task runqueue, each processor has a classqueue
- * a class enters the classqueue when the first task in this class local runqueue shows up
- * a class enters the classqueue when the last task in the local runqueue leaves
- * class local runqueues are ordered based their priority
- *
- * status:
- * hzheng: is 32bit base long enough?
+ * classqueue can be thought of as runqueue of lrq's (per cpu object of
+ * a CKRM class as task runqueue (instead of runqueue of tasks)
+ * - a class's local lrq is enqueued into the local classqueue when a
+ * first task is enqueued lrq.
+ * - a class's local lrq is removed from the local classqueue when the
+ * last task is dequeued from the lrq.
+ * - lrq's are ordered based on their priority (determined elsewhere)
+ * ( CKRM: caculated based on it's progress (cvt) and urgency (top_priority)
*/
+
struct classqueue_struct {
- struct cq_prio_array array;
+ int enabled; // support dynamic on/off
unsigned long base;
unsigned long base_offset;
+ struct cq_prio_array array;
};
/**
- * struct cq_node_struct - the link object between class local runqueue and classqueue
+ * struct cq_node_struct:
+ * - the link object between class local runqueue and classqueue
* @list: links the class local runqueue to classqueue
- * @prio: class priority, which is caculated based on it's progress (cvt) and urgency (top_priority)
+ * @prio: class priority
* @index: real index into the classqueue array, calculated based on priority
- *
- * NOTE: make sure list is empty when it's not in classqueue
*/
struct cq_node_struct {
struct list_head list;
int prio;
int index;
+ /*
+ * set when the class jump out of the class queue window
+ * class with this value set should be repositioned whenever classqueue slides window
+ * real_prio is valid when need_repos is set
+ */
+ int real_prio;
+ int need_repos;
};
typedef struct cq_node_struct cq_node_t;
-typedef unsigned long long CVT_t; // cummulative virtual time
-
static inline void cq_node_init(cq_node_t * node)
{
node->prio = 0;
node->index = -1;
+ node->real_prio = 0;
+ node->need_repos = 0;
INIT_LIST_HEAD(&node->list);
}
}
/*initialize the data structure*/
-int classqueue_init(struct classqueue_struct *cq);
+int classqueue_init(struct classqueue_struct *cq, int enabled);
-/*add the class to classqueue*/
-void classqueue_enqueue(struct classqueue_struct *cq, cq_node_t * node, int prio);
+/*add the class to classqueue at given priority */
+void classqueue_enqueue(struct classqueue_struct *cq,
+ cq_node_t * node, int prio);
-/**
- * classqueue_dequeue - remove the class from classqueue
- *
- * internal:
- * called when the last task is removed from the queue
- * checked on load balancing and schedule
- * hzheng: why don't I call it on class_dequeue_task?
- */
+/*remove the class from classqueue */
void classqueue_dequeue(struct classqueue_struct *cq, cq_node_t * node);
/*change the position of the class in classqueue*/
-void classqueue_update_prio(struct classqueue_struct *cq, cq_node_t * node, int new_prio);
+void classqueue_update_prio(struct classqueue_struct *cq,
+ cq_node_t * node, int new_prio);
/*return the first class in classqueue*/
cq_node_t *classqueue_get_head(struct classqueue_struct *cq);
/**
* class_compare_prio: compare the priority of this two nodes
*/
-static inline int class_compare_prio(struct cq_node_struct* node1, struct cq_node_struct* node2)
+static inline int class_compare_prio(struct cq_node_struct* node1,
+ struct cq_node_struct* node2)
{
return ( node1->prio - node2->prio);
}
* Copyright (C) Haoqiang Zheng, IBM Corp. 2004
* Copyright (C) Hubertus Franke, IBM Corp. 2004
*
- * Latest version, more details at http://ckrm.sf.net
- *
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
*
*/
+/*
+ * Overview:
+ * ---------
+ *
+ * Please read Documentation/ckrm/cpu_sched for a general overview of
+ * how the O(1) CKRM scheduler.
+ *
+ * ckrm_sched.h provides the definition for the per class local runqueue.
+ *
+ */
+
#ifndef _CKRM_SCHED_H
#define _CKRM_SCHED_H
struct list_head queue[MAX_PRIO];
};
-#ifdef CONFIG_CKRM_CPU_SCHEDULE
-#define rq_active(p,rq) (get_task_lrq(p)->active)
-#define rq_expired(p,rq) (get_task_lrq(p)->expired)
-int __init init_ckrm_sched_res(void);
-#else
+
+#ifndef CONFIG_CKRM_CPU_SCHEDULE
+
#define rq_active(p,rq) (rq->active)
#define rq_expired(p,rq) (rq->expired)
static inline void init_ckrm_sched_res(void) {}
static inline int ckrm_cpu_monitor_init(void) {return 0;}
-#endif
-#ifdef CONFIG_CKRM_CPU_SCHEDULE
+#else
+
+#define rq_active(p,rq) (get_task_lrq(p)->active)
+#define rq_expired(p,rq) (get_task_lrq(p)->expired)
+
+enum ckrm_sched_mode {
+ CKRM_SCHED_MODE_DISABLED, /* always use default linux scheduling */
+ /* effectively disables the ckrm scheduler */
+ CKRM_SCHED_MODE_ENABLED /* always uses ckrm scheduling behavior */
+};
+
+extern unsigned int ckrm_sched_mode; /* true internal sched_mode (DIS/EN ABLED) */
+
+int __init init_ckrm_sched_res(void);
+
+typedef unsigned long long CVT_t; // cummulative virtual time
+
struct ckrm_runqueue {
cq_node_t classqueue_linkobj; /*links in classqueue */
struct ckrm_cpu_class *cpu_class; // class it belongs to
reset to jiffies if expires
*/
unsigned long expired_timestamp;
+ int best_expired_prio;
/*
* highest priority of tasks in active
CVT_t local_cvt;
unsigned long lrq_load;
- int local_weight;
+ /* Three different weights are distinguished:
+ * local_weight, skewed_weight, over_weight:
+ *
+ * - local_weight: main weight to drive CVT progression
+ * - over_weight: weight to reduce savings when over its guarantee
+ * - skewed_weight: weight to use when local_weight to small
+ * avoids starvation problems.
+ */
+ int local_weight;
+ int over_weight;
+ int skewed_weight;
/*
- * unused CPU time accumulated while thoe class
+ * unused CPU time accumulated while the class
* is inactive goes to savings
*
* initialized to be 0
* a class can't accumulate more than SAVING_THRESHOLD of savings
*/
- unsigned long long savings;
+ CVT_t savings;
unsigned long magic; //for debugging
-};
+} ____cacheline_aligned_in_smp;
+
+#define CKRM_LRQ_MAGIC (0xACDC0702)
typedef struct ckrm_runqueue ckrm_lrq_t;
+#define ckrm_cpu_disabled() (ckrm_sched_mode == CKRM_SCHED_MODE_DISABLED)
+#define ckrm_cpu_enabled() (ckrm_sched_mode == CKRM_SCHED_MODE_ENABLED)
+
/**
* ckrm_cpu_class_stat - cpu usage statistics maintained for each class
*
*/
int eshare;
int meshare;
+
+ /* a boolean indicates if the class has savings or not */
+ int has_savings;
+
+ /*
+ * a temporary value used by reorder_surplus_queue
+ */
+ int demand_per_share;
};
#define CKRM_CPU_CLASS_MAGIC 0x7af2abe3
-#define USAGE_SAMPLE_FREQ HZ //sample every 1 seconds
-#define NS_PER_SAMPLE (USAGE_SAMPLE_FREQ*(NSEC_PER_SEC/HZ))
-#define USAGE_WINDOW_SIZE 60 //keep the last 60 sample
+#define USAGE_SAMPLE_FREQ (HZ) //sample every 1 seconds
+#define USAGE_MAX_HISTORY (60) // keep the last 60 usage samples
+#define NS_PER_SAMPLE (USAGE_SAMPLE_FREQ*(NSEC_PER_SEC/HZ))
struct ckrm_usage {
- unsigned long samples[USAGE_WINDOW_SIZE]; //record usages
- unsigned long sample_pointer; //pointer for the sliding window
- unsigned long long last_ns; //ns for last sample
- long long last_sample_jiffies; //in number of jiffies
+ unsigned long samples[USAGE_MAX_HISTORY]; //record usages
+ unsigned long sample_pointer; // pointer for the sliding window
+ unsigned long long last_ns; // ns for last sample
+ long long last_sample_jiffies; // in number of jiffies
};
/*
- * manages the class status
- * there should be only one instance of this object for each class in the whole system
+ * CPU controller object allocated for each CLASS
*/
struct ckrm_cpu_class {
struct ckrm_core_class *core;
spinlock_t cnt_lock; // always grab parent's lock first and then child's
struct ckrm_cpu_class_stat stat;
struct list_head links; // for linking up in cpu classes
- ckrm_lrq_t local_queues[NR_CPUS]; // runqueues
+ struct list_head surplus_queue; //used for surplus allocation
+ ckrm_lrq_t* local_queues[NR_CPUS]; // runqueues
struct ckrm_usage usage;
unsigned long magic; //for debugging
+#ifdef __SIMULATOR__
+ int class_id;
+#endif
};
-#define cpu_class_weight(cls) (cls->stat.meshare)
+#define cpu_class_weight(cls) (SHARE_TO_WEIGHT(cls->stat.meshare))
#define local_class_weight(lrq) (lrq->local_weight)
static inline int valid_cpu_class(struct ckrm_cpu_class * cls)
{
int i;
- for (i=0; i < USAGE_WINDOW_SIZE; i++)
+ for (i=0; i < USAGE_MAX_HISTORY; i++)
usage->samples[i] = 0;
usage->sample_pointer = 0;
usage->last_ns = 0;
// printk("sample = %llu jiffies=%lu \n",cur_sample, jiffies);
usage->sample_pointer ++;
- if (usage->sample_pointer >= USAGE_WINDOW_SIZE)
+ if (usage->sample_pointer >= USAGE_MAX_HISTORY)
usage->sample_pointer = 0;
}
-//duration is specified in number of jiffies
-//return the usage in percentage
-static inline int get_ckrm_usage(struct ckrm_cpu_class* clsptr, int duration)
-{
- int nr_samples = duration/USAGE_SAMPLE_FREQ?:1;
- struct ckrm_usage* usage = &clsptr->usage;
- unsigned long long total = 0;
- int i, idx;
-
- if (nr_samples > USAGE_WINDOW_SIZE)
- nr_samples = USAGE_WINDOW_SIZE;
-
- idx = usage->sample_pointer;
- for (i = 0; i< nr_samples; i++) {
- if (! idx)
- idx = USAGE_WINDOW_SIZE;
- idx --;
- total += usage->samples[idx];
- }
- total *= 100;
- do_div(total,nr_samples);
- do_div(total,NS_PER_SAMPLE);
- do_div(total,cpus_weight(cpu_online_map));
- return total;
-}
-
-
#define lrq_nr_running(lrq) \
(lrq->active->nr_active + lrq->expired->nr_active)
-static inline ckrm_lrq_t *
-get_ckrm_lrq(struct ckrm_cpu_class*cls, int cpu)
+static inline ckrm_lrq_t *get_ckrm_lrq(struct ckrm_cpu_class*cls, int cpu)
{
- return &(cls->local_queues[cpu]);
+ return cls->local_queues[cpu];
}
static inline ckrm_lrq_t *get_task_lrq(struct task_struct *p)
{
- return &(p->cpu_class->local_queues[task_cpu(p)]);
+ return p->cpu_class->local_queues[task_cpu(p)];
}
#define task_list_entry(list) list_entry(list,struct task_struct,run_list)
void init_cpu_class(struct ckrm_cpu_class *cls,ckrm_shares_t* shares);
void ckrm_cpu_change_class(void *task, void *old, void *new);
-
#define CPU_DEMAND_ENQUEUE 0
#define CPU_DEMAND_DEQUEUE 1
#define CPU_DEMAND_DESCHEDULE 2
#define CPU_DEMAND_INIT 3
/*functions exported by ckrm_cpu_monitor.c*/
+int update_effectives(void);
void ckrm_cpu_monitor(int check_min);
int ckrm_cpu_monitor_init(void);
-void ckrm_cpu_stat_init(struct ckrm_cpu_class_stat *stat);
+void ckrm_cpu_stat_init(struct ckrm_cpu_class_stat *stat, int eshares);
void cpu_demand_event(struct ckrm_cpu_demand_stat* local_stat, int event, unsigned long long len);
void adjust_local_weight(void);
*
*******************************************************************/
-#define CLASS_QUANTIZER 16 //shift from ns to increase class bonus
-#define PRIORITY_QUANTIZER 2 //controls how much a high prio task can borrow
-
-#define CKRM_SHARE_ACCURACY 10
-#define NSEC_PER_MS 1000000
-#define NSEC_PER_JIFFIES (NSEC_PER_SEC/HZ)
-
-
-#define MAX_SAVINGS_ABSOLUTE (10LLU*NSEC_PER_SEC) // 10 seconds
-
-#define CVT_UPDATE_TICK ((HZ/2)?:1)
-
-// ABSOLUTE_CKRM_TUNING determines whether classes can make up
-// lost time in absolute time or in relative values
-
-#define ABSOLUTE_CKRM_TUNING // preferred due to more predictable behavior
-
-#ifdef ABSOLUTE_CKRM_TUNING
-
-#define MAX_SAVINGS MAX_SAVINGS_ABSOLUTE
-//an absolute bonus of 200ms for classes when reactivated
-#define INTERACTIVE_BONUS(lrq) ((200*NSEC_PER_MS)/local_class_weight(lrq))
-#define SAVINGS_LEAK_SPEED (CVT_UPDATE_TICK/10*NSEC_PER_JIFFIES)
-
-#define scale_cvt(val,lrq) ((val)*local_class_weight(lrq))
-#define unscale_cvt(val,lrq) (do_div(val,local_class_weight(lrq)))
-
-#else
-
-#define MAX_SAVINGS (MAX_SAVINGS_ABSOLUTE >> CKRM_SHARE_ACCURACY)
/*
- * to improve system responsiveness
- * an inactive class is put a little bit ahead of the current class when it wakes up
- * the amount is set in normalized term to simplify the calculation
- * for class with 100% share, it can be 2s ahead
- * while for class with 10% share, it can be 200ms ahead
+ * The class priority is biasd toward classes with high priority tasks.
+ * But we need to prevent this bias from starving other classes.
+ * If a class has nice value of -20, how much it can starve the default class?
+ * priority bonus = (120-100) >> PRIORITY_QUANTIZER,
+ * if PRIORITY_QUANTIZER = 2, then it's 5 steps ahead
+ * A class without bonus thus can't get to run until:
+ * bonus * CKRM_MAX_WEIGHT * CVT_INC_PERSHARE = (120-100) >> PRIORITY_QUANTIZER
+ * (1 << CKRM_WEIGHT_SHIFT)
+ * (1 << CLASS_QUANTIZER)
+*/
+
+/*
+ * CKRM_WEIGHT_SHIFT and CLASS_QUANTIZER control how much a class with
+ * high priority task can starve a normal priority class, so it should
+ * be constant CLASS_QUANTIZER should not be too small otherwise we
+ * don't have enough bins in the classqueue.
+ * The ideal value of CLASS_QUANTIZER is 20, but a little smaller is acceptable
*/
-#define INTERACTIVE_BONUS(lrq) (2*NSEC_PER_MS)
-/*
- * normalized savings can't be more than MAX_NORMALIZED_SAVINGS
- * based on the current configuration
- * this means that a class with share 100% will accumulate 10s at most
- * while a class with 1% of the share can only accumulate 100ms
+#define CLASS_QUANTIZER (18)// shift from ns to increase class bonus
+#define PRIORITY_QUANTIZER (2) // how much a high prio task can borrow
+#define CKRM_WEIGHT_SHIFT (8) // 1/2^x == finest weight granularity
+#define CKRM_MAX_WEIGHT (1<<CKRM_WEIGHT_SHIFT) // - " -
+
+/* SHARES:
+ * shares are set in a hierarchical path. Since specified share settings
+ * of a class (c) are relative to the parent (p) and its totals
+ * the shares can get very small, dependent on how many classes are
+ * specified.
*/
+
+#define CKRM_SHARE_SHIFT (13)
+#define CKRM_SHARE_MAX (1 << CKRM_SHARE_SHIFT)
-//a class with share 100% can get 100ms every 500ms
-//while a class with share 10% can only get 10ms every 500ms
-#define SAVINGS_LEAK_SPEED ((CVT_UPDATE_TICK/5*NSEC_PER_JIFFIES) >> CKRM_SHARE_ACCURACY)
+#define SHARE_TO_WEIGHT(x) ((x) >> (CKRM_SHARE_SHIFT - CKRM_WEIGHT_SHIFT))
+#define WEIGHT_TO_SHARE(x) ((x) << (CKRM_SHARE_SHIFT - CKRM_WEIGHT_SHIFT))
-#define scale_cvt(val,lrq) (val)
-#define unscale_cvt(val,lrq) (val)
+/* Other constants */
-#endif
+#define NSEC_PER_MS (1000000)
+#define NSEC_PER_JIFFIES (NSEC_PER_SEC/HZ)
+#define MAX_SAVINGS_ABSOLUTE (4LLU*NSEC_PER_SEC) // 4 seconds
+#define CVT_UPDATE_TICK ((HZ/2)?:1)
+#define MAX_SAVINGS MAX_SAVINGS_ABSOLUTE
+#define SAVINGS_LEAK_SPEED (CVT_UPDATE_TICK/10*NSEC_PER_JIFFIES)
/**
* get_effective_prio: return the effective priority of a class local queue
}
/*
- * runqueue load is the local_weight of all the classes on this cpu
- * must be called with class_list_lock held
+ * moved to ckrm_sched.c
+ * but may need to make it static inline to improve performance
*/
-static inline unsigned long ckrm_cpu_load(int cpu)
-{
- struct ckrm_cpu_class *clsptr;
- ckrm_lrq_t* lrq;
- struct ckrm_cpu_demand_stat* l_stat;
- int total_load = 0;
- int load;
-
- list_for_each_entry(clsptr,&active_cpu_classes,links) {
- lrq = get_ckrm_lrq(clsptr,cpu);
- l_stat = get_cls_local_stat(clsptr,cpu);
- load = lrq->local_weight;
- if (l_stat->cpu_demand < load)
- load = l_stat->cpu_demand;
- total_load += load;
- }
- return total_load;
-}
-
-static inline void class_enqueue_task(struct task_struct *p,
- prio_array_t * array)
-{
- ckrm_lrq_t *lrq;
- int effective_prio;
-
- lrq = get_task_lrq(p);
-
- cpu_demand_event(&p->demand_stat,CPU_DEMAND_ENQUEUE,0);
- lrq->lrq_load += task_load(p);
-
- if ((p->prio < lrq->top_priority) && (array == lrq->active))
- set_top_priority(lrq, p->prio);
-
- if (! cls_in_classqueue(&lrq->classqueue_linkobj)) {
- cpu_demand_event(get_task_lrq_stat(p),CPU_DEMAND_ENQUEUE,0);
- effective_prio = get_effective_prio(lrq);
- classqueue_enqueue(lrq->classqueue, &lrq->classqueue_linkobj, effective_prio);
- }
-
-}
-
-static inline void class_dequeue_task(struct task_struct *p,
- prio_array_t * array)
-{
- ckrm_lrq_t *lrq = get_task_lrq(p);
- unsigned long load = task_load(p);
-
- BUG_ON(lrq->lrq_load < load);
- lrq->lrq_load -= load;
-
- cpu_demand_event(&p->demand_stat,CPU_DEMAND_DEQUEUE,0);
-
- if ((array == lrq->active) && (p->prio == lrq->top_priority)
- && list_empty(&(array->queue[p->prio])))
- set_top_priority(lrq,
- find_next_bit(array->bitmap, MAX_PRIO,
- p->prio));
-}
-
-/*
- * called after a task is switched out. Update the local cvt accounting
- * we need to stick with long instead of long long due to nonexistent 64-bit division
- */
-static inline void update_local_cvt(struct task_struct *p, unsigned long nsec)
-{
- ckrm_lrq_t * lrq = get_task_lrq(p);
-
- unsigned long cvt_inc = nsec / local_class_weight(lrq);
-
- lrq->local_cvt += cvt_inc;
- lrq->uncounted_ns += nsec;
-
- update_class_priority(lrq);
-}
-
+void update_local_cvt(struct task_struct *p, unsigned long nsec);
+
static inline int class_preempts_curr(struct task_struct * p, struct task_struct* curr)
{
struct cq_node_struct* node1 = &(get_task_lrq(p)->classqueue_linkobj);
return rand;
}
-void update_class_cputime(int this_cpu);
+void update_class_cputime(int this_cpu, int idle);
/**********************************************/
/* PID_LOAD_BALANCING */
/**********************************************/
+
+#define CPU_PID_CTRL_TICK 32
+
struct ckrm_load_struct {
unsigned long load_p; /*propotional*/
unsigned long load_i; /*integral */
}
void ckrm_load_sample(ckrm_load_t* ckrm_load,int cpu);
-long pid_get_pressure(ckrm_load_t* ckrm_load, int local_group);
+long ckrm_get_pressure(ckrm_load_t* ckrm_load, int local_group);
#define rq_ckrm_load(rq) (&((rq)->ckrm_load))
-static inline void ckrm_sched_tick(unsigned long j,int this_cpu,struct ckrm_load_struct* ckrm_load)
-{
- read_lock(&class_list_lock);
-
-#ifdef CONFIG_SMP
- ckrm_load_sample(ckrm_load,this_cpu);
-#endif
-
- if (! (j % CVT_UPDATE_TICK)) {
- // printk("ckrm_sched j=%lu\n",j);
- classqueue_update_base(get_cpu_classqueue(this_cpu));
- update_class_cputime(this_cpu);
- }
-
- read_unlock(&class_list_lock);
-}
#endif /*CONFIG_CKRM_CPU_SCHEDULE */
#endif
+
+
+
git://git.onelab.eu / linux-2.6.git / commitdiff