#include <linux/thread_info.h>
#include <linux/time.h>
#include <linux/jiffies.h>
+#include <linux/posix-timers.h>
#include <linux/cpu.h>
+#include <linux/syscalls.h>
+#include <linux/vs_cvirt.h>
+#include <linux/vserver/sched.h>
#include <asm/uaccess.h>
+#include <asm/unistd.h>
#include <asm/div64.h>
#include <asm/timex.h>
+#include <asm/io.h>
+
+#ifdef CONFIG_TIME_INTERPOLATION
+static void time_interpolator_update(long delta_nsec);
+#else
+#define time_interpolator_update(x)
+#endif
/*
* per-CPU timer vector definitions:
*/
-#define TVN_BITS 6
-#define TVR_BITS 8
+
+#define TVN_BITS (CONFIG_BASE_SMALL ? 4 : 6)
+#define TVR_BITS (CONFIG_BASE_SMALL ? 6 : 8)
#define TVN_SIZE (1 << TVN_BITS)
#define TVR_SIZE (1 << TVR_BITS)
#define TVN_MASK (TVN_SIZE - 1)
spin_unlock_irqrestore(&base->lock, flags);
}
+
/***
* mod_timer - modify a timer's timeout
* @timer: the timer to be modified
goto repeat;
}
list_del(&timer->entry);
+ /* Need to make sure that anybody who sees a NULL base also sees the list ops */
+ smp_wmb();
timer->base = NULL;
spin_unlock_irqrestore(&base->lock, flags);
*
* Synchronization rules: callers must prevent restarting of the timer,
* otherwise this function is meaningless. It must not be called from
- * interrupt contexts. Upon exit the timer is not queued and the handler
- * is not running on any CPU.
+ * interrupt contexts. The caller must not hold locks which would prevent
+ * completion of the timer's handler. Upon exit the timer is not queued and
+ * the handler is not running on any CPU.
*
* The function returns whether it has deactivated a pending timer or not.
+ *
+ * del_timer_sync() is slow and complicated because it copes with timer
+ * handlers which re-arm the timer (periodic timers). If the timer handler
+ * is known to not do this (a single shot timer) then use
+ * del_singleshot_timer_sync() instead.
*/
int del_timer_sync(struct timer_list *timer)
{
del_again:
ret += del_timer(timer);
- for_each_cpu(i) {
+ for_each_online_cpu(i) {
base = &per_cpu(tvec_bases, i);
if (base->running_timer == timer) {
while (base->running_timer == timer) {
return ret;
}
-
EXPORT_SYMBOL(del_timer_sync);
+
+/***
+ * del_singleshot_timer_sync - deactivate a non-recursive timer
+ * @timer: the timer to be deactivated
+ *
+ * This function is an optimization of del_timer_sync for the case where the
+ * caller can guarantee the timer does not reschedule itself in its timer
+ * function.
+ *
+ * Synchronization rules: callers must prevent restarting of the timer,
+ * otherwise this function is meaningless. It must not be called from
+ * interrupt contexts. The caller must not hold locks which wold prevent
+ * completion of the timer's handler. Upon exit the timer is not queued and
+ * the handler is not running on any CPU.
+ *
+ * The function returns whether it has deactivated a pending timer or not.
+ */
+int del_singleshot_timer_sync(struct timer_list *timer)
+{
+ int ret = del_timer(timer);
+
+ if (!ret) {
+ ret = del_timer_sync(timer);
+ BUG_ON(ret);
+ }
+
+ return ret;
+}
+EXPORT_SYMBOL(del_singleshot_timer_sync);
#endif
static int cascade(tvec_base_t *base, tvec_t *tv, int index)
smp_wmb();
timer->base = NULL;
spin_unlock_irq(&base->lock);
- fn(data);
+ {
+ u32 preempt_count = preempt_count();
+ fn(data);
+ if (preempt_count != preempt_count()) {
+ printk("huh, entered %p with %08x, exited with %08x?\n", fn, preempt_count, preempt_count());
+ BUG();
+ }
+ }
spin_lock_irq(&base->lock);
goto repeat;
}
/*
* The current time
* wall_to_monotonic is what we need to add to xtime (or xtime corrected
- * for sub jiffie times) to get to monotonic time. Monotonic is pegged at zero
+ * for sub jiffie times) to get to monotonic time. Monotonic is pegged
* at zero at system boot time, so wall_to_monotonic will be negative,
* however, we will ALWAYS keep the tv_nsec part positive so we can use
* the usual normalization.
long time_precision = 1; /* clock precision (us) */
long time_maxerror = NTP_PHASE_LIMIT; /* maximum error (us) */
long time_esterror = NTP_PHASE_LIMIT; /* estimated error (us) */
-long time_phase; /* phase offset (scaled us) */
+static long time_phase; /* phase offset (scaled us) */
long time_freq = (((NSEC_PER_SEC + HZ/2) % HZ - HZ/2) << SHIFT_USEC) / NSEC_PER_USEC;
/* frequency offset (scaled ppm)*/
-long time_adj; /* tick adjust (scaled 1 / HZ) */
+static long time_adj; /* tick adjust (scaled 1 / HZ) */
long time_reftime; /* time at last adjustment (s) */
long time_adjust;
long time_next_adjust;
if (xtime.tv_sec % 86400 == 0) {
xtime.tv_sec--;
wall_to_monotonic.tv_sec++;
+ /* The timer interpolator will make time change gradually instead
+ * of an immediate jump by one second.
+ */
time_interpolator_update(-NSEC_PER_SEC);
time_state = TIME_OOP;
clock_was_set();
if ((xtime.tv_sec + 1) % 86400 == 0) {
xtime.tv_sec++;
wall_to_monotonic.tv_sec--;
+ /* Use of time interpolator for a gradual change of time */
time_interpolator_update(NSEC_PER_SEC);
time_state = TIME_WAIT;
clock_was_set();
if (ltemp > (MAXPHASE / MINSEC) << SHIFT_UPDATE)
ltemp = (MAXPHASE / MINSEC) << SHIFT_UPDATE;
time_offset += ltemp;
+ #if SHIFT_SCALE - SHIFT_HZ - SHIFT_UPDATE > 0
time_adj = -ltemp << (SHIFT_SCALE - SHIFT_HZ - SHIFT_UPDATE);
+ #else
+ time_adj = -ltemp >> (SHIFT_HZ + SHIFT_UPDATE - SHIFT_SCALE);
+ #endif
} else {
ltemp = time_offset;
if (!(time_status & STA_FLL))
if (ltemp > (MAXPHASE / MINSEC) << SHIFT_UPDATE)
ltemp = (MAXPHASE / MINSEC) << SHIFT_UPDATE;
time_offset -= ltemp;
+ #if SHIFT_SCALE - SHIFT_HZ - SHIFT_UPDATE > 0
time_adj = ltemp << (SHIFT_SCALE - SHIFT_HZ - SHIFT_UPDATE);
+ #else
+ time_adj = ltemp >> (SHIFT_HZ + SHIFT_UPDATE - SHIFT_SCALE);
+ #endif
}
/*
do {
ticks--;
update_wall_time_one_tick();
- } while (ticks);
-
- if (xtime.tv_nsec >= 1000000000) {
- xtime.tv_nsec -= 1000000000;
- xtime.tv_sec++;
- second_overflow();
- }
-}
-
-static inline void do_process_times(struct task_struct *p,
- unsigned long user, unsigned long system)
-{
- unsigned long psecs;
-
- psecs = (p->utime += user);
- psecs += (p->stime += system);
- if (psecs / HZ > p->rlim[RLIMIT_CPU].rlim_cur) {
- /* Send SIGXCPU every second.. */
- if (!(psecs % HZ))
- send_sig(SIGXCPU, p, 1);
- /* and SIGKILL when we go over max.. */
- if (psecs / HZ > p->rlim[RLIMIT_CPU].rlim_max)
- send_sig(SIGKILL, p, 1);
- }
-}
-
-static inline void do_it_virt(struct task_struct * p, unsigned long ticks)
-{
- unsigned long it_virt = p->it_virt_value;
-
- if (it_virt) {
- it_virt -= ticks;
- if (!it_virt) {
- it_virt = p->it_virt_incr;
- send_sig(SIGVTALRM, p, 1);
+ if (xtime.tv_nsec >= 1000000000) {
+ xtime.tv_nsec -= 1000000000;
+ xtime.tv_sec++;
+ second_overflow();
}
- p->it_virt_value = it_virt;
- }
-}
-
-static inline void do_it_prof(struct task_struct *p)
-{
- unsigned long it_prof = p->it_prof_value;
-
- if (it_prof) {
- if (--it_prof == 0) {
- it_prof = p->it_prof_incr;
- send_sig(SIGPROF, p, 1);
- }
- p->it_prof_value = it_prof;
- }
+ } while (ticks);
}
-void update_one_process(struct task_struct *p, unsigned long user,
- unsigned long system, int cpu)
-{
- do_process_times(p, user, system);
- do_it_virt(p, user);
- do_it_prof(p);
-}
-
/*
* Called from the timer interrupt handler to charge one tick to the current
* process. user_tick is 1 if the tick is user time, 0 for system.
void update_process_times(int user_tick)
{
struct task_struct *p = current;
- int cpu = smp_processor_id(), system = user_tick ^ 1;
+ int cpu = smp_processor_id();
- update_one_process(p, user_tick, system, cpu);
+ /* Note: this timer irq context must be accounted for as well. */
+ if (user_tick)
+ account_user_time(p, jiffies_to_cputime(1));
+ else
+ account_system_time(p, HARDIRQ_OFFSET, jiffies_to_cputime(1));
run_local_timers();
- scheduler_tick(user_tick, system);
+ if (rcu_pending(cpu))
+ rcu_check_callbacks(cpu, user_tick);
+ scheduler_tick();
+ run_posix_cpu_timers(p);
}
/*
*/
unsigned long avenrun[3];
+EXPORT_SYMBOL(avenrun);
+
/*
* calc_load - given tick count, update the avenrun load estimates.
* This is called while holding a write_lock on xtime_lock.
void do_timer(struct pt_regs *regs)
{
jiffies_64++;
-#ifndef CONFIG_SMP
- /* SMP process accounting uses the local APIC timer */
-
- update_process_times(user_mode(regs));
-#endif
update_times();
}
-#if !defined(__alpha__) && !defined(__ia64__)
+#ifdef __ARCH_WANT_SYS_ALARM
/*
* For backwards compatibility? This can be done in libc so Alpha
#endif
-#ifndef __alpha__
-
-/*
- * The Alpha uses getxpid, getxuid, and getxgid instead. Maybe this
- * should be moved into arch/i386 instead?
- */
/**
* sys_getpid - return the thread group id of the current process
*/
asmlinkage long sys_getpid(void)
{
- return current->tgid;
+ return vx_map_tgid(current->tgid);
}
/*
* Make sure we read the pid before re-reading the
* parent pointer:
*/
- rmb();
+ smp_rmb();
parent = me->group_leader->real_parent;
if (old != parent)
continue;
#endif
break;
}
- return pid;
+ return vx_map_pid(pid);
}
+#ifdef __alpha__
+
+/*
+ * The Alpha uses getxpid, getxuid, and getxgid instead.
+ */
+
+asmlinkage long do_getxpid(long *ppid)
+{
+ *ppid = sys_getppid();
+ return sys_getpid();
+}
+
+#else /* _alpha_ */
+
asmlinkage long sys_getuid(void)
{
/* Only we change this so SMP safe */
add_timer(&timer);
schedule();
- del_timer_sync(&timer);
+ del_singleshot_timer_sync(&timer);
timeout = expire - jiffies;
* too.
*/
- do_gettimeofday((struct timeval *)&tp);
- tp.tv_nsec *= NSEC_PER_USEC;
+ getnstimeofday(&tp);
tp.tv_sec += wall_to_monotonic.tv_sec;
tp.tv_nsec += wall_to_monotonic.tv_nsec;
if (tp.tv_nsec - NSEC_PER_SEC >= 0) {
tp.tv_nsec = tp.tv_nsec - NSEC_PER_SEC;
tp.tv_sec++;
}
+ if (vx_flags(VXF_VIRT_UPTIME, 0))
+ vx_vsi_uptime(&tp, NULL);
val.uptime = tp.tv_sec + (tp.tv_nsec ? 1 : 0);
val.loads[0] = avenrun[0] << (SI_LOAD_SHIFT - FSHIFT);
}
#ifdef CONFIG_TIME_INTERPOLATION
-volatile unsigned long last_nsec_offset;
-#ifndef __HAVE_ARCH_CMPXCHG
-spinlock_t last_nsec_offset_lock = SPIN_LOCK_UNLOCKED;
-#endif
struct time_interpolator *time_interpolator;
static struct time_interpolator *time_interpolator_list;
-static spinlock_t time_interpolator_lock = SPIN_LOCK_UNLOCKED;
+static DEFINE_SPINLOCK(time_interpolator_lock);
+
+static inline u64 time_interpolator_get_cycles(unsigned int src)
+{
+ unsigned long (*x)(void);
+
+ switch (src)
+ {
+ case TIME_SOURCE_FUNCTION:
+ x = time_interpolator->addr;
+ return x();
+
+ case TIME_SOURCE_MMIO64 :
+ return readq((void __iomem *) time_interpolator->addr);
+
+ case TIME_SOURCE_MMIO32 :
+ return readl((void __iomem *) time_interpolator->addr);
+
+ default: return get_cycles();
+ }
+}
+
+static inline u64 time_interpolator_get_counter(void)
+{
+ unsigned int src = time_interpolator->source;
+
+ if (time_interpolator->jitter)
+ {
+ u64 lcycle;
+ u64 now;
+
+ do {
+ lcycle = time_interpolator->last_cycle;
+ now = time_interpolator_get_cycles(src);
+ if (lcycle && time_after(lcycle, now))
+ return lcycle;
+ /* Keep track of the last timer value returned. The use of cmpxchg here
+ * will cause contention in an SMP environment.
+ */
+ } while (unlikely(cmpxchg(&time_interpolator->last_cycle, lcycle, now) != lcycle));
+ return now;
+ }
+ else
+ return time_interpolator_get_cycles(src);
+}
+
+void time_interpolator_reset(void)
+{
+ time_interpolator->offset = 0;
+ time_interpolator->last_counter = time_interpolator_get_counter();
+}
+
+#define GET_TI_NSECS(count,i) (((((count) - i->last_counter) & (i)->mask) * (i)->nsec_per_cyc) >> (i)->shift)
+
+unsigned long time_interpolator_get_offset(void)
+{
+ /* If we do not have a time interpolator set up then just return zero */
+ if (!time_interpolator)
+ return 0;
+
+ return time_interpolator->offset +
+ GET_TI_NSECS(time_interpolator_get_counter(), time_interpolator);
+}
+
+#define INTERPOLATOR_ADJUST 65536
+#define INTERPOLATOR_MAX_SKIP 10*INTERPOLATOR_ADJUST
+
+static void time_interpolator_update(long delta_nsec)
+{
+ u64 counter;
+ unsigned long offset;
+
+ /* If there is no time interpolator set up then do nothing */
+ if (!time_interpolator)
+ return;
+
+ /* The interpolator compensates for late ticks by accumulating
+ * the late time in time_interpolator->offset. A tick earlier than
+ * expected will lead to a reset of the offset and a corresponding
+ * jump of the clock forward. Again this only works if the
+ * interpolator clock is running slightly slower than the regular clock
+ * and the tuning logic insures that.
+ */
+
+ counter = time_interpolator_get_counter();
+ offset = time_interpolator->offset + GET_TI_NSECS(counter, time_interpolator);
+
+ if (delta_nsec < 0 || (unsigned long) delta_nsec < offset)
+ time_interpolator->offset = offset - delta_nsec;
+ else {
+ time_interpolator->skips++;
+ time_interpolator->ns_skipped += delta_nsec - offset;
+ time_interpolator->offset = 0;
+ }
+ time_interpolator->last_counter = counter;
+
+ /* Tuning logic for time interpolator invoked every minute or so.
+ * Decrease interpolator clock speed if no skips occurred and an offset is carried.
+ * Increase interpolator clock speed if we skip too much time.
+ */
+ if (jiffies % INTERPOLATOR_ADJUST == 0)
+ {
+ if (time_interpolator->skips == 0 && time_interpolator->offset > TICK_NSEC)
+ time_interpolator->nsec_per_cyc--;
+ if (time_interpolator->ns_skipped > INTERPOLATOR_MAX_SKIP && time_interpolator->offset == 0)
+ time_interpolator->nsec_per_cyc++;
+ time_interpolator->skips = 0;
+ time_interpolator->ns_skipped = 0;
+ }
+}
static inline int
is_better_time_interpolator(struct time_interpolator *new)
void
register_time_interpolator(struct time_interpolator *ti)
{
+ unsigned long flags;
+
+ /* Sanity check */
+ if (ti->frequency == 0 || ti->mask == 0)
+ BUG();
+
+ ti->nsec_per_cyc = ((u64)NSEC_PER_SEC << ti->shift) / ti->frequency;
spin_lock(&time_interpolator_lock);
- write_seqlock_irq(&xtime_lock);
- if (is_better_time_interpolator(ti))
+ write_seqlock_irqsave(&xtime_lock, flags);
+ if (is_better_time_interpolator(ti)) {
time_interpolator = ti;
- write_sequnlock_irq(&xtime_lock);
+ time_interpolator_reset();
+ }
+ write_sequnlock_irqrestore(&xtime_lock, flags);
ti->next = time_interpolator_list;
time_interpolator_list = ti;
unregister_time_interpolator(struct time_interpolator *ti)
{
struct time_interpolator *curr, **prev;
+ unsigned long flags;
spin_lock(&time_interpolator_lock);
prev = &time_interpolator_list;
prev = &curr->next;
}
- write_seqlock_irq(&xtime_lock);
+ write_seqlock_irqsave(&xtime_lock, flags);
if (ti == time_interpolator) {
/* we lost the best time-interpolator: */
time_interpolator = NULL;
for (curr = time_interpolator_list; curr; curr = curr->next)
if (is_better_time_interpolator(curr))
time_interpolator = curr;
+ time_interpolator_reset();
}
- write_sequnlock_irq(&xtime_lock);
+ write_sequnlock_irqrestore(&xtime_lock, flags);
spin_unlock(&time_interpolator_lock);
}
#endif /* CONFIG_TIME_INTERPOLATION */
+
+/**
+ * msleep - sleep safely even with waitqueue interruptions
+ * @msecs: Time in milliseconds to sleep for
+ */
+void msleep(unsigned int msecs)
+{
+ unsigned long timeout = msecs_to_jiffies(msecs) + 1;
+
+ while (timeout) {
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ timeout = schedule_timeout(timeout);
+ }
+}
+
+EXPORT_SYMBOL(msleep);
+
+/**
+ * msleep_interruptible - sleep waiting for waitqueue interruptions
+ * @msecs: Time in milliseconds to sleep for
+ */
+unsigned long msleep_interruptible(unsigned int msecs)
+{
+ unsigned long timeout = msecs_to_jiffies(msecs) + 1;
+
+ while (timeout && !signal_pending(current)) {
+ set_current_state(TASK_INTERRUPTIBLE);
+ timeout = schedule_timeout(timeout);
+ }
+ return jiffies_to_msecs(timeout);
+}
+
+EXPORT_SYMBOL(msleep_interruptible);
git://git.onelab.eu / linux-2.6.git / blobdiff