+
+
+#ifdef HAVE_PERCPU_SB
+/*
+ * Per-cpu incore superblock counters
+ *
+ * Simple concept, difficult implementation
+ *
+ * Basically, replace the incore superblock counters with a distributed per cpu
+ * counter for contended fields (e.g. free block count).
+ *
+ * Difficulties arise in that the incore sb is used for ENOSPC checking, and
+ * hence needs to be accurately read when we are running low on space. Hence
+ * there is a method to enable and disable the per-cpu counters based on how
+ * much "stuff" is available in them.
+ *
+ * Basically, a counter is enabled if there is enough free resource to justify
+ * running a per-cpu fast-path. If the per-cpu counter runs out (i.e. a local
+ * ENOSPC), then we disable the counters to synchronise all callers and
+ * re-distribute the available resources.
+ *
+ * If, once we redistributed the available resources, we still get a failure,
+ * we disable the per-cpu counter and go through the slow path.
+ *
+ * The slow path is the current xfs_mod_incore_sb() function. This means that
+ * when we disable a per-cpu counter, we need to drain it's resources back to
+ * the global superblock. We do this after disabling the counter to prevent
+ * more threads from queueing up on the counter.
+ *
+ * Essentially, this means that we still need a lock in the fast path to enable
+ * synchronisation between the global counters and the per-cpu counters. This
+ * is not a problem because the lock will be local to a CPU almost all the time
+ * and have little contention except when we get to ENOSPC conditions.
+ *
+ * Basically, this lock becomes a barrier that enables us to lock out the fast
+ * path while we do things like enabling and disabling counters and
+ * synchronising the counters.
+ *
+ * Locking rules:
+ *
+ * 1. XFS_SB_LOCK() before picking up per-cpu locks
+ * 2. per-cpu locks always picked up via for_each_online_cpu() order
+ * 3. accurate counter sync requires XFS_SB_LOCK + per cpu locks
+ * 4. modifying per-cpu counters requires holding per-cpu lock
+ * 5. modifying global counters requires holding XFS_SB_LOCK
+ * 6. enabling or disabling a counter requires holding the XFS_SB_LOCK
+ * and _none_ of the per-cpu locks.
+ *
+ * Disabled counters are only ever re-enabled by a balance operation
+ * that results in more free resources per CPU than a given threshold.
+ * To ensure counters don't remain disabled, they are rebalanced when
+ * the global resource goes above a higher threshold (i.e. some hysteresis
+ * is present to prevent thrashing).
+ */
+
+#ifdef CONFIG_HOTPLUG_CPU
+/*
+ * hot-plug CPU notifier support.
+ *
+ * We need a notifier per filesystem as we need to be able to identify
+ * the filesystem to balance the counters out. This is achieved by
+ * having a notifier block embedded in the xfs_mount_t and doing pointer
+ * magic to get the mount pointer from the notifier block address.
+ */
+STATIC int
+xfs_icsb_cpu_notify(
+ struct notifier_block *nfb,
+ unsigned long action,
+ void *hcpu)
+{
+ xfs_icsb_cnts_t *cntp;
+ xfs_mount_t *mp;
+ int s;
+
+ mp = (xfs_mount_t *)container_of(nfb, xfs_mount_t, m_icsb_notifier);
+ cntp = (xfs_icsb_cnts_t *)
+ per_cpu_ptr(mp->m_sb_cnts, (unsigned long)hcpu);
+ switch (action) {
+ case CPU_UP_PREPARE:
+ /* Easy Case - initialize the area and locks, and
+ * then rebalance when online does everything else for us. */
+ memset(cntp, 0, sizeof(xfs_icsb_cnts_t));
+ break;
+ case CPU_ONLINE:
+ xfs_icsb_balance_counter(mp, XFS_SBS_ICOUNT, 0);
+ xfs_icsb_balance_counter(mp, XFS_SBS_IFREE, 0);
+ xfs_icsb_balance_counter(mp, XFS_SBS_FDBLOCKS, 0);
+ break;
+ case CPU_DEAD:
+ /* Disable all the counters, then fold the dead cpu's
+ * count into the total on the global superblock and
+ * re-enable the counters. */
+ s = XFS_SB_LOCK(mp);
+ xfs_icsb_disable_counter(mp, XFS_SBS_ICOUNT);
+ xfs_icsb_disable_counter(mp, XFS_SBS_IFREE);
+ xfs_icsb_disable_counter(mp, XFS_SBS_FDBLOCKS);
+
+ mp->m_sb.sb_icount += cntp->icsb_icount;
+ mp->m_sb.sb_ifree += cntp->icsb_ifree;
+ mp->m_sb.sb_fdblocks += cntp->icsb_fdblocks;
+
+ memset(cntp, 0, sizeof(xfs_icsb_cnts_t));
+
+ xfs_icsb_balance_counter(mp, XFS_SBS_ICOUNT, XFS_ICSB_SB_LOCKED);
+ xfs_icsb_balance_counter(mp, XFS_SBS_IFREE, XFS_ICSB_SB_LOCKED);
+ xfs_icsb_balance_counter(mp, XFS_SBS_FDBLOCKS, XFS_ICSB_SB_LOCKED);
+ XFS_SB_UNLOCK(mp, s);
+ break;
+ }
+
+ return NOTIFY_OK;
+}
+#endif /* CONFIG_HOTPLUG_CPU */
+
+int
+xfs_icsb_init_counters(
+ xfs_mount_t *mp)
+{
+ xfs_icsb_cnts_t *cntp;
+ int i;
+
+ mp->m_sb_cnts = alloc_percpu(xfs_icsb_cnts_t);
+ if (mp->m_sb_cnts == NULL)
+ return -ENOMEM;
+
+#ifdef CONFIG_HOTPLUG_CPU
+ mp->m_icsb_notifier.notifier_call = xfs_icsb_cpu_notify;
+ mp->m_icsb_notifier.priority = 0;
+ register_hotcpu_notifier(&mp->m_icsb_notifier);
+#endif /* CONFIG_HOTPLUG_CPU */
+
+ for_each_online_cpu(i) {
+ cntp = (xfs_icsb_cnts_t *)per_cpu_ptr(mp->m_sb_cnts, i);
+ memset(cntp, 0, sizeof(xfs_icsb_cnts_t));
+ }
+ /*
+ * start with all counters disabled so that the
+ * initial balance kicks us off correctly
+ */
+ mp->m_icsb_counters = -1;
+ return 0;
+}
+
+STATIC void
+xfs_icsb_destroy_counters(
+ xfs_mount_t *mp)
+{
+ if (mp->m_sb_cnts) {
+ unregister_hotcpu_notifier(&mp->m_icsb_notifier);
+ free_percpu(mp->m_sb_cnts);
+ }
+}
+
+STATIC inline void
+xfs_icsb_lock_cntr(
+ xfs_icsb_cnts_t *icsbp)
+{
+ while (test_and_set_bit(XFS_ICSB_FLAG_LOCK, &icsbp->icsb_flags)) {
+ ndelay(1000);
+ }
+}
+
+STATIC inline void
+xfs_icsb_unlock_cntr(
+ xfs_icsb_cnts_t *icsbp)
+{
+ clear_bit(XFS_ICSB_FLAG_LOCK, &icsbp->icsb_flags);
+}
+
+
+STATIC inline void
+xfs_icsb_lock_all_counters(
+ xfs_mount_t *mp)
+{
+ xfs_icsb_cnts_t *cntp;
+ int i;
+
+ for_each_online_cpu(i) {
+ cntp = (xfs_icsb_cnts_t *)per_cpu_ptr(mp->m_sb_cnts, i);
+ xfs_icsb_lock_cntr(cntp);
+ }
+}
+
+STATIC inline void
+xfs_icsb_unlock_all_counters(
+ xfs_mount_t *mp)
+{
+ xfs_icsb_cnts_t *cntp;
+ int i;
+
+ for_each_online_cpu(i) {
+ cntp = (xfs_icsb_cnts_t *)per_cpu_ptr(mp->m_sb_cnts, i);
+ xfs_icsb_unlock_cntr(cntp);
+ }
+}
+
+STATIC void
+xfs_icsb_count(
+ xfs_mount_t *mp,
+ xfs_icsb_cnts_t *cnt,
+ int flags)
+{
+ xfs_icsb_cnts_t *cntp;
+ int i;
+
+ memset(cnt, 0, sizeof(xfs_icsb_cnts_t));
+
+ if (!(flags & XFS_ICSB_LAZY_COUNT))
+ xfs_icsb_lock_all_counters(mp);
+
+ for_each_online_cpu(i) {
+ cntp = (xfs_icsb_cnts_t *)per_cpu_ptr(mp->m_sb_cnts, i);
+ cnt->icsb_icount += cntp->icsb_icount;
+ cnt->icsb_ifree += cntp->icsb_ifree;
+ cnt->icsb_fdblocks += cntp->icsb_fdblocks;
+ }
+
+ if (!(flags & XFS_ICSB_LAZY_COUNT))
+ xfs_icsb_unlock_all_counters(mp);
+}
+
+STATIC int
+xfs_icsb_counter_disabled(
+ xfs_mount_t *mp,
+ xfs_sb_field_t field)
+{
+ ASSERT((field >= XFS_SBS_ICOUNT) && (field <= XFS_SBS_FDBLOCKS));
+ return test_bit(field, &mp->m_icsb_counters);
+}
+
+STATIC int
+xfs_icsb_disable_counter(
+ xfs_mount_t *mp,
+ xfs_sb_field_t field)
+{
+ xfs_icsb_cnts_t cnt;
+
+ ASSERT((field >= XFS_SBS_ICOUNT) && (field <= XFS_SBS_FDBLOCKS));
+
+ xfs_icsb_lock_all_counters(mp);
+ if (!test_and_set_bit(field, &mp->m_icsb_counters)) {
+ /* drain back to superblock */
+
+ xfs_icsb_count(mp, &cnt, XFS_ICSB_SB_LOCKED|XFS_ICSB_LAZY_COUNT);
+ switch(field) {
+ case XFS_SBS_ICOUNT:
+ mp->m_sb.sb_icount = cnt.icsb_icount;
+ break;
+ case XFS_SBS_IFREE:
+ mp->m_sb.sb_ifree = cnt.icsb_ifree;
+ break;
+ case XFS_SBS_FDBLOCKS:
+ mp->m_sb.sb_fdblocks = cnt.icsb_fdblocks;
+ break;
+ default:
+ BUG();
+ }
+ }
+
+ xfs_icsb_unlock_all_counters(mp);
+
+ return 0;
+}
+
+STATIC void
+xfs_icsb_enable_counter(
+ xfs_mount_t *mp,
+ xfs_sb_field_t field,
+ uint64_t count,
+ uint64_t resid)
+{
+ xfs_icsb_cnts_t *cntp;
+ int i;
+
+ ASSERT((field >= XFS_SBS_ICOUNT) && (field <= XFS_SBS_FDBLOCKS));
+
+ xfs_icsb_lock_all_counters(mp);
+ for_each_online_cpu(i) {
+ cntp = per_cpu_ptr(mp->m_sb_cnts, i);
+ switch (field) {
+ case XFS_SBS_ICOUNT:
+ cntp->icsb_icount = count + resid;
+ break;
+ case XFS_SBS_IFREE:
+ cntp->icsb_ifree = count + resid;
+ break;
+ case XFS_SBS_FDBLOCKS:
+ cntp->icsb_fdblocks = count + resid;
+ break;
+ default:
+ BUG();
+ break;
+ }
+ resid = 0;
+ }
+ clear_bit(field, &mp->m_icsb_counters);
+ xfs_icsb_unlock_all_counters(mp);
+}
+
+STATIC void
+xfs_icsb_sync_counters_int(
+ xfs_mount_t *mp,
+ int flags)
+{
+ xfs_icsb_cnts_t cnt;
+ int s;
+
+ /* Pass 1: lock all counters */
+ if ((flags & XFS_ICSB_SB_LOCKED) == 0)
+ s = XFS_SB_LOCK(mp);
+
+ xfs_icsb_count(mp, &cnt, flags);
+
+ /* Step 3: update mp->m_sb fields */
+ if (!xfs_icsb_counter_disabled(mp, XFS_SBS_ICOUNT))
+ mp->m_sb.sb_icount = cnt.icsb_icount;
+ if (!xfs_icsb_counter_disabled(mp, XFS_SBS_IFREE))
+ mp->m_sb.sb_ifree = cnt.icsb_ifree;
+ if (!xfs_icsb_counter_disabled(mp, XFS_SBS_FDBLOCKS))
+ mp->m_sb.sb_fdblocks = cnt.icsb_fdblocks;
+
+ if ((flags & XFS_ICSB_SB_LOCKED) == 0)
+ XFS_SB_UNLOCK(mp, s);
+}
+
+/*
+ * Accurate update of per-cpu counters to incore superblock
+ */
+STATIC void
+xfs_icsb_sync_counters(
+ xfs_mount_t *mp)
+{
+ xfs_icsb_sync_counters_int(mp, 0);
+}
+
+/*
+ * lazy addition used for things like df, background sb syncs, etc
+ */
+void
+xfs_icsb_sync_counters_lazy(
+ xfs_mount_t *mp)
+{
+ xfs_icsb_sync_counters_int(mp, XFS_ICSB_LAZY_COUNT);
+}
+
+/*
+ * Balance and enable/disable counters as necessary.
+ *
+ * Thresholds for re-enabling counters are somewhat magic.
+ * inode counts are chosen to be the same number as single
+ * on disk allocation chunk per CPU, and free blocks is
+ * something far enough zero that we aren't going thrash
+ * when we get near ENOSPC.
+ */
+#define XFS_ICSB_INO_CNTR_REENABLE 64
+#define XFS_ICSB_FDBLK_CNTR_REENABLE(mp) \
+ (512 + XFS_ALLOC_SET_ASIDE(mp))
+STATIC void
+xfs_icsb_balance_counter(
+ xfs_mount_t *mp,
+ xfs_sb_field_t field,
+ int flags)
+{
+ uint64_t count, resid;
+ int weight = num_online_cpus();
+ int s;
+
+ if (!(flags & XFS_ICSB_SB_LOCKED))
+ s = XFS_SB_LOCK(mp);
+
+ /* disable counter and sync counter */
+ xfs_icsb_disable_counter(mp, field);
+
+ /* update counters - first CPU gets residual*/
+ switch (field) {
+ case XFS_SBS_ICOUNT:
+ count = mp->m_sb.sb_icount;
+ resid = do_div(count, weight);
+ if (count < XFS_ICSB_INO_CNTR_REENABLE)
+ goto out;
+ break;
+ case XFS_SBS_IFREE:
+ count = mp->m_sb.sb_ifree;
+ resid = do_div(count, weight);
+ if (count < XFS_ICSB_INO_CNTR_REENABLE)
+ goto out;
+ break;
+ case XFS_SBS_FDBLOCKS:
+ count = mp->m_sb.sb_fdblocks;
+ resid = do_div(count, weight);
+ if (count < XFS_ICSB_FDBLK_CNTR_REENABLE(mp))
+ goto out;
+ break;
+ default:
+ BUG();
+ count = resid = 0; /* quiet, gcc */
+ break;
+ }
+
+ xfs_icsb_enable_counter(mp, field, count, resid);
+out:
+ if (!(flags & XFS_ICSB_SB_LOCKED))
+ XFS_SB_UNLOCK(mp, s);
+}
+
+STATIC int
+xfs_icsb_modify_counters_int(
+ xfs_mount_t *mp,
+ xfs_sb_field_t field,
+ int delta,
+ int rsvd,
+ int flags)
+{
+ xfs_icsb_cnts_t *icsbp;
+ long long lcounter; /* long counter for 64 bit fields */
+ int cpu, s, locked = 0;
+ int ret = 0, balance_done = 0;
+
+again:
+ cpu = get_cpu();
+ icsbp = (xfs_icsb_cnts_t *)per_cpu_ptr(mp->m_sb_cnts, cpu),
+ xfs_icsb_lock_cntr(icsbp);
+ if (unlikely(xfs_icsb_counter_disabled(mp, field)))
+ goto slow_path;
+
+ switch (field) {
+ case XFS_SBS_ICOUNT:
+ lcounter = icsbp->icsb_icount;
+ lcounter += delta;
+ if (unlikely(lcounter < 0))
+ goto slow_path;
+ icsbp->icsb_icount = lcounter;
+ break;
+
+ case XFS_SBS_IFREE:
+ lcounter = icsbp->icsb_ifree;
+ lcounter += delta;
+ if (unlikely(lcounter < 0))
+ goto slow_path;
+ icsbp->icsb_ifree = lcounter;
+ break;
+
+ case XFS_SBS_FDBLOCKS:
+ BUG_ON((mp->m_resblks - mp->m_resblks_avail) != 0);
+
+ lcounter = icsbp->icsb_fdblocks - XFS_ALLOC_SET_ASIDE(mp);
+ lcounter += delta;
+ if (unlikely(lcounter < 0))
+ goto slow_path;
+ icsbp->icsb_fdblocks = lcounter + XFS_ALLOC_SET_ASIDE(mp);
+ break;
+ default:
+ BUG();
+ break;
+ }
+ xfs_icsb_unlock_cntr(icsbp);
+ put_cpu();
+ if (locked)
+ XFS_SB_UNLOCK(mp, s);
+ return 0;
+
+ /*
+ * The slow path needs to be run with the SBLOCK
+ * held so that we prevent other threads from
+ * attempting to run this path at the same time.
+ * this provides exclusion for the balancing code,
+ * and exclusive fallback if the balance does not
+ * provide enough resources to continue in an unlocked
+ * manner.
+ */
+slow_path:
+ xfs_icsb_unlock_cntr(icsbp);
+ put_cpu();
+
+ /* need to hold superblock incase we need
+ * to disable a counter */
+ if (!(flags & XFS_ICSB_SB_LOCKED)) {
+ s = XFS_SB_LOCK(mp);
+ locked = 1;
+ flags |= XFS_ICSB_SB_LOCKED;
+ }
+ if (!balance_done) {
+ xfs_icsb_balance_counter(mp, field, flags);
+ balance_done = 1;
+ goto again;
+ } else {
+ /*
+ * we might not have enough on this local
+ * cpu to allocate for a bulk request.
+ * We need to drain this field from all CPUs
+ * and disable the counter fastpath
+ */
+ xfs_icsb_disable_counter(mp, field);
+ }
+
+ ret = xfs_mod_incore_sb_unlocked(mp, field, delta, rsvd);
+
+ if (locked)
+ XFS_SB_UNLOCK(mp, s);
+ return ret;
+}
+
+STATIC int
+xfs_icsb_modify_counters(
+ xfs_mount_t *mp,
+ xfs_sb_field_t field,
+ int delta,
+ int rsvd)
+{
+ return xfs_icsb_modify_counters_int(mp, field, delta, rsvd, 0);
+}
+
+/*
+ * Called when superblock is already locked
+ */
+STATIC int
+xfs_icsb_modify_counters_locked(
+ xfs_mount_t *mp,
+ xfs_sb_field_t field,
+ int delta,
+ int rsvd)
+{
+ return xfs_icsb_modify_counters_int(mp, field, delta,
+ rsvd, XFS_ICSB_SB_LOCKED);
+}
+#endif