X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=net%2Fsched%2Fsch_generic.c;h=bc116bd6937c01ffba6adcb142161f38b05b7e8c;hb=97bf2856c6014879bd04983a3e9dfcdac1e7fe85;hp=138ea92ed268457b99d9c6b44a1a9580cc7c016a;hpb=43bc926fffd92024b46cafaf7350d669ba9ca884;p=linux-2.6.git diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 138ea92ed..bc116bd69 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -46,11 +45,10 @@ The idea is the following: - enqueue, dequeue are serialized via top level device spinlock dev->queue_lock. - - tree walking is protected by read_lock_bh(qdisc_tree_lock) + - tree walking is protected by read_lock(qdisc_tree_lock) and this lock is used only in process context. - - updates to tree are made under rtnl semaphore or - from softirq context (__qdisc_destroy rcu-callback) - hence this lock needs local bh disabling. + - updates to tree are made only under rtnl semaphore, + hence this lock may be made without local bh disabling. qdisc_tree_lock must be grabbed BEFORE dev->queue_lock! */ @@ -58,23 +56,23 @@ DEFINE_RWLOCK(qdisc_tree_lock); void qdisc_lock_tree(struct net_device *dev) { - write_lock_bh(&qdisc_tree_lock); + write_lock(&qdisc_tree_lock); spin_lock_bh(&dev->queue_lock); } void qdisc_unlock_tree(struct net_device *dev) { spin_unlock_bh(&dev->queue_lock); - write_unlock_bh(&qdisc_tree_lock); + write_unlock(&qdisc_tree_lock); } /* dev->queue_lock serializes queue accesses for this device AND dev->qdisc pointer itself. - dev->xmit_lock serializes accesses to device driver. + netif_tx_lock serializes accesses to device driver. - dev->queue_lock and dev->xmit_lock are mutually exclusive, + dev->queue_lock and netif_tx_lock are mutually exclusive, if one is grabbed, another must be free. */ @@ -90,14 +88,17 @@ void qdisc_unlock_tree(struct net_device *dev) NOTE: Called under dev->queue_lock with locally disabled BH. */ -int qdisc_restart(struct net_device *dev) +static inline int qdisc_restart(struct net_device *dev) { struct Qdisc *q = dev->qdisc; struct sk_buff *skb; /* Dequeue packet */ - if ((skb = q->dequeue(q)) != NULL) { + if (((skb = dev->gso_skb)) || ((skb = q->dequeue(q)))) { unsigned nolock = (dev->features & NETIF_F_LLTX); + + dev->gso_skb = NULL; + /* * When the driver has LLTX set it does its own locking * in start_xmit. No need to add additional overhead by @@ -108,7 +109,7 @@ int qdisc_restart(struct net_device *dev) * will be requeued. */ if (!nolock) { - if (!spin_trylock(&dev->xmit_lock)) { + if (!netif_tx_trylock(dev)) { collision: /* So, someone grabbed the driver. */ @@ -126,8 +127,6 @@ int qdisc_restart(struct net_device *dev) __get_cpu_var(netdev_rx_stat).cpu_collision++; goto requeue; } - /* Remember that the driver is grabbed by us. */ - dev->xmit_lock_owner = smp_processor_id(); } { @@ -136,14 +135,11 @@ int qdisc_restart(struct net_device *dev) if (!netif_queue_stopped(dev)) { int ret; - if (netdev_nit) - dev_queue_xmit_nit(skb, dev); - ret = dev->hard_start_xmit(skb, dev); + ret = dev_hard_start_xmit(skb, dev); if (ret == NETDEV_TX_OK) { if (!nolock) { - dev->xmit_lock_owner = -1; - spin_unlock(&dev->xmit_lock); + netif_tx_unlock(dev); } spin_lock(&dev->queue_lock); return -1; @@ -157,8 +153,7 @@ int qdisc_restart(struct net_device *dev) /* NETDEV_TX_BUSY - we need to requeue */ /* Release the driver */ if (!nolock) { - dev->xmit_lock_owner = -1; - spin_unlock(&dev->xmit_lock); + netif_tx_unlock(dev); } spin_lock(&dev->queue_lock); q = dev->qdisc; @@ -175,7 +170,10 @@ int qdisc_restart(struct net_device *dev) */ requeue: - q->ops->requeue(skb, q); + if (skb->next) + dev->gso_skb = skb; + else + q->ops->requeue(skb, q); netif_schedule(dev); return 1; } @@ -183,11 +181,23 @@ requeue: return q->q.qlen; } +void __qdisc_run(struct net_device *dev) +{ + if (unlikely(dev->qdisc == &noop_qdisc)) + goto out; + + while (qdisc_restart(dev) < 0 && !netif_queue_stopped(dev)) + /* NOTHING */; + +out: + clear_bit(__LINK_STATE_QDISC_RUNNING, &dev->state); +} + static void dev_watchdog(unsigned long arg) { struct net_device *dev = (struct net_device *)arg; - spin_lock(&dev->xmit_lock); + netif_tx_lock(dev); if (dev->qdisc != &noop_qdisc) { if (netif_device_present(dev) && netif_running(dev) && @@ -203,7 +213,7 @@ static void dev_watchdog(unsigned long arg) dev_hold(dev); } } - spin_unlock(&dev->xmit_lock); + netif_tx_unlock(dev); dev_put(dev); } @@ -227,17 +237,15 @@ void __netdev_watchdog_up(struct net_device *dev) static void dev_watchdog_up(struct net_device *dev) { - spin_lock_bh(&dev->xmit_lock); __netdev_watchdog_up(dev); - spin_unlock_bh(&dev->xmit_lock); } static void dev_watchdog_down(struct net_device *dev) { - spin_lock_bh(&dev->xmit_lock); + netif_tx_lock_bh(dev); if (del_timer(&dev->watchdog_timer)) dev_put(dev); - spin_unlock_bh(&dev->xmit_lock); + netif_tx_unlock_bh(dev); } void netif_carrier_on(struct net_device *dev) @@ -421,10 +429,9 @@ struct Qdisc *qdisc_alloc(struct net_device *dev, struct Qdisc_ops *ops) size = QDISC_ALIGN(sizeof(*sch)); size += ops->priv_size + (QDISC_ALIGNTO - 1); - p = kmalloc(size, GFP_KERNEL); + p = kzalloc(size, GFP_KERNEL); if (!p) goto errout; - memset(p, 0, size); sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p); sch->padded = (char *) sch - (char *) p; @@ -443,13 +450,15 @@ errout: return ERR_PTR(-err); } -struct Qdisc * qdisc_create_dflt(struct net_device *dev, struct Qdisc_ops *ops) +struct Qdisc * qdisc_create_dflt(struct net_device *dev, struct Qdisc_ops *ops, + unsigned int parentid) { struct Qdisc *sch; sch = qdisc_alloc(dev, ops); if (IS_ERR(sch)) goto errout; + sch->parent = parentid; if (!ops->init || ops->init(sch, NULL) == 0) return sch; @@ -475,20 +484,6 @@ void qdisc_reset(struct Qdisc *qdisc) static void __qdisc_destroy(struct rcu_head *head) { struct Qdisc *qdisc = container_of(head, struct Qdisc, q_rcu); - struct Qdisc_ops *ops = qdisc->ops; - -#ifdef CONFIG_NET_ESTIMATOR - gen_kill_estimator(&qdisc->bstats, &qdisc->rate_est); -#endif - write_lock(&qdisc_tree_lock); - if (ops->reset) - ops->reset(qdisc); - if (ops->destroy) - ops->destroy(qdisc); - write_unlock(&qdisc_tree_lock); - module_put(ops->owner); - - dev_put(qdisc->dev); kfree((char *) qdisc - qdisc->padded); } @@ -496,32 +491,23 @@ static void __qdisc_destroy(struct rcu_head *head) void qdisc_destroy(struct Qdisc *qdisc) { - struct list_head cql = LIST_HEAD_INIT(cql); - struct Qdisc *cq, *q, *n; + struct Qdisc_ops *ops = qdisc->ops; if (qdisc->flags & TCQ_F_BUILTIN || - !atomic_dec_and_test(&qdisc->refcnt)) + !atomic_dec_and_test(&qdisc->refcnt)) return; - if (!list_empty(&qdisc->list)) { - if (qdisc->ops->cl_ops == NULL) - list_del(&qdisc->list); - else - list_move(&qdisc->list, &cql); - } - - /* unlink inner qdiscs from dev->qdisc_list immediately */ - list_for_each_entry(cq, &cql, list) - list_for_each_entry_safe(q, n, &qdisc->dev->qdisc_list, list) - if (TC_H_MAJ(q->parent) == TC_H_MAJ(cq->handle)) { - if (q->ops->cl_ops == NULL) - list_del_init(&q->list); - else - list_move_tail(&q->list, &cql); - } - list_for_each_entry_safe(cq, n, &cql, list) - list_del_init(&cq->list); + list_del(&qdisc->list); +#ifdef CONFIG_NET_ESTIMATOR + gen_kill_estimator(&qdisc->bstats, &qdisc->rate_est); +#endif + if (ops->reset) + ops->reset(qdisc); + if (ops->destroy) + ops->destroy(qdisc); + module_put(ops->owner); + dev_put(qdisc->dev); call_rcu(&qdisc->q_rcu, __qdisc_destroy); } @@ -536,20 +522,21 @@ void dev_activate(struct net_device *dev) if (dev->qdisc_sleeping == &noop_qdisc) { struct Qdisc *qdisc; if (dev->tx_queue_len) { - qdisc = qdisc_create_dflt(dev, &pfifo_fast_ops); + qdisc = qdisc_create_dflt(dev, &pfifo_fast_ops, + TC_H_ROOT); if (qdisc == NULL) { printk(KERN_INFO "%s: activation failed\n", dev->name); return; } - write_lock_bh(&qdisc_tree_lock); + write_lock(&qdisc_tree_lock); list_add_tail(&qdisc->list, &dev->qdisc_list); - write_unlock_bh(&qdisc_tree_lock); + write_unlock(&qdisc_tree_lock); } else { qdisc = &noqueue_qdisc; } - write_lock_bh(&qdisc_tree_lock); + write_lock(&qdisc_tree_lock); dev->qdisc_sleeping = qdisc; - write_unlock_bh(&qdisc_tree_lock); + write_unlock(&qdisc_tree_lock); } if (!netif_carrier_ok(dev)) @@ -579,10 +566,17 @@ void dev_deactivate(struct net_device *dev) dev_watchdog_down(dev); - while (test_bit(__LINK_STATE_SCHED, &dev->state)) + /* Wait for outstanding dev_queue_xmit calls. */ + synchronize_rcu(); + + /* Wait for outstanding qdisc_run calls. */ + while (test_bit(__LINK_STATE_QDISC_RUNNING, &dev->state)) yield(); - spin_unlock_wait(&dev->xmit_lock); + if (dev->gso_skb) { + kfree_skb(dev->gso_skb); + dev->gso_skb = NULL; + } } void dev_init_scheduler(struct net_device *dev) @@ -615,15 +609,11 @@ void dev_shutdown(struct net_device *dev) qdisc_unlock_tree(dev); } -EXPORT_SYMBOL(__netdev_watchdog_up); EXPORT_SYMBOL(netif_carrier_on); EXPORT_SYMBOL(netif_carrier_off); EXPORT_SYMBOL(noop_qdisc); -EXPORT_SYMBOL(noop_qdisc_ops); EXPORT_SYMBOL(qdisc_create_dflt); -EXPORT_SYMBOL(qdisc_alloc); EXPORT_SYMBOL(qdisc_destroy); EXPORT_SYMBOL(qdisc_reset); -EXPORT_SYMBOL(qdisc_restart); EXPORT_SYMBOL(qdisc_lock_tree); EXPORT_SYMBOL(qdisc_unlock_tree);