#include <asm/system.h>
#include <linux/bitops.h>
#include <linux/capability.h>
-#include <linux/config.h>
#include <linux/cpu.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/seq_file.h>
#include <linux/stat.h>
#include <linux/if_bridge.h>
-#include <linux/divert.h>
#include <net/dst.h>
#include <net/pkt_sched.h>
#include <net/checksum.h>
#include <net/iw_handler.h>
#include <asm/current.h>
#include <linux/audit.h>
+#include <linux/dmaengine.h>
#include <linux/err.h>
+#include <linux/ctype.h>
+#include <linux/vs_context.h> /* remove with NXF_HIDE_NETIF */
#include <linux/vs_network.h>
#ifdef CONFIG_XEN
static struct list_head ptype_base[16]; /* 16 way hashed list */
static struct list_head ptype_all; /* Taps */
+#ifdef CONFIG_NET_DMA
+static struct dma_client *net_dma_client;
+static unsigned int net_dma_count;
+static spinlock_t net_dma_event_lock;
+#endif
+
/*
* The @dev_base list is protected by @dev_base_lock and the rtnl
* semaphore.
* For efficiency
*/
-int netdev_nit;
+static int netdev_nit;
/*
* Add a protocol ID to the list. Now that the input handler is
* @name: name string
*
* Network device names need to be valid file names to
- * to allow sysfs to work
+ * to allow sysfs to work. We also disallow any kind of
+ * whitespace.
*/
int dev_valid_name(const char *name)
{
- return !(*name == '\0'
- || !strcmp(name, ".")
- || !strcmp(name, "..")
- || strchr(name, '/'));
+ if (*name == '\0')
+ return 0;
+ if (strlen(name) >= IFNAMSIZ)
+ return 0;
+ if (!strcmp(name, ".") || !strcmp(name, ".."))
+ return 0;
+
+ while (*name) {
+ if (*name == '/' || isspace(*name))
+ return 0;
+ name++;
+ }
+ return 1;
}
/**
* Invalidate hardware checksum when packet is to be mangled, and
* complete checksum manually on outgoing path.
*/
-int skb_checksum_help(struct sk_buff *skb, int inward)
+int skb_checksum_help(struct sk_buff *skb)
{
- unsigned int csum;
+ __wsum csum;
int ret = 0, offset = skb->h.raw - skb->data;
- if (inward) {
- skb->ip_summed = CHECKSUM_NONE;
- goto out;
+ if (skb->ip_summed == CHECKSUM_COMPLETE)
+ goto out_set_summed;
+
+ if (unlikely(skb_shinfo(skb)->gso_size)) {
+ /* Let GSO fix up the checksum. */
+ goto out_set_summed;
}
if (skb_cloned(skb)) {
offset = skb->tail - skb->h.raw;
BUG_ON(offset <= 0);
- BUG_ON(skb->csum + 2 > offset);
+ BUG_ON(skb->csum_offset + 2 > offset);
- *(u16*)(skb->h.raw + skb->csum) = csum_fold(csum);
+ *(__sum16*)(skb->h.raw + skb->csum_offset) = csum_fold(csum);
+
+out_set_summed:
skb->ip_summed = CHECKSUM_NONE;
out:
return ret;
{
struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
struct packet_type *ptype;
- int type = skb->protocol;
+ __be16 type = skb->protocol;
+ int err;
BUG_ON(skb_shinfo(skb)->frag_list);
- BUG_ON(skb->ip_summed != CHECKSUM_HW);
skb->mac.raw = skb->data;
skb->mac_len = skb->nh.raw - skb->data;
__skb_pull(skb, skb->mac_len);
+ if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
+ if (skb_header_cloned(skb) &&
+ (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
+ return ERR_PTR(err);
+ }
+
rcu_read_lock();
list_for_each_entry_rcu(ptype, &ptype_base[ntohs(type) & 15], list) {
if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
+ if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
+ err = ptype->gso_send_check(skb);
+ segs = ERR_PTR(err);
+ if (err || skb_gso_ok(skb, features))
+ break;
+ __skb_push(skb, skb->data - skb->nh.raw);
+ }
segs = ptype->gso_segment(skb, features);
break;
}
EXPORT_SYMBOL(netdev_rx_csum_fault);
#endif
-#ifdef CONFIG_HIGHMEM
/* Actually, we should eliminate this check as soon as we know, that:
* 1. IOMMU is present and allows to map all the memory.
* 2. No high memory really exists on this machine.
static inline int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
{
+#ifdef CONFIG_HIGHMEM
int i;
if (dev->features & NETIF_F_HIGHDMA)
if (PageHighMem(skb_shinfo(skb)->frags[i].page))
return 1;
+#endif
return 0;
}
-#else
-#define illegal_highdma(dev, skb) (0)
-#endif
struct dev_gso_cb {
void (*destructor)(struct sk_buff *skb);
}
if ((skb->h.raw + skb->csum + 2) > skb->tail)
goto out;
- skb->ip_summed = CHECKSUM_HW;
+ skb->ip_summed = CHECKSUM_PARTIAL;
skb->proto_csum_blank = 0;
}
return 0;
/* If packet is not checksummed and device does not support
* checksumming for this protocol, complete checksumming here.
*/
- if (skb->ip_summed == CHECKSUM_HW &&
+ if (skb->ip_summed == CHECKSUM_PARTIAL &&
(!(dev->features & NETIF_F_GEN_CSUM) &&
(!(dev->features & NETIF_F_IP_CSUM) ||
skb->protocol != htons(ETH_P_IP))))
- if (skb_checksum_help(skb, 0))
+ if (skb_checksum_help(skb))
goto out_kfree_skb;
gso:
if (q->enqueue) {
/* Grab device queue */
spin_lock(&dev->queue_lock);
+ q = dev->qdisc;
+ if (q->enqueue) {
+ rc = q->enqueue(skb, q);
+ qdisc_run(dev);
+ spin_unlock(&dev->queue_lock);
- rc = q->enqueue(skb, q);
-
- qdisc_run(dev);
-
+ rc = rc == NET_XMIT_BYPASS ? NET_XMIT_SUCCESS : rc;
+ goto out;
+ }
spin_unlock(&dev->queue_lock);
- rc = rc == NET_XMIT_BYPASS ? NET_XMIT_SUCCESS : rc;
- goto out;
}
/* The device has no queue. Common case for software devices:
struct net_device *dev = skb->dev;
if (dev->master) {
- /*
- * On bonding slaves other than the currently active
- * slave, suppress duplicates except for 802.3ad
- * ETH_P_SLOW and alb non-mcast/bcast.
- */
- if (dev->priv_flags & IFF_SLAVE_INACTIVE) {
- if (dev->master->priv_flags & IFF_MASTER_ALB) {
- if (skb->pkt_type != PACKET_BROADCAST &&
- skb->pkt_type != PACKET_MULTICAST)
- goto keep;
- }
-
- if (dev->master->priv_flags & IFF_MASTER_8023AD &&
- skb->protocol == __constant_htons(ETH_P_SLOW))
- goto keep;
-
+ if (skb_bond_should_drop(skb)) {
kfree_skb(skb);
return NULL;
}
-keep:
skb->dev = dev->master;
}
if (dev->qdisc_ingress) {
__u32 ttl = (__u32) G_TC_RTTL(skb->tc_verd);
if (MAX_RED_LOOP < ttl++) {
- printk("Redir loop detected Dropping packet (%s->%s)\n",
- skb->input_dev->name, skb->dev->name);
+ printk(KERN_WARNING "Redir loop detected Dropping packet (%d->%d)\n",
+ skb->iif, skb->dev->ifindex);
return TC_ACT_SHOT;
}
skb->tc_verd = SET_TC_AT(skb->tc_verd,AT_INGRESS);
- spin_lock(&dev->ingress_lock);
+ spin_lock(&dev->queue_lock);
if ((q = dev->qdisc_ingress) != NULL)
result = q->enqueue(skb, q);
- spin_unlock(&dev->ingress_lock);
+ spin_unlock(&dev->queue_lock);
}
struct packet_type *ptype, *pt_prev;
struct net_device *orig_dev;
int ret = NET_RX_DROP;
- unsigned short type;
+ __be16 type;
/* if we've gotten here through NAPI, check netpoll */
if (skb->dev->poll && netpoll_rx(skb))
if (!skb->tstamp.off_sec)
net_timestamp(skb);
- if (!skb->input_dev)
- skb->input_dev = skb->dev;
+ if (!skb->iif)
+ skb->iif = skb->dev->ifindex;
orig_dev = skb_bond(skb);
case CHECKSUM_UNNECESSARY:
skb->proto_data_valid = 1;
break;
- case CHECKSUM_HW:
+ case CHECKSUM_PARTIAL:
/* XXX Implement me. */
default:
skb->proto_data_valid = 0;
ncls:
#endif
- handle_diverter(skb);
-
if (handle_bridge(&skb, &pt_prev, &ret, orig_dev))
goto out;
}
}
out:
+#ifdef CONFIG_NET_DMA
+ /*
+ * There may not be any more sk_buffs coming right now, so push
+ * any pending DMA copies to hardware
+ */
+ if (net_dma_client) {
+ struct dma_chan *chan;
+ rcu_read_lock();
+ list_for_each_entry_rcu(chan, &net_dma_client->channels, client_node)
+ dma_async_memcpy_issue_pending(chan);
+ rcu_read_unlock();
+ }
+#endif
local_irq_enable();
return;
spin_lock_init(&dev->ingress_lock);
#endif
- ret = alloc_divert_blk(dev);
- if (ret)
- goto out;
-
dev->iflink = -1;
/* Init, if this function is available */
if (ret) {
if (ret > 0)
ret = -EIO;
- goto out_err;
+ goto out;
}
}
if (!dev_valid_name(dev->name)) {
ret = -EINVAL;
- goto out_err;
+ goto out;
}
dev->ifindex = dev_new_index();
= hlist_entry(p, struct net_device, name_hlist);
if (!strncmp(d->name, dev->name, IFNAMSIZ)) {
ret = -EEXIST;
- goto out_err;
+ goto out;
}
}
/* Fix illegal SG+CSUM combinations. */
if ((dev->features & NETIF_F_SG) &&
!(dev->features & NETIF_F_ALL_CSUM)) {
- printk("%s: Dropping NETIF_F_SG since no checksum feature.\n",
+ printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no checksum feature.\n",
dev->name);
dev->features &= ~NETIF_F_SG;
}
/* TSO requires that SG is present as well. */
if ((dev->features & NETIF_F_TSO) &&
!(dev->features & NETIF_F_SG)) {
- printk("%s: Dropping NETIF_F_TSO since no SG feature.\n",
+ printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no SG feature.\n",
dev->name);
dev->features &= ~NETIF_F_TSO;
}
ret = netdev_register_sysfs(dev);
if (ret)
- goto out_err;
+ goto out;
dev->reg_state = NETREG_REGISTERED;
/*
out:
return ret;
-out_err:
- free_divert_blk(dev);
- goto out;
}
/**
goto out;
}
- /*
- * Back compatibility hook. Kill this one in 2.5
- */
- if (dev->name[0] == 0 || dev->name[0] == ' ') {
- err = dev_alloc_name(dev, "eth%d");
- if (err < 0)
- goto out;
- }
-
err = register_netdevice(dev);
out:
rtnl_unlock();
static DEFINE_MUTEX(net_todo_run_mutex);
void netdev_run_todo(void)
{
- struct list_head list = LIST_HEAD_INIT(list);
+ struct list_head list;
/* Need to guard against multiple cpu's getting out of order. */
mutex_lock(&net_todo_run_mutex);
/* Snapshot list, allow later requests */
spin_lock(&net_todo_list_lock);
- list_splice_init(&net_todo_list, &list);
+ list_replace_init(&net_todo_list, &list);
spin_unlock(&net_todo_list_lock);
-
+
while (!list_empty(&list)) {
struct net_device *dev
= list_entry(list.next, struct net_device, todo_list);
struct net_device *dev;
int alloc_size;
+ BUG_ON(strlen(name) >= sizeof(dev->name));
+
/* ensure 32-byte alignment of both the device and private area */
alloc_size = (sizeof(*dev) + NETDEV_ALIGN_CONST) & ~NETDEV_ALIGN_CONST;
alloc_size += sizeof_priv + NETDEV_ALIGN_CONST;
p = kzalloc(alloc_size, GFP_KERNEL);
if (!p) {
- printk(KERN_ERR "alloc_dev: Unable to allocate device.\n");
+ printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
return NULL;
}
/* Notifier chain MUST detach us from master device. */
BUG_TRAP(!dev->master);
- free_divert_blk(dev);
-
/* Finish processing unregister after unlock */
net_set_todo(dev);
EXPORT_SYMBOL(unregister_netdev);
-#ifdef CONFIG_HOTPLUG_CPU
static int dev_cpu_callback(struct notifier_block *nfb,
unsigned long action,
void *ocpu)
return NOTIFY_OK;
}
-#endif /* CONFIG_HOTPLUG_CPU */
+#ifdef CONFIG_NET_DMA
+/**
+ * net_dma_rebalance -
+ * This is called when the number of channels allocated to the net_dma_client
+ * changes. The net_dma_client tries to have one DMA channel per CPU.
+ */
+static void net_dma_rebalance(void)
+{
+ unsigned int cpu, i, n;
+ struct dma_chan *chan;
+
+ if (net_dma_count == 0) {
+ for_each_online_cpu(cpu)
+ rcu_assign_pointer(per_cpu(softnet_data, cpu).net_dma, NULL);
+ return;
+ }
+
+ i = 0;
+ cpu = first_cpu(cpu_online_map);
+
+ rcu_read_lock();
+ list_for_each_entry(chan, &net_dma_client->channels, client_node) {
+ n = ((num_online_cpus() / net_dma_count)
+ + (i < (num_online_cpus() % net_dma_count) ? 1 : 0));
+
+ while(n) {
+ per_cpu(softnet_data, cpu).net_dma = chan;
+ cpu = next_cpu(cpu, cpu_online_map);
+ n--;
+ }
+ i++;
+ }
+ rcu_read_unlock();
+}
+
+/**
+ * netdev_dma_event - event callback for the net_dma_client
+ * @client: should always be net_dma_client
+ * @chan: DMA channel for the event
+ * @event: event type
+ */
+static void netdev_dma_event(struct dma_client *client, struct dma_chan *chan,
+ enum dma_event event)
+{
+ spin_lock(&net_dma_event_lock);
+ switch (event) {
+ case DMA_RESOURCE_ADDED:
+ net_dma_count++;
+ net_dma_rebalance();
+ break;
+ case DMA_RESOURCE_REMOVED:
+ net_dma_count--;
+ net_dma_rebalance();
+ break;
+ default:
+ break;
+ }
+ spin_unlock(&net_dma_event_lock);
+}
+
+/**
+ * netdev_dma_regiser - register the networking subsystem as a DMA client
+ */
+static int __init netdev_dma_register(void)
+{
+ spin_lock_init(&net_dma_event_lock);
+ net_dma_client = dma_async_client_register(netdev_dma_event);
+ if (net_dma_client == NULL)
+ return -ENOMEM;
+
+ dma_async_client_chan_request(net_dma_client, num_online_cpus());
+ return 0;
+}
+
+#else
+static int __init netdev_dma_register(void) { return -ENODEV; }
+#endif /* CONFIG_NET_DMA */
/*
* Initialize the DEV module. At boot time this walks the device list and
BUG_ON(!dev_boot_phase);
- net_random_init();
-
if (dev_proc_init())
goto out;
atomic_set(&queue->backlog_dev.refcnt, 1);
}
+ netdev_dma_register();
+
dev_boot_phase = 0;
open_softirq(NET_TX_SOFTIRQ, net_tx_action, NULL);
git://git.onelab.eu / linux-2.6.git / blobdiff