Previously, we tracked status changes for ofports on a per-device basis.
Each time in the main thread's loop, we would inspect every ofport
to determine whether the status had changed for corresponding devices.
This patch replaces the per-netdev change_seq with a global 'struct seq'
which tracks status change for all ports. In the average case where
ports are not constantly going up or down, this allows us to check the
sequence once per main loop and not poll any ports. In the worst case,
execution is expected to be similar to how it is currently.
In a test environment of 5000 internal ports and 50 tunnel ports with
bfd, this reduces average CPU usage of the main thread from about 40% to
about 35%.
Signed-off-by: Joe Stringer <joestringer@nicira.com>
Signed-off-by: Ethan Jackson <ethan@nicira.com>
Acked-by: Ethan Jackson <ethan@nicira.com>
lib/command-line.c \
lib/command-line.h \
lib/compiler.h \
+ lib/connectivity.c \
+ lib/connectivity.h \
lib/coverage.c \
lib/coverage.h \
lib/crc32c.c \
--- /dev/null
+/*
+ * Copyright (c) 2013 Nicira, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+#include "connectivity.h"
+#include "ovs-thread.h"
+#include "seq.h"
+
+static struct seq *connectivity_seq;
+
+/* Provides a global seq for connectivity changes.
+ *
+ * Connectivity monitoring modules should call seq_change() on the returned
+ * object whenever the status of a port changes, whether the cause is local or
+ * remote.
+ *
+ * Clients can seq_wait() on this object for changes to netdev flags, features,
+ * ethernet addresses, carrier changes, and bfd/cfm/lacp/stp status. */
+struct seq *
+connectivity_seq_get(void)
+{
+ static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER;
+
+ if (ovsthread_once_start(&once)) {
+ connectivity_seq = seq_create();
+ ovsthread_once_done(&once);
+ }
+
+ return connectivity_seq;
+}
--- /dev/null
+/*
+ * Copyright (c) 2013 Nicira, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CONNECTIVITY_H
+#define CONNECTIVITY_H 1
+
+#include <stdint.h>
+
+/* For tracking connectivity changes globally. */
+struct seq *connectivity_seq_get(void);
+
+#endif /* connectivity.h */
#endif
#include "rtbsd.h"
+#include "connectivity.h"
#include "coverage.h"
#include "dynamic-string.h"
#include "fatal-signal.h"
#include "ovs-thread.h"
#include "packets.h"
#include "poll-loop.h"
-#include "socket-util.h"
+#include "seq.h"
#include "shash.h"
+#include "socket-util.h"
#include "svec.h"
#include "util.h"
#include "vlog.h"
struct ovs_mutex mutex;
unsigned int cache_valid;
- unsigned int change_seq;
int ifindex;
uint8_t etheraddr[ETH_ADDR_LEN];
rtbsd_notifier_wait();
}
-static void
-netdev_bsd_changed(struct netdev_bsd *dev)
-{
- dev->change_seq++;
- if (!dev->change_seq) {
- dev->change_seq++;
- }
-}
-
/* Invalidate cache in case of interface status change. */
static void
netdev_bsd_cache_cb(const struct rtbsd_change *change,
if (is_netdev_bsd_class(netdev_class)) {
dev = netdev_bsd_cast(base_dev);
dev->cache_valid = 0;
- netdev_bsd_changed(dev);
+ seq_change(connectivity_seq_get());
}
netdev_close(base_dev);
}
struct netdev *netdev = node->data;
dev = netdev_bsd_cast(netdev);
dev->cache_valid = 0;
- netdev_bsd_changed(dev);
+ seq_change(connectivity_seq_get());
netdev_close(netdev);
}
shash_destroy(&device_shash);
}
ovs_mutex_init(&netdev->mutex);
- netdev->change_seq = 1;
netdev->tap_fd = -1;
netdev->kernel_name = xstrdup(netdev_->name);
* to retrieve the name of the tap device. */
ovs_mutex_init(&netdev->mutex);
netdev->tap_fd = open("/dev/tap", O_RDWR);
- netdev->change_seq = 1;
if (netdev->tap_fd < 0) {
error = errno;
VLOG_WARN("opening \"/dev/tap\" failed: %s", ovs_strerror(error));
ovs_mutex_lock(&netdev->mutex);
error = netdev_bsd_open_pcap(netdev_get_kernel_name(netdev_),
&rx->pcap_handle, &rx->fd);
- if (!error) {
- netdev_bsd_changed(netdev);
- }
ovs_mutex_unlock(&netdev->mutex);
}
if (!error) {
netdev->cache_valid |= VALID_ETHERADDR;
memcpy(netdev->etheraddr, mac, ETH_ADDR_LEN);
- netdev_bsd_changed(netdev);
+ seq_change(connectivity_seq_get());
}
}
ovs_mutex_unlock(&netdev->mutex);
netdev->netmask = mask;
}
}
- netdev_bsd_changed(netdev);
+ seq_change(connectivity_seq_get());
}
ovs_mutex_unlock(&netdev->mutex);
new_flags = (old_flags & ~nd_to_iff_flags(off)) | nd_to_iff_flags(on);
if (new_flags != old_flags) {
error = set_flags(netdev_get_kernel_name(netdev_), new_flags);
- netdev_bsd_changed(netdev);
+ seq_change(connectivity_seq_get());
}
}
return error;
}
-static unsigned int
-netdev_bsd_change_seq(const struct netdev *netdev)
-{
- return netdev_bsd_cast(netdev)->change_seq;
-}
-
const struct netdev_class netdev_bsd_class = {
"system",
netdev_bsd_update_flags,
- netdev_bsd_change_seq,
-
netdev_bsd_rx_alloc,
netdev_bsd_rx_construct,
netdev_bsd_rx_destruct,
netdev_bsd_update_flags,
- netdev_bsd_change_seq,
-
netdev_bsd_rx_alloc,
netdev_bsd_rx_construct,
netdev_bsd_rx_destruct,
#include <errno.h>
+#include "connectivity.h"
#include "flow.h"
#include "list.h"
#include "netdev-provider.h"
#include "packets.h"
#include "pcap-file.h"
#include "poll-loop.h"
+#include "seq.h"
#include "shash.h"
#include "sset.h"
#include "stream.h"
int mtu OVS_GUARDED;
struct netdev_stats stats OVS_GUARDED;
enum netdev_flags flags OVS_GUARDED;
- unsigned int change_seq OVS_GUARDED;
int ifindex OVS_GUARDED;
struct pstream *pstream OVS_GUARDED;
static unixctl_cb_func netdev_dummy_set_admin_state;
static int netdev_dummy_construct(struct netdev *);
-static void netdev_dummy_changed(struct netdev_dummy *netdev)
- OVS_REQUIRES(netdev->mutex);
static void netdev_dummy_queue_packet(struct netdev_dummy *, struct ofpbuf *);
static void dummy_stream_close(struct dummy_stream *);
netdev->hwaddr[5] = n;
netdev->mtu = 1500;
netdev->flags = 0;
- netdev->change_seq = 1;
netdev->ifindex = -EOPNOTSUPP;
netdev->pstream = NULL;
ovs_mutex_lock(&dev->mutex);
if (!eth_addr_equals(dev->hwaddr, mac)) {
memcpy(dev->hwaddr, mac, ETH_ADDR_LEN);
- netdev_dummy_changed(dev);
+ seq_change(connectivity_seq_get());
}
ovs_mutex_unlock(&dev->mutex);
netdev->flags |= on;
netdev->flags &= ~off;
if (*old_flagsp != netdev->flags) {
- netdev_dummy_changed(netdev);
+ seq_change(connectivity_seq_get());
}
return 0;
return error;
}
-
-static unsigned int
-netdev_dummy_change_seq(const struct netdev *netdev_)
-{
- struct netdev_dummy *netdev = netdev_dummy_cast(netdev_);
- unsigned int change_seq;
-
- ovs_mutex_lock(&netdev->mutex);
- change_seq = netdev->change_seq;
- ovs_mutex_unlock(&netdev->mutex);
-
- return change_seq;
-}
\f
/* Helper functions. */
-static void
-netdev_dummy_changed(struct netdev_dummy *dev)
-{
- dev->change_seq++;
- if (!dev->change_seq) {
- dev->change_seq++;
- }
-}
-
static const struct netdev_class dummy_class = {
"dummy",
NULL, /* init */
netdev_dummy_update_flags,
- netdev_dummy_change_seq,
-
netdev_dummy_rx_alloc,
netdev_dummy_rx_construct,
netdev_dummy_rx_destruct,
#include <string.h>
#include <unistd.h>
+#include "connectivity.h"
#include "coverage.h"
#include "dpif-linux.h"
#include "dynamic-string.h"
#include "packets.h"
#include "poll-loop.h"
#include "rtnetlink-link.h"
+#include "seq.h"
#include "shash.h"
#include "socket-util.h"
#include "sset.h"
struct ovs_mutex mutex;
unsigned int cache_valid;
- unsigned int change_seq;
bool miimon; /* Link status of last poll. */
long long int miimon_interval; /* Miimon Poll rate. Disabled if <= 0. */
unsigned int ifi_flags, unsigned int mask)
OVS_REQUIRES(dev->mutex)
{
- dev->change_seq++;
- if (!dev->change_seq) {
- dev->change_seq++;
- }
+ seq_change(connectivity_seq_get());
if ((dev->ifi_flags ^ ifi_flags) & IFF_RUNNING) {
dev->carrier_resets++;
netdev_linux_common_construct(struct netdev_linux *netdev)
{
ovs_mutex_init(&netdev->mutex);
- netdev->change_seq = 1;
}
/* Creates system and internal devices. */
return error;
}
-static unsigned int
-netdev_linux_change_seq(const struct netdev *netdev_)
-{
- struct netdev_linux *netdev = netdev_linux_cast(netdev_);
- unsigned int change_seq;
-
- ovs_mutex_lock(&netdev->mutex);
- change_seq = netdev->change_seq;
- ovs_mutex_unlock(&netdev->mutex);
-
- return change_seq;
-}
-
#define NETDEV_LINUX_CLASS(NAME, CONSTRUCT, GET_STATS, SET_STATS, \
GET_FEATURES, GET_STATUS) \
{ \
\
netdev_linux_update_flags, \
\
- netdev_linux_change_seq, \
- \
netdev_linux_rx_alloc, \
netdev_linux_rx_construct, \
netdev_linux_rx_destruct, \
* Each "dealloc" function frees raw memory that was allocated by the the
* "alloc" function. The memory's base and derived members might not have ever
* been initialized (but if "construct" returned successfully, then it has been
- * "destruct"ed already). The "dealloc" function is not allowed to fail. */
+ * "destruct"ed already). The "dealloc" function is not allowed to fail.
+ *
+ *
+ * Device Change Notification
+ * ==========================
+ *
+ * Minimally, implementations are required to report changes to netdev flags,
+ * features, ethernet address or carrier through connectivity_seq. Changes to
+ * other properties are allowed to cause notification through this interface,
+ * although implementations should try to avoid this. connectivity_seq_get()
+ * can be used to acquire a reference to the struct seq. The interface is
+ * described in detail in seq.h. */
struct netdev_class {
/* Type of netdevs in this class, e.g. "system", "tap", "gre", etc.
*
int (*update_flags)(struct netdev *netdev, enum netdev_flags off,
enum netdev_flags on, enum netdev_flags *old_flags);
- /* Returns a sequence number which indicates changes in one of 'netdev''s
- * properties. The returned sequence number must be nonzero so that
- * callers have a value which they may use as a reset when tracking
- * 'netdev'.
- *
- * Minimally, the returned sequence number is required to change whenever
- * 'netdev''s flags, features, ethernet address, or carrier changes. The
- * returned sequence number is allowed to change even when 'netdev' doesn't
- * change, although implementations should try to avoid this. */
- unsigned int (*change_seq)(const struct netdev *netdev);
-
/* ## ------------------- ## */
/* ## netdev_rx Functions ## */
/* ## ------------------- ## */
#include <sys/ioctl.h>
#include "byte-order.h"
+#include "connectivity.h"
#include "daemon.h"
#include "dirs.h"
#include "dpif.h"
#include "ofpbuf.h"
#include "packets.h"
#include "route-table.h"
+#include "seq.h"
#include "shash.h"
#include "socket-util.h"
#include "vlog.h"
/* Protects all members below. */
struct ovs_mutex mutex;
- unsigned int change_seq;
uint8_t etheraddr[ETH_ADDR_LEN];
struct netdev_stats stats;
static int netdev_vport_construct(struct netdev *);
static int get_patch_config(const struct netdev *netdev, struct smap *args);
static int get_tunnel_config(const struct netdev *, struct smap *args);
-static void netdev_vport_changed(struct netdev_vport *netdev)
- OVS_REQUIRES(netdev->mutex);
static bool
is_vport_class(const struct netdev_class *class)
struct netdev_vport *netdev = netdev_vport_cast(netdev_);
ovs_mutex_init(&netdev->mutex);
- netdev->change_seq = 1;
eth_addr_random(netdev->etheraddr);
route_table_register();
ovs_mutex_lock(&netdev->mutex);
memcpy(netdev->etheraddr, mac, ETH_ADDR_LEN);
- netdev_vport_changed(netdev);
ovs_mutex_unlock(&netdev->mutex);
+ seq_change(connectivity_seq_get());
return 0;
}
return 0;
}
-static unsigned int
-netdev_vport_change_seq(const struct netdev *netdev)
-{
- return netdev_vport_cast(netdev)->change_seq;
-}
-
static void
netdev_vport_run(void)
{
route_table_wait();
}
\f
-/* Helper functions. */
-
-static void
-netdev_vport_changed(struct netdev_vport *ndv)
-{
- ndv->change_seq++;
- if (!ndv->change_seq) {
- ndv->change_seq++;
- }
-}
-\f
/* Code specific to tunnel types. */
static ovs_be64
ovs_mutex_lock(&dev->mutex);
dev->tnl_cfg = tnl_cfg;
- netdev_vport_changed(dev);
+ seq_change(connectivity_seq_get());
ovs_mutex_unlock(&dev->mutex);
return 0;
ovs_mutex_lock(&dev->mutex);
free(dev->peer);
dev->peer = xstrdup(peer);
- netdev_vport_changed(dev);
+ seq_change(connectivity_seq_get());
ovs_mutex_unlock(&dev->mutex);
return 0;
\
netdev_vport_update_flags, \
\
- netdev_vport_change_seq, \
- \
NULL, /* rx_alloc */ \
NULL, /* rx_construct */ \
NULL, /* rx_destruct */ \
#include <string.h>
#include <unistd.h>
+#include "connectivity.h"
#include "coverage.h"
#include "dpif.h"
#include "dynamic-string.h"
#include "openflow/openflow.h"
#include "packets.h"
#include "poll-loop.h"
+#include "seq.h"
#include "shash.h"
#include "smap.h"
#include "sset.h"
int old_ref_cnt;
atomic_add(&rc->ref_cnt, 1, &old_ref_cnt);
+ seq_change(connectivity_seq_get());
} else {
free(netdev->name);
ovs_assert(list_is_empty(&netdev->saved_flags_list));
: EOPNOTSUPP);
}
-/* Returns a sequence number which indicates changes in one of 'netdev''s
- * properties. The returned sequence will be nonzero so that callers have a
- * value which they may use as a reset when tracking 'netdev'.
- *
- * The returned sequence number will change whenever 'netdev''s flags,
- * features, ethernet address, or carrier changes. It may change for other
- * reasons as well, or no reason at all. */
-unsigned int
-netdev_change_seq(const struct netdev *netdev)
-{
- return netdev->netdev_class->change_seq(netdev);
-}
\f
/* Returns the class type of 'netdev'.
*
int netdev_dump_queue_stats(const struct netdev *,
netdev_dump_queue_stats_cb *, void *aux);
-unsigned int netdev_change_seq(const struct netdev *netdev);
-
#ifdef __cplusplus
}
#endif
#include <stdlib.h>
#include <math.h>
+#include "connectivity.h"
#include "coverage.h"
#include "dynamic-string.h"
#include "flow.h"
#include "ofpbuf.h"
#include "packets.h"
#include "poll-loop.h"
+#include "seq.h"
#include "shash.h"
#include "timeval.h"
#include "unixctl.h"
/* Enable slaves based on link status and LACP feedback. */
HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) {
bond_link_status_update(slave);
- slave->change_seq = netdev_change_seq(slave->netdev);
+ slave->change_seq = seq_read(connectivity_seq_get());
}
if (!bond->active_slave || !bond->active_slave->enabled) {
bond_choose_active_slave(bond);
poll_timer_wait_until(slave->delay_expires);
}
- if (slave->change_seq != netdev_change_seq(slave->netdev)) {
- poll_immediate_wake();
- }
+ seq_wait(connectivity_seq_get(), slave->change_seq);
}
if (bond->next_fake_iface_update != LLONG_MAX) {
uint16_t alloc_port_no; /* Last allocated OpenFlow port number. */
uint16_t max_ports; /* Max possible OpenFlow port num, plus one. */
struct hmap ofport_usage; /* Map ofport to last used time. */
+ uint64_t change_seq; /* Change sequence for netdev status. */
/* Flow tables. */
long long int eviction_group_timer; /* For rate limited reheapification. */
struct netdev *netdev;
struct ofputil_phy_port pp;
ofp_port_t ofp_port; /* OpenFlow port number. */
- unsigned int change_seq;
long long int created; /* Time created, in msec. */
int mtu;
};
#include "bitmap.h"
#include "byte-order.h"
#include "classifier.h"
+#include "connectivity.h"
#include "connmgr.h"
#include "coverage.h"
#include "dynamic-string.h"
#include "pktbuf.h"
#include "poll-loop.h"
#include "random.h"
+#include "seq.h"
#include "shash.h"
#include "simap.h"
#include "smap.h"
int
ofproto_run(struct ofproto *p)
{
- struct sset changed_netdevs;
- const char *changed_netdev;
- struct ofport *ofport;
int error;
+ uint64_t new_seq;
error = p->ofproto_class->run(p);
if (error && error != EAGAIN) {
}
}
- /* Update OpenFlow port status for any port whose netdev has changed.
- *
- * Refreshing a given 'ofport' can cause an arbitrary ofport to be
- * destroyed, so it's not safe to update ports directly from the
- * HMAP_FOR_EACH loop, or even to use HMAP_FOR_EACH_SAFE. Instead, we
- * need this two-phase approach. */
- sset_init(&changed_netdevs);
- HMAP_FOR_EACH (ofport, hmap_node, &p->ports) {
- unsigned int change_seq = netdev_change_seq(ofport->netdev);
- if (ofport->change_seq != change_seq) {
- ofport->change_seq = change_seq;
- sset_add(&changed_netdevs, netdev_get_name(ofport->netdev));
+ new_seq = seq_read(connectivity_seq_get());
+ if (new_seq != p->change_seq) {
+ struct sset devnames;
+ const char *devname;
+ struct ofport *ofport;
+
+ /* Update OpenFlow port status for any port whose netdev has changed.
+ *
+ * Refreshing a given 'ofport' can cause an arbitrary ofport to be
+ * destroyed, so it's not safe to update ports directly from the
+ * HMAP_FOR_EACH loop, or even to use HMAP_FOR_EACH_SAFE. Instead, we
+ * need this two-phase approach. */
+ sset_init(&devnames);
+ HMAP_FOR_EACH (ofport, hmap_node, &p->ports) {
+ sset_add(&devnames, netdev_get_name(ofport->netdev));
}
+ SSET_FOR_EACH (devname, &devnames) {
+ update_port(p, devname);
+ }
+ sset_destroy(&devnames);
+
+ p->change_seq = new_seq;
}
- SSET_FOR_EACH (changed_netdev, &changed_netdevs) {
- update_port(p, changed_netdev);
- }
- sset_destroy(&changed_netdevs);
switch (p->state) {
case S_OPENFLOW:
void
ofproto_wait(struct ofproto *p)
{
- struct ofport *ofport;
-
p->ofproto_class->wait(p);
if (p->ofproto_class->port_poll_wait) {
p->ofproto_class->port_poll_wait(p);
}
-
- HMAP_FOR_EACH (ofport, hmap_node, &p->ports) {
- if (ofport->change_seq != netdev_change_seq(ofport->netdev)) {
- poll_immediate_wake();
- }
- }
+ seq_wait(connectivity_seq_get(), p->change_seq);
switch (p->state) {
case S_OPENFLOW:
}
ofport->ofproto = p;
ofport->netdev = netdev;
- ofport->change_seq = netdev_change_seq(netdev);
ofport->pp = *pp;
ofport->ofp_port = pp->port_no;
ofport->created = time_msec();
* Don't close the old netdev yet in case port_modified has to
* remove a retained reference to it.*/
port->netdev = netdev;
- port->change_seq = netdev_change_seq(netdev);
if (port->ofproto->ofproto_class->port_modified) {
port->ofproto->ofproto_class->port_modified(port);
#include <errno.h>
#include "byte-order.h"
+#include "connectivity.h"
#include "dynamic-string.h"
#include "hash.h"
#include "hmap.h"
#include "netdev.h"
#include "odp-util.h"
#include "packets.h"
+#include "seq.h"
#include "smap.h"
#include "socket-util.h"
#include "tunnel.h"
struct hmap_node match_node;
const struct ofport_dpif *ofport;
- unsigned int netdev_seq;
+ unsigned int change_seq;
struct netdev *netdev;
struct tnl_match match;
tnl_port = xzalloc(sizeof *tnl_port);
tnl_port->ofport = ofport;
tnl_port->netdev = netdev_ref(netdev);
- tnl_port->netdev_seq = netdev_change_seq(tnl_port->netdev);
+ tnl_port->change_seq = seq_read(connectivity_seq_get());
tnl_port->match.in_key = cfg->in_key;
tnl_port->match.ip_src = cfg->ip_src;
changed = tnl_port_add__(ofport, netdev, odp_port, false);
} else if (tnl_port->netdev != netdev
|| tnl_port->match.odp_port != odp_port
- || tnl_port->netdev_seq != netdev_change_seq(netdev)) {
+ || tnl_port->change_seq != seq_read(connectivity_seq_get())) {
VLOG_DBG("reconfiguring %s", tnl_port_get_name(tnl_port));
tnl_port_del__(ofport);
tnl_port_add__(ofport, netdev, odp_port, true);