X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=lib%2Fbond.c;h=9caa3f2efb745c6bd33cf87722c4d07f252c8349;hb=b2fda3effc787f265b5ad5dfa967ac00627bd075;hp=3ec1ba2075ed793225cd6b7f1045397d70b0d4fe;hpb=5827ce14d087a18ede839c39ce3438e0b0e6f717;p=sliver-openvswitch.git diff --git a/lib/bond.c b/lib/bond.c index 3ec1ba207..9caa3f2ef 100644 --- a/lib/bond.c +++ b/lib/bond.c @@ -26,7 +26,6 @@ #include "dynamic-string.h" #include "flow.h" #include "hmap.h" -#include "lacp.h" #include "list.h" #include "netdev.h" #include "odp-util.h" @@ -66,6 +65,7 @@ struct bond_slave { long long delay_expires; /* Time after which 'enabled' may change. */ bool up; /* Last link status read from netdev. */ bool enabled; /* May be chosen for flows? */ + bool lacp_may_enable; /* LACP considers this interface bondable. */ tag_type tag; /* Tag associated with this slave. */ /* Rebalancing info. Used only by bond_rebalance(). */ @@ -74,9 +74,7 @@ struct bond_slave { uint64_t tx_bytes; /* Sum across 'tx_bytes' of entries. */ /* BM_STABLE specific bonding info. */ - size_t stb_idx; /* Index in 'bond''s 'stb_slaves' array. - Undefined value if participating in a - BTM_STABLE bond or not enabled. */ + uint32_t stb_id; /* ID used for 'stb_slaves' ordering. */ }; /* A bond, that is, a set of network devices grouped to improve performance or @@ -93,6 +91,9 @@ struct bond { struct bond_slave *active_slave; tag_type no_slaves_tag; /* Tag for flows when all slaves disabled. */ int updelay, downdelay; /* Delay before slave goes up/down, in ms. */ + bool lacp_negotiated; /* LACP negotiations were successful. */ + bool bond_revalidate; /* True if flows need revalidation. */ + uint32_t basis; /* Basis for flow hash function. */ /* SLB specific bonding info. */ struct bond_entry *hash; /* An array of (BOND_MASK + 1) elements. */ @@ -101,13 +102,7 @@ struct bond { bool send_learning_packets; /* BM_STABLE specific bonding info. */ - struct bond_slave **stb_slaves; /* Ordered list of enabled slaves. */ - size_t n_stb_slaves; /* Number of slaves in 'stb_slaves'. */ - size_t len_stb_slaves; /* Slaves allocated in 'stb_slaves'. */ - bool stb_need_sort; /* True if stb_slaves is not sorted. */ - - /* LACP. */ - const struct lacp *lacp; /* Client provided LACP handle. */ + tag_type stb_tag; /* Tag associated with this bond. */ /* Monitoring. */ enum bond_detect_mode detect; /* Link status mode, one of BLSM_*. */ @@ -131,14 +126,13 @@ static struct bond_slave *bond_slave_lookup(struct bond *, const void *slave_); static bool bond_is_link_up(struct bond *, struct netdev *); static void bond_enable_slave(struct bond_slave *, bool enable, struct tag_set *); -static bool bond_stb_sort(struct bond *); -static void bond_stb_enable_slave(struct bond_slave *); static void bond_link_status_update(struct bond_slave *, struct tag_set *); static void bond_choose_active_slave(struct bond *, struct tag_set *); static bool bond_is_tcp_hash(const struct bond *); static unsigned int bond_hash_src(const uint8_t mac[ETH_ADDR_LEN], - uint16_t vlan); -static unsigned int bond_hash_tcp(const struct flow *, uint16_t vlan); + uint16_t vlan, uint32_t basis); +static unsigned int bond_hash_tcp(const struct flow *, uint16_t vlan, + uint32_t basis); static struct bond_entry *lookup_bond_entry(const struct bond *, const struct flow *, uint16_t vlan); @@ -226,6 +220,7 @@ bond_create(const struct bond_settings *s) bond = xzalloc(sizeof *bond); hmap_init(&bond->slaves); bond->no_slaves_tag = tag_create_random(); + bond->stb_tag = tag_create_random(); bond->miimon_next_update = LLONG_MAX; bond->next_fake_iface_update = LLONG_MAX; @@ -268,9 +263,7 @@ bond_destroy(struct bond *bond) * * The caller should register each slave on 'bond' by calling * bond_slave_register(). This is optional if none of the slaves' - * configuration has changed, except that it is mandatory if 's' enables LACP - * and 'bond' previously didn't have LACP enabled. In any case it can't - * hurt. + * configuration has changed. In any case it can't hurt. * * Returns true if the configuration has changed in such a way that requires * flow revalidation. @@ -289,7 +282,6 @@ bond_reconfigure(struct bond *bond, const struct bond_settings *s) hmap_insert(&all_bonds, &bond->hmap_node, hash_string(bond->name, 0)); } - bond->lacp = s->lacp; bond->detect = s->detect; bond->miimon_interval = s->miimon_interval; bond->updelay = s->up_delay; @@ -301,6 +293,11 @@ bond_reconfigure(struct bond *bond, const struct bond_settings *s) revalidate = true; } + if (bond->basis != s->basis) { + bond->basis = s->basis; + revalidate = true; + } + if (bond->detect == BLSM_CARRIER) { struct bond_slave *slave; @@ -328,21 +325,9 @@ bond_reconfigure(struct bond *bond, const struct bond_settings *s) bond->next_fake_iface_update = LLONG_MAX; } - if (bond->balance != BM_STABLE) { - free(bond->stb_slaves); - bond->stb_slaves = NULL; - } else if (!bond->stb_slaves) { - struct bond_slave *slave; - - bond->n_stb_slaves = 0; - bond->len_stb_slaves = 0; - bond->stb_slaves = NULL; - - HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) { - if (slave->enabled) { - bond_stb_enable_slave(slave); - } - } + if (bond->bond_revalidate) { + revalidate = true; + bond->bond_revalidate = false; } if (bond->balance == BM_AB || !bond->hash || revalidate) { @@ -352,19 +337,37 @@ bond_reconfigure(struct bond *bond, const struct bond_settings *s) return revalidate; } +static void +bond_slave_set_netdev__(struct bond *bond, struct bond_slave *slave, + struct netdev *netdev) +{ + if (slave->netdev != netdev) { + if (bond->monitor) { + if (slave->netdev) { + netdev_monitor_remove(bond->monitor, slave->netdev); + } + netdev_monitor_add(bond->monitor, netdev); + } + slave->netdev = netdev; + } +} + /* Registers 'slave_' as a slave of 'bond'. The 'slave_' pointer is an * arbitrary client-provided pointer that uniquely identifies a slave within a * bond. If 'slave_' already exists within 'bond' then this function * reconfigures the existing slave. * + * 'stb_id' is used in BM_STABLE bonds to guarantee consistent slave choices + * across restarts and distributed vswitch instances. It should be unique per + * slave, and preferably consistent across restarts and reconfigurations. + * * 'netdev' must be the network device that 'slave_' represents. It is owned * by the client, so the client must not close it before either unregistering * 'slave_' or destroying 'bond'. - * - * If 'bond' has a LACP configuration then 'slave_' should be the same handle - * used in the LACP module. */ + */ void -bond_slave_register(struct bond *bond, void *slave_, struct netdev *netdev) +bond_slave_register(struct bond *bond, void *slave_, uint32_t stb_id, + struct netdev *netdev) { struct bond_slave *slave = bond_slave_lookup(bond, slave_); @@ -376,15 +379,38 @@ bond_slave_register(struct bond *bond, void *slave_, struct netdev *netdev) slave->aux = slave_; slave->delay_expires = LLONG_MAX; slave->up = bond_is_link_up(bond, netdev); + slave->name = xstrdup(netdev_get_name(netdev)); + bond->bond_revalidate = true; + slave->enabled = false; bond_enable_slave(slave, slave->up, NULL); } - slave->netdev = netdev; + if (slave->stb_id != stb_id) { + slave->stb_id = stb_id; + bond->bond_revalidate = true; + } + + bond_slave_set_netdev__(bond, slave, netdev); + free(slave->name); slave->name = xstrdup(netdev_get_name(netdev)); } +/* Updates the network device to be used with 'slave_' to 'netdev'. + * + * This is useful if the caller closes and re-opens the network device + * registered with bond_slave_register() but doesn't need to change anything + * else. */ +void +bond_slave_set_netdev(struct bond *bond, void *slave_, struct netdev *netdev) +{ + struct bond_slave *slave = bond_slave_lookup(bond, slave_); + if (slave) { + bond_slave_set_netdev__(bond, slave, netdev); + } +} + /* Unregisters 'slave_' from 'bond'. If 'bond' does not contain such a slave * then this function has no effect. * @@ -399,6 +425,10 @@ bond_slave_unregister(struct bond *bond, const void *slave_) return; } + if (bond->monitor) { + netdev_monitor_remove(bond->monitor, slave->netdev); + } + bond_enable_slave(slave, false, NULL); del_active = bond->active_slave == slave; @@ -426,16 +456,27 @@ bond_slave_unregister(struct bond *bond, const void *slave_) } } +/* Should be called on each slave in 'bond' before bond_run() to indicate the + * results of lacp_slave_may_enable() on 'slave_'. */ +void +bond_slave_set_lacp_may_enable(struct bond *bond, void *slave_, + bool may_enable) +{ + bond_slave_lookup(bond, slave_)->lacp_may_enable = may_enable; +} + /* Performs periodic maintenance on 'bond'. The caller must provide 'tags' to * allow tagged flows to be invalidated. * * The caller should check bond_should_send_learning_packets() afterward. */ void -bond_run(struct bond *bond, struct tag_set *tags) +bond_run(struct bond *bond, struct tag_set *tags, bool lacp_negotiated) { struct bond_slave *slave; bool is_tcp_hash = bond_is_tcp_hash(bond); + bond->lacp_negotiated = lacp_negotiated; + /* Update link status. */ if (bond->detect == BLSM_CARRIER || time_msec() >= bond->miimon_next_update) @@ -446,6 +487,10 @@ bond_run(struct bond *bond, struct tag_set *tags) bond->miimon_next_update = time_msec() + bond->miimon_interval; } + if (bond->monitor) { + netdev_monitor_flush(bond->monitor); + } + /* Enable slaves based on link status and LACP feedback. */ HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) { bond_link_status_update(slave, tags); @@ -460,13 +505,24 @@ bond_run(struct bond *bond, struct tag_set *tags) bond->next_fake_iface_update = time_msec() + 1000; } - if (bond_stb_sort(bond) || is_tcp_hash != bond_is_tcp_hash(bond)) { - struct bond_slave *slave; + if (is_tcp_hash != bond_is_tcp_hash(bond)) { + bond->bond_revalidate = true; + } + + if (bond->bond_revalidate) { + bond->bond_revalidate = false; bond_entry_reset(bond); - HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) { - tag_set_add(tags, slave->tag); + if (bond->balance != BM_STABLE) { + struct bond_slave *slave; + + HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) { + tag_set_add(tags, slave->tag); + } + } else { + tag_set_add(tags, bond->stb_tag); } + tag_set_add(tags, bond->no_slaves_tag); } /* Invalidate any tags required by */ @@ -512,7 +568,7 @@ bond_wait(struct bond *bond) static bool may_send_learning_packets(const struct bond *bond) { - return !lacp_negotiated(bond->lacp) && bond->balance != BM_AB; + return !bond->lacp_negotiated && bond->balance != BM_AB; } /* Returns true if 'bond' needs the client to send out packets to assist with @@ -589,7 +645,7 @@ bond_check_admissibility(struct bond *bond, const void *slave_, { /* Admit all packets if LACP has been negotiated, because that means that * the remote switch is aware of the bond and will "do the right thing". */ - if (lacp_negotiated(bond->lacp)) { + if (bond->lacp_negotiated) { return BV_ACCEPT; } @@ -627,7 +683,7 @@ bond_choose_output_slave(struct bond *bond, const struct flow *flow, { struct bond_slave *slave = choose_output_slave(bond, flow, vlan); if (slave) { - *tags |= slave->tag; + *tags |= bond->balance == BM_STABLE ? bond->stb_tag : slave->tag; return slave->aux; } else { *tags |= bond->no_slaves_tag; @@ -949,18 +1005,13 @@ bond_unixctl_show(struct unixctl_conn *conn, ds_put_format(&ds, "bond_mode: %s\n", bond_mode_to_string(bond->balance)); - if (bond->lacp) { - ds_put_format(&ds, "lacp: %s\n", - lacp_is_active(bond->lacp) ? "active" : "passive"); - } else { - ds_put_cstr(&ds, "lacp: off\n"); - } - if (bond->balance != BM_AB) { ds_put_format(&ds, "bond-hash-algorithm: %s\n", bond_is_tcp_hash(bond) ? "balance-tcp" : "balance-slb"); } + ds_put_format(&ds, "bond-hash-basis: %"PRIu32"\n", bond->basis); + ds_put_format(&ds, "bond-detect-mode: %s\n", bond->monitor ? "carrier" : "miimon"); @@ -977,6 +1028,9 @@ bond_unixctl_show(struct unixctl_conn *conn, bond->next_rebalance - time_msec()); } + ds_put_format(&ds, "lacp_negotiated: %s\n", + bond->lacp_negotiated ? "true" : "false"); + HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) { struct bond_entry *be; struct flow flow; @@ -993,6 +1047,9 @@ bond_unixctl_show(struct unixctl_conn *conn, slave->delay_expires - time_msec()); } + ds_put_format(&ds, "\tlacp_may_enable: %s\n", + slave->lacp_may_enable ? "true" : "false"); + if (!bond_is_balanced(bond)) { continue; } @@ -1183,11 +1240,13 @@ bond_unixctl_hash(struct unixctl_conn *conn, const char *args_, uint8_t hash; char *hash_cstr; unsigned int vlan; - char *mac_s, *vlan_s; + uint32_t basis; + char *mac_s, *vlan_s, *basis_s; char *save_ptr = NULL; mac_s = strtok_r(args, " ", &save_ptr); vlan_s = strtok_r(NULL, " ", &save_ptr); + basis_s = strtok_r(NULL, " ", &save_ptr); if (vlan_s) { if (sscanf(vlan_s, "%u", &vlan) != 1) { @@ -1198,9 +1257,18 @@ bond_unixctl_hash(struct unixctl_conn *conn, const char *args_, vlan = OFP_VLAN_NONE; } + if (basis_s) { + if (sscanf(basis_s, "%"PRIu32, &basis) != 1) { + unixctl_command_reply(conn, 501, "invalid basis"); + return; + } + } else { + basis = 0; + } + if (sscanf(mac_s, ETH_ADDR_SCAN_FMT, ETH_ADDR_SCAN_ARGS(mac)) == ETH_ADDR_SCAN_COUNT) { - hash = bond_hash_src(mac, vlan) & BOND_MASK; + hash = bond_hash_src(mac, vlan, basis) & BOND_MASK; hash_cstr = xasprintf("%u", hash); unixctl_command_reply(conn, 200, hash_cstr); @@ -1266,77 +1334,10 @@ bond_is_link_up(struct bond *bond, struct netdev *netdev) : netdev_get_miimon(netdev)); } -static int -bond_stb_sort_cmp__(const void *a_, const void *b_) -{ - const struct bond_slave *const *ap = a_; - const struct bond_slave *const *bp = b_; - const struct bond_slave *a = *ap; - const struct bond_slave *b = *bp; - const struct lacp *lacp = a->bond->lacp; - int a_id, b_id; - - if (lacp) { - a_id = lacp_slave_get_port_id(lacp, a->aux); - b_id = lacp_slave_get_port_id(lacp, b->aux); - } else { - a_id = netdev_get_ifindex(a->netdev); - b_id = netdev_get_ifindex(b->netdev); - } - - return (a_id == b_id ? 0 : (a_id < b_id ? -1 : 1)); -} - -static bool -bond_stb_sort(struct bond *bond) -{ - size_t i; - - if (!bond->stb_slaves || !bond->stb_need_sort) { - return false; - } - bond->stb_need_sort = false; - - qsort(bond->stb_slaves, bond->n_stb_slaves, sizeof *bond->stb_slaves, - bond_stb_sort_cmp__); - - for (i = 0; i < bond->n_stb_slaves; i++) { - bond->stb_slaves[i]->stb_idx = i; - } - - return true; -} - -static void -bond_stb_enable_slave(struct bond_slave *slave) -{ - struct bond *bond = slave->bond; - - if (!bond->stb_slaves) { - return; - } - - bond->stb_need_sort = true; - - if (slave->enabled) { - if (bond->len_stb_slaves <= bond->n_stb_slaves) { - bond->stb_slaves = x2nrealloc(bond->stb_slaves, - &bond->len_stb_slaves, - sizeof *bond->stb_slaves); - } - - slave->stb_idx = bond->n_stb_slaves++; - bond->stb_slaves[slave->stb_idx] = slave; - } else { - size_t index = slave->stb_idx; - bond->stb_slaves[index] = bond->stb_slaves[--bond->n_stb_slaves]; - bond->stb_slaves[index]->stb_idx = index; - } -} - static void bond_enable_slave(struct bond_slave *slave, bool enable, struct tag_set *tags) { + struct bond *bond = slave->bond; slave->delay_expires = LLONG_MAX; if (enable != slave->enabled) { slave->enabled = enable; @@ -1349,7 +1350,10 @@ bond_enable_slave(struct bond_slave *slave, bool enable, struct tag_set *tags) VLOG_WARN("interface %s: enabled", slave->name); slave->tag = tag_create_random(); } - bond_stb_enable_slave(slave); + + if (bond->balance == BM_STABLE) { + bond->bond_revalidate = true; + } } } @@ -1359,7 +1363,7 @@ bond_link_status_update(struct bond_slave *slave, struct tag_set *tags) struct bond *bond = slave->bond; bool up; - up = slave->up && lacp_slave_may_enable(bond->lacp, slave->aux); + up = slave->up && slave->lacp_may_enable; if ((up == slave->enabled) != (slave->delay_expires == LLONG_MAX)) { static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20); VLOG_INFO_RL(&rl, "interface %s: link state %s", @@ -1369,7 +1373,7 @@ bond_link_status_update(struct bond_slave *slave, struct tag_set *tags) VLOG_INFO_RL(&rl, "interface %s: will not be %s", slave->name, up ? "disabled" : "enabled"); } else { - int delay = (lacp_negotiated(bond->lacp) ? 0 + int delay = (bond->lacp_negotiated ? 0 : up ? bond->updelay : bond->downdelay); slave->delay_expires = time_msec() + delay; if (delay) { @@ -1392,25 +1396,25 @@ static bool bond_is_tcp_hash(const struct bond *bond) { return (bond->balance == BM_TCP || bond->balance == BM_STABLE) - && lacp_negotiated(bond->lacp); + && bond->lacp_negotiated; } static unsigned int -bond_hash_src(const uint8_t mac[ETH_ADDR_LEN], uint16_t vlan) +bond_hash_src(const uint8_t mac[ETH_ADDR_LEN], uint16_t vlan, uint32_t basis) { - return hash_bytes(mac, ETH_ADDR_LEN, vlan); + return hash_3words(hash_bytes(mac, ETH_ADDR_LEN, 0), vlan, basis); } static unsigned int -bond_hash_tcp(const struct flow *flow, uint16_t vlan) +bond_hash_tcp(const struct flow *flow, uint16_t vlan, uint32_t basis) { struct flow hash_flow = *flow; - hash_flow.vlan_tci = vlan; + hash_flow.vlan_tci = htons(vlan); /* The symmetric quality of this hash function is not required, but * flow_hash_symmetric_l4 already exists, and is sufficient for our * purposes, so we use it out of convenience. */ - return flow_hash_symmetric_l4(&hash_flow, 0); + return flow_hash_symmetric_l4(&hash_flow, basis); } static unsigned int @@ -1419,8 +1423,8 @@ bond_hash(const struct bond *bond, const struct flow *flow, uint16_t vlan) assert(bond->balance != BM_AB); return (bond_is_tcp_hash(bond) - ? bond_hash_tcp(flow, vlan) - : bond_hash_src(flow->dl_src, vlan)); + ? bond_hash_tcp(flow, vlan, bond->basis) + : bond_hash_src(flow->dl_src, vlan, bond->basis)); } static struct bond_entry * @@ -1430,6 +1434,37 @@ lookup_bond_entry(const struct bond *bond, const struct flow *flow, return &bond->hash[bond_hash(bond, flow, vlan) & BOND_MASK]; } +/* This function uses Highest Random Weight hashing to choose an output slave. + * This approach only reassigns a minimal number of flows when slaves are + * enabled or disabled. Unfortunately, it has O(n) performance against the + * number of slaves. There exist algorithms which are O(1), but have slightly + * more complex implementations and require the use of memory. This may need + * to be reimplemented if it becomes a performance bottleneck. */ +static struct bond_slave * +choose_stb_slave(const struct bond *bond, const struct flow *flow, + uint16_t vlan) +{ + struct bond_slave *best, *slave; + uint32_t best_hash, flow_hash; + + best = NULL; + best_hash = 0; + flow_hash = bond_hash(bond, flow, vlan); + HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) { + if (slave->enabled) { + uint32_t hash; + + hash = hash_2words(flow_hash, slave->stb_id); + if (!best || hash > best_hash) { + best = slave; + best_hash = hash; + } + } + } + + return best; +} + static struct bond_slave * choose_output_slave(const struct bond *bond, const struct flow *flow, uint16_t vlan) @@ -1441,13 +1476,7 @@ choose_output_slave(const struct bond *bond, const struct flow *flow, return bond->active_slave; case BM_STABLE: - if (bond->n_stb_slaves) { - return bond->stb_slaves[bond_hash(bond, flow, vlan) - % bond->n_stb_slaves]; - } else { - return bond->active_slave; - } - + return choose_stb_slave(bond, flow, vlan); case BM_SLB: case BM_TCP: e = lookup_bond_entry(bond, flow, vlan); @@ -1483,7 +1512,7 @@ bond_choose_slave(const struct bond *bond) best = NULL; HMAP_FOR_EACH (slave, hmap_node, &bond->slaves) { if (slave->delay_expires != LLONG_MAX - && lacp_slave_may_enable(bond->lacp, slave->aux) + && slave->lacp_may_enable && (!best || slave->delay_expires < best->delay_expires)) { best = slave; }