From 01d18a3affec8249863053b99bc646545ca8171c Mon Sep 17 00:00:00 2001 From: Alex Wang Date: Thu, 22 Aug 2013 15:44:23 -0700 Subject: [PATCH] bfd: Implement forwarding_if_rx. This commit adds a new boolean option "forwarding_if_rx" to bfd. When forwarding_if_rx is true the interface will be considered capable of packet I/O as long as there is packet received at interface. This is important in that when link becomes temporarily conjested, consecutive BFD control packets can be lost. And the forwarding_if_rx can prevent link failover by detecting non-control packets received at interface. Signed-off-by: Alex Wang --- lib/bfd.c | 74 +++++++++++-- tests/bfd.at | 254 ++++++++++++++++++++++++++++++++++++++++++- vswitchd/vswitch.xml | 8 ++ 3 files changed, 326 insertions(+), 10 deletions(-) diff --git a/lib/bfd.c b/lib/bfd.c index 47e67dfbd..7da6fd989 100644 --- a/lib/bfd.c +++ b/lib/bfd.c @@ -191,12 +191,19 @@ struct bfd { atomic_bool check_tnl_key; /* Verify tunnel key of inbound packets? */ atomic_int ref_cnt; + /* When forward_if_rx is true, bfd_forwarding() will return + * true as long as there are incoming packets received. + * Note, forwarding_override still has higher priority. */ + bool forwarding_if_rx; + long long int forwarding_if_rx_detect_time; + /* BFD decay related variables. */ bool in_decay; /* True when bfd is in decay. */ int decay_min_rx; /* min_rx is set to decay_min_rx when */ /* in decay. */ int decay_rx_ctl; /* Count bfd packets received within decay */ /* detect interval. */ + uint64_t decay_rx_packets; /* Packets received by 'netdev'. */ long long int decay_detect_time; /* Decay detection time. */ }; @@ -223,6 +230,8 @@ static void bfd_put_details(struct ds *, const struct bfd *) static uint64_t bfd_rx_packets(const struct bfd *) OVS_REQUIRES(mutex); static void bfd_try_decay(struct bfd *) OVS_REQUIRES(mutex); static void bfd_decay_update(struct bfd *) OVS_REQUIRES(mutex); +static void bfd_check_rx(struct bfd *) OVS_REQUIRES(mutex); +static void bfd_forwarding_if_rx_update(struct bfd *) OVS_REQUIRES(mutex); static void bfd_unixctl_show(struct unixctl_conn *, int argc, const char *argv[], void *aux OVS_UNUSED); static void bfd_unixctl_set_forwarding_override(struct unixctl_conn *, @@ -280,7 +289,7 @@ bfd_configure(struct bfd *bfd, const char *name, const struct smap *cfg, long long int min_tx, min_rx; bool need_poll = false; bool cfg_min_rx_changed = false; - bool cpath_down; + bool cpath_down, forwarding_if_rx; const char *hwaddr; uint8_t ea[ETH_ADDR_LEN]; @@ -311,6 +320,7 @@ bfd_configure(struct bfd *bfd, const char *name, const struct smap *cfg, bfd->mult = 3; atomic_init(&bfd->ref_cnt, 1); bfd->netdev = netdev_ref(netdev); + bfd->rx_packets = bfd_rx_packets(bfd); bfd->in_decay = false; /* RFC 5881 section 4 @@ -384,6 +394,16 @@ bfd_configure(struct bfd *bfd, const char *name, const struct smap *cfg, bfd->eth_dst_set = false; } + forwarding_if_rx = smap_get_bool(cfg, "forwarding_if_rx", false); + if (bfd->forwarding_if_rx != forwarding_if_rx) { + bfd->forwarding_if_rx = forwarding_if_rx; + if (bfd->state == STATE_UP && bfd->forwarding_if_rx) { + bfd_forwarding_if_rx_update(bfd); + } else { + bfd->forwarding_if_rx_detect_time = 0; + } + } + if (need_poll) { bfd_poll(bfd); } @@ -458,6 +478,9 @@ bfd_run(struct bfd *bfd) OVS_EXCLUDED(mutex) bfd_try_decay(bfd); } + /* Always checks the reception of any packet. */ + bfd_check_rx(bfd); + if (bfd->min_tx != bfd->cfg_min_tx || (bfd->min_rx != bfd->cfg_min_rx && bfd->min_rx != bfd->decay_min_rx) || bfd->in_decay != old_in_decay) { @@ -752,9 +775,13 @@ bfd_set_netdev(struct bfd *bfd, const struct netdev *netdev) if (bfd->netdev != netdev) { netdev_close(bfd->netdev); bfd->netdev = netdev_ref(netdev); - if (bfd->decay_min_rx) { + if (bfd->decay_min_rx && bfd->state == STATE_UP) { bfd_decay_update(bfd); } + if (bfd->forwarding_if_rx && bfd->state == STATE_UP) { + bfd_forwarding_if_rx_update(bfd); + } + bfd->rx_packets = bfd_rx_packets(bfd); } ovs_mutex_unlock(&mutex); } @@ -763,14 +790,18 @@ bfd_set_netdev(struct bfd *bfd, const struct netdev *netdev) static bool bfd_forwarding__(const struct bfd *bfd) OVS_REQUIRES(mutex) { + long long int time; + if (bfd->forwarding_override != -1) { return bfd->forwarding_override == 1; } - return bfd->state == STATE_UP - && bfd->rmt_diag != DIAG_PATH_DOWN - && bfd->rmt_diag != DIAG_CPATH_DOWN - && bfd->rmt_diag != DIAG_RCPATH_DOWN; + time = bfd->forwarding_if_rx_detect_time; + return (bfd->state == STATE_UP + || (bfd->forwarding_if_rx && time > time_msec())) + && bfd->rmt_diag != DIAG_PATH_DOWN + && bfd->rmt_diag != DIAG_CPATH_DOWN + && bfd->rmt_diag != DIAG_RCPATH_DOWN; } /* Helpers. */ @@ -1001,7 +1032,7 @@ bfd_try_decay(struct bfd *bfd) OVS_REQUIRES(mutex) * asynchronously to the bfd_rx_packets() function, the 'diff' value * can be jittered. Thusly, we double the decay_rx_ctl to provide * more wiggle room. */ - diff = bfd_rx_packets(bfd) - bfd->rx_packets; + diff = bfd_rx_packets(bfd) - bfd->decay_rx_packets; expect = 2 * MAX(bfd->decay_rx_ctl, 1); bfd->in_decay = diff <= expect ? true : false; bfd_decay_update(bfd); @@ -1011,11 +1042,38 @@ bfd_try_decay(struct bfd *bfd) OVS_REQUIRES(mutex) static void bfd_decay_update(struct bfd * bfd) OVS_REQUIRES(mutex) { - bfd->rx_packets = bfd_rx_packets(bfd); + bfd->decay_rx_packets = bfd_rx_packets(bfd); bfd->decay_rx_ctl = 0; bfd->decay_detect_time = MAX(bfd->decay_min_rx, 2000) + time_msec(); } +/* Checks if there are packets received during the time since last call. + * If forwarding_if_rx is enabled and packets are received, updates the + * forwarding_if_rx_detect_time. */ +static void +bfd_check_rx(struct bfd *bfd) OVS_REQUIRES(mutex) +{ + uint64_t rx_packets = bfd_rx_packets(bfd); + int64_t diff; + + diff = rx_packets - bfd->rx_packets; + bfd->rx_packets = rx_packets; + if (diff < 0) { + VLOG_INFO_RL(&rl, "rx_packets count is smaller than last time."); + } + if (bfd->forwarding_if_rx && diff > 0) { + bfd_forwarding_if_rx_update(bfd); + } +} + +/* Updates the forwarding_if_rx_detect_time. */ +static void +bfd_forwarding_if_rx_update(struct bfd *bfd) OVS_REQUIRES(mutex) +{ + int64_t incr = bfd_rx_interval(bfd) * bfd->mult; + bfd->forwarding_if_rx_detect_time = MAX(incr, 2000) + time_msec(); +} + static uint32_t generate_discriminator(void) { diff --git a/tests/bfd.at b/tests/bfd.at index da45b27e1..0b2b7ccf4 100644 --- a/tests/bfd.at +++ b/tests/bfd.at @@ -261,14 +261,14 @@ OVS_VSWITCHD_START([add-br br1 -- set bridge br1 datapath-type=dummy -- \ ovs-appctl time/stop # wait for local session state to go from down to up. -for i in `seq 0 2`; do ovs-appctl time/warp 500; done +for i in `seq 0 1`; do ovs-appctl time/warp 500; done BFD_CHECK([p0], [true], [false], [none], [up], [No Diagnostic], [none], [init], [No Diagnostic]) # Test-1 BFD decay: decay to decay_min_rx # bfd:decay_min_rx is set to 3000ms after the local state of p0 goes up, # so for the first 2500ms, there should be no change. -for i in `seq 0 3`; do ovs-appctl time/warp 500; done +for i in `seq 0 4`; do ovs-appctl time/warp 500; done BFD_CHECK([p0], [true], [false], [none], [up], [No Diagnostic], [none], [up], [No Diagnostic]) BFD_CHECK_TX([p0], [500ms], [300ms], [500ms]) BFD_CHECK_RX([p0], [500ms], [300ms], [500ms]) @@ -507,4 +507,254 @@ BFD_CHECK_RX([p0], [3000ms], [3000ms], [500ms]) # End of Test-8 ################################################################ OVS_VSWITCHD_STOP +AT_CLEANUP + +# Tests below are for bfd forwarding_if_rx feature. +# forwarding_if_rx Test1: bfd is enabled on one end of link. +AT_SETUP([bfd - bfd forwarding_if_rx 1]) +OVS_VSWITCHD_START([add-br br1 -- set bridge br1 datapath-type=dummy -- \ + add-port br1 p1 -- set Interface p1 type=patch \ + options:peer=p0 ofport_request=2 -- \ + add-port br0 p0 -- set Interface p0 type=patch \ + options:peer=p1 ofport_request=1 -- \ + set Interface p0 bfd:enable=true bfd:min_tx=500 bfd:min_rx=500 -- \ + add-port br1 p2 -- set Interface p2 type=internal ofport_request=3]) + +ovs-appctl time/stop +# check the inital status. +BFD_CHECK([p0], [false], [false], [none], [down], [No Diagnostic], [none], [down], [No Diagnostic]) +BFD_CHECK_TX([p0], [1000ms], [1000ms], [0ms]) +BFD_CHECK_RX([p0], [500ms], [500ms], [1ms]) + +# enable forwarding_if_rx. +AT_CHECK([ovs-vsctl set Interface p0 bfd:forwarding_if_rx=true], [0]) + +# there should be no change of forwarding flag, since +# there is no traffic. +for i in `seq 0 3` +do + ovs-appctl time/warp 500 + BFD_CHECK([p0], [false], [false], [none], [down], [No Diagnostic], [none], [down], [No Diagnostic]) +done + +# receive one packet. +AT_CHECK([ovs-ofctl packet-out br1 3 2 "90e2ba01475000101856b2e80806000108000604000100101856b2e80202020300000000000002020202"], + [0], [stdout], []) +for i in `seq 0 14` +do + ovs-appctl time/warp 100 + # the forwarding flag should be true, since there is data received. + BFD_CHECK([p0], [true], [false], [none], [down], [No Diagnostic], [none], [down], [No Diagnostic]) + BFD_CHECK_TX([p0], [1000ms], [1000ms], [0ms]) + BFD_CHECK_RX([p0], [500ms], [500ms], [1ms]) +done + +# Stop sending packets for 1000ms. +for i in `seq 0 9`; do ovs-appctl time/warp 100; done +BFD_CHECK([p0], [false], [false], [none], [down], [No Diagnostic], [none], [down], [No Diagnostic]) +BFD_CHECK_TX([p0], [1000ms], [1000ms], [0ms]) +BFD_CHECK_RX([p0], [500ms], [500ms], [1ms]) + +# receive packet at 1/100ms rate for 1000ms. +for i in `seq 0 9` +do + ovs-appctl time/warp 100 + AT_CHECK([ovs-ofctl packet-out br1 3 2 "90e2ba01475000101856b2e80806000108000604000100101856b2e80202020300000000000002020202"], + [0], [stdout], []) +done +# the forwarding flag should be true, since there is data received. +BFD_CHECK([p0], [true], [false], [none], [down], [No Diagnostic], [none], [down], [No Diagnostic]) +BFD_CHECK_TX([p0], [1000ms], [1000ms], [0ms]) +BFD_CHECK_RX([p0], [500ms], [500ms], [1ms]) + +# reset bfd forwarding_if_rx. +AT_CHECK([ovs-vsctl set Interface p0 bfd:forwarding_if_rx=false], [0]) +# forwarding flag should turn to false since the STATE is DOWN. +BFD_CHECK([p0], [false], [false], [none], [down], [No Diagnostic], [none], [down], [No Diagnostic]) +BFD_CHECK_TX([p0], [1000ms], [1000ms], [0ms]) +BFD_CHECK_RX([p0], [500ms], [500ms], [1ms]) + +AT_CHECK([ovs-vsctl del-br br1], [0], [ignore]) +AT_CLEANUP + +# forwarding_if_rx Test2: bfd is enabled on both ends of link. +AT_SETUP([bfd - bfd forwarding_if_rx 2]) +OVS_VSWITCHD_START([add-br br1 -- set bridge br1 datapath-type=dummy -- \ + add-port br1 p1 -- set Interface p1 type=patch \ + options:peer=p0 ofport_request=2 -- \ + add-port br0 p0 -- set Interface p0 type=patch \ + options:peer=p1 ofport_request=1 -- \ + set Interface p0 bfd:enable=true bfd:min_tx=500 bfd:min_rx=500 -- \ + set Interface p1 bfd:enable=true bfd:min_tx=300 bfd:min_rx=300 -- \ + add-port br1 p2 -- set Interface p2 type=internal ofport_request=3]) + +ovs-appctl time/stop +# advance the clock, to stablize the states. +for i in `seq 0 9`; do ovs-appctl time/warp 500; done + +# enable forwarding_if_rx. +AT_CHECK([ovs-vsctl set Interface p0 bfd:forwarding_if_rx=true], [0]) + +# there should be no change of the forwarding flag, since +# the bfd on both ends is already up. +for i in `seq 0 5` +do + ovs-appctl time/warp 500 + BFD_CHECK([p0], [true], [false], [none], [up], [No Diagnostic], [none], [up], [No Diagnostic]) +done + +# stop the bfd on one side. +AT_CHECK([ovs-vsctl set Interface p1 bfd:enable=false], [0]) +# for within 1500ms, the detection timer is not out. +# there is no change to status. +for i in `seq 0 1` +do + ovs-appctl time/warp 500 + BFD_CHECK([p0], [true], [false], [none], [up], [No Diagnostic], [none], [up], [No Diagnostic]) + for i in `seq 0 5` + do + AT_CHECK([ovs-ofctl packet-out br1 3 2 "90e2ba01475000101856b2e80806000108000604000100101856b2e80202020300000000000002020202"], + [0], [stdout], []) + done +done + +# at 1500ms, the STATE should go DOWN, due to Control Detection Time Expired. +# but forwarding flag should be still true. +ovs-appctl time/warp 500 +BFD_CHECK([p0], [true], [false], [none], [down], [Control Detection Time Expired], [none], [down], [No Diagnostic]) + +# receive packet at 1/100ms rate for 1000ms. +for i in `seq 0 9` +do + AT_CHECK([ovs-ofctl packet-out br1 3 2 "90e2ba01475000101856b2e80806000108000604000100101856b2e80202020300000000000002020202"], + [0], [stdout], []) + ovs-appctl time/warp 100 + # the forwarding flag should always be true during this time. + BFD_CHECK([p0], [true], [false], [none], [down], [Control Detection Time Expired], [none], [down], [No Diagnostic]) +done + +# reset bfd forwarding_if_rx. +AT_CHECK([ovs-vsctl set Interface p0 bfd:forwarding_if_rx=false], [0]) +# forwarding flag should turn to false since the STATE is DOWN. +BFD_CHECK([p0], [false], [false], [none], [down], [Control Detection Time Expired], [none], [down], [No Diagnostic]) +BFD_CHECK_TX([p0], [1000ms], [1000ms], [0ms]) +BFD_CHECK_RX([p0], [500ms], [500ms], [1ms]) + +# re-enable bfd on the other end. the states should be up. +AT_CHECK([ovs-vsctl set Interface p1 bfd:enable=true bfd:min_tx=300 bfd:min_rx=300]) +# advance the clock, to stablize the states. +for i in `seq 0 9`; do ovs-appctl time/warp 500; done +BFD_CHECK([p0], [true], [false], [none], [up], [Control Detection Time Expired], [none], [up], [No Diagnostic]) +BFD_CHECK([p1], [true], [false], [none], [up], [No Diagnostic], [none], [up], [Control Detection Time Expired]) +BFD_CHECK_TX([p0], [500ms], [500ms], [300ms]) +BFD_CHECK_RX([p0], [500ms], [500ms], [300ms]) + +AT_CHECK([ovs-vsctl del-br br1], [0], [ignore]) +AT_CLEANUP + +# forwarding_if_rx Test3: bfd is enabled on both ends of link and decay is enabled. +AT_SETUP([bfd - bfd forwarding_if_rx 3]) +OVS_VSWITCHD_START([add-br br1 -- set bridge br1 datapath-type=dummy -- \ + add-port br1 p1 -- set Interface p1 type=patch \ + options:peer=p0 ofport_request=2 -- \ + add-port br0 p0 -- set Interface p0 type=patch \ + options:peer=p1 ofport_request=1 -- \ + set Interface p0 bfd:enable=true bfd:min_tx=300 bfd:min_rx=300 bfd:decay_min_rx=3000 -- \ + set Interface p1 bfd:enable=true bfd:min_tx=500 bfd:min_rx=500]) + +ovs-appctl time/stop +# advance the clock, to stablize the states. +for i in `seq 0 19`; do ovs-appctl time/warp 500; done +BFD_CHECK([p0], [true], [false], [none], [up], [No Diagnostic], [none], [up], [No Diagnostic]) +BFD_CHECK([p1], [true], [false], [none], [up], [No Diagnostic], [none], [up], [No Diagnostic]) +BFD_CHECK_TX([p0], [500ms], [300ms], [500ms]) +BFD_CHECK_RX([p0], [3000ms], [3000ms], [500ms]) + +# enable forwarding_if_rx. +AT_CHECK([ovs-vsctl set Interface p0 bfd:forwarding_if_rx=true], [0]) + +# there should be no change of the forwarding flag, since +# the bfd on both ends is already up. +for i in `seq 0 9` +do + ovs-appctl time/warp 500 + BFD_CHECK([p0], [true], [false], [none], [up], [No Diagnostic], [none], [up], [No Diagnostic]) +done + +# reconfigure the decay_min_rx to 1000ms. check the poll sequence. +AT_CHECK([ovs-vsctl set interface p0 bfd:decay_min_rx=1000]) +BFD_CHECK([p0], [true], [false], [none], [up], [No Diagnostic], [final], [up], [No Diagnostic]) +BFD_CHECK([p1], [true], [false], [none], [up], [No Diagnostic], [poll], [up], [No Diagnostic]) +BFD_CHECK_TX([p0], [500ms], [300ms], [500ms]) +BFD_CHECK_RX([p0], [500ms], [300ms], [500ms]) + +# wait for 2000ms to decay. +for i in `seq 0 3`; do ovs-appctl time/warp 500; done +BFD_CHECK([p0], [true], [false], [none], [up], [No Diagnostic], [final], [up], [No Diagnostic]) +BFD_CHECK([p1], [true], [false], [none], [up], [No Diagnostic], [poll], [up], [No Diagnostic]) +BFD_CHECK_TX([p0], [500ms], [300ms], [500ms]) +BFD_CHECK_RX([p0], [1000ms], [1000ms], [500ms]) + +# wait for 1000ms, so that the flags will go back to none. +for i in `seq 0 1`; do ovs-appctl time/warp 500; done +BFD_CHECK([p0], [true], [false], [none], [up], [No Diagnostic], [none], [up], [No Diagnostic]) +BFD_CHECK([p1], [true], [false], [none], [up], [No Diagnostic], [none], [up], [No Diagnostic]) +BFD_CHECK_TX([p0], [500ms], [300ms], [500ms]) +BFD_CHECK_RX([p0], [1000ms], [1000ms], [500ms]) + +# stop the bfd on one side. +AT_CHECK([ovs-vsctl set Interface p1 bfd:enable=false], [0]) +# for within 2500ms, the detection timer is not out. +# there is no change to status. +for i in `seq 0 4` +do + ovs-appctl time/warp 500 + BFD_CHECK([p0], [true], [false], [none], [up], [No Diagnostic], [none], [up], [No Diagnostic]) + AT_CHECK([ovs-ofctl packet-out br1 3 2 "90e2ba01475000101856b2e80806000108000604000100101856b2e80202020300000000000002020202"], + [0], [stdout], []) +done + +# at 3000ms, the STATE should go DOWN, due to Control Detection Time Expired. +# but forwarding flag should be still true. +ovs-appctl time/warp 500 +BFD_CHECK([p0], [true], [false], [none], [down], [Control Detection Time Expired], [none], [down], [No Diagnostic]) + +# receive packet at 1/100ms rate for 1000ms. +for i in `seq 0 9` +do + AT_CHECK([ovs-ofctl packet-out br1 3 2 "90e2ba01475000101856b2e80806000108000604000100101856b2e80202020300000000000002020202"], + [0], [stdout], []) + ovs-appctl time/warp 100 + # the forwarding flag should always be true during this time. + BFD_CHECK([p0], [true], [false], [none], [down], [Control Detection Time Expired], [none], [down], [No Diagnostic]) +done + +# stop receiving for 2000ms. +for i in `seq 0 19`; do ovs-appctl time/warp 100; done +BFD_CHECK([p0], [false], [false], [none], [down], [Control Detection Time Expired], [none], [down], [No Diagnostic]) + +# reset bfd forwarding_if_rx. +AT_CHECK([ovs-vsctl set Interface p0 bfd:forwarding_if_rx=false]) +# forwarding flag should turn to false since the STATE is DOWN. +BFD_CHECK([p0], [false], [false], [none], [down], [Control Detection Time Expired], [none], [down], [No Diagnostic]) +BFD_CHECK_TX([p0], [1000ms], [1000ms], [0ms]) +BFD_CHECK_RX([p0], [300ms], [300ms], [1ms]) + +# re-enable bfd forwarding_if_rx. +AT_CHECK([ovs-vsctl set Interface p0 bfd:forwarding_if_rx=true]) +# there should be no change. +BFD_CHECK([p0], [false], [false], [none], [down], [Control Detection Time Expired], [none], [down], [No Diagnostic]) +BFD_CHECK_TX([p0], [1000ms], [1000ms], [0ms]) +BFD_CHECK_RX([p0], [300ms], [300ms], [1ms]) + +# re-enable bfd on the other end. the states should be up. +AT_CHECK([ovs-vsctl set Interface p1 bfd:enable=true bfd:min_tx=300 bfd:min_rx=300]) +# advance the clock, to stablize the states. +for i in `seq 0 9`; do ovs-appctl time/warp 500; done +BFD_CHECK([p0], [true], [false], [none], [up], [Control Detection Time Expired], [none], [up], [No Diagnostic]) +BFD_CHECK([p1], [true], [false], [none], [up], [No Diagnostic], [none], [up], [Control Detection Time Expired]) +BFD_CHECK_TX([p0], [300ms], [300ms], [300ms]) +BFD_CHECK_RX([p0], [1000ms], [1000ms], [300ms]) + +AT_CHECK([ovs-vsctl del-br br1], [0], [ignore]) AT_CLEANUP \ No newline at end of file diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml index 65d4c3f81..e62594097 100644 --- a/vswitchd/vswitch.xml +++ b/vswitchd/vswitch.xml @@ -1905,6 +1905,14 @@ reconfigured. + + When forwarding_if_rx is true the interface will be + considered capable of packet I/O as long as there is packet + received at interface. This is important in that when link becomes + temporarily conjested, consecutive BFD control packets can be lost. + And the forwarding_if_rx can prevent link failover by + detecting non-control packets received at interface. + Concatenated path down may be used when the local system should not -- 2.43.0