X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=ofproto%2Fofproto-dpif-sflow.c;h=cb1f8308552e597543743a385c7882ef1a8cdd9d;hb=HEAD;hp=68eb804d6d4857c9f1d32fa3cc39e917e7132f46;hpb=975a704c827a8a5d5c2264441f3a4ebcf097720c;p=sliver-openvswitch.git diff --git a/ofproto/ofproto-dpif-sflow.c b/ofproto/ofproto-dpif-sflow.c index 68eb804d6..cb1f83085 100644 --- a/ofproto/ofproto-dpif-sflow.c +++ b/ofproto/ofproto-dpif-sflow.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2010, 2011 Nicira Networks. + * Copyright (c) 2009, 2010, 2011, 2012, 2013, 2014 Nicira, Inc. * Copyright (c) 2009 InMon Corp. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,8 @@ #include #include "ofproto-dpif-sflow.h" #include +#include +#include #include #include "collectors.h" #include "compiler.h" @@ -30,30 +32,34 @@ #include "ofproto.h" #include "packets.h" #include "poll-loop.h" +#include "route-table.h" #include "sflow_api.h" #include "socket-util.h" #include "timeval.h" #include "vlog.h" #include "lib/odp-util.h" +#include "ofproto-provider.h" VLOG_DEFINE_THIS_MODULE(sflow); +static struct ovs_mutex mutex; + struct dpif_sflow_port { struct hmap_node hmap_node; /* In struct dpif_sflow's "ports" hmap. */ - struct netdev *netdev; /* Underlying network device, for stats. */ SFLDataSource_instance dsi; /* sFlow library's notion of port number. */ - uint16_t odp_port; /* Datapath port number. */ + struct ofport *ofport; /* To retrive port stats. */ + odp_port_t odp_port; }; struct dpif_sflow { - struct ofproto *ofproto; struct collectors *collectors; SFLAgent *sflow_agent; struct ofproto_sflow_options *options; - struct dpif *dpif; time_t next_tick; size_t n_flood, n_all; struct hmap ports; /* Contains "struct dpif_sflow_port"s. */ + uint32_t probability; + struct ovs_refcount ref_cnt; }; static void dpif_sflow_del_port__(struct dpif_sflow *, @@ -139,12 +145,13 @@ sflow_agent_send_packet_cb(void *ds_, SFLAgent *agent OVS_UNUSED, } static struct dpif_sflow_port * -dpif_sflow_find_port(const struct dpif_sflow *ds, uint16_t odp_port) +dpif_sflow_find_port(const struct dpif_sflow *ds, odp_port_t odp_port) + OVS_REQUIRES(mutex) { struct dpif_sflow_port *dsp; - HMAP_FOR_EACH_IN_BUCKET (dsp, hmap_node, - hash_int(odp_port, 0), &ds->ports) { + HMAP_FOR_EACH_IN_BUCKET (dsp, hmap_node, hash_odp_port(odp_port), + &ds->ports) { if (dsp->odp_port == odp_port) { return dsp; } @@ -155,16 +162,17 @@ dpif_sflow_find_port(const struct dpif_sflow *ds, uint16_t odp_port) static void sflow_agent_get_counters(void *ds_, SFLPoller *poller, SFL_COUNTERS_SAMPLE_TYPE *cs) + OVS_REQUIRES(mutex) { struct dpif_sflow *ds = ds_; SFLCounters_sample_element elem; + enum netdev_features current; struct dpif_sflow_port *dsp; SFLIf_counters *counters; struct netdev_stats stats; enum netdev_flags flags; - uint32_t current; - dsp = dpif_sflow_find_port(ds, poller->bridgePort); + dsp = dpif_sflow_find_port(ds, u32_to_odp(poller->bridgePort)); if (!dsp) { return; } @@ -173,19 +181,19 @@ sflow_agent_get_counters(void *ds_, SFLPoller *poller, counters = &elem.counterBlock.generic; counters->ifIndex = SFL_DS_INDEX(poller->dsi); counters->ifType = 6; - if (!netdev_get_features(dsp->netdev, ¤t, NULL, NULL, NULL)) { + if (!netdev_get_features(dsp->ofport->netdev, ¤t, NULL, NULL, NULL)) { /* The values of ifDirection come from MAU MIB (RFC 2668): 0 = unknown, 1 = full-duplex, 2 = half-duplex, 3 = in, 4=out */ - counters->ifSpeed = netdev_features_to_bps(current); + counters->ifSpeed = netdev_features_to_bps(current, 0); counters->ifDirection = (netdev_features_is_full_duplex(current) ? 1 : 2); } else { counters->ifSpeed = 100000000; counters->ifDirection = 0; } - if (!netdev_get_flags(dsp->netdev, &flags) && flags & NETDEV_UP) { + if (!netdev_get_flags(dsp->ofport->netdev, &flags) && flags & NETDEV_UP) { counters->ifStatus = 1; /* ifAdminStatus up. */ - if (netdev_get_carrier(dsp->netdev)) { + if (netdev_get_carrier(dsp->ofport->netdev)) { counters->ifStatus |= 2; /* ifOperStatus us. */ } } else { @@ -197,7 +205,7 @@ sflow_agent_get_counters(void *ds_, SFLPoller *poller, 2. Does the multicast counter include broadcasts? 3. Does the rx_packets counter include multicasts/broadcasts? */ - netdev_get_stats(dsp->netdev, &stats); + ofproto_port_get_stats(dsp->ofport, &stats); counters->ifInOctets = stats.rx_bytes; counters->ifInUcastPkts = stats.rx_packets; counters->ifInMulticastPkts = stats.multicast; @@ -223,23 +231,36 @@ sflow_agent_get_counters(void *ds_, SFLPoller *poller, * The sFlow agent address should be a local IP address that is persistent and * reachable over the network, if possible. The IP address associated with * 'agent_device' is used if it has one, and otherwise 'control_ip', the IP - * address used to talk to the controller. */ + * address used to talk to the controller. If the agent device is not + * specified then it is figured out by taking a look at the routing table based + * on 'targets'. */ static bool -sflow_choose_agent_address(const char *agent_device, const char *control_ip, +sflow_choose_agent_address(const char *agent_device, + const struct sset *targets, + const char *control_ip, SFLAddress *agent_addr) { + const char *target; struct in_addr in4; memset(agent_addr, 0, sizeof *agent_addr); agent_addr->type = SFLADDRESSTYPE_IP_V4; if (agent_device) { - struct netdev *netdev; + if (!netdev_get_in4_by_name(agent_device, &in4)) { + goto success; + } + } - if (!netdev_open(agent_device, "system", &netdev)) { - int error = netdev_get_in4(netdev, &in4, NULL); - netdev_close(netdev); - if (!error) { + SSET_FOR_EACH (target, targets) { + struct sockaddr_storage ss; + char name[IFNAMSIZ]; + + if (inet_parse_active(target, SFL_DEFAULT_COLLECTOR_PORT, &ss) + && ss.ss_family == AF_INET) { + struct sockaddr_in *sin = (struct sockaddr_in *) &ss; + if (route_table_get_name(sin->sin_addr.s_addr, name) + && !netdev_get_in4_by_name(name, &in4)) { goto success; } } @@ -257,11 +278,12 @@ success: return true; } -void -dpif_sflow_clear(struct dpif_sflow *ds) +static void +dpif_sflow_clear__(struct dpif_sflow *ds) OVS_REQUIRES(mutex) { if (ds->sflow_agent) { sfl_agent_release(ds->sflow_agent); + free(ds->sflow_agent); ds->sflow_agent = NULL; } collectors_destroy(ds->collectors); @@ -270,33 +292,79 @@ dpif_sflow_clear(struct dpif_sflow *ds) ds->options = NULL; /* Turn off sampling to save CPU cycles. */ - dpif_set_sflow_probability(ds->dpif, 0); + ds->probability = 0; +} + +void +dpif_sflow_clear(struct dpif_sflow *ds) OVS_EXCLUDED(mutex) +{ + ovs_mutex_lock(&mutex); + dpif_sflow_clear__(ds); + ovs_mutex_unlock(&mutex); } bool -dpif_sflow_is_enabled(const struct dpif_sflow *ds) +dpif_sflow_is_enabled(const struct dpif_sflow *ds) OVS_EXCLUDED(mutex) { - return ds->collectors != NULL; + bool enabled; + + ovs_mutex_lock(&mutex); + enabled = ds->collectors != NULL; + ovs_mutex_unlock(&mutex); + return enabled; } struct dpif_sflow * -dpif_sflow_create(struct dpif *dpif) +dpif_sflow_create(void) { + static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER; struct dpif_sflow *ds; + if (ovsthread_once_start(&once)) { + ovs_mutex_init_recursive(&mutex); + ovsthread_once_done(&once); + } + ds = xcalloc(1, sizeof *ds); - ds->dpif = dpif; ds->next_tick = time_now() + 1; hmap_init(&ds->ports); + ds->probability = 0; + route_table_register(); + ovs_refcount_init(&ds->ref_cnt); + return ds; } -void -dpif_sflow_destroy(struct dpif_sflow *ds) +struct dpif_sflow * +dpif_sflow_ref(const struct dpif_sflow *ds_) { + struct dpif_sflow *ds = CONST_CAST(struct dpif_sflow *, ds_); if (ds) { + ovs_refcount_ref(&ds->ref_cnt); + } + return ds; +} + +/* 32-bit fraction of packets to sample with. A value of 0 samples no packets, + * a value of %UINT32_MAX samples all packets and intermediate values sample + * intermediate fractions of packets. */ +uint32_t +dpif_sflow_get_probability(const struct dpif_sflow *ds) OVS_EXCLUDED(mutex) +{ + uint32_t probability; + ovs_mutex_lock(&mutex); + probability = ds->probability; + ovs_mutex_unlock(&mutex); + return probability; +} + +void +dpif_sflow_unref(struct dpif_sflow *ds) OVS_EXCLUDED(mutex) +{ + if (ds && ovs_refcount_unref(&ds->ref_cnt) == 1) { struct dpif_sflow_port *dsp, *next; + route_table_unregister(); dpif_sflow_clear(ds); HMAP_FOR_EACH_SAFE (dsp, next, hmap_node, &ds->ports) { dpif_sflow_del_port__(ds, dsp); @@ -307,98 +375,95 @@ dpif_sflow_destroy(struct dpif_sflow *ds) } static void -dpif_sflow_add_poller(struct dpif_sflow *ds, - struct dpif_sflow_port *dsp, uint16_t odp_port) +dpif_sflow_add_poller(struct dpif_sflow *ds, struct dpif_sflow_port *dsp) + OVS_REQUIRES(mutex) { SFLPoller *poller = sfl_agent_addPoller(ds->sflow_agent, &dsp->dsi, ds, sflow_agent_get_counters); sfl_poller_set_sFlowCpInterval(poller, ds->options->polling_interval); sfl_poller_set_sFlowCpReceiver(poller, RECEIVER_INDEX); - sfl_poller_set_bridgePort(poller, odp_port); -} - -static void -dpif_sflow_add_sampler(struct dpif_sflow *ds, struct dpif_sflow_port *dsp) -{ - SFLSampler *sampler = sfl_agent_addSampler(ds->sflow_agent, &dsp->dsi); - sfl_sampler_set_sFlowFsPacketSamplingRate(sampler, ds->options->sampling_rate); - sfl_sampler_set_sFlowFsMaximumHeaderSize(sampler, ds->options->header_len); - sfl_sampler_set_sFlowFsReceiver(sampler, RECEIVER_INDEX); + sfl_poller_set_bridgePort(poller, odp_to_u32(dsp->odp_port)); } void -dpif_sflow_add_port(struct dpif_sflow *ds, uint16_t odp_port, - const char *netdev_name) +dpif_sflow_add_port(struct dpif_sflow *ds, struct ofport *ofport, + odp_port_t odp_port) OVS_EXCLUDED(mutex) { struct dpif_sflow_port *dsp; - struct netdev *netdev; - uint32_t ifindex; - int error; + int ifindex; + ovs_mutex_lock(&mutex); dpif_sflow_del_port(ds, odp_port); - /* Open network device. */ - error = netdev_open(netdev_name, "system", &netdev); - if (error) { - VLOG_WARN_RL(&rl, "failed to open network device \"%s\": %s", - netdev_name, strerror(error)); - return; + ifindex = netdev_get_ifindex(ofport->netdev); + + if (ifindex <= 0) { + /* Not an ifindex port, so do not add a cross-reference to it here */ + goto out; } /* Add to table of ports. */ dsp = xmalloc(sizeof *dsp); - dsp->netdev = netdev; - ifindex = netdev_get_ifindex(netdev); - if (ifindex <= 0) { - ifindex = (ds->sflow_agent->subId << 16) + odp_port; - } - SFL_DS_SET(dsp->dsi, 0, ifindex, 0); + dsp->ofport = ofport; dsp->odp_port = odp_port; - hmap_insert(&ds->ports, &dsp->hmap_node, hash_int(odp_port, 0)); + SFL_DS_SET(dsp->dsi, SFL_DSCLASS_IFINDEX, ifindex, 0); + hmap_insert(&ds->ports, &dsp->hmap_node, hash_odp_port(odp_port)); - /* Add poller and sampler. */ + /* Add poller. */ if (ds->sflow_agent) { - dpif_sflow_add_poller(ds, dsp, odp_port); - dpif_sflow_add_sampler(ds, dsp); + dpif_sflow_add_poller(ds, dsp); } + +out: + ovs_mutex_unlock(&mutex); } static void dpif_sflow_del_port__(struct dpif_sflow *ds, struct dpif_sflow_port *dsp) + OVS_REQUIRES(mutex) { if (ds->sflow_agent) { sfl_agent_removePoller(ds->sflow_agent, &dsp->dsi); sfl_agent_removeSampler(ds->sflow_agent, &dsp->dsi); } - netdev_close(dsp->netdev); hmap_remove(&ds->ports, &dsp->hmap_node); free(dsp); } void -dpif_sflow_del_port(struct dpif_sflow *ds, uint16_t odp_port) +dpif_sflow_del_port(struct dpif_sflow *ds, odp_port_t odp_port) + OVS_EXCLUDED(mutex) { - struct dpif_sflow_port *dsp = dpif_sflow_find_port(ds, odp_port); + struct dpif_sflow_port *dsp; + + ovs_mutex_lock(&mutex); + dsp = dpif_sflow_find_port(ds, odp_port); if (dsp) { dpif_sflow_del_port__(ds, dsp); } + ovs_mutex_unlock(&mutex); } void dpif_sflow_set_options(struct dpif_sflow *ds, const struct ofproto_sflow_options *options) + OVS_EXCLUDED(mutex) { struct dpif_sflow_port *dsp; bool options_changed; SFLReceiver *receiver; SFLAddress agentIP; time_t now; + SFLDataSource_instance dsi; + uint32_t dsIndex; + SFLSampler *sampler; + ovs_mutex_lock(&mutex); if (sset_is_empty(&options->targets) || !options->sampling_rate) { /* No point in doing any work if there are no targets or nothing to * sample. */ - dpif_sflow_clear(ds); - return; + dpif_sflow_clear__(ds); + goto out; } options_changed = (!ds->options @@ -415,25 +480,26 @@ dpif_sflow_set_options(struct dpif_sflow *ds, if (ds->collectors == NULL) { VLOG_WARN_RL(&rl, "no collectors could be initialized, " "sFlow disabled"); - dpif_sflow_clear(ds); - return; + dpif_sflow_clear__(ds); + goto out; } } + /* Choose agent IP address and agent device (if not yet setup) */ + if (!sflow_choose_agent_address(options->agent_device, + &options->targets, + options->control_ip, &agentIP)) { + dpif_sflow_clear__(ds); + goto out; + } + /* Avoid reconfiguring if options didn't change. */ if (!options_changed) { - return; + goto out; } ofproto_sflow_options_destroy(ds->options); ds->options = ofproto_sflow_options_clone(options); - /* Choose agent IP address. */ - if (!sflow_choose_agent_address(options->agent_device, - options->control_ip, &agentIP)) { - dpif_sflow_clear(ds); - return; - } - /* Create agent. */ VLOG_INFO("creating sFlow agent %d", options->sub_id); if (ds->sflow_agent) { @@ -457,133 +523,135 @@ dpif_sflow_set_options(struct dpif_sflow *ds, sfl_receiver_set_sFlowRcvrTimeout(receiver, 0xffffffff); /* Set the sampling_rate down in the datapath. */ - dpif_set_sflow_probability(ds->dpif, - MAX(1, UINT32_MAX / options->sampling_rate)); + ds->probability = MAX(1, UINT32_MAX / ds->options->sampling_rate); + + /* Add a single sampler for the bridge. This appears as a PHYSICAL_ENTITY + because it is associated with the hypervisor, and interacts with the server + hardware directly. The sub_id is used to distinguish this sampler from + others on other bridges within the same agent. */ + dsIndex = 1000 + options->sub_id; + SFL_DS_SET(dsi, SFL_DSCLASS_PHYSICAL_ENTITY, dsIndex, 0); + sampler = sfl_agent_addSampler(ds->sflow_agent, &dsi); + sfl_sampler_set_sFlowFsPacketSamplingRate(sampler, ds->options->sampling_rate); + sfl_sampler_set_sFlowFsMaximumHeaderSize(sampler, ds->options->header_len); + sfl_sampler_set_sFlowFsReceiver(sampler, RECEIVER_INDEX); - /* Add samplers and pollers for the currently known ports. */ + /* Add pollers for the currently known ifindex-ports */ HMAP_FOR_EACH (dsp, hmap_node, &ds->ports) { - dpif_sflow_add_poller(ds, dsp, dsp->odp_port); - dpif_sflow_add_sampler(ds, dsp); + dpif_sflow_add_poller(ds, dsp); } + + +out: + ovs_mutex_unlock(&mutex); } -static int +int dpif_sflow_odp_port_to_ifindex(const struct dpif_sflow *ds, - uint16_t odp_port) + odp_port_t odp_port) OVS_EXCLUDED(mutex) { - struct dpif_sflow_port *dsp = dpif_sflow_find_port(ds, odp_port); - return dsp ? SFL_DS_INDEX(dsp->dsi) : 0; + struct dpif_sflow_port *dsp; + int ret; + + ovs_mutex_lock(&mutex); + dsp = dpif_sflow_find_port(ds, odp_port); + ret = dsp ? SFL_DS_INDEX(dsp->dsi) : 0; + ovs_mutex_unlock(&mutex); + return ret; } void -dpif_sflow_received(struct dpif_sflow *ds, const struct dpif_upcall *upcall, - const struct flow *flow) +dpif_sflow_received(struct dpif_sflow *ds, struct ofpbuf *packet, + const struct flow *flow, odp_port_t odp_in_port, + const union user_action_cookie *cookie) + OVS_EXCLUDED(mutex) { SFL_FLOW_SAMPLE_TYPE fs; SFLFlow_sample_element hdrElem; SFLSampled_header *header; SFLFlow_sample_element switchElem; SFLSampler *sampler; - unsigned int left; - struct nlattr *a; - size_t n_outputs; + struct dpif_sflow_port *in_dsp; + ovs_be16 vlan_tci; - /* Build a flow sample */ - memset(&fs, 0, sizeof fs); - fs.input = dpif_sflow_odp_port_to_ifindex(ds, - ofp_port_to_odp_port(flow->in_port)); - fs.output = 0; /* Filled in correctly below. */ - fs.sample_pool = upcall->sample_pool; - - /* We are going to give it to the sampler that represents this input port. - * By implementing "ingress-only" sampling like this we ensure that we - * never have to offer the same sample to more than one sampler. */ - sampler = sfl_agent_getSamplerByIfIndex(ds->sflow_agent, fs.input); + ovs_mutex_lock(&mutex); + sampler = ds->sflow_agent->samplers; if (!sampler) { - VLOG_WARN_RL(&rl, "no sampler for input ifIndex (%"PRIu32")", - fs.input); - return; + goto out; + } + + /* Build a flow sample. */ + memset(&fs, 0, sizeof fs); + + /* Look up the input ifIndex if this port has one. Otherwise just + * leave it as 0 (meaning 'unknown') and continue. */ + in_dsp = dpif_sflow_find_port(ds, odp_in_port); + if (in_dsp) { + fs.input = SFL_DS_INDEX(in_dsp->dsi); } + /* Make the assumption that the random number generator in the datapath converges + * to the configured mean, and just increment the samplePool by the configured + * sampling rate every time. */ + sampler->samplePool += sfl_sampler_get_sFlowFsPacketSamplingRate(sampler); + /* Sampled header. */ memset(&hdrElem, 0, sizeof hdrElem); hdrElem.tag = SFLFLOW_HEADER; header = &hdrElem.flowType.header; header->header_protocol = SFLHEADER_ETHERNET_ISO8023; /* The frame_length should include the Ethernet FCS (4 bytes), - but it has already been stripped, so we need to add 4 here. */ - header->frame_length = upcall->packet->size + 4; + * but it has already been stripped, so we need to add 4 here. */ + header->frame_length = ofpbuf_size(packet) + 4; /* Ethernet FCS stripped off. */ header->stripped = 4; - header->header_length = MIN(upcall->packet->size, + header->header_length = MIN(ofpbuf_size(packet), sampler->sFlowFsMaximumHeaderSize); - header->header_bytes = upcall->packet->data; + header->header_bytes = ofpbuf_data(packet); /* Add extended switch element. */ memset(&switchElem, 0, sizeof(switchElem)); switchElem.tag = SFLFLOW_EX_SWITCH; switchElem.flowType.sw.src_vlan = vlan_tci_to_vid(flow->vlan_tci); switchElem.flowType.sw.src_priority = vlan_tci_to_pcp(flow->vlan_tci); - /* Initialize the output VLAN and priority to be the same as the input, - but these fields can be overriden below if affected by an action. */ - switchElem.flowType.sw.dst_vlan = switchElem.flowType.sw.src_vlan; - switchElem.flowType.sw.dst_priority = switchElem.flowType.sw.src_priority; - - /* Figure out the output ports. */ - n_outputs = 0; - NL_ATTR_FOR_EACH_UNSAFE (a, left, upcall->actions, upcall->actions_len) { - ovs_be16 tci; - - switch (nl_attr_type(a)) { - case OVS_ACTION_ATTR_OUTPUT: - fs.output = dpif_sflow_odp_port_to_ifindex(ds, nl_attr_get_u32(a)); - n_outputs++; - break; - - case OVS_ACTION_ATTR_PUSH_VLAN: - tci = nl_attr_get_be16(a); - switchElem.flowType.sw.dst_vlan = vlan_tci_to_vid(tci); - switchElem.flowType.sw.dst_priority = vlan_tci_to_pcp(tci); - break; - - default: - break; - } - } - /* Set output port, as defined by http://www.sflow.org/sflow_version_5.txt - (search for "Input/output port information"). */ - if (!n_outputs) { - /* This value indicates that the packet was dropped for an unknown - * reason. */ - fs.output = 0x40000000 | 256; - } else if (n_outputs > 1 || !fs.output) { - /* Setting the high bit means "multiple output ports". */ - fs.output = 0x80000000 | n_outputs; - } + /* Retrieve data from user_action_cookie. */ + vlan_tci = cookie->sflow.vlan_tci; + switchElem.flowType.sw.dst_vlan = vlan_tci_to_vid(vlan_tci); + switchElem.flowType.sw.dst_priority = vlan_tci_to_pcp(vlan_tci); + + fs.output = cookie->sflow.output; /* Submit the flow sample to be encoded into the next datagram. */ SFLADD_ELEMENT(&fs, &hdrElem); SFLADD_ELEMENT(&fs, &switchElem); sfl_sampler_writeFlowSample(sampler, &fs); + +out: + ovs_mutex_unlock(&mutex); } void -dpif_sflow_run(struct dpif_sflow *ds) +dpif_sflow_run(struct dpif_sflow *ds) OVS_EXCLUDED(mutex) { - if (dpif_sflow_is_enabled(ds)) { + ovs_mutex_lock(&mutex); + if (ds->collectors != NULL) { time_t now = time_now(); + route_table_run(); if (now >= ds->next_tick) { sfl_agent_tick(ds->sflow_agent, time_wall()); ds->next_tick = now + 1; } } + ovs_mutex_unlock(&mutex); } void -dpif_sflow_wait(struct dpif_sflow *ds) +dpif_sflow_wait(struct dpif_sflow *ds) OVS_EXCLUDED(mutex) { - if (dpif_sflow_is_enabled(ds)) { + ovs_mutex_lock(&mutex); + if (ds->collectors != NULL) { poll_timer_wait_until(ds->next_tick * 1000LL); } + ovs_mutex_unlock(&mutex); }