2 * Copyright (c) 2008, 2009, 2010, 2011 Nicira, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
19 #include <arpa/inet.h>
23 #include "byte-order.h"
24 #include "collectors.h"
27 #include "lib/netflow.h"
30 #include "ofproto/netflow.h"
32 #include "poll-loop.h"
33 #include "socket-util.h"
38 VLOG_DEFINE_THIS_MODULE(netflow);
41 uint8_t engine_type; /* Value of engine_type to use. */
42 uint8_t engine_id; /* Value of engine_id to use. */
43 long long int boot_time; /* Time when netflow_create() was called. */
44 struct collectors *collectors; /* NetFlow collectors. */
45 bool add_id_to_iface; /* Put the 7 least significiant bits of
46 * 'engine_id' into the most significant
47 * bits of the interface fields. */
48 uint32_t netflow_cnt; /* Flow sequence number for NetFlow. */
49 struct ofpbuf packet; /* NetFlow packet being accumulated. */
50 long long int active_timeout; /* Timeout for flows that are still active. */
51 long long int next_timeout; /* Next scheduled active timeout. */
52 long long int reconfig_time; /* When we reconfigured the timeouts. */
54 struct hmap flows; /* Contains 'netflow_flows'. */
60 struct hmap_node hmap_node;
62 long long int last_expired; /* Time this flow last timed out. */
63 long long int created; /* Time flow was created since time out. */
65 ofp_port_t output_iface; /* Output interface index. */
66 uint16_t tcp_flags; /* Bitwise-OR of all TCP flags seen. */
68 ofp_port_t in_port; /* Input port. */
69 ovs_be32 nw_src; /* IPv4 source address. */
70 ovs_be32 nw_dst; /* IPv4 destination address. */
71 uint8_t nw_tos; /* IP ToS (including DSCP and ECN). */
72 uint8_t nw_proto; /* IP protocol. */
73 ovs_be16 tp_src; /* TCP/UDP/SCTP source port. */
74 ovs_be16 tp_dst; /* TCP/UDP/SCTP destination port. */
76 uint64_t packet_count; /* Packets from subrules. */
77 uint64_t byte_count; /* Bytes from subrules. */
78 long long int used; /* Last-used time (0 if never used). */
81 static struct ovs_mutex mutex = OVS_MUTEX_INITIALIZER;
82 static atomic_uint netflow_count = ATOMIC_VAR_INIT(0);
84 static struct netflow_flow *netflow_flow_lookup(const struct netflow *,
87 static uint32_t netflow_flow_hash(const struct flow *);
88 static void netflow_expire__(struct netflow *, struct netflow_flow *)
90 static void netflow_run__(struct netflow *) OVS_REQUIRES(mutex);
93 netflow_mask_wc(struct flow *flow, struct flow_wildcards *wc)
95 if (flow->dl_type != htons(ETH_TYPE_IP)) {
98 memset(&wc->masks.nw_proto, 0xff, sizeof wc->masks.nw_proto);
99 memset(&wc->masks.nw_src, 0xff, sizeof wc->masks.nw_src);
100 memset(&wc->masks.nw_dst, 0xff, sizeof wc->masks.nw_dst);
101 memset(&wc->masks.tp_src, 0xff, sizeof wc->masks.tp_src);
102 memset(&wc->masks.tp_dst, 0xff, sizeof wc->masks.tp_dst);
103 wc->masks.nw_tos |= IP_DSCP_MASK;
107 gen_netflow_rec(struct netflow *nf, struct netflow_flow *nf_flow,
108 uint32_t packet_count, uint32_t byte_count)
111 struct netflow_v5_header *nf_hdr;
112 struct netflow_v5_record *nf_rec;
114 if (!nf->packet.size) {
117 time_wall_timespec(&now);
119 nf_hdr = ofpbuf_put_zeros(&nf->packet, sizeof *nf_hdr);
120 nf_hdr->version = htons(NETFLOW_V5_VERSION);
121 nf_hdr->count = htons(0);
122 nf_hdr->sysuptime = htonl(time_msec() - nf->boot_time);
123 nf_hdr->unix_secs = htonl(now.tv_sec);
124 nf_hdr->unix_nsecs = htonl(now.tv_nsec);
125 nf_hdr->flow_seq = htonl(nf->netflow_cnt++);
126 nf_hdr->engine_type = nf->engine_type;
127 nf_hdr->engine_id = nf->engine_id;
128 nf_hdr->sampling_interval = htons(0);
131 nf_hdr = nf->packet.data;
132 nf_hdr->count = htons(ntohs(nf_hdr->count) + 1);
134 nf_rec = ofpbuf_put_zeros(&nf->packet, sizeof *nf_rec);
135 nf_rec->src_addr = nf_flow->nw_src;
136 nf_rec->dst_addr = nf_flow->nw_dst;
137 nf_rec->nexthop = htonl(0);
138 if (nf->add_id_to_iface) {
139 uint16_t iface = (nf->engine_id & 0x7f) << 9;
140 nf_rec->input = htons(iface | (ofp_to_u16(nf_flow->in_port) & 0x1ff));
141 nf_rec->output = htons(iface
142 | (ofp_to_u16(nf_flow->output_iface) & 0x1ff));
144 nf_rec->input = htons(ofp_to_u16(nf_flow->in_port));
145 nf_rec->output = htons(ofp_to_u16(nf_flow->output_iface));
147 nf_rec->packet_count = htonl(packet_count);
148 nf_rec->byte_count = htonl(byte_count);
149 nf_rec->init_time = htonl(nf_flow->created - nf->boot_time);
150 nf_rec->used_time = htonl(MAX(nf_flow->created, nf_flow->used)
152 if (nf_flow->nw_proto == IPPROTO_ICMP) {
153 /* In NetFlow, the ICMP type and code are concatenated and
154 * placed in the 'dst_port' field. */
155 uint8_t type = ntohs(nf_flow->tp_src);
156 uint8_t code = ntohs(nf_flow->tp_dst);
157 nf_rec->src_port = htons(0);
158 nf_rec->dst_port = htons((type << 8) | code);
160 nf_rec->src_port = nf_flow->tp_src;
161 nf_rec->dst_port = nf_flow->tp_dst;
163 nf_rec->tcp_flags = (uint8_t) nf_flow->tcp_flags;
164 nf_rec->ip_proto = nf_flow->nw_proto;
165 nf_rec->ip_tos = nf_flow->nw_tos & IP_DSCP_MASK;
167 /* NetFlow messages are limited to 30 records. */
168 if (ntohs(nf_hdr->count) >= 30) {
174 netflow_flow_update(struct netflow *nf, struct flow *flow,
175 ofp_port_t output_iface,
176 const struct dpif_flow_stats *stats)
179 struct netflow_flow *nf_flow;
182 /* NetFlow only reports on IP packets. */
183 if (flow->dl_type != htons(ETH_TYPE_IP)) {
187 ovs_mutex_lock(&mutex);
188 nf_flow = netflow_flow_lookup(nf, flow);
190 nf_flow = xzalloc(sizeof *nf_flow);
191 nf_flow->in_port = flow->in_port.ofp_port;
192 nf_flow->nw_src = flow->nw_src;
193 nf_flow->nw_dst = flow->nw_dst;
194 nf_flow->nw_tos = flow->nw_tos;
195 nf_flow->nw_proto = flow->nw_proto;
196 nf_flow->tp_src = flow->tp_src;
197 nf_flow->tp_dst = flow->tp_dst;
198 nf_flow->created = stats->used;
199 nf_flow->output_iface = output_iface;
200 hmap_insert(&nf->flows, &nf_flow->hmap_node, netflow_flow_hash(flow));
203 if (nf_flow->output_iface != output_iface) {
204 netflow_expire__(nf, nf_flow);
205 nf_flow->created = stats->used;
206 nf_flow->output_iface = output_iface;
209 nf_flow->packet_count += stats->n_packets;
210 nf_flow->byte_count += stats->n_bytes;
211 nf_flow->tcp_flags |= stats->tcp_flags;
213 used = MAX(nf_flow->used, stats->used);
214 if (nf_flow->used != used) {
215 nf_flow->used = used;
216 if (!nf->active_timeout || !nf_flow->last_expired
217 || nf->reconfig_time > nf_flow->last_expired) {
218 /* Keep the time updated to prevent a flood of expiration in
220 nf_flow->last_expired = time_msec();
224 ovs_mutex_unlock(&mutex);
228 netflow_expire__(struct netflow *nf, struct netflow_flow *nf_flow)
231 uint64_t pkts, bytes;
233 pkts = nf_flow->packet_count;
234 bytes = nf_flow->byte_count;
236 nf_flow->last_expired += nf->active_timeout;
242 if ((bytes >> 32) <= 175) {
243 /* NetFlow v5 records are limited to 32-bit counters. If we've wrapped
244 * a counter, send as multiple records so we don't lose track of any
245 * traffic. We try to evenly distribute the packet and byte counters,
246 * so that the bytes-per-packet lengths don't look wonky across the
249 int n_recs = (bytes + UINT32_MAX - 1) / UINT32_MAX;
250 uint32_t pkt_count = pkts / n_recs;
251 uint32_t byte_count = bytes / n_recs;
253 gen_netflow_rec(nf, nf_flow, pkt_count, byte_count);
259 /* In 600 seconds, a 10GbE link can theoretically transmit 75 * 10**10
260 * == 175 * 2**32 bytes. The byte counter is bigger than that, so it's
261 * probably a bug--for example, the netdev code uses UINT64_MAX to
262 * report "unknown value", and perhaps that has leaked through to here.
264 * We wouldn't want to hit the loop above in this case, because it
265 * would try to send up to UINT32_MAX netflow records, which would take
268 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1);
270 VLOG_WARN_RL(&rl, "impossible byte counter %"PRIu64, bytes);
273 /* Update flow tracking data. */
274 nf_flow->created = 0;
275 nf_flow->packet_count = 0;
276 nf_flow->byte_count = 0;
277 nf_flow->tcp_flags = 0;
281 netflow_expire(struct netflow *nf, struct flow *flow) OVS_EXCLUDED(mutex)
283 struct netflow_flow *nf_flow;
285 ovs_mutex_lock(&mutex);
286 nf_flow = netflow_flow_lookup(nf, flow);
288 netflow_expire__(nf, nf_flow);
290 ovs_mutex_unlock(&mutex);
294 netflow_flow_clear(struct netflow *nf, struct flow *flow) OVS_EXCLUDED(mutex)
296 struct netflow_flow *nf_flow;
298 ovs_mutex_lock(&mutex);
299 nf_flow = netflow_flow_lookup(nf, flow);
301 ovs_assert(!nf_flow->packet_count);
302 ovs_assert(!nf_flow->byte_count);
303 hmap_remove(&nf->flows, &nf_flow->hmap_node);
306 ovs_mutex_unlock(&mutex);
309 /* Returns true if it's time to send out a round of NetFlow active timeouts,
310 * false otherwise. */
312 netflow_run__(struct netflow *nf) OVS_REQUIRES(mutex)
314 long long int now = time_msec();
315 struct netflow_flow *nf_flow, *next;
317 if (nf->packet.size) {
318 collectors_send(nf->collectors, nf->packet.data, nf->packet.size);
322 if (!nf->active_timeout || now < nf->next_timeout) {
326 nf->next_timeout = now + 1000;
328 HMAP_FOR_EACH_SAFE (nf_flow, next, hmap_node, &nf->flows) {
329 if (now > nf_flow->last_expired + nf->active_timeout) {
330 bool idle = nf_flow->used < nf_flow->last_expired;
331 netflow_expire__(nf, nf_flow);
334 /* If the netflow_flow hasn't been used in a while, it's
335 * possible the upper layer lost track of it. */
336 hmap_remove(&nf->flows, &nf_flow->hmap_node);
344 netflow_run(struct netflow *nf)
346 ovs_mutex_lock(&mutex);
348 ovs_mutex_unlock(&mutex);
352 netflow_wait(struct netflow *nf) OVS_EXCLUDED(mutex)
354 ovs_mutex_lock(&mutex);
355 if (nf->active_timeout) {
356 poll_timer_wait_until(nf->next_timeout);
358 if (nf->packet.size) {
359 poll_immediate_wake();
361 ovs_mutex_unlock(&mutex);
365 netflow_set_options(struct netflow *nf,
366 const struct netflow_options *nf_options)
370 long long int old_timeout;
372 ovs_mutex_lock(&mutex);
373 nf->engine_type = nf_options->engine_type;
374 nf->engine_id = nf_options->engine_id;
375 nf->add_id_to_iface = nf_options->add_id_to_iface;
377 collectors_destroy(nf->collectors);
378 collectors_create(&nf_options->collectors, 0, &nf->collectors);
380 old_timeout = nf->active_timeout;
381 if (nf_options->active_timeout >= 0) {
382 nf->active_timeout = nf_options->active_timeout;
384 nf->active_timeout = NF_ACTIVE_TIMEOUT_DEFAULT;
386 nf->active_timeout *= 1000;
387 if (old_timeout != nf->active_timeout) {
388 nf->reconfig_time = time_msec();
389 nf->next_timeout = time_msec();
391 ovs_mutex_unlock(&mutex);
399 struct netflow *nf = xzalloc(sizeof *nf);
404 nf->boot_time = time_msec();
405 nf->collectors = NULL;
406 nf->add_id_to_iface = false;
408 hmap_init(&nf->flows);
409 atomic_init(&nf->ref_cnt, 1);
410 ofpbuf_init(&nf->packet, 1500);
411 atomic_add(&netflow_count, 1, &junk);
416 netflow_ref(const struct netflow *nf_)
418 struct netflow *nf = CONST_CAST(struct netflow *, nf_);
421 atomic_add(&nf->ref_cnt, 1, &orig);
422 ovs_assert(orig > 0);
428 netflow_unref(struct netflow *nf)
436 atomic_sub(&nf->ref_cnt, 1, &orig);
437 ovs_assert(orig > 0);
439 atomic_sub(&netflow_count, 1, &orig);
440 collectors_destroy(nf->collectors);
441 ofpbuf_uninit(&nf->packet);
446 /* Returns true if there exist any netflow objects, false otherwise. */
452 atomic_read(&netflow_count, &n);
458 static struct netflow_flow *
459 netflow_flow_lookup(const struct netflow *nf, const struct flow *flow)
462 struct netflow_flow *nf_flow;
464 HMAP_FOR_EACH_WITH_HASH (nf_flow, hmap_node, netflow_flow_hash(flow),
466 if (flow->in_port.ofp_port == nf_flow->in_port
467 && flow->nw_src == nf_flow->nw_src
468 && flow->nw_dst == nf_flow->nw_dst
469 && flow->nw_tos == nf_flow->nw_tos
470 && flow->nw_proto == nf_flow->nw_proto
471 && flow->tp_src == nf_flow->tp_src
472 && flow->tp_dst == nf_flow->tp_dst) {
481 netflow_flow_hash(const struct flow *flow)
485 hash = mhash_add(hash, (OVS_FORCE uint32_t) flow->in_port.ofp_port);
486 hash = mhash_add(hash, ntohl(flow->nw_src));
487 hash = mhash_add(hash, ntohl(flow->nw_dst));
488 hash = mhash_add(hash, flow->nw_tos);
489 hash = mhash_add(hash, flow->nw_proto);
490 hash = mhash_add(hash, ntohs(flow->tp_src));
491 hash = mhash_add(hash, ntohs(flow->tp_dst));
493 return mhash_finish(hash, 28);