datapath/table-hash.c

   1 /*
   2  * Distributed under the terms of the GNU GPL version 2.
   3  * Copyright (c) 2007, 2008 The Board of Trustees of The Leland
   4  * Stanford Junior University
   5  */
   6
   7 #include "table.h"
   8 #include "crc32.h"
   9 #include "flow.h"
  10 #include "datapath.h"
  11
  12 #include <linux/slab.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/mm.h>
  15 #include <linux/highmem.h>
  16 #include <asm/pgtable.h>
  17
  18 static void *kmem_alloc(size_t);
  19 static void *kmem_zalloc(size_t);
  20 static void kmem_free(void *, size_t);
  21
  22 struct sw_table_hash {
  23         struct sw_table swt;
  24         struct crc32 crc32;
  25         unsigned int n_flows;
  26         unsigned int bucket_mask; /* Number of buckets minus 1. */
  27         struct sw_flow **buckets;
  28 };
  29
  30 static struct sw_flow **find_bucket(struct sw_table *swt,
  31                                                                         const struct sw_flow_key *key)
  32 {
  33         struct sw_table_hash *th = (struct sw_table_hash *) swt;
  34         unsigned int crc = crc32_calculate(&th->crc32, key,
  35                                 offsetof(struct sw_flow_key, wildcards));
  36         return &th->buckets[crc & th->bucket_mask];
  37 }
  38
  39 static struct sw_flow *table_hash_lookup(struct sw_table *swt,
  40                                                                                  const struct sw_flow_key *key)
  41 {
  42         struct sw_flow *flow = *find_bucket(swt, key);
  43         return flow && flow_keys_equal(&flow->key, key) ? flow : NULL;
  44 }
  45
  46 static int table_hash_insert(struct sw_table *swt, struct sw_flow *flow)
  47 {
  48         struct sw_table_hash *th = (struct sw_table_hash *) swt;
  49         struct sw_flow **bucket;
  50         int retval;
  51
  52         if (flow->key.wildcards != 0)
  53                 return 0;
  54
  55         bucket = find_bucket(swt, &flow->key);
  56         if (*bucket == NULL) {
  57                 th->n_flows++;
  58                 rcu_assign_pointer(*bucket, flow);
  59                 retval = 1;
  60         } else {
  61                 struct sw_flow *old_flow = *bucket;
  62                 if (flow_keys_equal(&old_flow->key, &flow->key)) {
  63                         rcu_assign_pointer(*bucket, flow);
  64                         flow_deferred_free(old_flow);
  65                         retval = 1;
  66                 } else {
  67                         retval = 0;
  68                 }
  69         }
  70         return retval;
  71 }
  72
  73 static int table_hash_modify(struct sw_table *swt,
  74                 const struct sw_flow_key *key, uint16_t priority, int strict,
  75                 const struct ofp_action_header *actions, size_t actions_len)
  76 {
  77         struct sw_table_hash *th = (struct sw_table_hash *) swt;
  78         unsigned int count = 0;
  79
  80         if (key->wildcards == 0) {
  81                 struct sw_flow **bucket = find_bucket(swt, key);
  82                 struct sw_flow *flow = *bucket;
  83                 if (flow && flow_matches_desc(&flow->key, key, strict)
  84                                 && (!strict || (flow->priority == priority))) {
  85                         flow_replace_acts(flow, actions, actions_len);
  86                         count = 1;
  87                 }
  88         } else {
  89                 unsigned int i;
  90
  91                 for (i = 0; i <= th->bucket_mask; i++) {
  92                         struct sw_flow **bucket = &th->buckets[i];
  93                         struct sw_flow *flow = *bucket;
  94                         if (flow && flow_matches_desc(&flow->key, key, strict)
  95                                         && (!strict || (flow->priority == priority))) {
  96                                 flow_replace_acts(flow, actions, actions_len);
  97                                 count++;
  98                         }
  99                 }
 100         }
 101         return count;
 102 }
 103
 104 /* Caller must update n_flows. */
 105 static int do_delete(struct sw_flow **bucket, struct sw_flow *flow)
 106 {
 107         rcu_assign_pointer(*bucket, NULL);
 108         flow_deferred_free(flow);
 109         return 1;
 110 }
 111
 112 /* Returns number of deleted flows.  We can ignore the priority
 113  * argument, since all exact-match entries are the same (highest)
 114  * priority. */
 115 static int table_hash_delete(struct sw_table *swt,
 116                                         const struct sw_flow_key *key,  uint16_t out_port,
 117                                         uint16_t priority, int strict)
 118 {
 119         struct sw_table_hash *th = (struct sw_table_hash *) swt;
 120         unsigned int count = 0;
 121
 122         if (key->wildcards == 0) {
 123                 struct sw_flow **bucket = find_bucket(swt, key);
 124                 struct sw_flow *flow = *bucket;
 125                 if (flow && flow_keys_equal(&flow->key, key)
 126                                 && flow_has_out_port(flow, out_port))
 127                         count = do_delete(bucket, flow);
 128         } else {
 129                 unsigned int i;
 130
 131                 for (i = 0; i <= th->bucket_mask; i++) {
 132                         struct sw_flow **bucket = &th->buckets[i];
 133                         struct sw_flow *flow = *bucket;
 134                         if (flow && flow_matches_desc(&flow->key, key, strict)
 135                                         && flow_has_out_port(flow, out_port))
 136                                 count += do_delete(bucket, flow);
 137                 }
 138         }
 139         th->n_flows -= count;
 140         return count;
 141 }
 142
 143 static int table_hash_timeout(struct datapath *dp, struct sw_table *swt)
 144 {
 145         struct sw_table_hash *th = (struct sw_table_hash *) swt;
 146         unsigned int i;
 147         int count = 0;
 148
 149         mutex_lock(&dp_mutex);
 150         for (i = 0; i <= th->bucket_mask; i++) {
 151                 struct sw_flow **bucket = &th->buckets[i];
 152                 struct sw_flow *flow = *bucket;
 153                 if (flow) {
 154                         int reason = flow_timeout(flow);
 155                         if (reason >= 0) {
 156                                 count += do_delete(bucket, flow);
 157                                 dp_send_flow_expired(dp, flow, reason);
 158                         }
 159                 }
 160         }
 161         th->n_flows -= count;
 162         mutex_unlock(&dp_mutex);
 163
 164         return count;
 165 }
 166
 167 static void table_hash_destroy(struct sw_table *swt)
 168 {
 169         struct sw_table_hash *th = (struct sw_table_hash *) swt;
 170         unsigned int i;
 171         for (i = 0; i <= th->bucket_mask; i++)
 172         if (th->buckets[i])
 173                 flow_free(th->buckets[i]);
 174         kmem_free(th->buckets, (th->bucket_mask + 1) * sizeof *th->buckets);
 175         kfree(th);
 176 }
 177
 178 static int table_hash_iterate(struct sw_table *swt,
 179                               const struct sw_flow_key *key, uint16_t out_port,
 180                               struct sw_table_position *position,
 181                               int (*callback)(struct sw_flow *, void *private),
 182                               void *private)
 183 {
 184         struct sw_table_hash *th = (struct sw_table_hash *) swt;
 185
 186         if (position->private[0] > th->bucket_mask)
 187                 return 0;
 188
 189         if (key->wildcards == 0) {
 190                 struct sw_flow *flow;
 191                 int error;
 192
 193                 flow = table_hash_lookup(swt, key);
 194                 if (!flow || !flow_has_out_port(flow, out_port))
 195                         return 0;
 196
 197                 error = callback(flow, private);
 198                 if (!error)
 199                         position->private[0] = -1;
 200                 return error;
 201         } else {
 202                 int i;
 203
 204                 for (i = position->private[0]; i <= th->bucket_mask; i++) {
 205                         struct sw_flow *flow = th->buckets[i];
 206                         if (flow && flow_matches_1wild(&flow->key, key)
 207                                         && flow_has_out_port(flow, out_port)) {
 208                                 int error = callback(flow, private);
 209                                 if (error) {
 210                                         position->private[0] = i;
 211                                         return error;
 212                                 }
 213                         }
 214                 }
 215                 return 0;
 216         }
 217 }
 218 static void table_hash_stats(struct sw_table *swt,
 219                                  struct sw_table_stats *stats)
 220 {
 221         struct sw_table_hash *th = (struct sw_table_hash *) swt;
 222         stats->name = "hash";
 223         stats->wildcards = 0;          /* No wildcards are supported. */
 224         stats->n_flows   = th->n_flows;
 225         stats->max_flows = th->bucket_mask + 1;
 226         stats->n_lookup  = swt->n_lookup;
 227         stats->n_matched = swt->n_matched;
 228 }
 229
 230 struct sw_table *table_hash_create(unsigned int polynomial,
 231                         unsigned int n_buckets)
 232 {
 233         struct sw_table_hash *th;
 234         struct sw_table *swt;
 235
 236         th = kzalloc(sizeof *th, GFP_KERNEL);
 237         if (th == NULL)
 238                 return NULL;
 239
 240         BUG_ON(n_buckets & (n_buckets - 1));
 241         th->buckets = kmem_zalloc(n_buckets * sizeof *th->buckets);
 242         if (th->buckets == NULL) {
 243                 printk("failed to allocate %u buckets\n", n_buckets);
 244                 kfree(th);
 245                 return NULL;
 246         }
 247         th->bucket_mask = n_buckets - 1;
 248
 249         swt = &th->swt;
 250         swt->lookup = table_hash_lookup;
 251         swt->insert = table_hash_insert;
 252         swt->delete = table_hash_delete;
 253         swt->timeout = table_hash_timeout;
 254         swt->destroy = table_hash_destroy;
 255         swt->iterate = table_hash_iterate;
 256         swt->stats = table_hash_stats;
 257
 258         crc32_init(&th->crc32, polynomial);
 259         th->n_flows = 0;
 260
 261         return swt;
 262 }
 263
 264 /* Double-hashing table. */
 265
 266 struct sw_table_hash2 {
 267         struct sw_table swt;
 268         struct sw_table *subtable[2];
 269 };
 270
 271 static struct sw_flow *table_hash2_lookup(struct sw_table *swt,
 272                                                                                   const struct sw_flow_key *key)
 273 {
 274         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 275         int i;
 276
 277         for (i = 0; i < 2; i++) {
 278                 struct sw_flow *flow = *find_bucket(t2->subtable[i], key);
 279                 if (flow && flow_keys_equal(&flow->key, key))
 280                         return flow;
 281         }
 282         return NULL;
 283 }
 284
 285 static int table_hash2_insert(struct sw_table *swt, struct sw_flow *flow)
 286 {
 287         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 288
 289         if (table_hash_insert(t2->subtable[0], flow))
 290                 return 1;
 291         return table_hash_insert(t2->subtable[1], flow);
 292 }
 293
 294 static int table_hash2_modify(struct sw_table *swt,
 295                 const struct sw_flow_key *key, uint16_t priority, int strict,
 296                 const struct ofp_action_header *actions, size_t actions_len)
 297 {
 298         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 299         return (table_hash_modify(t2->subtable[0], key, priority, strict,
 300                                         actions, actions_len)
 301                         + table_hash_modify(t2->subtable[1], key, priority, strict,
 302                                         actions, actions_len));
 303 }
 304
 305 static int table_hash2_delete(struct sw_table *swt,
 306                                                           const struct sw_flow_key *key,
 307                                                           uint16_t out_port,
 308                                                           uint16_t priority, int strict)
 309 {
 310         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 311         return (table_hash_delete(t2->subtable[0], key, out_port, priority, strict)
 312                         + table_hash_delete(t2->subtable[1], key, out_port,
 313                                 priority, strict));
 314 }
 315
 316 static int table_hash2_timeout(struct datapath *dp, struct sw_table *swt)
 317 {
 318         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 319         return (table_hash_timeout(dp, t2->subtable[0])
 320                         + table_hash_timeout(dp, t2->subtable[1]));
 321 }
 322
 323 static void table_hash2_destroy(struct sw_table *swt)
 324 {
 325         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 326         table_hash_destroy(t2->subtable[0]);
 327         table_hash_destroy(t2->subtable[1]);
 328         kfree(t2);
 329 }
 330
 331 static int table_hash2_iterate(struct sw_table *swt,
 332                                const struct sw_flow_key *key, uint16_t out_port,
 333                                struct sw_table_position *position,
 334                                int (*callback)(struct sw_flow *, void *),
 335                                void *private)
 336 {
 337         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 338         int i;
 339
 340         for (i = position->private[1]; i < 2; i++) {
 341                 int error = table_hash_iterate(t2->subtable[i], key, out_port,
 342                                 position, callback, private);
 343                 if (error) {
 344                         return error;
 345                 }
 346                 position->private[0] = 0;
 347                 position->private[1]++;
 348         }
 349         return 0;
 350 }
 351
 352 static void table_hash2_stats(struct sw_table *swt,
 353                                  struct sw_table_stats *stats)
 354 {
 355         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 356         struct sw_table_stats substats[2];
 357         int i;
 358
 359         for (i = 0; i < 2; i++)
 360                 table_hash_stats(t2->subtable[i], &substats[i]);
 361         stats->name = "hash2";
 362         stats->wildcards = 0;          /* No wildcards are supported. */
 363         stats->n_flows   = substats[0].n_flows + substats[1].n_flows;
 364         stats->max_flows = substats[0].max_flows + substats[1].max_flows;
 365         stats->n_lookup  = swt->n_lookup;
 366         stats->n_matched = swt->n_matched;
 367 }
 368
 369 struct sw_table *table_hash2_create(unsigned int poly0, unsigned int buckets0,
 370                                                                         unsigned int poly1, unsigned int buckets1)
 371
 372 {
 373         struct sw_table_hash2 *t2;
 374         struct sw_table *swt;
 375
 376         t2 = kzalloc(sizeof *t2, GFP_KERNEL);
 377         if (t2 == NULL)
 378                 return NULL;
 379
 380         t2->subtable[0] = table_hash_create(poly0, buckets0);
 381         if (t2->subtable[0] == NULL)
 382                 goto out_free_t2;
 383
 384         t2->subtable[1] = table_hash_create(poly1, buckets1);
 385         if (t2->subtable[1] == NULL)
 386                 goto out_free_subtable0;
 387
 388         swt = &t2->swt;
 389         swt->lookup = table_hash2_lookup;
 390         swt->insert = table_hash2_insert;
 391         swt->modify = table_hash2_modify;
 392         swt->delete = table_hash2_delete;
 393         swt->timeout = table_hash2_timeout;
 394         swt->destroy = table_hash2_destroy;
 395         swt->iterate = table_hash2_iterate;
 396         swt->stats = table_hash2_stats;
 397
 398         return swt;
 399
 400 out_free_subtable0:
 401         table_hash_destroy(t2->subtable[0]);
 402 out_free_t2:
 403         kfree(t2);
 404         return NULL;
 405 }
 406
 407 /* From fs/xfs/linux-2.4/kmem.c. */
 408
 409 static void *
 410 kmem_alloc(size_t size)
 411 {
 412         void *ptr;
 413
 414 #ifdef KMALLOC_MAX_SIZE
 415         if (size > KMALLOC_MAX_SIZE)
 416                 return NULL;
 417 #endif
 418         ptr = kmalloc(size, GFP_KERNEL);
 419         if (!ptr) {
 420                 ptr = vmalloc(size);
 421                 if (ptr)
 422                         printk("openflow: used vmalloc for %lu bytes\n",
 423                                         (unsigned long)size);
 424         }
 425         return ptr;
 426 }
 427
 428 static void *
 429 kmem_zalloc(size_t size)
 430 {
 431         void *ptr = kmem_alloc(size);
 432         if (ptr)
 433                 memset(ptr, 0, size);
 434         return ptr;
 435 }
 436
 437 static void
 438 kmem_free(void *ptr, size_t size)
 439 {
 440         if (((unsigned long)ptr < VMALLOC_START) ||
 441                 ((unsigned long)ptr >= VMALLOC_END)) {
 442                 kfree(ptr);
 443         } else {
 444                 vfree(ptr);
 445         }
 446 }