datapath/table-hash.c

   1 /*
   2  * Distributed under the terms of the GNU GPL version 2.
   3  * Copyright (c) 2007, 2008 The Board of Trustees of The Leland
   4  * Stanford Junior University
   5  */
   6
   7 #include "table.h"
   8 #include "crc32.h"
   9 #include "flow.h"
  10 #include "datapath.h"
  11
  12 #include <linux/slab.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/mm.h>
  15 #include <linux/highmem.h>
  16 #include <asm/pgtable.h>
  17
  18 static void *kmem_alloc(size_t);
  19 static void *kmem_zalloc(size_t);
  20 static void kmem_free(void *, size_t);
  21
  22 struct sw_table_hash {
  23         struct sw_table swt;
  24         struct crc32 crc32;
  25         unsigned int n_flows;
  26         unsigned int bucket_mask; /* Number of buckets minus 1. */
  27         struct sw_flow **buckets;
  28 };
  29
  30 static struct sw_flow **find_bucket(struct sw_table *swt,
  31                                                                         const struct sw_flow_key *key)
  32 {
  33         struct sw_table_hash *th = (struct sw_table_hash *) swt;
  34         unsigned int crc = crc32_calculate(&th->crc32, key,
  35                                 offsetof(struct sw_flow_key, wildcards));
  36         return &th->buckets[crc & th->bucket_mask];
  37 }
  38
  39 static struct sw_flow *table_hash_lookup(struct sw_table *swt,
  40                                                                                  const struct sw_flow_key *key)
  41 {
  42         struct sw_flow *flow = *find_bucket(swt, key);
  43         return flow && flow_keys_equal(&flow->key, key) ? flow : NULL;
  44 }
  45
  46 static int table_hash_insert(struct sw_table *swt, struct sw_flow *flow)
  47 {
  48         struct sw_table_hash *th = (struct sw_table_hash *) swt;
  49         struct sw_flow **bucket;
  50         int retval;
  51
  52         if (flow->key.wildcards != 0)
  53                 return 0;
  54
  55         bucket = find_bucket(swt, &flow->key);
  56         if (*bucket == NULL) {
  57                 th->n_flows++;
  58                 rcu_assign_pointer(*bucket, flow);
  59                 retval = 1;
  60         } else {
  61                 struct sw_flow *old_flow = *bucket;
  62                 if (flow_keys_equal(&old_flow->key, &flow->key)) {
  63                         rcu_assign_pointer(*bucket, flow);
  64                         flow_deferred_free(old_flow);
  65                         retval = 1;
  66                 } else {
  67                         retval = 0;
  68                 }
  69         }
  70         return retval;
  71 }
  72
  73 /* Caller must update n_flows. */
  74 static int do_delete(struct sw_flow **bucket, struct sw_flow *flow)
  75 {
  76         rcu_assign_pointer(*bucket, NULL);
  77         flow_deferred_free(flow);
  78         return 1;
  79 }
  80
  81 /* Returns number of deleted flows.  We can ignore the priority
  82  * argument, since all exact-match entries are the same (highest)
  83  * priority. */
  84 static int table_hash_delete(struct sw_table *swt,
  85                                                          const struct sw_flow_key *key,
  86                                                          uint16_t priority, int strict)
  87 {
  88         struct sw_table_hash *th = (struct sw_table_hash *) swt;
  89         unsigned int count = 0;
  90
  91         if (key->wildcards == 0) {
  92                 struct sw_flow **bucket = find_bucket(swt, key);
  93                 struct sw_flow *flow = *bucket;
  94                 if (flow && flow_keys_equal(&flow->key, key))
  95                         count = do_delete(bucket, flow);
  96         } else {
  97                 unsigned int i;
  98
  99                 for (i = 0; i <= th->bucket_mask; i++) {
 100                         struct sw_flow **bucket = &th->buckets[i];
 101                         struct sw_flow *flow = *bucket;
 102                         if (flow && flow_del_matches(&flow->key, key, strict))
 103                                 count += do_delete(bucket, flow);
 104                 }
 105         }
 106         th->n_flows -= count;
 107         return count;
 108 }
 109
 110 static int table_hash_timeout(struct datapath *dp, struct sw_table *swt)
 111 {
 112         struct sw_table_hash *th = (struct sw_table_hash *) swt;
 113         unsigned int i;
 114         int count = 0;
 115
 116         mutex_lock(&dp_mutex);
 117         for (i = 0; i <= th->bucket_mask; i++) {
 118                 struct sw_flow **bucket = &th->buckets[i];
 119                 struct sw_flow *flow = *bucket;
 120                 if (flow) {
 121                         int reason = flow_timeout(flow);
 122                         if (reason >= 0) {
 123                                 count += do_delete(bucket, flow);
 124                                 dp_send_flow_expired(dp, flow, reason);
 125                         }
 126                 }
 127         }
 128         th->n_flows -= count;
 129         mutex_unlock(&dp_mutex);
 130
 131         return count;
 132 }
 133
 134 static void table_hash_destroy(struct sw_table *swt)
 135 {
 136         struct sw_table_hash *th = (struct sw_table_hash *) swt;
 137         unsigned int i;
 138         for (i = 0; i <= th->bucket_mask; i++)
 139         if (th->buckets[i])
 140                 flow_free(th->buckets[i]);
 141         kmem_free(th->buckets, (th->bucket_mask + 1) * sizeof *th->buckets);
 142         kfree(th);
 143 }
 144
 145 static int table_hash_iterate(struct sw_table *swt,
 146                               const struct sw_flow_key *key,
 147                               struct sw_table_position *position,
 148                               int (*callback)(struct sw_flow *, void *private),
 149                               void *private)
 150 {
 151         struct sw_table_hash *th = (struct sw_table_hash *) swt;
 152
 153         if (position->private[0] > th->bucket_mask)
 154                 return 0;
 155
 156         if (key->wildcards == 0) {
 157                 struct sw_flow *flow;
 158                 int error;
 159
 160                 flow = table_hash_lookup(swt, key);
 161                 if (!flow)
 162                         return 0;
 163
 164                 error = callback(flow, private);
 165                 if (!error)
 166                         position->private[0] = -1;
 167                 return error;
 168         } else {
 169                 int i;
 170
 171                 for (i = position->private[0]; i <= th->bucket_mask; i++) {
 172                         struct sw_flow *flow = th->buckets[i];
 173                         if (flow && flow_matches_1wild(&flow->key, key)) {
 174                                 int error = callback(flow, private);
 175                                 if (error) {
 176                                         position->private[0] = i;
 177                                         return error;
 178                                 }
 179                         }
 180                 }
 181                 return 0;
 182         }
 183 }
 184 static void table_hash_stats(struct sw_table *swt,
 185                                  struct sw_table_stats *stats)
 186 {
 187         struct sw_table_hash *th = (struct sw_table_hash *) swt;
 188         stats->name = "hash";
 189         stats->wildcards = 0;          /* No wildcards are supported. */
 190         stats->n_flows   = th->n_flows;
 191         stats->max_flows = th->bucket_mask + 1;
 192         stats->n_matched = swt->n_matched;
 193 }
 194
 195 struct sw_table *table_hash_create(unsigned int polynomial,
 196                         unsigned int n_buckets)
 197 {
 198         struct sw_table_hash *th;
 199         struct sw_table *swt;
 200
 201         th = kzalloc(sizeof *th, GFP_KERNEL);
 202         if (th == NULL)
 203                 return NULL;
 204
 205         BUG_ON(n_buckets & (n_buckets - 1));
 206         th->buckets = kmem_zalloc(n_buckets * sizeof *th->buckets);
 207         if (th->buckets == NULL) {
 208                 printk("failed to allocate %u buckets\n", n_buckets);
 209                 kfree(th);
 210                 return NULL;
 211         }
 212         th->bucket_mask = n_buckets - 1;
 213
 214         swt = &th->swt;
 215         swt->lookup = table_hash_lookup;
 216         swt->insert = table_hash_insert;
 217         swt->delete = table_hash_delete;
 218         swt->timeout = table_hash_timeout;
 219         swt->destroy = table_hash_destroy;
 220         swt->iterate = table_hash_iterate;
 221         swt->stats = table_hash_stats;
 222
 223         crc32_init(&th->crc32, polynomial);
 224         th->n_flows = 0;
 225
 226         return swt;
 227 }
 228
 229 /* Double-hashing table. */
 230
 231 struct sw_table_hash2 {
 232         struct sw_table swt;
 233         struct sw_table *subtable[2];
 234 };
 235
 236 static struct sw_flow *table_hash2_lookup(struct sw_table *swt,
 237                                                                                   const struct sw_flow_key *key)
 238 {
 239         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 240         int i;
 241
 242         for (i = 0; i < 2; i++) {
 243                 struct sw_flow *flow = *find_bucket(t2->subtable[i], key);
 244                 if (flow && flow_keys_equal(&flow->key, key))
 245                         return flow;
 246         }
 247         return NULL;
 248 }
 249
 250 static int table_hash2_insert(struct sw_table *swt, struct sw_flow *flow)
 251 {
 252         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 253
 254         if (table_hash_insert(t2->subtable[0], flow))
 255                 return 1;
 256         return table_hash_insert(t2->subtable[1], flow);
 257 }
 258
 259 static int table_hash2_delete(struct sw_table *swt,
 260                                                           const struct sw_flow_key *key,
 261                                                           uint16_t priority, int strict)
 262 {
 263         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 264         return (table_hash_delete(t2->subtable[0], key, priority, strict)
 265                         + table_hash_delete(t2->subtable[1], key, priority, strict));
 266 }
 267
 268 static int table_hash2_timeout(struct datapath *dp, struct sw_table *swt)
 269 {
 270         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 271         return (table_hash_timeout(dp, t2->subtable[0])
 272                         + table_hash_timeout(dp, t2->subtable[1]));
 273 }
 274
 275 static void table_hash2_destroy(struct sw_table *swt)
 276 {
 277         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 278         table_hash_destroy(t2->subtable[0]);
 279         table_hash_destroy(t2->subtable[1]);
 280         kfree(t2);
 281 }
 282
 283 static int table_hash2_iterate(struct sw_table *swt,
 284                                const struct sw_flow_key *key,
 285                                struct sw_table_position *position,
 286                                int (*callback)(struct sw_flow *, void *),
 287                                void *private)
 288 {
 289         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 290         int i;
 291
 292         for (i = position->private[1]; i < 2; i++) {
 293                 int error = table_hash_iterate(t2->subtable[i], key, position,
 294                                                callback, private);
 295                 if (error) {
 296                         return error;
 297                 }
 298                 position->private[0] = 0;
 299                 position->private[1]++;
 300         }
 301         return 0;
 302 }
 303
 304 static void table_hash2_stats(struct sw_table *swt,
 305                                  struct sw_table_stats *stats)
 306 {
 307         struct sw_table_hash2 *t2 = (struct sw_table_hash2 *) swt;
 308         struct sw_table_stats substats[2];
 309         int i;
 310
 311         for (i = 0; i < 2; i++)
 312                 table_hash_stats(t2->subtable[i], &substats[i]);
 313         stats->name = "hash2";
 314         stats->wildcards = 0;          /* No wildcards are supported. */
 315         stats->n_flows   = substats[0].n_flows + substats[1].n_flows;
 316         stats->max_flows = substats[0].max_flows + substats[1].max_flows;
 317         stats->n_matched = swt->n_matched;
 318 }
 319
 320 struct sw_table *table_hash2_create(unsigned int poly0, unsigned int buckets0,
 321                                                                         unsigned int poly1, unsigned int buckets1)
 322
 323 {
 324         struct sw_table_hash2 *t2;
 325         struct sw_table *swt;
 326
 327         t2 = kzalloc(sizeof *t2, GFP_KERNEL);
 328         if (t2 == NULL)
 329                 return NULL;
 330
 331         t2->subtable[0] = table_hash_create(poly0, buckets0);
 332         if (t2->subtable[0] == NULL)
 333                 goto out_free_t2;
 334
 335         t2->subtable[1] = table_hash_create(poly1, buckets1);
 336         if (t2->subtable[1] == NULL)
 337                 goto out_free_subtable0;
 338
 339         swt = &t2->swt;
 340         swt->lookup = table_hash2_lookup;
 341         swt->insert = table_hash2_insert;
 342         swt->delete = table_hash2_delete;
 343         swt->timeout = table_hash2_timeout;
 344         swt->destroy = table_hash2_destroy;
 345         swt->iterate = table_hash2_iterate;
 346         swt->stats = table_hash2_stats;
 347
 348         return swt;
 349
 350 out_free_subtable0:
 351         table_hash_destroy(t2->subtable[0]);
 352 out_free_t2:
 353         kfree(t2);
 354         return NULL;
 355 }
 356
 357 /* From fs/xfs/linux-2.4/kmem.c. */
 358
 359 static void *
 360 kmem_alloc(size_t size)
 361 {
 362         void *ptr;
 363
 364 #ifdef KMALLOC_MAX_SIZE
 365         if (size > KMALLOC_MAX_SIZE)
 366                 return NULL;
 367 #endif
 368         ptr = kmalloc(size, GFP_KERNEL);
 369         if (!ptr) {
 370                 ptr = vmalloc(size);
 371                 if (ptr)
 372                         printk("openflow: used vmalloc for %lu bytes\n",
 373                                         (unsigned long)size);
 374         }
 375         return ptr;
 376 }
 377
 378 static void *
 379 kmem_zalloc(size_t size)
 380 {
 381         void *ptr = kmem_alloc(size);
 382         if (ptr)
 383                 memset(ptr, 0, size);
 384         return ptr;
 385 }
 386
 387 static void
 388 kmem_free(void *ptr, size_t size)
 389 {
 390         if (((unsigned long)ptr < VMALLOC_START) ||
 391                 ((unsigned long)ptr >= VMALLOC_END)) {
 392                 kfree(ptr);
 393         } else {
 394                 vfree(ptr);
 395         }
 396 }