net/tux/accept.c

   1 /*
   2  * TUX - Integrated Application Protocols Layer and Object Cache
   3  *
   4  * Copyright (C) 2000, 2001, Ingo Molnar <mingo@redhat.com>
   5  *
   6  * accept.c: accept new connections, allocate requests
   7  */
   8
   9 #include <net/tux.h>
  10
  11 /****************************************************************
  12  *      This program is free software; you can redistribute it and/or modify
  13  *      it under the terms of the GNU General Public License as published by
  14  *      the Free Software Foundation; either version 2, or (at your option)
  15  *      any later version.
  16  *
  17  *      This program is distributed in the hope that it will be useful,
  18  *      but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20  *      GNU General Public License for more details.
  21  *
  22  *      You should have received a copy of the GNU General Public License
  23  *      along with this program; if not, write to the Free Software
  24  *      Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  25  *
  26  ****************************************************************/
  27
  28 unsigned int tux_ack_pingpong = 1;
  29 unsigned int tux_push_all = 0;
  30 unsigned int tux_zerocopy_parse = 1;
  31
  32 static int __idle_event (tux_req_t *req);
  33 static int __output_space_event (tux_req_t *req);
  34
  35 struct socket * start_listening(tux_socket_t *listen, int nr)
  36 {
  37         struct sockaddr_in sin;
  38         struct socket *sock = NULL;
  39         struct sock *sk;
  40         struct tcp_sock *tp;
  41         struct inet_connection_sock *icsk;
  42         int err;
  43         u16 port = listen->port;
  44         u32 addr = listen->ip;
  45         tux_proto_t *proto = listen->proto;
  46
  47         /* Create a listening socket: */
  48
  49         err = sock_create_kern(PF_INET, SOCK_STREAM, IPPROTO_TCP, &sock);
  50         if (err) {
  51                 printk(KERN_ERR "TUX: error %d creating socket.\n", err);
  52                 goto error;
  53         }
  54
  55         /* Bind the socket: */
  56
  57         sin.sin_family = AF_INET;
  58         sin.sin_addr.s_addr = htonl(addr);
  59         sin.sin_port = htons(port);
  60
  61         sk = sock->sk;
  62         icsk = inet_csk(sk);
  63         sk->sk_reuse = 1;
  64         sock_set_flag(sk, SOCK_URGINLINE);
  65
  66         err = sock->ops->bind(sock, (struct sockaddr*)&sin, sizeof(sin));
  67         if (err) {
  68                 printk(KERN_ERR "TUX: error %d binding socket. This means that probably some other process is (or was a short time ago) using addr %s://%d.%d.%d.%d:%d.\n",
  69                         err, proto->name, HIPQUAD(addr), port);
  70                 goto error;
  71         }
  72
  73         tp = tcp_sk(sk);
  74         Dprintk("listen sk accept_queue: %d.\n",
  75                 !reqsk_queue_empty(&icsk->icsk_accept_queue));
  76         icsk->icsk_ack.pingpong = tux_ack_pingpong;
  77
  78         sock_reset_flag(sk, SOCK_LINGER);
  79         sk->sk_lingertime = 0;
  80         tp->linger2 = tux_keepalive_timeout * HZ;
  81
  82         if (proto->defer_accept && !tux_keepalive_timeout && tux_defer_accept)
  83                 icsk->icsk_accept_queue.rskq_defer_accept = 1;
  84
  85         /* Now, start listening on the socket */
  86
  87         err = sock->ops->listen(sock, tux_max_backlog);
  88         if (err) {
  89                 printk(KERN_ERR "TUX: error %d listening on socket.\n", err);
  90                 goto error;
  91         }
  92
  93         printk(KERN_NOTICE "TUX: thread %d listens on %s://%d.%d.%d.%d:%d.\n",
  94                 nr, proto->name, HIPQUAD(addr), port);
  95         return sock;
  96
  97 error:
  98         if (sock)
  99                 sock_release(sock);
 100         return NULL;
 101 }
 102
 103 static inline void __kfree_req (tux_req_t *req, threadinfo_t * ti)
 104 {
 105         list_del(&req->all);
 106         DEBUG_DEL_LIST(&req->all);
 107         ti->nr_requests--;
 108         kfree(req);
 109 }
 110
 111 int flush_freequeue (threadinfo_t * ti)
 112 {
 113         struct list_head *tmp;
 114         unsigned long flags;
 115         tux_req_t *req;
 116         int count = 0;
 117
 118         spin_lock_irqsave(&ti->free_requests_lock,flags);
 119         while (ti->nr_free_requests) {
 120                 ti->nr_free_requests--;
 121                 tmp = ti->free_requests.next;
 122                 req = list_entry(tmp, tux_req_t, free);
 123                 list_del(tmp);
 124                 DEBUG_DEL_LIST(tmp);
 125                 DEC_STAT(nr_free_pending);
 126                 __kfree_req(req, ti);
 127                 count++;
 128         }
 129         spin_unlock_irqrestore(&ti->free_requests_lock,flags);
 130
 131         return count;
 132 }
 133
 134 static tux_req_t * kmalloc_req (threadinfo_t * ti)
 135 {
 136         struct list_head *tmp;
 137         unsigned long flags;
 138         tux_req_t *req;
 139
 140         spin_lock_irqsave(&ti->free_requests_lock, flags);
 141         if (ti->nr_free_requests) {
 142                 ti->nr_free_requests--;
 143                 tmp = ti->free_requests.next;
 144                 req = list_entry(tmp, tux_req_t, free);
 145                 list_del(tmp);
 146                 DEBUG_DEL_LIST(tmp);
 147                 DEC_STAT(nr_free_pending);
 148                 req->magic = TUX_MAGIC;
 149                 spin_unlock_irqrestore(&ti->free_requests_lock, flags);
 150         } else {
 151                 spin_unlock_irqrestore(&ti->free_requests_lock, flags);
 152                 req = tux_kmalloc(sizeof(*req));
 153                 ti->nr_requests++;
 154                 memset (req, 0, sizeof(*req));
 155                 list_add(&req->all, &ti->all_requests);
 156         }
 157         req->magic = TUX_MAGIC;
 158         INC_STAT(nr_allocated);
 159         init_waitqueue_entry(&req->sleep, current);
 160         init_waitqueue_entry(&req->ftp_sleep, current);
 161         INIT_LIST_HEAD(&req->work);
 162         INIT_LIST_HEAD(&req->free);
 163         INIT_LIST_HEAD(&req->lru);
 164         req->ti = ti;
 165         req->total_bytes = 0;
 166         SET_TIMESTAMP(req->accept_timestamp);
 167         req->first_timestamp = jiffies;
 168         req->fd = -1;
 169         init_timer(&req->keepalive_timer);
 170         init_timer(&req->output_timer);
 171
 172         Dprintk("allocated NEW req %p.\n", req);
 173         return req;
 174 }
 175
 176 void kfree_req (tux_req_t *req)
 177 {
 178         threadinfo_t * ti = req->ti;
 179         unsigned long flags;
 180
 181         Dprintk("freeing req %p.\n", req);
 182
 183         if (req->magic != TUX_MAGIC)
 184                 TUX_BUG();
 185         spin_lock_irqsave(&ti->free_requests_lock,flags);
 186         req->magic = 0;
 187         DEC_STAT(nr_allocated);
 188         if (req->sock || req->dentry || req->private)
 189                 TUX_BUG();
 190         if (ti->nr_free_requests > tux_max_free_requests)
 191                 __kfree_req(req, ti);
 192         else {
 193                 req->error = 0;
 194                 ti->nr_free_requests++;
 195
 196                 // the free requests queue is LIFO
 197                 list_add(&req->free, &ti->free_requests);
 198                 INC_STAT(nr_free_pending);
 199         }
 200         spin_unlock_irqrestore(&ti->free_requests_lock,flags);
 201 }
 202
 203 static void __add_req_to_workqueue (tux_req_t *req)
 204 {
 205         threadinfo_t *ti = req->ti;
 206
 207         if (!list_empty(&req->work))
 208                 TUX_BUG();
 209         Dprintk("work-queueing request %p at %p/%p.\n", req, __builtin_return_address(0), __builtin_return_address(1));
 210         if (connection_too_fast(req))
 211                 list_add_tail(&req->work, &ti->work_pending);
 212         else
 213                 list_add(&req->work, &ti->work_pending);
 214         INC_STAT(nr_work_pending);
 215         wake_up_process(ti->thread);
 216         return;
 217 }
 218
 219 void add_req_to_workqueue (tux_req_t *req)
 220 {
 221         unsigned long flags;
 222         threadinfo_t *ti = req->ti;
 223
 224         spin_lock_irqsave(&ti->work_lock, flags);
 225         __add_req_to_workqueue(req);
 226         spin_unlock_irqrestore(&ti->work_lock, flags);
 227 }
 228
 229 void del_output_timer (tux_req_t *req)
 230 {
 231 #ifdef CONFIG_SMP
 232         if (!spin_is_locked(&req->ti->work_lock))
 233                 TUX_BUG();
 234 #endif
 235         if (!list_empty(&req->lru)) {
 236                 list_del(&req->lru);
 237                 DEBUG_DEL_LIST(&req->lru);
 238                 req->ti->nr_lru--;
 239         }
 240         Dprintk("del output timeout for req %p.\n", req);
 241         del_timer(&req->output_timer);
 242 }
 243
 244 static void output_timeout_fn (unsigned long data);
 245
 246 #define OUTPUT_TIMEOUT HZ
 247
 248 static void add_output_timer (tux_req_t *req)
 249 {
 250         struct timer_list *timer = &req->output_timer;
 251
 252         timer->data = (unsigned long) req;
 253         timer->function = &output_timeout_fn;
 254         mod_timer(timer, jiffies + OUTPUT_TIMEOUT);
 255 }
 256
 257 static void output_timeout_fn (unsigned long data)
 258 {
 259         tux_req_t *req = (tux_req_t *)data;
 260
 261         if (connection_too_fast(req)) {
 262                 add_output_timer(req);
 263 //              mod_timer(&req->output_timer, jiffies + OUTPUT_TIMEOUT);
 264                 return;
 265         }
 266         output_space_event(req);
 267 }
 268
 269 void output_timeout (tux_req_t *req)
 270 {
 271         Dprintk("output timeout for req %p.\n", req);
 272         if (test_and_set_bit(0, &req->wait_output_space))
 273                 TUX_BUG();
 274         INC_STAT(nr_output_space_pending);
 275         add_output_timer(req);
 276 }
 277
 278 void __del_keepalive_timer (tux_req_t *req)
 279 {
 280 #ifdef CONFIG_SMP
 281         if (!spin_is_locked(&req->ti->work_lock))
 282                 TUX_BUG();
 283 #endif
 284         if (!list_empty(&req->lru)) {
 285                 list_del(&req->lru);
 286                 DEBUG_DEL_LIST(&req->lru);
 287                 req->ti->nr_lru--;
 288         }
 289         Dprintk("del keepalive timeout for req %p.\n", req);
 290         del_timer(&req->keepalive_timer);
 291 }
 292
 293 static void keepalive_timeout_fn (unsigned long data)
 294 {
 295         tux_req_t *req = (tux_req_t *)data;
 296
 297 #ifdef CONFIG_TUX_DEBUG
 298         Dprintk("req %p timed out after %d sec!\n", req, tux_keepalive_timeout);
 299         if (tux_Dprintk)
 300                 print_req(req);
 301 #endif
 302         Dprintk("req->error = TUX_ERROR_CONN_TIMEOUT!\n");
 303         req->error = TUX_ERROR_CONN_TIMEOUT;
 304         if (!idle_event(req))
 305                 output_space_event(req);
 306 }
 307
 308 void __add_keepalive_timer (tux_req_t *req)
 309 {
 310         struct timer_list *timer = &req->keepalive_timer;
 311
 312         if (!tux_keepalive_timeout)
 313                 TUX_BUG();
 314 #ifdef CONFIG_SMP
 315         if (!spin_is_locked(&req->ti->work_lock))
 316                 TUX_BUG();
 317 #endif
 318
 319         if (!list_empty(&req->lru))
 320                 TUX_BUG();
 321         if (req->ti->nr_lru > tux_max_keepalives) {
 322                 struct list_head *head, *last;
 323                 tux_req_t *last_req;
 324
 325                 head = &req->ti->lru;
 326                 last = head->prev;
 327                 if (last == head)
 328                         TUX_BUG();
 329                 last_req = list_entry(last, tux_req_t, lru);
 330                 list_del(last);
 331                 DEBUG_DEL_LIST(last);
 332                 req->ti->nr_lru--;
 333
 334                 Dprintk("LRU-aging req %p!\n", last_req);
 335                 last_req->error = TUX_ERROR_CONN_TIMEOUT;
 336                 if (!__idle_event(last_req))
 337                         __output_space_event(last_req);
 338         }
 339         list_add(&req->lru, &req->ti->lru);
 340         req->ti->nr_lru++;
 341
 342         timer->expires = jiffies + tux_keepalive_timeout * HZ;
 343         timer->data = (unsigned long) req;
 344         timer->function = &keepalive_timeout_fn;
 345         add_timer(timer);
 346 }
 347
 348 static int __output_space_event (tux_req_t *req)
 349 {
 350         if (!req || (req->magic != TUX_MAGIC))
 351                 TUX_BUG();
 352
 353         if (!test_and_clear_bit(0, &req->wait_output_space)) {
 354                 Dprintk("output space ready event at <%p>, on non-idle %p.\n", __builtin_return_address(0), req);
 355                 return 0;
 356         }
 357
 358         Dprintk("output space ready event at <%p>, %p was waiting!\n", __builtin_return_address(0), req);
 359         DEC_STAT(nr_output_space_pending);
 360
 361         del_keepalive_timer(req);
 362         del_output_timer(req);
 363
 364         __add_req_to_workqueue(req);
 365         return 1;
 366 }
 367
 368 int output_space_event (tux_req_t *req)
 369 {
 370         int ret;
 371         unsigned long flags;
 372
 373         spin_lock_irqsave(&req->ti->work_lock, flags);
 374         ret = __output_space_event(req);
 375         spin_unlock_irqrestore(&req->ti->work_lock, flags);
 376
 377         return ret;
 378 }
 379
 380 static int __idle_event (tux_req_t *req)
 381 {
 382         struct inet_connection_sock *icsk;
 383         threadinfo_t *ti;
 384
 385         if (!req || (req->magic != TUX_MAGIC))
 386                 TUX_BUG();
 387         ti = req->ti;
 388
 389         if (!test_and_clear_bit(0, &req->idle_input)) {
 390                 Dprintk("data ready event at <%p>, on non-idle %p.\n", __builtin_return_address(0), req);
 391                 return 0;
 392         }
 393
 394         Dprintk("data ready event at <%p>, %p was idle!\n", __builtin_return_address(0), req);
 395         del_keepalive_timer(req);
 396         del_output_timer(req);
 397         DEC_STAT(nr_idle_input_pending);
 398
 399         icsk = inet_csk(req->sock->sk);
 400
 401         icsk->icsk_ack.pingpong = tux_ack_pingpong;
 402         SET_TIMESTAMP(req->accept_timestamp);
 403
 404         __add_req_to_workqueue(req);
 405
 406         return 1;
 407 }
 408
 409 int idle_event (tux_req_t *req)
 410 {
 411         int ret;
 412         unsigned long flags;
 413
 414         spin_lock_irqsave(&req->ti->work_lock, flags);
 415         ret = __idle_event(req);
 416         spin_unlock_irqrestore(&req->ti->work_lock, flags);
 417
 418         return ret;
 419 }
 420
 421 #define HANDLE_CALLBACK_1(callback, tux_name, real_name, param...)      \
 422         tux_req_t *req;                                 \
 423                                                         \
 424         read_lock(&sk->sk_callback_lock);               \
 425         req = sk->sk_user_data;                         \
 426                                                         \
 427         Dprintk("callback "#callback"(%p) req %p.\n",   \
 428                 sk->sk_##callback, req);                \
 429                                                         \
 430         if (!req) {                                     \
 431                 if (sk->sk_##callback == tux_name) {    \
 432                         printk("BUG: "#callback" "#tux_name" "#real_name" no req!"); \
 433                         TUX_BUG();                      \
 434                 }                                       \
 435                 read_unlock(&sk->sk_callback_lock);     \
 436                 if (sk->sk_##callback)                  \
 437                         sk->sk_##callback(param);       \
 438                 return;                                 \
 439         }                                               \
 440
 441 #define HANDLE_CALLBACK_2(callback, tux_name, real_name, param...)      \
 442         Dprintk(#tux_name"() on %p.\n", req);           \
 443         if (req->magic != TUX_MAGIC)                    \
 444                 TUX_BUG();                              \
 445         if (req->real_name)                             \
 446                 req->real_name(param);
 447
 448 #define HANDLE_CALLBACK(callback, tux_name, real_name, param...)        \
 449         HANDLE_CALLBACK_1(callback,tux_name,real_name,param)    \
 450         HANDLE_CALLBACK_2(callback,tux_name,real_name,param)
 451
 452 static void tux_data_ready (struct sock *sk, int len)
 453 {
 454         HANDLE_CALLBACK_1(data_ready, tux_data_ready, real_data_ready, sk, len);
 455
 456         if (!idle_event(req))
 457                 output_space_event(req);
 458         read_unlock(&sk->sk_callback_lock);
 459 }
 460
 461 static void tux_write_space (struct sock *sk)
 462 {
 463         HANDLE_CALLBACK(write_space, tux_write_space, real_write_space, sk);
 464
 465         Dprintk("sk->sk_wmem_queued: %d, sk->sk_sndbuf: %d.\n",
 466                 sk->sk_wmem_queued, sk->sk_sndbuf);
 467
 468         if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) {
 469                 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 470                 if (!idle_event(req))
 471                         output_space_event(req);
 472         }
 473         read_unlock(&sk->sk_callback_lock);
 474 }
 475
 476 static void tux_error_report (struct sock *sk)
 477 {
 478         HANDLE_CALLBACK(error_report, tux_error_report, real_error_report, sk);
 479
 480         req->error = TUX_ERROR_CONN_CLOSE;
 481         if (!idle_event(req))
 482                 output_space_event(req);
 483         read_unlock(&sk->sk_callback_lock);
 484 }
 485
 486 static void tux_state_change (struct sock *sk)
 487 {
 488         HANDLE_CALLBACK(state_change, tux_state_change, real_state_change, sk);
 489
 490         if (req->sock && req->sock->sk &&
 491                                 (req->sock->sk->sk_state > TCP_ESTABLISHED)) {
 492                 Dprintk("req %p changed to TCP non-established!\n", req);
 493                 Dprintk("req->sock: %p\n", req->sock);
 494                 if (req->sock)
 495                         Dprintk("req->sock->sk: %p\n", req->sock->sk);
 496                 if (req->sock && req->sock->sk)
 497                         Dprintk("TCP state: %d\n", req->sock->sk->sk_state);
 498                 Dprintk("req->error = TUX_ERROR_CONN_CLOSE!\n");
 499                 req->error = TUX_ERROR_CONN_CLOSE;
 500         }
 501         if (!idle_event(req))
 502                 output_space_event(req);
 503         read_unlock(&sk->sk_callback_lock);
 504 }
 505
 506 static void tux_destruct (struct sock *sk)
 507 {
 508         BUG();
 509 }
 510
 511 static void tux_ftp_data_ready (struct sock *sk, int len)
 512 {
 513         HANDLE_CALLBACK_1(data_ready, tux_ftp_data_ready,
 514                                 ftp_real_data_ready, sk, len);
 515         if (!idle_event(req))
 516                 output_space_event(req);
 517         read_unlock(&sk->sk_callback_lock);
 518 }
 519
 520 static void tux_ftp_write_space (struct sock *sk)
 521 {
 522         HANDLE_CALLBACK_1(write_space, tux_ftp_write_space,
 523                                 ftp_real_write_space, sk);
 524
 525         Dprintk("sk->sk_wmem_queued: %d, sk->sk_sndbuf: %d.\n",
 526                 sk->sk_wmem_queued, sk->sk_sndbuf);
 527
 528         if (sk_stream_wspace(sk) >= sk->sk_sndbuf/10*8) {
 529                 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 530                 if (!idle_event(req))
 531                         output_space_event(req);
 532         }
 533         read_unlock(&sk->sk_callback_lock);
 534 }
 535
 536 static void tux_ftp_error_report (struct sock *sk)
 537 {
 538         HANDLE_CALLBACK(error_report, tux_ftp_error_report,
 539                 ftp_real_error_report, sk);
 540
 541         TDprintk("req %p sock %p got TCP errors on FTP data connection!\n", req, sk);
 542         TDprintk("req->error = TUX_ERROR_CONN_CLOSE!\n");
 543         req->error = TUX_ERROR_CONN_CLOSE;
 544         if (!idle_event(req))
 545                 output_space_event(req);
 546         read_unlock(&sk->sk_callback_lock);
 547 }
 548
 549 static void tux_ftp_state_change (struct sock *sk)
 550 {
 551         HANDLE_CALLBACK(state_change, tux_ftp_state_change,
 552                         ftp_real_state_change, sk);
 553
 554         if (req->sock && req->sock->sk &&
 555                         (req->sock->sk->sk_state > TCP_ESTABLISHED)) {
 556                 Dprintk("req %p FTP control sock changed to TCP non-established!\n", req);
 557                 Dprintk("req->sock: %p\n", req->sock);
 558                 TDprintk("req->error = TUX_ERROR_CONN_CLOSE!\n");
 559
 560                 req->error = TUX_ERROR_CONN_CLOSE;
 561         }
 562         if (!idle_event(req))
 563                 output_space_event(req);
 564         read_unlock(&sk->sk_callback_lock);
 565 }
 566
 567 static void tux_ftp_create_child (struct sock *sk, struct sock *newsk)
 568 {
 569         HANDLE_CALLBACK(create_child, tux_ftp_create_child,
 570                         ftp_real_create_child, sk, newsk);
 571
 572         newsk->sk_user_data = NULL;
 573         newsk->sk_data_ready = req->ftp_real_data_ready;
 574         newsk->sk_state_change = req->ftp_real_state_change;
 575         newsk->sk_write_space = req->ftp_real_write_space;
 576         newsk->sk_error_report = req->ftp_real_error_report;
 577         newsk->sk_create_child = req->ftp_real_create_child;
 578         newsk->sk_destruct = req->ftp_real_destruct;
 579
 580         if (!idle_event(req))
 581                 output_space_event(req);
 582         read_unlock(&sk->sk_callback_lock);
 583 }
 584
 585 static void tux_ftp_destruct (struct sock *sk)
 586 {
 587         BUG();
 588 }
 589
 590 static void link_tux_socket (tux_req_t *req, struct socket *sock)
 591 {
 592         struct sock *sk = sock->sk;
 593
 594         if (req->sock)
 595                 TUX_BUG();
 596         if (sk->sk_destruct == tux_destruct)
 597                 TUX_BUG();
 598         /*
 599          * (No need to lock the socket, we just want to
 600          * make sure that events from now on go through
 601          * tux_data_ready())
 602          */
 603         write_lock_irq(&sk->sk_callback_lock);
 604
 605         req->sock = sock;
 606         sk->sk_user_data = req;
 607
 608         req->real_data_ready = sk->sk_data_ready;
 609         req->real_state_change = sk->sk_state_change;
 610         req->real_write_space = sk->sk_write_space;
 611         req->real_error_report = sk->sk_error_report;
 612         req->real_destruct = sk->sk_destruct;
 613
 614         sk->sk_data_ready = tux_data_ready;
 615         sk->sk_state_change = tux_state_change;
 616         sk->sk_write_space = tux_write_space;
 617         sk->sk_error_report = tux_error_report;
 618         sk->sk_destruct = tux_destruct;
 619
 620         write_unlock_irq(&sk->sk_callback_lock);
 621
 622         if (req->real_destruct == tux_destruct)
 623                 TUX_BUG();
 624         req->client_addr = inet_sk(sk)->daddr;
 625         req->client_port = inet_sk(sk)->dport;
 626
 627         add_wait_queue(sk->sk_sleep, &req->sleep);
 628 }
 629
 630 void __link_data_socket (tux_req_t *req, struct socket *sock,
 631                                                 struct sock *sk)
 632 {
 633         /*
 634          * (No need to lock the socket, we just want to
 635          * make sure that events from now on go through
 636          * tux_data_ready())
 637          */
 638         write_lock_irq(&sk->sk_callback_lock);
 639
 640         req->data_sock = sock;
 641         sk->sk_user_data = req;
 642
 643         req->ftp_real_data_ready = sk->sk_data_ready;
 644         req->ftp_real_state_change = sk->sk_state_change;
 645         req->ftp_real_write_space = sk->sk_write_space;
 646         req->ftp_real_error_report = sk->sk_error_report;
 647         req->ftp_real_create_child = sk->sk_create_child;
 648         req->ftp_real_destruct = sk->sk_destruct;
 649
 650         sk->sk_data_ready = tux_ftp_data_ready;
 651         sk->sk_state_change = tux_ftp_state_change;
 652         sk->sk_write_space = tux_ftp_write_space;
 653         sk->sk_error_report = tux_ftp_error_report;
 654         sk->sk_create_child = tux_ftp_create_child;
 655         sk->sk_destruct = tux_ftp_destruct;
 656
 657         if (req->ftp_real_destruct == tux_ftp_destruct)
 658                 TUX_BUG();
 659
 660         write_unlock_irq(&sk->sk_callback_lock);
 661
 662         add_wait_queue(sk->sk_sleep, &req->ftp_sleep);
 663 }
 664
 665 void link_tux_data_socket (tux_req_t *req, struct socket *sock)
 666 {
 667         struct sock *sk = sock->sk;
 668
 669         if (req->data_sock)
 670                 TUX_BUG();
 671         if (sk->sk_destruct == tux_ftp_destruct)
 672                 TUX_BUG();
 673         __link_data_socket(req, sock, sk);
 674 }
 675
 676 void unlink_tux_socket (tux_req_t *req)
 677 {
 678         struct sock *sk;
 679
 680         if (!req->sock || !req->sock->sk)
 681                 return;
 682         sk = req->sock->sk;
 683
 684         write_lock_irq(&sk->sk_callback_lock);
 685         if (!sk->sk_user_data)
 686                 TUX_BUG();
 687         if (req->real_destruct == tux_destruct)
 688                 TUX_BUG();
 689
 690         sk->sk_user_data = NULL;
 691
 692         sk->sk_data_ready = req->real_data_ready;
 693         sk->sk_state_change = req->real_state_change;
 694         sk->sk_write_space = req->real_write_space;
 695         sk->sk_error_report = req->real_error_report;
 696         sk->sk_destruct = req->real_destruct;
 697
 698         if (sk->sk_destruct == tux_destruct)
 699                 TUX_BUG();
 700
 701         req->real_data_ready = NULL;
 702         req->real_state_change = NULL;
 703         req->real_write_space = NULL;
 704         req->real_error_report = NULL;
 705         req->real_destruct = NULL;
 706
 707         write_unlock_irq(&sk->sk_callback_lock);
 708
 709         remove_wait_queue(sk->sk_sleep, &req->sleep);
 710 }
 711
 712 void unlink_tux_data_socket (tux_req_t *req)
 713 {
 714         struct sock *sk;
 715
 716         if (!req->data_sock || !req->data_sock->sk)
 717                 return;
 718         sk = req->data_sock->sk;
 719
 720         write_lock_irq(&sk->sk_callback_lock);
 721
 722         if (req->real_destruct == tux_ftp_destruct)
 723                 TUX_BUG();
 724
 725         sk->sk_user_data = NULL;
 726         sk->sk_data_ready = req->ftp_real_data_ready;
 727         sk->sk_state_change = req->ftp_real_state_change;
 728         sk->sk_write_space = req->ftp_real_write_space;
 729         sk->sk_error_report = req->ftp_real_error_report;
 730         sk->sk_create_child = req->ftp_real_create_child;
 731         sk->sk_destruct = req->ftp_real_destruct;
 732
 733         req->ftp_real_data_ready = NULL;
 734         req->ftp_real_state_change = NULL;
 735         req->ftp_real_write_space = NULL;
 736         req->ftp_real_error_report = NULL;
 737         req->ftp_real_create_child = NULL;
 738         req->ftp_real_destruct = NULL;
 739
 740         write_unlock_irq(&sk->sk_callback_lock);
 741
 742         if (sk->sk_destruct == tux_ftp_destruct)
 743                 TUX_BUG();
 744
 745         remove_wait_queue(sk->sk_sleep, &req->ftp_sleep);
 746 }
 747
 748 void add_tux_atom (tux_req_t *req, atom_func_t *atom)
 749 {
 750         Dprintk("adding TUX atom %p to req %p, atom_idx: %d, at %p/%p.\n",
 751                 atom, req, req->atom_idx, __builtin_return_address(0), __builtin_return_address(1));
 752         if (req->atom_idx == MAX_TUX_ATOMS)
 753                 TUX_BUG();
 754         req->atoms[req->atom_idx] = atom;
 755         req->atom_idx++;
 756 }
 757
 758 void del_tux_atom (tux_req_t *req)
 759 {
 760         if (!req->atom_idx)
 761                 TUX_BUG();
 762         req->atom_idx--;
 763         Dprintk("removing TUX atom %p to req %p, atom_idx: %d, at %p.\n",
 764                 req->atoms[req->atom_idx], req, req->atom_idx, __builtin_return_address(0));
 765 }
 766
 767 void tux_schedule_atom (tux_req_t *req, int cachemiss)
 768 {
 769         if (!list_empty(&req->work))
 770                 TUX_BUG();
 771         if (!req->atom_idx)
 772                 TUX_BUG();
 773         req->atom_idx--;
 774         Dprintk("DOING TUX atom %p, req %p, atom_idx: %d, at %p.\n",
 775                 req->atoms[req->atom_idx], req, req->atom_idx, __builtin_return_address(0));
 776         might_sleep();
 777         req->atoms[req->atom_idx](req, cachemiss);
 778         might_sleep();
 779         Dprintk("DONE TUX atom %p, req %p, atom_idx: %d, at %p.\n",
 780                 req->atoms[req->atom_idx], req, req->atom_idx, __builtin_return_address(0));
 781 }
 782
 783 /*
 784  * Puts newly accepted connections into the inputqueue. This is the
 785  * first step in the life of a TUX request.
 786  */
 787 int accept_requests (threadinfo_t *ti)
 788 {
 789         int count = 0, last_count = 0, error, socknr = 0;
 790         struct socket *sock, *new_sock;
 791         struct tcp_sock *tp2;
 792         struct inet_connection_sock *icsk1, *icsk2;
 793         tux_req_t *req;
 794
 795         if (ti->nr_requests > tux_max_connect)
 796                 goto out;
 797
 798 repeat:
 799         for (socknr = 0; socknr < CONFIG_TUX_NUMSOCKETS; socknr++) {
 800                 tux_listen_t *tux_listen;
 801
 802                 tux_listen = ti->listen + socknr;
 803                 sock = tux_listen->sock;
 804                 if (!sock)
 805                         break;
 806                 if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
 807                         break;
 808
 809                 icsk1 = inet_csk(sock->sk);
 810                 /*
 811                  * Quick test to see if there are connections on the queue.
 812                  * This is cheaper than accept() itself because this saves us
 813                  * the allocation of a new socket. (Which doesn't seem to be
 814                  * used anyway)
 815                  */
 816                 if (!reqsk_queue_empty(&icsk1->icsk_accept_queue)) {
 817                         tux_proto_t *proto;
 818
 819                         if (!count++)
 820                                 __set_task_state(current, TASK_RUNNING);
 821
 822                         new_sock = sock_alloc();
 823                         if (!new_sock)
 824                                 goto out;
 825
 826                         new_sock->type = sock->type;
 827                         new_sock->ops = sock->ops;
 828
 829                         error = sock->ops->accept(sock, new_sock, O_NONBLOCK);
 830                         if (error < 0)
 831                                 goto err;
 832                         if (new_sock->sk->sk_state != TCP_ESTABLISHED)
 833                                 goto err;
 834
 835                         tp2 = tcp_sk(new_sock->sk);
 836                         icsk2 = inet_csk(new_sock->sk);
 837                         tp2->nonagle = 2;
 838                         icsk2->icsk_ack.pingpong = tux_ack_pingpong;
 839                         new_sock->sk->sk_reuse = 1;
 840                         sock_set_flag(new_sock->sk, SOCK_URGINLINE);
 841
 842                         /* Allocate a request-entry for the connection */
 843                         req = kmalloc_req(ti);
 844                         if (!req)
 845                                 BUG();
 846                         link_tux_socket(req, new_sock);
 847
 848                         proto = req->proto = tux_listen->proto;
 849
 850                         proto->got_request(req);
 851                 }
 852         }
 853         if (count != last_count) {
 854                 last_count = count;
 855                 goto repeat;
 856         }
 857 out:
 858         return count;
 859 err:
 860         sock_release(new_sock);
 861         goto out;
 862 }
 863