net/ipv4/tcp.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              Implementation of the Transmission Control Protocol(TCP).
   7  *
   8  * Version:     $Id: tcp.c,v 1.216 2002/02/01 22:01:04 davem Exp $
   9  *
  10  * Authors:     Ross Biro, <bir7@leland.Stanford.Edu>
  11  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  12  *              Mark Evans, <evansmp@uhura.aston.ac.uk>
  13  *              Corey Minyard <wf-rch!minyard@relay.EU.net>
  14  *              Florian La Roche, <flla@stud.uni-sb.de>
  15  *              Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
  16  *              Linus Torvalds, <torvalds@cs.helsinki.fi>
  17  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  18  *              Matthew Dillon, <dillon@apollo.west.oic.com>
  19  *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  20  *              Jorge Cwik, <jorge@laser.satlink.net>
  21  *
  22  * Fixes:
  23  *              Alan Cox        :       Numerous verify_area() calls
  24  *              Alan Cox        :       Set the ACK bit on a reset
  25  *              Alan Cox        :       Stopped it crashing if it closed while
  26  *                                      sk->inuse=1 and was trying to connect
  27  *                                      (tcp_err()).
  28  *              Alan Cox        :       All icmp error handling was broken
  29  *                                      pointers passed where wrong and the
  30  *                                      socket was looked up backwards. Nobody
  31  *                                      tested any icmp error code obviously.
  32  *              Alan Cox        :       tcp_err() now handled properly. It
  33  *                                      wakes people on errors. poll
  34  *                                      behaves and the icmp error race
  35  *                                      has gone by moving it into sock.c
  36  *              Alan Cox        :       tcp_send_reset() fixed to work for
  37  *                                      everything not just packets for
  38  *                                      unknown sockets.
  39  *              Alan Cox        :       tcp option processing.
  40  *              Alan Cox        :       Reset tweaked (still not 100%) [Had
  41  *                                      syn rule wrong]
  42  *              Herp Rosmanith  :       More reset fixes
  43  *              Alan Cox        :       No longer acks invalid rst frames.
  44  *                                      Acking any kind of RST is right out.
  45  *              Alan Cox        :       Sets an ignore me flag on an rst
  46  *                                      receive otherwise odd bits of prattle
  47  *                                      escape still
  48  *              Alan Cox        :       Fixed another acking RST frame bug.
  49  *                                      Should stop LAN workplace lockups.
  50  *              Alan Cox        :       Some tidyups using the new skb list
  51  *                                      facilities
  52  *              Alan Cox        :       sk->keepopen now seems to work
  53  *              Alan Cox        :       Pulls options out correctly on accepts
  54  *              Alan Cox        :       Fixed assorted sk->rqueue->next errors
  55  *              Alan Cox        :       PSH doesn't end a TCP read. Switched a
  56  *                                      bit to skb ops.
  57  *              Alan Cox        :       Tidied tcp_data to avoid a potential
  58  *                                      nasty.
  59  *              Alan Cox        :       Added some better commenting, as the
  60  *                                      tcp is hard to follow
  61  *              Alan Cox        :       Removed incorrect check for 20 * psh
  62  *      Michael O'Reilly        :       ack < copied bug fix.
  63  *      Johannes Stille         :       Misc tcp fixes (not all in yet).
  64  *              Alan Cox        :       FIN with no memory -> CRASH
  65  *              Alan Cox        :       Added socket option proto entries.
  66  *                                      Also added awareness of them to accept.
  67  *              Alan Cox        :       Added TCP options (SOL_TCP)
  68  *              Alan Cox        :       Switched wakeup calls to callbacks,
  69  *                                      so the kernel can layer network
  70  *                                      sockets.
  71  *              Alan Cox        :       Use ip_tos/ip_ttl settings.
  72  *              Alan Cox        :       Handle FIN (more) properly (we hope).
  73  *              Alan Cox        :       RST frames sent on unsynchronised
  74  *                                      state ack error.
  75  *              Alan Cox        :       Put in missing check for SYN bit.
  76  *              Alan Cox        :       Added tcp_select_window() aka NET2E
  77  *                                      window non shrink trick.
  78  *              Alan Cox        :       Added a couple of small NET2E timer
  79  *                                      fixes
  80  *              Charles Hedrick :       TCP fixes
  81  *              Toomas Tamm     :       TCP window fixes
  82  *              Alan Cox        :       Small URG fix to rlogin ^C ack fight
  83  *              Charles Hedrick :       Rewrote most of it to actually work
  84  *              Linus           :       Rewrote tcp_read() and URG handling
  85  *                                      completely
  86  *              Gerhard Koerting:       Fixed some missing timer handling
  87  *              Matthew Dillon  :       Reworked TCP machine states as per RFC
  88  *              Gerhard Koerting:       PC/TCP workarounds
  89  *              Adam Caldwell   :       Assorted timer/timing errors
  90  *              Matthew Dillon  :       Fixed another RST bug
  91  *              Alan Cox        :       Move to kernel side addressing changes.
  92  *              Alan Cox        :       Beginning work on TCP fastpathing
  93  *                                      (not yet usable)
  94  *              Arnt Gulbrandsen:       Turbocharged tcp_check() routine.
  95  *              Alan Cox        :       TCP fast path debugging
  96  *              Alan Cox        :       Window clamping
  97  *              Michael Riepe   :       Bug in tcp_check()
  98  *              Matt Dillon     :       More TCP improvements and RST bug fixes
  99  *              Matt Dillon     :       Yet more small nasties remove from the
 100  *                                      TCP code (Be very nice to this man if
 101  *                                      tcp finally works 100%) 8)
 102  *              Alan Cox        :       BSD accept semantics.
 103  *              Alan Cox        :       Reset on closedown bug.
 104  *      Peter De Schrijver      :       ENOTCONN check missing in tcp_sendto().
 105  *              Michael Pall    :       Handle poll() after URG properly in
 106  *                                      all cases.
 107  *              Michael Pall    :       Undo the last fix in tcp_read_urg()
 108  *                                      (multi URG PUSH broke rlogin).
 109  *              Michael Pall    :       Fix the multi URG PUSH problem in
 110  *                                      tcp_readable(), poll() after URG
 111  *                                      works now.
 112  *              Michael Pall    :       recv(...,MSG_OOB) never blocks in the
 113  *                                      BSD api.
 114  *              Alan Cox        :       Changed the semantics of sk->socket to
 115  *                                      fix a race and a signal problem with
 116  *                                      accept() and async I/O.
 117  *              Alan Cox        :       Relaxed the rules on tcp_sendto().
 118  *              Yury Shevchuk   :       Really fixed accept() blocking problem.
 119  *              Craig I. Hagan  :       Allow for BSD compatible TIME_WAIT for
 120  *                                      clients/servers which listen in on
 121  *                                      fixed ports.
 122  *              Alan Cox        :       Cleaned the above up and shrank it to
 123  *                                      a sensible code size.
 124  *              Alan Cox        :       Self connect lockup fix.
 125  *              Alan Cox        :       No connect to multicast.
 126  *              Ross Biro       :       Close unaccepted children on master
 127  *                                      socket close.
 128  *              Alan Cox        :       Reset tracing code.
 129  *              Alan Cox        :       Spurious resets on shutdown.
 130  *              Alan Cox        :       Giant 15 minute/60 second timer error
 131  *              Alan Cox        :       Small whoops in polling before an
 132  *                                      accept.
 133  *              Alan Cox        :       Kept the state trace facility since
 134  *                                      it's handy for debugging.
 135  *              Alan Cox        :       More reset handler fixes.
 136  *              Alan Cox        :       Started rewriting the code based on
 137  *                                      the RFC's for other useful protocol
 138  *                                      references see: Comer, KA9Q NOS, and
 139  *                                      for a reference on the difference
 140  *                                      between specifications and how BSD
 141  *                                      works see the 4.4lite source.
 142  *              A.N.Kuznetsov   :       Don't time wait on completion of tidy
 143  *                                      close.
 144  *              Linus Torvalds  :       Fin/Shutdown & copied_seq changes.
 145  *              Linus Torvalds  :       Fixed BSD port reuse to work first syn
 146  *              Alan Cox        :       Reimplemented timers as per the RFC
 147  *                                      and using multiple timers for sanity.
 148  *              Alan Cox        :       Small bug fixes, and a lot of new
 149  *                                      comments.
 150  *              Alan Cox        :       Fixed dual reader crash by locking
 151  *                                      the buffers (much like datagram.c)
 152  *              Alan Cox        :       Fixed stuck sockets in probe. A probe
 153  *                                      now gets fed up of retrying without
 154  *                                      (even a no space) answer.
 155  *              Alan Cox        :       Extracted closing code better
 156  *              Alan Cox        :       Fixed the closing state machine to
 157  *                                      resemble the RFC.
 158  *              Alan Cox        :       More 'per spec' fixes.
 159  *              Jorge Cwik      :       Even faster checksumming.
 160  *              Alan Cox        :       tcp_data() doesn't ack illegal PSH
 161  *                                      only frames. At least one pc tcp stack
 162  *                                      generates them.
 163  *              Alan Cox        :       Cache last socket.
 164  *              Alan Cox        :       Per route irtt.
 165  *              Matt Day        :       poll()->select() match BSD precisely on error
 166  *              Alan Cox        :       New buffers
 167  *              Marc Tamsky     :       Various sk->prot->retransmits and
 168  *                                      sk->retransmits misupdating fixed.
 169  *                                      Fixed tcp_write_timeout: stuck close,
 170  *                                      and TCP syn retries gets used now.
 171  *              Mark Yarvis     :       In tcp_read_wakeup(), don't send an
 172  *                                      ack if state is TCP_CLOSED.
 173  *              Alan Cox        :       Look up device on a retransmit - routes may
 174  *                                      change. Doesn't yet cope with MSS shrink right
 175  *                                      but it's a start!
 176  *              Marc Tamsky     :       Closing in closing fixes.
 177  *              Mike Shaver     :       RFC1122 verifications.
 178  *              Alan Cox        :       rcv_saddr errors.
 179  *              Alan Cox        :       Block double connect().
 180  *              Alan Cox        :       Small hooks for enSKIP.
 181  *              Alexey Kuznetsov:       Path MTU discovery.
 182  *              Alan Cox        :       Support soft errors.
 183  *              Alan Cox        :       Fix MTU discovery pathological case
 184  *                                      when the remote claims no mtu!
 185  *              Marc Tamsky     :       TCP_CLOSE fix.
 186  *              Colin (G3TNE)   :       Send a reset on syn ack replies in
 187  *                                      window but wrong (fixes NT lpd problems)
 188  *              Pedro Roque     :       Better TCP window handling, delayed ack.
 189  *              Joerg Reuter    :       No modification of locked buffers in
 190  *                                      tcp_do_retransmit()
 191  *              Eric Schenk     :       Changed receiver side silly window
 192  *                                      avoidance algorithm to BSD style
 193  *                                      algorithm. This doubles throughput
 194  *                                      against machines running Solaris,
 195  *                                      and seems to result in general
 196  *                                      improvement.
 197  *      Stefan Magdalinski      :       adjusted tcp_readable() to fix FIONREAD
 198  *      Willy Konynenberg       :       Transparent proxying support.
 199  *      Mike McLagan            :       Routing by source
 200  *              Keith Owens     :       Do proper merging with partial SKB's in
 201  *                                      tcp_do_sendmsg to avoid burstiness.
 202  *              Eric Schenk     :       Fix fast close down bug with
 203  *                                      shutdown() followed by close().
 204  *              Andi Kleen      :       Make poll agree with SIGIO
 205  *      Salvatore Sanfilippo    :       Support SO_LINGER with linger == 1 and
 206  *                                      lingertime == 0 (RFC 793 ABORT Call)
 207  *      Hirokazu Takahashi      :       Use copy_from_user() instead of
 208  *                                      csum_and_copy_from_user() if possible.
 209  *
 210  *              This program is free software; you can redistribute it and/or
 211  *              modify it under the terms of the GNU General Public License
 212  *              as published by the Free Software Foundation; either version
 213  *              2 of the License, or(at your option) any later version.
 214  *
 215  * Description of States:
 216  *
 217  *      TCP_SYN_SENT            sent a connection request, waiting for ack
 218  *
 219  *      TCP_SYN_RECV            received a connection request, sent ack,
 220  *                              waiting for final ack in three-way handshake.
 221  *
 222  *      TCP_ESTABLISHED         connection established
 223  *
 224  *      TCP_FIN_WAIT1           our side has shutdown, waiting to complete
 225  *                              transmission of remaining buffered data
 226  *
 227  *      TCP_FIN_WAIT2           all buffered data sent, waiting for remote
 228  *                              to shutdown
 229  *
 230  *      TCP_CLOSING             both sides have shutdown but we still have
 231  *                              data we have to finish sending
 232  *
 233  *      TCP_TIME_WAIT           timeout to catch resent junk before entering
 234  *                              closed, can only be entered from FIN_WAIT2
 235  *                              or CLOSING.  Required because the other end
 236  *                              may not have gotten our last ACK causing it
 237  *                              to retransmit the data packet (which we ignore)
 238  *
 239  *      TCP_CLOSE_WAIT          remote side has shutdown and is waiting for
 240  *                              us to finish writing our data and to shutdown
 241  *                              (we have to close() to move on to LAST_ACK)
 242  *
 243  *      TCP_LAST_ACK            out side has shutdown after remote has
 244  *                              shutdown.  There may still be data in our
 245  *                              buffer that we have to finish sending
 246  *
 247  *      TCP_CLOSE               socket is finished
 248  */
 249
 250 #include <linux/config.h>
 251 #include <linux/module.h>
 252 #include <linux/types.h>
 253 #include <linux/fcntl.h>
 254 #include <linux/poll.h>
 255 #include <linux/init.h>
 256 #include <linux/smp_lock.h>
 257 #include <linux/fs.h>
 258 #include <linux/random.h>
 259
 260 #include <net/icmp.h>
 261 #include <net/tcp.h>
 262 #include <net/xfrm.h>
 263 #include <net/ip.h>
 264
 265
 266 #include <asm/uaccess.h>
 267 #include <asm/ioctls.h>
 268
 269 int sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
 270
 271 DEFINE_SNMP_STAT(struct tcp_mib, tcp_statistics);
 272
 273 kmem_cache_t *tcp_openreq_cachep;
 274 kmem_cache_t *tcp_bucket_cachep;
 275 kmem_cache_t *tcp_timewait_cachep;
 276
 277 atomic_t tcp_orphan_count = ATOMIC_INIT(0);
 278
 279 int sysctl_tcp_default_win_scale = 7;
 280
 281 int sysctl_tcp_mem[3];
 282 int sysctl_tcp_wmem[3] = { 4 * 1024, 16 * 1024, 128 * 1024 };
 283 int sysctl_tcp_rmem[3] = { 4 * 1024, 87380, 87380 * 2 };
 284
 285 EXPORT_SYMBOL(sysctl_tcp_mem);
 286 EXPORT_SYMBOL(sysctl_tcp_rmem);
 287 EXPORT_SYMBOL(sysctl_tcp_wmem);
 288
 289 atomic_t tcp_memory_allocated;  /* Current allocated memory. */
 290 atomic_t tcp_sockets_allocated; /* Current number of TCP sockets. */
 291
 292 EXPORT_SYMBOL(tcp_memory_allocated);
 293 EXPORT_SYMBOL(tcp_sockets_allocated);
 294
 295 /*
 296  * Pressure flag: try to collapse.
 297  * Technical note: it is used by multiple contexts non atomically.
 298  * All the sk_stream_mem_schedule() is of this nature: accounting
 299  * is strict, actions are advisory and have some latency.
 300  */
 301 int tcp_memory_pressure;
 302
 303 EXPORT_SYMBOL(tcp_memory_pressure);
 304
 305 void tcp_enter_memory_pressure(void)
 306 {
 307         if (!tcp_memory_pressure) {
 308                 NET_INC_STATS(LINUX_MIB_TCPMEMORYPRESSURES);
 309                 tcp_memory_pressure = 1;
 310         }
 311 }
 312
 313 EXPORT_SYMBOL(tcp_enter_memory_pressure);
 314
 315 /*
 316  * LISTEN is a special case for poll..
 317  */
 318 static __inline__ unsigned int tcp_listen_poll(struct sock *sk,
 319                                                poll_table *wait)
 320 {
 321         return tcp_sk(sk)->accept_queue ? (POLLIN | POLLRDNORM) : 0;
 322 }
 323
 324 /*
 325  *      Wait for a TCP event.
 326  *
 327  *      Note that we don't need to lock the socket, as the upper poll layers
 328  *      take care of normal races (between the test and the event) and we don't
 329  *      go look at any of the socket buffers directly.
 330  */
 331 unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
 332 {
 333         unsigned int mask;
 334         struct sock *sk = sock->sk;
 335         struct tcp_opt *tp = tcp_sk(sk);
 336
 337         poll_wait(file, sk->sk_sleep, wait);
 338         if (sk->sk_state == TCP_LISTEN)
 339                 return tcp_listen_poll(sk, wait);
 340
 341         /* Socket is not locked. We are protected from async events
 342            by poll logic and correct handling of state changes
 343            made by another threads is impossible in any case.
 344          */
 345
 346         mask = 0;
 347         if (sk->sk_err)
 348                 mask = POLLERR;
 349
 350         /*
 351          * POLLHUP is certainly not done right. But poll() doesn't
 352          * have a notion of HUP in just one direction, and for a
 353          * socket the read side is more interesting.
 354          *
 355          * Some poll() documentation says that POLLHUP is incompatible
 356          * with the POLLOUT/POLLWR flags, so somebody should check this
 357          * all. But careful, it tends to be safer to return too many
 358          * bits than too few, and you can easily break real applications
 359          * if you don't tell them that something has hung up!
 360          *
 361          * Check-me.
 362          *
 363          * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and
 364          * our fs/select.c). It means that after we received EOF,
 365          * poll always returns immediately, making impossible poll() on write()
 366          * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP
 367          * if and only if shutdown has been made in both directions.
 368          * Actually, it is interesting to look how Solaris and DUX
 369          * solve this dilemma. I would prefer, if PULLHUP were maskable,
 370          * then we could set it on SND_SHUTDOWN. BTW examples given
 371          * in Stevens' books assume exactly this behaviour, it explains
 372          * why PULLHUP is incompatible with POLLOUT.    --ANK
 373          *
 374          * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
 375          * blocking on fresh not-connected or disconnected socket. --ANK
 376          */
 377         if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE)
 378                 mask |= POLLHUP;
 379         if (sk->sk_shutdown & RCV_SHUTDOWN)
 380                 mask |= POLLIN | POLLRDNORM;
 381
 382         /* Connected? */
 383         if ((1 << sk->sk_state) & ~(TCPF_SYN_SENT | TCPF_SYN_RECV)) {
 384                 /* Potential race condition. If read of tp below will
 385                  * escape above sk->sk_state, we can be illegally awaken
 386                  * in SYN_* states. */
 387                 if ((tp->rcv_nxt != tp->copied_seq) &&
 388                     (tp->urg_seq != tp->copied_seq ||
 389                      tp->rcv_nxt != tp->copied_seq + 1 ||
 390                      sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data))
 391                         mask |= POLLIN | POLLRDNORM;
 392
 393                 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
 394                         if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) {
 395                                 mask |= POLLOUT | POLLWRNORM;
 396                         } else {  /* send SIGIO later */
 397                                 set_bit(SOCK_ASYNC_NOSPACE,
 398                                         &sk->sk_socket->flags);
 399                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 400
 401                                 /* Race breaker. If space is freed after
 402                                  * wspace test but before the flags are set,
 403                                  * IO signal will be lost.
 404                                  */
 405                                 if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk))
 406                                         mask |= POLLOUT | POLLWRNORM;
 407                         }
 408                 }
 409
 410                 if (tp->urg_data & TCP_URG_VALID)
 411                         mask |= POLLPRI;
 412         }
 413         return mask;
 414 }
 415
 416 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
 417 {
 418         struct tcp_opt *tp = tcp_sk(sk);
 419         int answ;
 420
 421         switch (cmd) {
 422         case SIOCINQ:
 423                 if (sk->sk_state == TCP_LISTEN)
 424                         return -EINVAL;
 425
 426                 lock_sock(sk);
 427                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 428                         answ = 0;
 429                 else if (sock_flag(sk, SOCK_URGINLINE) ||
 430                          !tp->urg_data ||
 431                          before(tp->urg_seq, tp->copied_seq) ||
 432                          !before(tp->urg_seq, tp->rcv_nxt)) {
 433                         answ = tp->rcv_nxt - tp->copied_seq;
 434
 435                         /* Subtract 1, if FIN is in queue. */
 436                         if (answ && !skb_queue_empty(&sk->sk_receive_queue))
 437                                 answ -=
 438                        ((struct sk_buff *)sk->sk_receive_queue.prev)->h.th->fin;
 439                 } else
 440                         answ = tp->urg_seq - tp->copied_seq;
 441                 release_sock(sk);
 442                 break;
 443         case SIOCATMARK:
 444                 answ = tp->urg_data && tp->urg_seq == tp->copied_seq;
 445                 break;
 446         case SIOCOUTQ:
 447                 if (sk->sk_state == TCP_LISTEN)
 448                         return -EINVAL;
 449
 450                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 451                         answ = 0;
 452                 else
 453                         answ = tp->write_seq - tp->snd_una;
 454                 break;
 455         default:
 456                 return -ENOIOCTLCMD;
 457         };
 458
 459         return put_user(answ, (int __user *)arg);
 460 }
 461
 462
 463 int tcp_listen_start(struct sock *sk)
 464 {
 465         struct inet_opt *inet = inet_sk(sk);
 466         struct tcp_opt *tp = tcp_sk(sk);
 467         struct tcp_listen_opt *lopt;
 468
 469         sk->sk_max_ack_backlog = 0;
 470         sk->sk_ack_backlog = 0;
 471         tp->accept_queue = tp->accept_queue_tail = NULL;
 472         tp->syn_wait_lock = RW_LOCK_UNLOCKED;
 473         tcp_delack_init(tp);
 474
 475         lopt = kmalloc(sizeof(struct tcp_listen_opt), GFP_KERNEL);
 476         if (!lopt)
 477                 return -ENOMEM;
 478
 479         memset(lopt, 0, sizeof(struct tcp_listen_opt));
 480         for (lopt->max_qlen_log = 6; ; lopt->max_qlen_log++)
 481                 if ((1 << lopt->max_qlen_log) >= sysctl_max_syn_backlog)
 482                         break;
 483         get_random_bytes(&lopt->hash_rnd, 4);
 484
 485         write_lock_bh(&tp->syn_wait_lock);
 486         tp->listen_opt = lopt;
 487         write_unlock_bh(&tp->syn_wait_lock);
 488
 489         /* There is race window here: we announce ourselves listening,
 490          * but this transition is still not validated by get_port().
 491          * It is OK, because this socket enters to hash table only
 492          * after validation is complete.
 493          */
 494         sk->sk_state = TCP_LISTEN;
 495         if (!sk->sk_prot->get_port(sk, inet->num)) {
 496                 inet->sport = htons(inet->num);
 497
 498                 sk_dst_reset(sk);
 499                 sk->sk_prot->hash(sk);
 500
 501                 return 0;
 502         }
 503
 504         sk->sk_state = TCP_CLOSE;
 505         write_lock_bh(&tp->syn_wait_lock);
 506         tp->listen_opt = NULL;
 507         write_unlock_bh(&tp->syn_wait_lock);
 508         kfree(lopt);
 509         return -EADDRINUSE;
 510 }
 511
 512 /*
 513  *      This routine closes sockets which have been at least partially
 514  *      opened, but not yet accepted.
 515  */
 516
 517 static void tcp_listen_stop (struct sock *sk)
 518 {
 519         struct tcp_opt *tp = tcp_sk(sk);
 520         struct tcp_listen_opt *lopt = tp->listen_opt;
 521         struct open_request *acc_req = tp->accept_queue;
 522         struct open_request *req;
 523         int i;
 524
 525         tcp_delete_keepalive_timer(sk);
 526
 527         /* make all the listen_opt local to us */
 528         write_lock_bh(&tp->syn_wait_lock);
 529         tp->listen_opt = NULL;
 530         write_unlock_bh(&tp->syn_wait_lock);
 531         tp->accept_queue = tp->accept_queue_tail = NULL;
 532
 533         if (lopt->qlen) {
 534                 for (i = 0; i < TCP_SYNQ_HSIZE; i++) {
 535                         while ((req = lopt->syn_table[i]) != NULL) {
 536                                 lopt->syn_table[i] = req->dl_next;
 537                                 lopt->qlen--;
 538                                 tcp_openreq_free(req);
 539
 540                 /* Following specs, it would be better either to send FIN
 541                  * (and enter FIN-WAIT-1, it is normal close)
 542                  * or to send active reset (abort).
 543                  * Certainly, it is pretty dangerous while synflood, but it is
 544                  * bad justification for our negligence 8)
 545                  * To be honest, we are not able to make either
 546                  * of the variants now.                 --ANK
 547                  */
 548                         }
 549                 }
 550         }
 551         BUG_TRAP(!lopt->qlen);
 552
 553         kfree(lopt);
 554
 555         while ((req = acc_req) != NULL) {
 556                 struct sock *child = req->sk;
 557
 558                 acc_req = req->dl_next;
 559
 560                 local_bh_disable();
 561                 bh_lock_sock(child);
 562                 BUG_TRAP(!sock_owned_by_user(child));
 563                 sock_hold(child);
 564
 565                 tcp_disconnect(child, O_NONBLOCK);
 566
 567                 sock_orphan(child);
 568
 569                 atomic_inc(&tcp_orphan_count);
 570
 571                 tcp_destroy_sock(child);
 572
 573                 bh_unlock_sock(child);
 574                 local_bh_enable();
 575                 sock_put(child);
 576
 577                 sk_acceptq_removed(sk);
 578                 tcp_openreq_fastfree(req);
 579         }
 580         BUG_TRAP(!sk->sk_ack_backlog);
 581 }
 582
 583 static inline void tcp_mark_push(struct tcp_opt *tp, struct sk_buff *skb)
 584 {
 585         TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH;
 586         tp->pushed_seq = tp->write_seq;
 587 }
 588
 589 static inline int forced_push(struct tcp_opt *tp)
 590 {
 591         return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
 592 }
 593
 594 static inline void skb_entail(struct sock *sk, struct tcp_opt *tp,
 595                               struct sk_buff *skb)
 596 {
 597         skb->csum = 0;
 598         TCP_SKB_CB(skb)->seq = tp->write_seq;
 599         TCP_SKB_CB(skb)->end_seq = tp->write_seq;
 600         TCP_SKB_CB(skb)->flags = TCPCB_FLAG_ACK;
 601         TCP_SKB_CB(skb)->sacked = 0;
 602         __skb_queue_tail(&sk->sk_write_queue, skb);
 603         sk_charge_skb(sk, skb);
 604         if (!sk->sk_send_head)
 605                 sk->sk_send_head = skb;
 606         else if (tp->nonagle&TCP_NAGLE_PUSH)
 607                 tp->nonagle &= ~TCP_NAGLE_PUSH;
 608 }
 609
 610 static inline void tcp_mark_urg(struct tcp_opt *tp, int flags,
 611                                 struct sk_buff *skb)
 612 {
 613         if (flags & MSG_OOB) {
 614                 tp->urg_mode = 1;
 615                 tp->snd_up = tp->write_seq;
 616                 TCP_SKB_CB(skb)->sacked |= TCPCB_URG;
 617         }
 618 }
 619
 620 static inline void tcp_push(struct sock *sk, struct tcp_opt *tp, int flags,
 621                             int mss_now, int nonagle)
 622 {
 623         if (sk->sk_send_head) {
 624                 struct sk_buff *skb = sk->sk_write_queue.prev;
 625                 if (!(flags & MSG_MORE) || forced_push(tp))
 626                         tcp_mark_push(tp, skb);
 627                 tcp_mark_urg(tp, flags, skb);
 628                 __tcp_push_pending_frames(sk, tp, mss_now,
 629                                           (flags & MSG_MORE) ? TCP_NAGLE_CORK : nonagle);
 630         }
 631 }
 632
 633 static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffset,
 634                          size_t psize, int flags)
 635 {
 636         struct tcp_opt *tp = tcp_sk(sk);
 637         int mss_now;
 638         int err;
 639         ssize_t copied;
 640         long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 641
 642         /* Wait for a connection to finish. */
 643         if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
 644                 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
 645                         goto out_err;
 646
 647         clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 648
 649         mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 650         copied = 0;
 651
 652         err = -EPIPE;
 653         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 654                 goto do_error;
 655
 656         while (psize > 0) {
 657                 struct sk_buff *skb = sk->sk_write_queue.prev;
 658                 struct page *page = pages[poffset / PAGE_SIZE];
 659                 int copy, i;
 660                 int offset = poffset % PAGE_SIZE;
 661                 int size = min_t(size_t, psize, PAGE_SIZE - offset);
 662
 663                 if (!sk->sk_send_head || (copy = mss_now - skb->len) <= 0) {
 664 new_segment:
 665                         if (!sk_stream_memory_free(sk))
 666                                 goto wait_for_sndbuf;
 667
 668                         skb = sk_stream_alloc_pskb(sk, 0, tp->mss_cache,
 669                                                    sk->sk_allocation);
 670                         if (!skb)
 671                                 goto wait_for_memory;
 672
 673                         skb_entail(sk, tp, skb);
 674                         copy = mss_now;
 675                 }
 676
 677                 if (copy > size)
 678                         copy = size;
 679
 680                 i = skb_shinfo(skb)->nr_frags;
 681                 if (skb_can_coalesce(skb, i, page, offset)) {
 682                         skb_shinfo(skb)->frags[i - 1].size += copy;
 683                 } else if (i < MAX_SKB_FRAGS) {
 684                         get_page(page);
 685                         skb_fill_page_desc(skb, i, page, offset, copy);
 686                 } else {
 687                         tcp_mark_push(tp, skb);
 688                         goto new_segment;
 689                 }
 690
 691                 skb->len += copy;
 692                 skb->data_len += copy;
 693                 skb->ip_summed = CHECKSUM_HW;
 694                 tp->write_seq += copy;
 695                 TCP_SKB_CB(skb)->end_seq += copy;
 696
 697                 if (!copied)
 698                         TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH;
 699
 700                 copied += copy;
 701                 poffset += copy;
 702                 if (!(psize -= copy))
 703                         goto out;
 704
 705                 if (skb->len != mss_now || (flags & MSG_OOB))
 706                         continue;
 707
 708                 if (forced_push(tp)) {
 709                         tcp_mark_push(tp, skb);
 710                         __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH);
 711                 } else if (skb == sk->sk_send_head)
 712                         tcp_push_one(sk, mss_now);
 713                 continue;
 714
 715 wait_for_sndbuf:
 716                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 717 wait_for_memory:
 718                 if (copied)
 719                         tcp_push(sk, tp, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
 720
 721                 if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
 722                         goto do_error;
 723
 724                 mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 725         }
 726
 727 out:
 728         if (copied)
 729                 tcp_push(sk, tp, flags, mss_now, tp->nonagle);
 730         return copied;
 731
 732 do_error:
 733         if (copied)
 734                 goto out;
 735 out_err:
 736         return sk_stream_error(sk, flags, err);
 737 }
 738
 739 ssize_t tcp_sendpage(struct socket *sock, struct page *page, int offset,
 740                      size_t size, int flags)
 741 {
 742         ssize_t res;
 743         struct sock *sk = sock->sk;
 744
 745 #define TCP_ZC_CSUM_FLAGS (NETIF_F_IP_CSUM | NETIF_F_NO_CSUM | NETIF_F_HW_CSUM)
 746
 747         if (!(sk->sk_route_caps & NETIF_F_SG) ||
 748             !(sk->sk_route_caps & TCP_ZC_CSUM_FLAGS))
 749                 return sock_no_sendpage(sock, page, offset, size, flags);
 750
 751 #undef TCP_ZC_CSUM_FLAGS
 752
 753         lock_sock(sk);
 754         TCP_CHECK_TIMER(sk);
 755         res = do_tcp_sendpages(sk, &page, offset, size, flags);
 756         TCP_CHECK_TIMER(sk);
 757         release_sock(sk);
 758         return res;
 759 }
 760
 761 #define TCP_PAGE(sk)    (sk->sk_sndmsg_page)
 762 #define TCP_OFF(sk)     (sk->sk_sndmsg_off)
 763
 764 static inline int select_size(struct sock *sk, struct tcp_opt *tp)
 765 {
 766         int tmp = tp->mss_cache_std;
 767
 768         if (sk->sk_route_caps & NETIF_F_SG) {
 769                 int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER);
 770
 771                 if (tmp >= pgbreak &&
 772                     tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE)
 773                         tmp = pgbreak;
 774         }
 775         return tmp;
 776 }
 777
 778 int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
 779                 size_t size)
 780 {
 781         struct iovec *iov;
 782         struct tcp_opt *tp = tcp_sk(sk);
 783         struct sk_buff *skb;
 784         int iovlen, flags;
 785         int mss_now;
 786         int err, copied;
 787         long timeo;
 788
 789         lock_sock(sk);
 790         TCP_CHECK_TIMER(sk);
 791
 792         flags = msg->msg_flags;
 793         timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 794
 795         /* Wait for a connection to finish. */
 796         if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
 797                 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
 798                         goto out_err;
 799
 800         /* This should be in poll */
 801         clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 802
 803         mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 804
 805         /* Ok commence sending. */
 806         iovlen = msg->msg_iovlen;
 807         iov = msg->msg_iov;
 808         copied = 0;
 809
 810         err = -EPIPE;
 811         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 812                 goto do_error;
 813
 814         while (--iovlen >= 0) {
 815                 int seglen = iov->iov_len;
 816                 unsigned char __user *from = iov->iov_base;
 817
 818                 iov++;
 819
 820                 while (seglen > 0) {
 821                         int copy;
 822
 823                         skb = sk->sk_write_queue.prev;
 824
 825                         if (!sk->sk_send_head ||
 826                             (copy = mss_now - skb->len) <= 0) {
 827
 828 new_segment:
 829                                 /* Allocate new segment. If the interface is SG,
 830                                  * allocate skb fitting to single page.
 831                                  */
 832                                 if (!sk_stream_memory_free(sk))
 833                                         goto wait_for_sndbuf;
 834
 835                                 skb = sk_stream_alloc_pskb(sk, select_size(sk, tp),
 836                                                            0, sk->sk_allocation);
 837                                 if (!skb)
 838                                         goto wait_for_memory;
 839
 840                                 /*
 841                                  * Check whether we can use HW checksum.
 842                                  */
 843                                 if (sk->sk_route_caps &
 844                                     (NETIF_F_IP_CSUM | NETIF_F_NO_CSUM |
 845                                      NETIF_F_HW_CSUM))
 846                                         skb->ip_summed = CHECKSUM_HW;
 847
 848                                 skb_entail(sk, tp, skb);
 849                                 copy = mss_now;
 850                         }
 851
 852                         /* Try to append data to the end of skb. */
 853                         if (copy > seglen)
 854                                 copy = seglen;
 855
 856                         /* Where to copy to? */
 857                         if (skb_tailroom(skb) > 0) {
 858                                 /* We have some space in skb head. Superb! */
 859                                 if (copy > skb_tailroom(skb))
 860                                         copy = skb_tailroom(skb);
 861                                 if ((err = skb_add_data(skb, from, copy)) != 0)
 862                                         goto do_fault;
 863                         } else {
 864                                 int merge = 0;
 865                                 int i = skb_shinfo(skb)->nr_frags;
 866                                 struct page *page = TCP_PAGE(sk);
 867                                 int off = TCP_OFF(sk);
 868
 869                                 if (skb_can_coalesce(skb, i, page, off) &&
 870                                     off != PAGE_SIZE) {
 871                                         /* We can extend the last page
 872                                          * fragment. */
 873                                         merge = 1;
 874                                 } else if (i == MAX_SKB_FRAGS ||
 875                                            (!i &&
 876                                            !(sk->sk_route_caps & NETIF_F_SG))) {
 877                                         /* Need to add new fragment and cannot
 878                                          * do this because interface is non-SG,
 879                                          * or because all the page slots are
 880                                          * busy. */
 881                                         tcp_mark_push(tp, skb);
 882                                         goto new_segment;
 883                                 } else if (page) {
 884                                         /* If page is cached, align
 885                                          * offset to L1 cache boundary
 886                                          */
 887                                         off = (off + L1_CACHE_BYTES - 1) &
 888                                               ~(L1_CACHE_BYTES - 1);
 889                                         if (off == PAGE_SIZE) {
 890                                                 put_page(page);
 891                                                 TCP_PAGE(sk) = page = NULL;
 892                                         }
 893                                 }
 894
 895                                 if (!page) {
 896                                         /* Allocate new cache page. */
 897                                         if (!(page = sk_stream_alloc_page(sk)))
 898                                                 goto wait_for_memory;
 899                                         off = 0;
 900                                 }
 901
 902                                 if (copy > PAGE_SIZE - off)
 903                                         copy = PAGE_SIZE - off;
 904
 905                                 /* Time to copy data. We are close to
 906                                  * the end! */
 907                                 err = skb_copy_to_page(sk, from, skb, page,
 908                                                        off, copy);
 909                                 if (err) {
 910                                         /* If this page was new, give it to the
 911                                          * socket so it does not get leaked.
 912                                          */
 913                                         if (!TCP_PAGE(sk)) {
 914                                                 TCP_PAGE(sk) = page;
 915                                                 TCP_OFF(sk) = 0;
 916                                         }
 917                                         goto do_error;
 918                                 }
 919
 920                                 /* Update the skb. */
 921                                 if (merge) {
 922                                         skb_shinfo(skb)->frags[i - 1].size +=
 923                                                                         copy;
 924                                 } else {
 925                                         skb_fill_page_desc(skb, i, page, off, copy);
 926                                         if (TCP_PAGE(sk)) {
 927                                                 get_page(page);
 928                                         } else if (off + copy < PAGE_SIZE) {
 929                                                 get_page(page);
 930                                                 TCP_PAGE(sk) = page;
 931                                         }
 932                                 }
 933
 934                                 TCP_OFF(sk) = off + copy;
 935                         }
 936
 937                         if (!copied)
 938                                 TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH;
 939
 940                         tp->write_seq += copy;
 941                         TCP_SKB_CB(skb)->end_seq += copy;
 942
 943                         from += copy;
 944                         copied += copy;
 945                         if ((seglen -= copy) == 0 && iovlen == 0)
 946                                 goto out;
 947
 948                         if (skb->len != mss_now || (flags & MSG_OOB))
 949                                 continue;
 950
 951                         if (forced_push(tp)) {
 952                                 tcp_mark_push(tp, skb);
 953                                 __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH);
 954                         } else if (skb == sk->sk_send_head)
 955                                 tcp_push_one(sk, mss_now);
 956                         continue;
 957
 958 wait_for_sndbuf:
 959                         set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 960 wait_for_memory:
 961                         if (copied)
 962                                 tcp_push(sk, tp, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
 963
 964                         if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
 965                                 goto do_error;
 966
 967                         mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 968                 }
 969         }
 970
 971 out:
 972         if (copied)
 973                 tcp_push(sk, tp, flags, mss_now, tp->nonagle);
 974         TCP_CHECK_TIMER(sk);
 975         release_sock(sk);
 976         return copied;
 977
 978 do_fault:
 979         if (!skb->len) {
 980                 if (sk->sk_send_head == skb)
 981                         sk->sk_send_head = NULL;
 982                 __skb_unlink(skb, skb->list);
 983                 sk_stream_free_skb(sk, skb);
 984         }
 985
 986 do_error:
 987         if (copied)
 988                 goto out;
 989 out_err:
 990         err = sk_stream_error(sk, flags, err);
 991         TCP_CHECK_TIMER(sk);
 992         release_sock(sk);
 993         return err;
 994 }
 995
 996 /*
 997  *      Handle reading urgent data. BSD has very simple semantics for
 998  *      this, no blocking and very strange errors 8)
 999  */
1000
1001 static int tcp_recv_urg(struct sock *sk, long timeo,
1002                         struct msghdr *msg, int len, int flags,
1003                         int *addr_len)
1004 {
1005         struct tcp_opt *tp = tcp_sk(sk);
1006
1007         /* No URG data to read. */
1008         if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1009             tp->urg_data == TCP_URG_READ)
1010                 return -EINVAL; /* Yes this is right ! */
1011
1012         if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1013                 return -ENOTCONN;
1014
1015         if (tp->urg_data & TCP_URG_VALID) {
1016                 int err = 0;
1017                 char c = tp->urg_data;
1018
1019                 if (!(flags & MSG_PEEK))
1020                         tp->urg_data = TCP_URG_READ;
1021
1022                 /* Read urgent data. */
1023                 msg->msg_flags |= MSG_OOB;
1024
1025                 if (len > 0) {
1026                         if (!(flags & MSG_TRUNC))
1027                                 err = memcpy_toiovec(msg->msg_iov, &c, 1);
1028                         len = 1;
1029                 } else
1030                         msg->msg_flags |= MSG_TRUNC;
1031
1032                 return err ? -EFAULT : len;
1033         }
1034
1035         if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1036                 return 0;
1037
1038         /* Fixed the recv(..., MSG_OOB) behaviour.  BSD docs and
1039          * the available implementations agree in this case:
1040          * this call should never block, independent of the
1041          * blocking state of the socket.
1042          * Mike <pall@rz.uni-karlsruhe.de>
1043          */
1044         return -EAGAIN;
1045 }
1046
1047 /* Clean up the receive buffer for full frames taken by the user,
1048  * then send an ACK if necessary.  COPIED is the number of bytes
1049  * tcp_recvmsg has given to the user so far, it speeds up the
1050  * calculation of whether or not we must ACK for the sake of
1051  * a window update.
1052  */
1053 static void cleanup_rbuf(struct sock *sk, int copied)
1054 {
1055         struct tcp_opt *tp = tcp_sk(sk);
1056         int time_to_ack = 0;
1057
1058 #if TCP_DEBUG
1059         struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1060
1061         BUG_TRAP(!skb || before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq));
1062 #endif
1063
1064         if (tcp_ack_scheduled(tp)) {
1065                    /* Delayed ACKs frequently hit locked sockets during bulk
1066                     * receive. */
1067                 if (tp->ack.blocked ||
1068                     /* Once-per-two-segments ACK was not sent by tcp_input.c */
1069                     tp->rcv_nxt - tp->rcv_wup > tp->ack.rcv_mss ||
1070                     /*
1071                      * If this read emptied read buffer, we send ACK, if
1072                      * connection is not bidirectional, user drained
1073                      * receive buffer and there was a small segment
1074                      * in queue.
1075                      */
1076                     (copied > 0 && (tp->ack.pending & TCP_ACK_PUSHED) &&
1077                      !tp->ack.pingpong && !atomic_read(&sk->sk_rmem_alloc)))
1078                         time_to_ack = 1;
1079         }
1080
1081         /* We send an ACK if we can now advertise a non-zero window
1082          * which has been raised "significantly".
1083          *
1084          * Even if window raised up to infinity, do not send window open ACK
1085          * in states, where we will not receive more. It is useless.
1086          */
1087         if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1088                 __u32 rcv_window_now = tcp_receive_window(tp);
1089
1090                 /* Optimize, __tcp_select_window() is not cheap. */
1091                 if (2*rcv_window_now <= tp->window_clamp) {
1092                         __u32 new_window = __tcp_select_window(sk);
1093
1094                         /* Send ACK now, if this read freed lots of space
1095                          * in our buffer. Certainly, new_window is new window.
1096                          * We can advertise it now, if it is not less than current one.
1097                          * "Lots" means "at least twice" here.
1098                          */
1099                         if (new_window && new_window >= 2 * rcv_window_now)
1100                                 time_to_ack = 1;
1101                 }
1102         }
1103         if (time_to_ack)
1104                 tcp_send_ack(sk);
1105 }
1106
1107 static void tcp_prequeue_process(struct sock *sk)
1108 {
1109         struct sk_buff *skb;
1110         struct tcp_opt *tp = tcp_sk(sk);
1111
1112         NET_ADD_STATS_USER(LINUX_MIB_TCPPREQUEUED, skb_queue_len(&tp->ucopy.prequeue));
1113
1114         /* RX process wants to run with disabled BHs, though it is not
1115          * necessary */
1116         local_bh_disable();
1117         while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
1118                 sk->sk_backlog_rcv(sk, skb);
1119         local_bh_enable();
1120
1121         /* Clear memory counter. */
1122         tp->ucopy.memory = 0;
1123 }
1124
1125 static inline struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1126 {
1127         struct sk_buff *skb;
1128         u32 offset;
1129
1130         skb_queue_walk(&sk->sk_receive_queue, skb) {
1131                 offset = seq - TCP_SKB_CB(skb)->seq;
1132                 if (skb->h.th->syn)
1133                         offset--;
1134                 if (offset < skb->len || skb->h.th->fin) {
1135                         *off = offset;
1136                         return skb;
1137                 }
1138         }
1139         return NULL;
1140 }
1141
1142 /*
1143  * This routine provides an alternative to tcp_recvmsg() for routines
1144  * that would like to handle copying from skbuffs directly in 'sendfile'
1145  * fashion.
1146  * Note:
1147  *      - It is assumed that the socket was locked by the caller.
1148  *      - The routine does not block.
1149  *      - At present, there is no support for reading OOB data
1150  *        or for 'peeking' the socket using this routine
1151  *        (although both would be easy to implement).
1152  */
1153 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1154                   sk_read_actor_t recv_actor)
1155 {
1156         struct sk_buff *skb;
1157         struct tcp_opt *tp = tcp_sk(sk);
1158         u32 seq = tp->copied_seq;
1159         u32 offset;
1160         int copied = 0;
1161
1162         if (sk->sk_state == TCP_LISTEN)
1163                 return -ENOTCONN;
1164         while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1165                 if (offset < skb->len) {
1166                         size_t used, len;
1167
1168                         len = skb->len - offset;
1169                         /* Stop reading if we hit a patch of urgent data */
1170                         if (tp->urg_data) {
1171                                 u32 urg_offset = tp->urg_seq - seq;
1172                                 if (urg_offset < len)
1173                                         len = urg_offset;
1174                                 if (!len)
1175                                         break;
1176                         }
1177                         used = recv_actor(desc, skb, offset, len);
1178                         if (used <= len) {
1179                                 seq += used;
1180                                 copied += used;
1181                                 offset += used;
1182                         }
1183                         if (offset != skb->len)
1184                                 break;
1185                 }
1186                 if (skb->h.th->fin) {
1187                         sk_eat_skb(sk, skb);
1188                         ++seq;
1189                         break;
1190                 }
1191                 sk_eat_skb(sk, skb);
1192                 if (!desc->count)
1193                         break;
1194         }
1195         tp->copied_seq = seq;
1196
1197         tcp_rcv_space_adjust(sk);
1198
1199         /* Clean up data we have read: This will do ACK frames. */
1200         if (copied)
1201                 cleanup_rbuf(sk, copied);
1202         return copied;
1203 }
1204
1205 /*
1206  *      This routine copies from a sock struct into the user buffer.
1207  *
1208  *      Technical note: in 2.3 we work on _locked_ socket, so that
1209  *      tricks with *seq access order and skb->users are not required.
1210  *      Probably, code can be easily improved even more.
1211  */
1212
1213 int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
1214                 size_t len, int nonblock, int flags, int *addr_len)
1215 {
1216         struct tcp_opt *tp = tcp_sk(sk);
1217         int copied = 0;
1218         u32 peek_seq;
1219         u32 *seq;
1220         unsigned long used;
1221         int err;
1222         int target;             /* Read at least this many bytes */
1223         long timeo;
1224         struct task_struct *user_recv = NULL;
1225
1226         lock_sock(sk);
1227
1228         TCP_CHECK_TIMER(sk);
1229
1230         err = -ENOTCONN;
1231         if (sk->sk_state == TCP_LISTEN)
1232                 goto out;
1233
1234         timeo = sock_rcvtimeo(sk, nonblock);
1235
1236         /* Urgent data needs to be handled specially. */
1237         if (flags & MSG_OOB)
1238                 goto recv_urg;
1239
1240         seq = &tp->copied_seq;
1241         if (flags & MSG_PEEK) {
1242                 peek_seq = tp->copied_seq;
1243                 seq = &peek_seq;
1244         }
1245
1246         target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1247
1248         do {
1249                 struct sk_buff *skb;
1250                 u32 offset;
1251
1252                 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
1253                 if (tp->urg_data && tp->urg_seq == *seq) {
1254                         if (copied)
1255                                 break;
1256                         if (signal_pending(current)) {
1257                                 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
1258                                 break;
1259                         }
1260                 }
1261
1262                 /* Next get a buffer. */
1263
1264                 skb = skb_peek(&sk->sk_receive_queue);
1265                 do {
1266                         if (!skb)
1267                                 break;
1268
1269                         /* Now that we have two receive queues this
1270                          * shouldn't happen.
1271                          */
1272                         if (before(*seq, TCP_SKB_CB(skb)->seq)) {
1273                                 printk(KERN_INFO "recvmsg bug: copied %X "
1274                                        "seq %X\n", *seq, TCP_SKB_CB(skb)->seq);
1275                                 break;
1276                         }
1277                         offset = *seq - TCP_SKB_CB(skb)->seq;
1278                         if (skb->h.th->syn)
1279                                 offset--;
1280                         if (offset < skb->len)
1281                                 goto found_ok_skb;
1282                         if (skb->h.th->fin)
1283                                 goto found_fin_ok;
1284                         BUG_TRAP(flags & MSG_PEEK);
1285                         skb = skb->next;
1286                 } while (skb != (struct sk_buff *)&sk->sk_receive_queue);
1287
1288                 /* Well, if we have backlog, try to process it now yet. */
1289
1290                 if (copied >= target && !sk->sk_backlog.tail)
1291                         break;
1292
1293                 if (copied) {
1294                         if (sk->sk_err ||
1295                             sk->sk_state == TCP_CLOSE ||
1296                             (sk->sk_shutdown & RCV_SHUTDOWN) ||
1297                             !timeo ||
1298                             signal_pending(current) ||
1299                             (flags & MSG_PEEK))
1300                                 break;
1301                 } else {
1302                         if (sock_flag(sk, SOCK_DONE))
1303                                 break;
1304
1305                         if (sk->sk_err) {
1306                                 copied = sock_error(sk);
1307                                 break;
1308                         }
1309
1310                         if (sk->sk_shutdown & RCV_SHUTDOWN)
1311                                 break;
1312
1313                         if (sk->sk_state == TCP_CLOSE) {
1314                                 if (!sock_flag(sk, SOCK_DONE)) {
1315                                         /* This occurs when user tries to read
1316                                          * from never connected socket.
1317                                          */
1318                                         copied = -ENOTCONN;
1319                                         break;
1320                                 }
1321                                 break;
1322                         }
1323
1324                         if (!timeo) {
1325                                 copied = -EAGAIN;
1326                                 break;
1327                         }
1328
1329                         if (signal_pending(current)) {
1330                                 copied = sock_intr_errno(timeo);
1331                                 break;
1332                         }
1333                 }
1334
1335                 cleanup_rbuf(sk, copied);
1336
1337                 if (tp->ucopy.task == user_recv) {
1338                         /* Install new reader */
1339                         if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) {
1340                                 user_recv = current;
1341                                 tp->ucopy.task = user_recv;
1342                                 tp->ucopy.iov = msg->msg_iov;
1343                         }
1344
1345                         tp->ucopy.len = len;
1346
1347                         BUG_TRAP(tp->copied_seq == tp->rcv_nxt ||
1348                                  (flags & (MSG_PEEK | MSG_TRUNC)));
1349
1350                         /* Ugly... If prequeue is not empty, we have to
1351                          * process it before releasing socket, otherwise
1352                          * order will be broken at second iteration.
1353                          * More elegant solution is required!!!
1354                          *
1355                          * Look: we have the following (pseudo)queues:
1356                          *
1357                          * 1. packets in flight
1358                          * 2. backlog
1359                          * 3. prequeue
1360                          * 4. receive_queue
1361                          *
1362                          * Each queue can be processed only if the next ones
1363                          * are empty. At this point we have empty receive_queue.
1364                          * But prequeue _can_ be not empty after 2nd iteration,
1365                          * when we jumped to start of loop because backlog
1366                          * processing added something to receive_queue.
1367                          * We cannot release_sock(), because backlog contains
1368                          * packets arrived _after_ prequeued ones.
1369                          *
1370                          * Shortly, algorithm is clear --- to process all
1371                          * the queues in order. We could make it more directly,
1372                          * requeueing packets from backlog to prequeue, if
1373                          * is not empty. It is more elegant, but eats cycles,
1374                          * unfortunately.
1375                          */
1376                         if (skb_queue_len(&tp->ucopy.prequeue))
1377                                 goto do_prequeue;
1378
1379                         /* __ Set realtime policy in scheduler __ */
1380                 }
1381
1382                 if (copied >= target) {
1383                         /* Do not sleep, just process backlog. */
1384                         release_sock(sk);
1385                         lock_sock(sk);
1386                 } else
1387                         sk_wait_data(sk, &timeo);
1388
1389                 if (user_recv) {
1390                         int chunk;
1391
1392                         /* __ Restore normal policy in scheduler __ */
1393
1394                         if ((chunk = len - tp->ucopy.len) != 0) {
1395                                 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk);
1396                                 len -= chunk;
1397                                 copied += chunk;
1398                         }
1399
1400                         if (tp->rcv_nxt == tp->copied_seq &&
1401                             skb_queue_len(&tp->ucopy.prequeue)) {
1402 do_prequeue:
1403                                 tcp_prequeue_process(sk);
1404
1405                                 if ((chunk = len - tp->ucopy.len) != 0) {
1406                                         NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1407                                         len -= chunk;
1408                                         copied += chunk;
1409                                 }
1410                         }
1411                 }
1412                 if ((flags & MSG_PEEK) && peek_seq != tp->copied_seq) {
1413                         if (net_ratelimit())
1414                                 printk(KERN_DEBUG "TCP(%s:%d): Application bug, race in MSG_PEEK.\n",
1415                                        current->comm, current->pid);
1416                         peek_seq = tp->copied_seq;
1417                 }
1418                 continue;
1419
1420         found_ok_skb:
1421                 /* Ok so how much can we use? */
1422                 used = skb->len - offset;
1423                 if (len < used)
1424                         used = len;
1425
1426                 /* Do we have urgent data here? */
1427                 if (tp->urg_data) {
1428                         u32 urg_offset = tp->urg_seq - *seq;
1429                         if (urg_offset < used) {
1430                                 if (!urg_offset) {
1431                                         if (!sock_flag(sk, SOCK_URGINLINE)) {
1432                                                 ++*seq;
1433                                                 offset++;
1434                                                 used--;
1435                                                 if (!used)
1436                                                         goto skip_copy;
1437                                         }
1438                                 } else
1439                                         used = urg_offset;
1440                         }
1441                 }
1442
1443                 if (!(flags & MSG_TRUNC)) {
1444                         err = skb_copy_datagram_iovec(skb, offset,
1445                                                       msg->msg_iov, used);
1446                         if (err) {
1447                                 /* Exception. Bailout! */
1448                                 if (!copied)
1449                                         copied = -EFAULT;
1450                                 break;
1451                         }
1452                 }
1453
1454                 *seq += used;
1455                 copied += used;
1456                 len -= used;
1457
1458                 tcp_rcv_space_adjust(sk);
1459
1460 skip_copy:
1461                 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
1462                         tp->urg_data = 0;
1463                         tcp_fast_path_check(sk, tp);
1464                 }
1465                 if (used + offset < skb->len)
1466                         continue;
1467
1468                 if (skb->h.th->fin)
1469                         goto found_fin_ok;
1470                 if (!(flags & MSG_PEEK))
1471                         sk_eat_skb(sk, skb);
1472                 continue;
1473
1474         found_fin_ok:
1475                 /* Process the FIN. */
1476                 ++*seq;
1477                 if (!(flags & MSG_PEEK))
1478                         sk_eat_skb(sk, skb);
1479                 break;
1480         } while (len > 0);
1481
1482         if (user_recv) {
1483                 if (skb_queue_len(&tp->ucopy.prequeue)) {
1484                         int chunk;
1485
1486                         tp->ucopy.len = copied > 0 ? len : 0;
1487
1488                         tcp_prequeue_process(sk);
1489
1490                         if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) {
1491                                 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1492                                 len -= chunk;
1493                                 copied += chunk;
1494                         }
1495                 }
1496
1497                 tp->ucopy.task = NULL;
1498                 tp->ucopy.len = 0;
1499         }
1500
1501         /* According to UNIX98, msg_name/msg_namelen are ignored
1502          * on connected socket. I was just happy when found this 8) --ANK
1503          */
1504
1505         /* Clean up data we have read: This will do ACK frames. */
1506         cleanup_rbuf(sk, copied);
1507
1508         TCP_CHECK_TIMER(sk);
1509         release_sock(sk);
1510         return copied;
1511
1512 out:
1513         TCP_CHECK_TIMER(sk);
1514         release_sock(sk);
1515         return err;
1516
1517 recv_urg:
1518         err = tcp_recv_urg(sk, timeo, msg, len, flags, addr_len);
1519         goto out;
1520 }
1521
1522 /*
1523  *      State processing on a close. This implements the state shift for
1524  *      sending our FIN frame. Note that we only send a FIN for some
1525  *      states. A shutdown() may have already sent the FIN, or we may be
1526  *      closed.
1527  */
1528
1529 static unsigned char new_state[16] = {
1530   /* current state:        new state:      action:      */
1531   /* (Invalid)          */ TCP_CLOSE,
1532   /* TCP_ESTABLISHED    */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1533   /* TCP_SYN_SENT       */ TCP_CLOSE,
1534   /* TCP_SYN_RECV       */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1535   /* TCP_FIN_WAIT1      */ TCP_FIN_WAIT1,
1536   /* TCP_FIN_WAIT2      */ TCP_FIN_WAIT2,
1537   /* TCP_TIME_WAIT      */ TCP_CLOSE,
1538   /* TCP_CLOSE          */ TCP_CLOSE,
1539   /* TCP_CLOSE_WAIT     */ TCP_LAST_ACK  | TCP_ACTION_FIN,
1540   /* TCP_LAST_ACK       */ TCP_LAST_ACK,
1541   /* TCP_LISTEN         */ TCP_CLOSE,
1542   /* TCP_CLOSING        */ TCP_CLOSING,
1543 };
1544
1545 static int tcp_close_state(struct sock *sk)
1546 {
1547         int next = (int)new_state[sk->sk_state];
1548         int ns = next & TCP_STATE_MASK;
1549
1550         tcp_set_state(sk, ns);
1551
1552         return next & TCP_ACTION_FIN;
1553 }
1554
1555 /*
1556  *      Shutdown the sending side of a connection. Much like close except
1557  *      that we don't receive shut down or set_sock_flag(sk, SOCK_DEAD).
1558  */
1559
1560 void tcp_shutdown(struct sock *sk, int how)
1561 {
1562         /*      We need to grab some memory, and put together a FIN,
1563          *      and then put it into the queue to be sent.
1564          *              Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
1565          */
1566         if (!(how & SEND_SHUTDOWN))
1567                 return;
1568
1569         /* If we've already sent a FIN, or it's a closed state, skip this. */
1570         if ((1 << sk->sk_state) &
1571             (TCPF_ESTABLISHED | TCPF_SYN_SENT |
1572              TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
1573                 /* Clear out any half completed packets.  FIN if needed. */
1574                 if (tcp_close_state(sk))
1575                         tcp_send_fin(sk);
1576         }
1577 }
1578
1579 /*
1580  * At this point, there should be no process reference to this
1581  * socket, and thus no user references at all.  Therefore we
1582  * can assume the socket waitqueue is inactive and nobody will
1583  * try to jump onto it.
1584  */
1585 void tcp_destroy_sock(struct sock *sk)
1586 {
1587         BUG_TRAP(sk->sk_state == TCP_CLOSE);
1588         BUG_TRAP(sock_flag(sk, SOCK_DEAD));
1589
1590         /* It cannot be in hash table! */
1591         BUG_TRAP(sk_unhashed(sk));
1592
1593         /* If it has not 0 inet_sk(sk)->num, it must be bound */
1594         BUG_TRAP(!inet_sk(sk)->num || tcp_sk(sk)->bind_hash);
1595
1596 #ifdef TCP_DEBUG
1597         if (sk->sk_zapped) {
1598                 printk(KERN_DEBUG "TCP: double destroy sk=%p\n", sk);
1599                 sock_hold(sk);
1600         }
1601         sk->sk_zapped = 1;
1602 #endif
1603
1604         sk->sk_prot->destroy(sk);
1605
1606         sk_stream_kill_queues(sk);
1607
1608         xfrm_sk_free_policy(sk);
1609
1610 #ifdef INET_REFCNT_DEBUG
1611         if (atomic_read(&sk->sk_refcnt) != 1) {
1612                 printk(KERN_DEBUG "Destruction TCP %p delayed, c=%d\n",
1613                        sk, atomic_read(&sk->sk_refcnt));
1614         }
1615 #endif
1616
1617         atomic_dec(&tcp_orphan_count);
1618         sock_put(sk);
1619 }
1620
1621 void tcp_close(struct sock *sk, long timeout)
1622 {
1623         struct sk_buff *skb;
1624         int data_was_unread = 0;
1625
1626         lock_sock(sk);
1627         sk->sk_shutdown = SHUTDOWN_MASK;
1628
1629         if (sk->sk_state == TCP_LISTEN) {
1630                 tcp_set_state(sk, TCP_CLOSE);
1631
1632                 /* Special case. */
1633                 tcp_listen_stop(sk);
1634
1635                 goto adjudge_to_death;
1636         }
1637
1638         /*  We need to flush the recv. buffs.  We do this only on the
1639          *  descriptor close, not protocol-sourced closes, because the
1640          *  reader process may not have drained the data yet!
1641          */
1642         while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
1643                 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq -
1644                           skb->h.th->fin;
1645                 data_was_unread += len;
1646                 __kfree_skb(skb);
1647         }
1648
1649         sk_stream_mem_reclaim(sk);
1650
1651         /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section
1652          * 3.10, we send a RST here because data was lost.  To
1653          * witness the awful effects of the old behavior of always
1654          * doing a FIN, run an older 2.1.x kernel or 2.0.x, start
1655          * a bulk GET in an FTP client, suspend the process, wait
1656          * for the client to advertise a zero window, then kill -9
1657          * the FTP client, wheee...  Note: timeout is always zero
1658          * in such a case.
1659          */
1660         if (data_was_unread) {
1661                 /* Unread data was tossed, zap the connection. */
1662                 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONCLOSE);
1663                 tcp_set_state(sk, TCP_CLOSE);
1664                 tcp_send_active_reset(sk, GFP_KERNEL);
1665         } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
1666                 /* Check zero linger _after_ checking for unread data. */
1667                 sk->sk_prot->disconnect(sk, 0);
1668                 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONDATA);
1669         } else if (tcp_close_state(sk)) {
1670                 /* We FIN if the application ate all the data before
1671                  * zapping the connection.
1672                  */
1673
1674                 /* RED-PEN. Formally speaking, we have broken TCP state
1675                  * machine. State transitions:
1676                  *
1677                  * TCP_ESTABLISHED -> TCP_FIN_WAIT1
1678                  * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible)
1679                  * TCP_CLOSE_WAIT -> TCP_LAST_ACK
1680                  *
1681                  * are legal only when FIN has been sent (i.e. in window),
1682                  * rather than queued out of window. Purists blame.
1683                  *
1684                  * F.e. "RFC state" is ESTABLISHED,
1685                  * if Linux state is FIN-WAIT-1, but FIN is still not sent.
1686                  *
1687                  * The visible declinations are that sometimes
1688                  * we enter time-wait state, when it is not required really
1689                  * (harmless), do not send active resets, when they are
1690                  * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
1691                  * they look as CLOSING or LAST_ACK for Linux)
1692                  * Probably, I missed some more holelets.
1693                  *                                              --ANK
1694                  */
1695                 tcp_send_fin(sk);
1696         }
1697
1698         sk_stream_wait_close(sk, timeout);
1699
1700 adjudge_to_death:
1701         /* It is the last release_sock in its life. It will remove backlog. */
1702         release_sock(sk);
1703
1704
1705         /* Now socket is owned by kernel and we acquire BH lock
1706            to finish close. No need to check for user refs.
1707          */
1708         local_bh_disable();
1709         bh_lock_sock(sk);
1710         BUG_TRAP(!sock_owned_by_user(sk));
1711
1712         sock_hold(sk);
1713         sock_orphan(sk);
1714
1715         /*      This is a (useful) BSD violating of the RFC. There is a
1716          *      problem with TCP as specified in that the other end could
1717          *      keep a socket open forever with no application left this end.
1718          *      We use a 3 minute timeout (about the same as BSD) then kill
1719          *      our end. If they send after that then tough - BUT: long enough
1720          *      that we won't make the old 4*rto = almost no time - whoops
1721          *      reset mistake.
1722          *
1723          *      Nope, it was not mistake. It is really desired behaviour
1724          *      f.e. on http servers, when such sockets are useless, but
1725          *      consume significant resources. Let's do it with special
1726          *      linger2 option.                                 --ANK
1727          */
1728
1729         if (sk->sk_state == TCP_FIN_WAIT2) {
1730                 struct tcp_opt *tp = tcp_sk(sk);
1731                 if (tp->linger2 < 0) {
1732                         tcp_set_state(sk, TCP_CLOSE);
1733                         tcp_send_active_reset(sk, GFP_ATOMIC);
1734                         NET_INC_STATS_BH(LINUX_MIB_TCPABORTONLINGER);
1735                 } else {
1736                         int tmo = tcp_fin_time(tp);
1737
1738                         if (tmo > TCP_TIMEWAIT_LEN) {
1739                                 tcp_reset_keepalive_timer(sk, tcp_fin_time(tp));
1740                         } else {
1741                                 atomic_inc(&tcp_orphan_count);
1742                                 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
1743                                 goto out;
1744                         }
1745                 }
1746         }
1747         if (sk->sk_state != TCP_CLOSE) {
1748                 sk_stream_mem_reclaim(sk);
1749                 if (atomic_read(&tcp_orphan_count) > sysctl_tcp_max_orphans ||
1750                     (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
1751                      atomic_read(&tcp_memory_allocated) > sysctl_tcp_mem[2])) {
1752                         if (net_ratelimit())
1753                                 printk(KERN_INFO "TCP: too many of orphaned "
1754                                        "sockets\n");
1755                         tcp_set_state(sk, TCP_CLOSE);
1756                         tcp_send_active_reset(sk, GFP_ATOMIC);
1757                         NET_INC_STATS_BH(LINUX_MIB_TCPABORTONMEMORY);
1758                 }
1759         }
1760         atomic_inc(&tcp_orphan_count);
1761
1762         if (sk->sk_state == TCP_CLOSE)
1763                 tcp_destroy_sock(sk);
1764         /* Otherwise, socket is reprieved until protocol close. */
1765
1766 out:
1767         bh_unlock_sock(sk);
1768         local_bh_enable();
1769         sock_put(sk);
1770 }
1771
1772 /* These states need RST on ABORT according to RFC793 */
1773
1774 static inline int tcp_need_reset(int state)
1775 {
1776         return (1 << state) &
1777                (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
1778                 TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
1779 }
1780
1781 int tcp_disconnect(struct sock *sk, int flags)
1782 {
1783         struct inet_opt *inet = inet_sk(sk);
1784         struct tcp_opt *tp = tcp_sk(sk);
1785         int err = 0;
1786         int old_state = sk->sk_state;
1787
1788         if (old_state != TCP_CLOSE)
1789                 tcp_set_state(sk, TCP_CLOSE);
1790
1791         /* ABORT function of RFC793 */
1792         if (old_state == TCP_LISTEN) {
1793                 tcp_listen_stop(sk);
1794         } else if (tcp_need_reset(old_state) ||
1795                    (tp->snd_nxt != tp->write_seq &&
1796                     (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
1797                 /* The last check adjusts for discrepance of Linux wrt. RFC
1798                  * states
1799                  */
1800                 tcp_send_active_reset(sk, gfp_any());
1801                 sk->sk_err = ECONNRESET;
1802         } else if (old_state == TCP_SYN_SENT)
1803                 sk->sk_err = ECONNRESET;
1804
1805         tcp_clear_xmit_timers(sk);
1806         __skb_queue_purge(&sk->sk_receive_queue);
1807         sk_stream_writequeue_purge(sk);
1808         __skb_queue_purge(&tp->out_of_order_queue);
1809
1810         inet->dport = 0;
1811
1812         if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
1813                 inet_reset_saddr(sk);
1814
1815         sk->sk_shutdown = 0;
1816         sock_reset_flag(sk, SOCK_DONE);
1817         tp->srtt = 0;
1818         if ((tp->write_seq += tp->max_window + 2) == 0)
1819                 tp->write_seq = 1;
1820         tp->backoff = 0;
1821         tp->snd_cwnd = 2;
1822         tp->probes_out = 0;
1823         tp->packets_out = 0;
1824         tp->snd_ssthresh = 0x7fffffff;
1825         tp->snd_cwnd_cnt = 0;
1826         tcp_set_ca_state(tp, TCP_CA_Open);
1827         tcp_clear_retrans(tp);
1828         tcp_delack_init(tp);
1829         sk->sk_send_head = NULL;
1830         tp->saw_tstamp = 0;
1831         tcp_sack_reset(tp);
1832         __sk_dst_reset(sk);
1833
1834         BUG_TRAP(!inet->num || tp->bind_hash);
1835
1836         sk->sk_error_report(sk);
1837         return err;
1838 }
1839
1840 /*
1841  *      Wait for an incoming connection, avoid race
1842  *      conditions. This must be called with the socket locked.
1843  */
1844 static int wait_for_connect(struct sock *sk, long timeo)
1845 {
1846         struct tcp_opt *tp = tcp_sk(sk);
1847         DEFINE_WAIT(wait);
1848         int err;
1849
1850         /*
1851          * True wake-one mechanism for incoming connections: only
1852          * one process gets woken up, not the 'whole herd'.
1853          * Since we do not 'race & poll' for established sockets
1854          * anymore, the common case will execute the loop only once.
1855          *
1856          * Subtle issue: "add_wait_queue_exclusive()" will be added
1857          * after any current non-exclusive waiters, and we know that
1858          * it will always _stay_ after any new non-exclusive waiters
1859          * because all non-exclusive waiters are added at the
1860          * beginning of the wait-queue. As such, it's ok to "drop"
1861          * our exclusiveness temporarily when we get woken up without
1862          * having to remove and re-insert us on the wait queue.
1863          */
1864         for (;;) {
1865                 prepare_to_wait_exclusive(sk->sk_sleep, &wait,
1866                                           TASK_INTERRUPTIBLE);
1867                 release_sock(sk);
1868                 if (!tp->accept_queue)
1869                         timeo = schedule_timeout(timeo);
1870                 lock_sock(sk);
1871                 err = 0;
1872                 if (tp->accept_queue)
1873                         break;
1874                 err = -EINVAL;
1875                 if (sk->sk_state != TCP_LISTEN)
1876                         break;
1877                 err = sock_intr_errno(timeo);
1878                 if (signal_pending(current))
1879                         break;
1880                 err = -EAGAIN;
1881                 if (!timeo)
1882                         break;
1883         }
1884         finish_wait(sk->sk_sleep, &wait);
1885         return err;
1886 }
1887
1888 /*
1889  *      This will accept the next outstanding connection.
1890  */
1891
1892 struct sock *tcp_accept(struct sock *sk, int flags, int *err)
1893 {
1894         struct tcp_opt *tp = tcp_sk(sk);
1895         struct open_request *req;
1896         struct sock *newsk;
1897         int error;
1898
1899         lock_sock(sk);
1900
1901         /* We need to make sure that this socket is listening,
1902          * and that it has something pending.
1903          */
1904         error = -EINVAL;
1905         if (sk->sk_state != TCP_LISTEN)
1906                 goto out;
1907
1908         /* Find already established connection */
1909         if (!tp->accept_queue) {
1910                 long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1911
1912                 /* If this is a non blocking socket don't sleep */
1913                 error = -EAGAIN;
1914                 if (!timeo)
1915                         goto out;
1916
1917                 error = wait_for_connect(sk, timeo);
1918                 if (error)
1919                         goto out;
1920         }
1921
1922         req = tp->accept_queue;
1923         if ((tp->accept_queue = req->dl_next) == NULL)
1924                 tp->accept_queue_tail = NULL;
1925
1926         newsk = req->sk;
1927         sk_acceptq_removed(sk);
1928         tcp_openreq_fastfree(req);
1929         BUG_TRAP(newsk->sk_state != TCP_SYN_RECV);
1930         release_sock(sk);
1931         return newsk;
1932
1933 out:
1934         release_sock(sk);
1935         *err = error;
1936         return NULL;
1937 }
1938
1939 /*
1940  *      Socket option code for TCP.
1941  */
1942 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval,
1943                    int optlen)
1944 {
1945         struct tcp_opt *tp = tcp_sk(sk);
1946         int val;
1947         int err = 0;
1948
1949         if (level != SOL_TCP)
1950                 return tp->af_specific->setsockopt(sk, level, optname,
1951                                                    optval, optlen);
1952
1953         if (optlen < sizeof(int))
1954                 return -EINVAL;
1955
1956         if (get_user(val, (int __user *)optval))
1957                 return -EFAULT;
1958
1959         lock_sock(sk);
1960
1961         switch (optname) {
1962         case TCP_MAXSEG:
1963                 /* Values greater than interface MTU won't take effect. However
1964                  * at the point when this call is done we typically don't yet
1965                  * know which interface is going to be used */
1966                 if (val < 8 || val > MAX_TCP_WINDOW) {
1967                         err = -EINVAL;
1968                         break;
1969                 }
1970                 tp->user_mss = val;
1971                 break;
1972
1973         case TCP_NODELAY:
1974                 if (val) {
1975                         /* TCP_NODELAY is weaker than TCP_CORK, so that
1976                          * this option on corked socket is remembered, but
1977                          * it is not activated until cork is cleared.
1978                          *
1979                          * However, when TCP_NODELAY is set we make
1980                          * an explicit push, which overrides even TCP_CORK
1981                          * for currently queued segments.
1982                          */
1983                         tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
1984                         tcp_push_pending_frames(sk, tp);
1985                 } else {
1986                         tp->nonagle &= ~TCP_NAGLE_OFF;
1987                 }
1988                 break;
1989
1990         case TCP_CORK:
1991                 /* When set indicates to always queue non-full frames.
1992                  * Later the user clears this option and we transmit
1993                  * any pending partial frames in the queue.  This is
1994                  * meant to be used alongside sendfile() to get properly
1995                  * filled frames when the user (for example) must write
1996                  * out headers with a write() call first and then use
1997                  * sendfile to send out the data parts.
1998                  *
1999                  * TCP_CORK can be set together with TCP_NODELAY and it is
2000                  * stronger than TCP_NODELAY.
2001                  */
2002                 if (val) {
2003                         tp->nonagle |= TCP_NAGLE_CORK;
2004                 } else {
2005                         tp->nonagle &= ~TCP_NAGLE_CORK;
2006                         if (tp->nonagle&TCP_NAGLE_OFF)
2007                                 tp->nonagle |= TCP_NAGLE_PUSH;
2008                         tcp_push_pending_frames(sk, tp);
2009                 }
2010                 break;
2011
2012         case TCP_KEEPIDLE:
2013                 if (val < 1 || val > MAX_TCP_KEEPIDLE)
2014                         err = -EINVAL;
2015                 else {
2016                         tp->keepalive_time = val * HZ;
2017                         if (sock_flag(sk, SOCK_KEEPOPEN) &&
2018                             !((1 << sk->sk_state) &
2019                               (TCPF_CLOSE | TCPF_LISTEN))) {
2020                                 __u32 elapsed = tcp_time_stamp - tp->rcv_tstamp;
2021                                 if (tp->keepalive_time > elapsed)
2022                                         elapsed = tp->keepalive_time - elapsed;
2023                                 else
2024                                         elapsed = 0;
2025                                 tcp_reset_keepalive_timer(sk, elapsed);
2026                         }
2027                 }
2028                 break;
2029         case TCP_KEEPINTVL:
2030                 if (val < 1 || val > MAX_TCP_KEEPINTVL)
2031                         err = -EINVAL;
2032                 else
2033                         tp->keepalive_intvl = val * HZ;
2034                 break;
2035         case TCP_KEEPCNT:
2036                 if (val < 1 || val > MAX_TCP_KEEPCNT)
2037                         err = -EINVAL;
2038                 else
2039                         tp->keepalive_probes = val;
2040                 break;
2041         case TCP_SYNCNT:
2042                 if (val < 1 || val > MAX_TCP_SYNCNT)
2043                         err = -EINVAL;
2044                 else
2045                         tp->syn_retries = val;
2046                 break;
2047
2048         case TCP_LINGER2:
2049                 if (val < 0)
2050                         tp->linger2 = -1;
2051                 else if (val > sysctl_tcp_fin_timeout / HZ)
2052                         tp->linger2 = 0;
2053                 else
2054                         tp->linger2 = val * HZ;
2055                 break;
2056
2057         case TCP_DEFER_ACCEPT:
2058                 tp->defer_accept = 0;
2059                 if (val > 0) {
2060                         /* Translate value in seconds to number of
2061                          * retransmits */
2062                         while (tp->defer_accept < 32 &&
2063                                val > ((TCP_TIMEOUT_INIT / HZ) <<
2064                                        tp->defer_accept))
2065                                 tp->defer_accept++;
2066                         tp->defer_accept++;
2067                 }
2068                 break;
2069
2070         case TCP_WINDOW_CLAMP:
2071                 if (!val) {
2072                         if (sk->sk_state != TCP_CLOSE) {
2073                                 err = -EINVAL;
2074                                 break;
2075                         }
2076                         tp->window_clamp = 0;
2077                 } else
2078                         tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
2079                                                 SOCK_MIN_RCVBUF / 2 : val;
2080                 break;
2081
2082         case TCP_QUICKACK:
2083                 if (!val) {
2084                         tp->ack.pingpong = 1;
2085                 } else {
2086                         tp->ack.pingpong = 0;
2087                         if ((1 << sk->sk_state) &
2088                             (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
2089                             tcp_ack_scheduled(tp)) {
2090                                 tp->ack.pending |= TCP_ACK_PUSHED;
2091                                 cleanup_rbuf(sk, 1);
2092                                 if (!(val & 1))
2093                                         tp->ack.pingpong = 1;
2094                         }
2095                 }
2096                 break;
2097
2098         default:
2099                 err = -ENOPROTOOPT;
2100                 break;
2101         };
2102         release_sock(sk);
2103         return err;
2104 }
2105
2106 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
2107                    int __user *optlen)
2108 {
2109         struct tcp_opt *tp = tcp_sk(sk);
2110         int val, len;
2111
2112         if (level != SOL_TCP)
2113                 return tp->af_specific->getsockopt(sk, level, optname,
2114                                                    optval, optlen);
2115
2116         if (get_user(len, optlen))
2117                 return -EFAULT;
2118
2119         len = min_t(unsigned int, len, sizeof(int));
2120
2121         if (len < 0)
2122                 return -EINVAL;
2123
2124         switch (optname) {
2125         case TCP_MAXSEG:
2126                 val = tp->mss_cache_std;
2127                 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
2128                         val = tp->user_mss;
2129                 break;
2130         case TCP_NODELAY:
2131                 val = !!(tp->nonagle&TCP_NAGLE_OFF);
2132                 break;
2133         case TCP_CORK:
2134                 val = !!(tp->nonagle&TCP_NAGLE_CORK);
2135                 break;
2136         case TCP_KEEPIDLE:
2137                 val = (tp->keepalive_time ? : sysctl_tcp_keepalive_time) / HZ;
2138                 break;
2139         case TCP_KEEPINTVL:
2140                 val = (tp->keepalive_intvl ? : sysctl_tcp_keepalive_intvl) / HZ;
2141                 break;
2142         case TCP_KEEPCNT:
2143                 val = tp->keepalive_probes ? : sysctl_tcp_keepalive_probes;
2144                 break;
2145         case TCP_SYNCNT:
2146                 val = tp->syn_retries ? : sysctl_tcp_syn_retries;
2147                 break;
2148         case TCP_LINGER2:
2149                 val = tp->linger2;
2150                 if (val >= 0)
2151                         val = (val ? : sysctl_tcp_fin_timeout) / HZ;
2152                 break;
2153         case TCP_DEFER_ACCEPT:
2154                 val = !tp->defer_accept ? 0 : ((TCP_TIMEOUT_INIT / HZ) <<
2155                                                (tp->defer_accept - 1));
2156                 break;
2157         case TCP_WINDOW_CLAMP:
2158                 val = tp->window_clamp;
2159                 break;
2160         case TCP_INFO: {
2161                 struct tcp_info info;
2162
2163                 if (get_user(len, optlen))
2164                         return -EFAULT;
2165
2166                 tcp_get_info(sk, &info);
2167
2168                 len = min_t(unsigned int, len, sizeof(info));
2169                 if (put_user(len, optlen))
2170                         return -EFAULT;
2171                 if (copy_to_user(optval, &info, len))
2172                         return -EFAULT;
2173                 return 0;
2174         }
2175         case TCP_QUICKACK:
2176                 val = !tp->ack.pingpong;
2177                 break;
2178         default:
2179                 return -ENOPROTOOPT;
2180         };
2181
2182         if (put_user(len, optlen))
2183                 return -EFAULT;
2184         if (copy_to_user(optval, &val, len))
2185                 return -EFAULT;
2186         return 0;
2187 }
2188
2189
2190 extern void __skb_cb_too_small_for_tcp(int, int);
2191 extern void tcpdiag_init(void);
2192
2193 static __initdata unsigned long thash_entries;
2194 static int __init set_thash_entries(char *str)
2195 {
2196         if (!str)
2197                 return 0;
2198         thash_entries = simple_strtoul(str, &str, 0);
2199         return 1;
2200 }
2201 __setup("thash_entries=", set_thash_entries);
2202
2203 void __init tcp_init(void)
2204 {
2205         struct sk_buff *skb = NULL;
2206         unsigned long goal;
2207         int order, i;
2208
2209         if (sizeof(struct tcp_skb_cb) > sizeof(skb->cb))
2210                 __skb_cb_too_small_for_tcp(sizeof(struct tcp_skb_cb),
2211                                            sizeof(skb->cb));
2212
2213         tcp_openreq_cachep = kmem_cache_create("tcp_open_request",
2214                                                    sizeof(struct open_request),
2215                                                0, SLAB_HWCACHE_ALIGN,
2216                                                NULL, NULL);
2217         if (!tcp_openreq_cachep)
2218                 panic("tcp_init: Cannot alloc open_request cache.");
2219
2220         tcp_bucket_cachep = kmem_cache_create("tcp_bind_bucket",
2221                                               sizeof(struct tcp_bind_bucket),
2222                                               0, SLAB_HWCACHE_ALIGN,
2223                                               NULL, NULL);
2224         if (!tcp_bucket_cachep)
2225                 panic("tcp_init: Cannot alloc tcp_bind_bucket cache.");
2226
2227         tcp_timewait_cachep = kmem_cache_create("tcp_tw_bucket",
2228                                                 sizeof(struct tcp_tw_bucket),
2229                                                 0, SLAB_HWCACHE_ALIGN,
2230                                                 NULL, NULL);
2231         if (!tcp_timewait_cachep)
2232                 panic("tcp_init: Cannot alloc tcp_tw_bucket cache.");
2233
2234         /* Size and allocate the main established and bind bucket
2235          * hash tables.
2236          *
2237          * The methodology is similar to that of the buffer cache.
2238          */
2239         if (num_physpages >= (128 * 1024))
2240                 goal = num_physpages >> (21 - PAGE_SHIFT);
2241         else
2242                 goal = num_physpages >> (23 - PAGE_SHIFT);
2243
2244         if (thash_entries)
2245                 goal = (thash_entries * sizeof(struct tcp_ehash_bucket)) >> PAGE_SHIFT;
2246         for (order = 0; (1UL << order) < goal; order++)
2247                 ;
2248         do {
2249                 tcp_ehash_size = (1UL << order) * PAGE_SIZE /
2250                         sizeof(struct tcp_ehash_bucket);
2251                 tcp_ehash_size >>= 1;
2252                 while (tcp_ehash_size & (tcp_ehash_size - 1))
2253                         tcp_ehash_size--;
2254                 tcp_ehash = (struct tcp_ehash_bucket *)
2255                         __get_free_pages(GFP_ATOMIC, order);
2256         } while (!tcp_ehash && --order > 0);
2257
2258         if (!tcp_ehash)
2259                 panic("Failed to allocate TCP established hash table\n");
2260         for (i = 0; i < (tcp_ehash_size << 1); i++) {
2261                 tcp_ehash[i].lock = RW_LOCK_UNLOCKED;
2262                 INIT_HLIST_HEAD(&tcp_ehash[i].chain);
2263         }
2264
2265         do {
2266                 tcp_bhash_size = (1UL << order) * PAGE_SIZE /
2267                         sizeof(struct tcp_bind_hashbucket);
2268                 if ((tcp_bhash_size > (64 * 1024)) && order > 0)
2269                         continue;
2270                 tcp_bhash = (struct tcp_bind_hashbucket *)
2271                         __get_free_pages(GFP_ATOMIC, order);
2272         } while (!tcp_bhash && --order >= 0);
2273
2274         if (!tcp_bhash)
2275                 panic("Failed to allocate TCP bind hash table\n");
2276         for (i = 0; i < tcp_bhash_size; i++) {
2277                 tcp_bhash[i].lock = SPIN_LOCK_UNLOCKED;
2278                 INIT_HLIST_HEAD(&tcp_bhash[i].chain);
2279         }
2280
2281         /* Try to be a bit smarter and adjust defaults depending
2282          * on available memory.
2283          */
2284         if (order > 4) {
2285                 sysctl_local_port_range[0] = 32768;
2286                 sysctl_local_port_range[1] = 61000;
2287                 sysctl_tcp_max_tw_buckets = 180000;
2288                 sysctl_tcp_max_orphans = 4096 << (order - 4);
2289                 sysctl_max_syn_backlog = 1024;
2290         } else if (order < 3) {
2291                 sysctl_local_port_range[0] = 1024 * (3 - order);
2292                 sysctl_tcp_max_tw_buckets >>= (3 - order);
2293                 sysctl_tcp_max_orphans >>= (3 - order);
2294                 sysctl_max_syn_backlog = 128;
2295         }
2296         tcp_port_rover = sysctl_local_port_range[0] - 1;
2297
2298         sysctl_tcp_mem[0] =  768 << order;
2299         sysctl_tcp_mem[1] = 1024 << order;
2300         sysctl_tcp_mem[2] = 1536 << order;
2301
2302         if (order < 3) {
2303                 sysctl_tcp_wmem[2] = 64 * 1024;
2304                 sysctl_tcp_rmem[0] = PAGE_SIZE;
2305                 sysctl_tcp_rmem[1] = 43689;
2306                 sysctl_tcp_rmem[2] = 2 * 43689;
2307         }
2308
2309         printk(KERN_INFO "TCP: Hash tables configured "
2310                "(established %d bind %d)\n",
2311                tcp_ehash_size << 1, tcp_bhash_size);
2312
2313         tcpdiag_init();
2314 }
2315
2316 EXPORT_SYMBOL(tcp_accept);
2317 EXPORT_SYMBOL(tcp_close);
2318 EXPORT_SYMBOL(tcp_close_state);
2319 EXPORT_SYMBOL(tcp_destroy_sock);
2320 EXPORT_SYMBOL(tcp_disconnect);
2321 EXPORT_SYMBOL(tcp_getsockopt);
2322 EXPORT_SYMBOL(tcp_ioctl);
2323 EXPORT_SYMBOL(tcp_openreq_cachep);
2324 EXPORT_SYMBOL(tcp_poll);
2325 EXPORT_SYMBOL(tcp_read_sock);
2326 EXPORT_SYMBOL(tcp_recvmsg);
2327 EXPORT_SYMBOL(tcp_sendmsg);
2328 EXPORT_SYMBOL(tcp_sendpage);
2329 EXPORT_SYMBOL(tcp_setsockopt);
2330 EXPORT_SYMBOL(tcp_shutdown);
2331 EXPORT_SYMBOL(tcp_statistics);
2332 EXPORT_SYMBOL(tcp_timewait_cachep);