drivers/s390/cio/qdio.c

   1 /*
   2  *
   3  * linux/drivers/s390/cio/qdio.c
   4  *
   5  * Linux for S/390 QDIO base support, Hipersocket base support
   6  * version 2
   7  *
   8  * Copyright 2000,2002 IBM Corporation
   9  * Author(s):             Utz Bacher <utz.bacher@de.ibm.com>
  10  * 2.6 cio integration by Cornelia Huck <cohuck@de.ibm.com>
  11  *
  12  * Restriction: only 63 iqdio subchannels would have its own indicator,
  13  * after that, subsequent subchannels share one indicator
  14  *
  15  *
  16  *
  17  *
  18  * This program is free software; you can redistribute it and/or modify
  19  * it under the terms of the GNU General Public License as published by
  20  * the Free Software Foundation; either version 2, or (at your option)
  21  * any later version.
  22  *
  23  * This program is distributed in the hope that it will be useful,
  24  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  25  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  26  * GNU General Public License for more details.
  27  *
  28  * You should have received a copy of the GNU General Public License
  29  * along with this program; if not, write to the Free Software
  30  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  31  */
  32
  33 #include <linux/config.h>
  34 #include <linux/module.h>
  35 #include <linux/init.h>
  36
  37 #include <linux/slab.h>
  38 #include <linux/kernel.h>
  39 #include <linux/proc_fs.h>
  40 #include <linux/timer.h>
  41
  42 #include <asm/ccwdev.h>
  43 #include <asm/io.h>
  44 #include <asm/atomic.h>
  45 #include <asm/semaphore.h>
  46 #include <asm/timex.h>
  47
  48 #include <asm/debug.h>
  49 #include <asm/qdio.h>
  50
  51 #include "cio.h"
  52 #include "css.h"
  53 #include "device.h"
  54 #include "airq.h"
  55 #include "qdio.h"
  56 #include "ioasm.h"
  57 #include "chsc.h"
  58
  59 #define VERSION_QDIO_C "$Revision: 1.80 $"
  60
  61 /****************** MODULE PARAMETER VARIABLES ********************/
  62 MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>");
  63 MODULE_DESCRIPTION("QDIO base support version 2, " \
  64                    "Copyright 2000 IBM Corporation");
  65 MODULE_LICENSE("GPL");
  66
  67 /******************** HERE WE GO ***********************************/
  68
  69 static const char version[] = "QDIO base support version 2 ("
  70         VERSION_QDIO_C "/" VERSION_QDIO_H  "/" VERSION_CIO_QDIO_H ")";
  71
  72 #ifdef QDIO_PERFORMANCE_STATS
  73 static int proc_perf_file_registration;
  74 static unsigned long i_p_c, i_p_nc, o_p_c, o_p_nc, ii_p_c, ii_p_nc;
  75 static struct qdio_perf_stats perf_stats;
  76 #endif /* QDIO_PERFORMANCE_STATS */
  77
  78 static int hydra_thinints;
  79 static int omit_svs;
  80
  81 static int indicator_used[INDICATORS_PER_CACHELINE];
  82 static __u32 * volatile indicators;
  83 static __u32 volatile spare_indicator;
  84 static atomic_t spare_indicator_usecount;
  85
  86 static debug_info_t *qdio_dbf_setup;
  87 static debug_info_t *qdio_dbf_sbal;
  88 static debug_info_t *qdio_dbf_trace;
  89 static debug_info_t *qdio_dbf_sense;
  90 #ifdef QDIO_DBF_LIKE_HELL
  91 static debug_info_t *qdio_dbf_slsb_out;
  92 static debug_info_t *qdio_dbf_slsb_in;
  93 #endif /* QDIO_DBF_LIKE_HELL */
  94
  95 /* iQDIO stuff: */
  96 static volatile struct qdio_q *tiq_list=NULL; /* volatile as it could change
  97                                                  during a while loop */
  98 static spinlock_t ttiq_list_lock=SPIN_LOCK_UNLOCKED;
  99 static int register_thinint_result;
 100 static void tiqdio_tl(unsigned long);
 101 static DECLARE_TASKLET(tiqdio_tasklet,tiqdio_tl,0);
 102
 103 /* not a macro, as one of the arguments is atomic_read */
 104 static inline int
 105 qdio_min(int a,int b)
 106 {
 107         if (a<b)
 108                 return a;
 109         else
 110                 return b;
 111 }
 112
 113 /***************** SCRUBBER HELPER ROUTINES **********************/
 114
 115 static inline volatile __u64
 116 qdio_get_micros(void)
 117 {
 118         return (get_clock() >> 10); /* time>>12 is microseconds */
 119 }
 120
 121 /*
 122  * unfortunately, we can't just xchg the values; in do_QDIO we want to reserve
 123  * the q in any case, so that we'll not be interrupted when we are in
 124  * qdio_mark_tiq... shouldn't have a really bad impact, as reserving almost
 125  * ever works (last famous words)
 126  */
 127 static inline int
 128 qdio_reserve_q(struct qdio_q *q)
 129 {
 130         return atomic_add_return(1,&q->use_count) - 1;
 131 }
 132
 133 static inline void
 134 qdio_release_q(struct qdio_q *q)
 135 {
 136         atomic_dec(&q->use_count);
 137 }
 138
 139 static volatile inline void
 140 qdio_set_slsb(volatile char *slsb, unsigned char value)
 141 {
 142         xchg((char*)slsb,value);
 143 }
 144
 145 static inline int
 146 qdio_siga_sync(struct qdio_q *q, unsigned int gpr2,
 147                unsigned int gpr3)
 148 {
 149         int cc;
 150
 151         QDIO_DBF_TEXT4(0,trace,"sigasync");
 152         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 153
 154 #ifdef QDIO_PERFORMANCE_STATS
 155         perf_stats.siga_syncs++;
 156 #endif /* QDIO_PERFORMANCE_STATS */
 157
 158         cc = do_siga_sync(q->irq, gpr2, gpr3);
 159         if (cc)
 160                 QDIO_DBF_HEX3(0,trace,&cc,sizeof(int*));
 161
 162         return cc;
 163 }
 164
 165 static inline int
 166 qdio_siga_sync_q(struct qdio_q *q)
 167 {
 168         if (q->is_input_q)
 169                 return qdio_siga_sync(q, 0, q->mask);
 170         return qdio_siga_sync(q, q->mask, 0);
 171 }
 172
 173 /*
 174  * returns QDIO_SIGA_ERROR_ACCESS_EXCEPTION as cc, when SIGA returns
 175  * an access exception
 176  */
 177 static inline int
 178 qdio_siga_output(struct qdio_q *q)
 179 {
 180         int cc;
 181         __u32 busy_bit;
 182         __u64 start_time=0;
 183
 184 #ifdef QDIO_PERFORMANCE_STATS
 185         perf_stats.siga_outs++;
 186 #endif /* QDIO_PERFORMANCE_STATS */
 187
 188         QDIO_DBF_TEXT4(0,trace,"sigaout");
 189         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 190
 191         for (;;) {
 192                 cc = do_siga_output(q->irq, q->mask, &busy_bit);
 193 //QDIO_PRINT_ERR("cc=%x, busy=%x\n",cc,busy_bit);
 194                 if ((cc==2) && (busy_bit) && (q->is_iqdio_q)) {
 195                         if (!start_time)
 196                                 start_time=NOW;
 197                         if ((NOW-start_time)>QDIO_BUSY_BIT_PATIENCE)
 198                                 break;
 199                 } else
 200                         break;
 201         }
 202
 203         if ((cc==2) && (busy_bit))
 204                 cc |= QDIO_SIGA_ERROR_B_BIT_SET;
 205
 206         if (cc)
 207                 QDIO_DBF_HEX3(0,trace,&cc,sizeof(int*));
 208
 209         return cc;
 210 }
 211
 212 static inline int
 213 qdio_siga_input(struct qdio_q *q)
 214 {
 215         int cc;
 216
 217         QDIO_DBF_TEXT4(0,trace,"sigain");
 218         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 219
 220 #ifdef QDIO_PERFORMANCE_STATS
 221         perf_stats.siga_ins++;
 222 #endif /* QDIO_PERFORMANCE_STATS */
 223
 224         cc = do_siga_input(q->irq, q->mask);
 225
 226         if (cc)
 227                 QDIO_DBF_HEX3(0,trace,&cc,sizeof(int*));
 228
 229         return cc;
 230 }
 231
 232 /* locked by the locks in qdio_activate and qdio_cleanup */
 233 static __u32 * volatile
 234 qdio_get_indicator(void)
 235 {
 236         int i;
 237
 238         for (i=1;i<INDICATORS_PER_CACHELINE;i++)
 239                 if (!indicator_used[i]) {
 240                         indicator_used[i]=1;
 241                         return indicators+i;
 242                 }
 243         atomic_inc(&spare_indicator_usecount);
 244         return (__u32 * volatile) &spare_indicator;
 245 }
 246
 247 /* locked by the locks in qdio_activate and qdio_cleanup */
 248 static void
 249 qdio_put_indicator(__u32 *addr)
 250 {
 251         int i;
 252
 253         if ( (addr) && (addr!=&spare_indicator) ) {
 254                 i=addr-indicators;
 255                 indicator_used[i]=0;
 256         }
 257         if (addr == &spare_indicator)
 258                 atomic_dec(&spare_indicator_usecount);
 259 }
 260
 261 static inline volatile void
 262 tiqdio_clear_summary_bit(__u32 *location)
 263 {
 264         QDIO_DBF_TEXT5(0,trace,"clrsummb");
 265         QDIO_DBF_HEX5(0,trace,&location,sizeof(void*));
 266
 267         xchg(location,0);
 268 }
 269
 270 static inline volatile void
 271 tiqdio_set_summary_bit(__u32 *location)
 272 {
 273         QDIO_DBF_TEXT5(0,trace,"setsummb");
 274         QDIO_DBF_HEX5(0,trace,&location,sizeof(void*));
 275
 276         xchg(location,-1);
 277 }
 278
 279 static inline void
 280 tiqdio_sched_tl(void)
 281 {
 282         tasklet_hi_schedule(&tiqdio_tasklet);
 283 }
 284
 285 static inline void
 286 qdio_mark_tiq(struct qdio_q *q)
 287 {
 288         unsigned long flags;
 289
 290         QDIO_DBF_TEXT4(0,trace,"mark iq");
 291         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 292
 293         spin_lock_irqsave(&ttiq_list_lock,flags);
 294         if (unlikely(atomic_read(&q->is_in_shutdown)))
 295                 goto out_unlock;
 296
 297         if (!q->is_input_q)
 298                 goto out_unlock;
 299
 300         if ((q->list_prev) || (q->list_next))
 301                 goto out_unlock;
 302
 303         if (!tiq_list) {
 304                 tiq_list=q;
 305                 q->list_prev=q;
 306                 q->list_next=q;
 307         } else {
 308                 q->list_next=tiq_list;
 309                 q->list_prev=tiq_list->list_prev;
 310                 tiq_list->list_prev->list_next=q;
 311                 tiq_list->list_prev=q;
 312         }
 313         spin_unlock_irqrestore(&ttiq_list_lock,flags);
 314
 315         tiqdio_set_summary_bit((__u32*)q->dev_st_chg_ind);
 316         tiqdio_sched_tl();
 317         return;
 318 out_unlock:
 319         spin_unlock_irqrestore(&ttiq_list_lock,flags);
 320         return;
 321 }
 322
 323 static inline void
 324 qdio_mark_q(struct qdio_q *q)
 325 {
 326         QDIO_DBF_TEXT4(0,trace,"mark q");
 327         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 328
 329         if (unlikely(atomic_read(&q->is_in_shutdown)))
 330                 return;
 331
 332         tasklet_schedule(&q->tasklet);
 333 }
 334
 335 static inline int
 336 qdio_stop_polling(struct qdio_q *q)
 337 {
 338 #ifdef QDIO_USE_PROCESSING_STATE
 339         int gsf;
 340
 341         if (!atomic_swap(&q->polling,0))
 342                 return 1;
 343
 344         QDIO_DBF_TEXT4(0,trace,"stoppoll");
 345         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 346
 347         /* show the card that we are not polling anymore */
 348         if (!q->is_input_q)
 349                 return 1;
 350
 351         gsf=GET_SAVED_FRONTIER(q);
 352         set_slsb(&q->slsb.acc.val[(gsf+QDIO_MAX_BUFFERS_PER_Q-1)&
 353                                   (QDIO_MAX_BUFFERS_PER_Q-1)],
 354                  SLSB_P_INPUT_NOT_INIT);
 355         /*
 356          * we don't issue this SYNC_MEMORY, as we trust Rick T and
 357          * moreover will not use the PROCESSING state, so q->polling was 0
 358          */
 359         /*SYNC_MEMORY;*/
 360         if (q->slsb.acc.val[gsf]!=SLSB_P_INPUT_PRIMED)
 361                 return 1;
 362         /*
 363          * set our summary bit again, as otherwise there is a
 364          * small window we can miss between resetting it and
 365          * checking for PRIMED state
 366          */
 367         if (q->is_iqdio_q)
 368                 tiqdio_set_summary_bit((__u32*)q->dev_st_chg_ind);
 369         return 0;
 370
 371 #else /* QDIO_USE_PROCESSING_STATE */
 372         return 1;
 373 #endif /* QDIO_USE_PROCESSING_STATE */
 374 }
 375
 376 /*
 377  * see the comment in do_QDIO and before qdio_reserve_q about the
 378  * sophisticated locking outside of unmark_q, so that we don't need to
 379  * disable the interrupts :-)
 380 */
 381 static inline void
 382 qdio_unmark_q(struct qdio_q *q)
 383 {
 384         unsigned long flags;
 385
 386         QDIO_DBF_TEXT4(0,trace,"unmark q");
 387         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 388
 389         if ((!q->list_prev)||(!q->list_next))
 390                 return;
 391
 392         if ((q->is_thinint_q)&&(q->is_input_q)) {
 393                 /* iQDIO */
 394                 spin_lock_irqsave(&ttiq_list_lock,flags);
 395                 /* in case cleanup has done this already and simultanously
 396                  * qdio_unmark_q is called from the interrupt handler, we've
 397                  * got to check this in this specific case again */
 398                 if ((!q->list_prev)||(!q->list_next))
 399                         goto out;
 400                 if (q->list_next==q) {
 401                         /* q was the only interesting q */
 402                         tiq_list=NULL;
 403                         q->list_next=NULL;
 404                         q->list_prev=NULL;
 405                 } else {
 406                         q->list_next->list_prev=q->list_prev;
 407                         q->list_prev->list_next=q->list_next;
 408                         tiq_list=q->list_next;
 409                         q->list_next=NULL;
 410                         q->list_prev=NULL;
 411                 }
 412 out:
 413                 spin_unlock_irqrestore(&ttiq_list_lock,flags);
 414         }
 415 }
 416
 417 static inline unsigned long
 418 tiqdio_clear_global_summary(void)
 419 {
 420         unsigned long time;
 421
 422         QDIO_DBF_TEXT5(0,trace,"clrglobl");
 423
 424         time = do_clear_global_summary();
 425
 426         QDIO_DBF_HEX5(0,trace,&time,sizeof(unsigned long));
 427
 428         return time;
 429 }
 430
 431
 432 /************************* OUTBOUND ROUTINES *******************************/
 433
 434 inline static int
 435 qdio_get_outbound_buffer_frontier(struct qdio_q *q)
 436 {
 437         int f,f_mod_no;
 438         volatile char *slsb;
 439         int first_not_to_check;
 440         char dbf_text[15];
 441
 442         QDIO_DBF_TEXT4(0,trace,"getobfro");
 443         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 444
 445         slsb=&q->slsb.acc.val[0];
 446         f_mod_no=f=q->first_to_check;
 447         /*
 448          * f points to already processed elements, so f+no_used is correct...
 449          * ... but: we don't check 128 buffers, as otherwise
 450          * qdio_has_outbound_q_moved would return 0
 451          */
 452         first_not_to_check=f+qdio_min(atomic_read(&q->number_of_buffers_used),
 453                                       (QDIO_MAX_BUFFERS_PER_Q-1));
 454
 455         if ((!q->is_iqdio_q)&&(!q->hydra_gives_outbound_pcis))
 456                 SYNC_MEMORY;
 457
 458 check_next:
 459         if (f==first_not_to_check)
 460                 goto out;
 461
 462         switch(slsb[f_mod_no]) {
 463
 464         /* the adapter has not fetched the output yet */
 465         case SLSB_CU_OUTPUT_PRIMED:
 466                 QDIO_DBF_TEXT5(0,trace,"outpprim");
 467                 break;
 468
 469         /* the adapter got it */
 470         case SLSB_P_OUTPUT_EMPTY:
 471                 atomic_dec(&q->number_of_buffers_used);
 472                 f++;
 473                 f_mod_no=f&(QDIO_MAX_BUFFERS_PER_Q-1);
 474                 QDIO_DBF_TEXT5(0,trace,"outpempt");
 475                 goto check_next;
 476
 477         case SLSB_P_OUTPUT_ERROR:
 478                 QDIO_DBF_TEXT3(0,trace,"outperr");
 479                 sprintf(dbf_text,"%x-%x-%x",f_mod_no,
 480                         q->sbal[f_mod_no]->element[14].sbalf.value,
 481                         q->sbal[f_mod_no]->element[15].sbalf.value);
 482                 QDIO_DBF_TEXT3(1,trace,dbf_text);
 483                 QDIO_DBF_HEX2(1,sbal,q->sbal[f_mod_no],256);
 484
 485                 /* kind of process the buffer */
 486                 set_slsb(&q->slsb.acc.val[f_mod_no], SLSB_P_OUTPUT_NOT_INIT);
 487
 488                 /*
 489                  * we increment the frontier, as this buffer
 490                  * was processed obviously
 491                  */
 492                 atomic_dec(&q->number_of_buffers_used);
 493                 f_mod_no=(f_mod_no+1)&(QDIO_MAX_BUFFERS_PER_Q-1);
 494
 495                 if (q->qdio_error)
 496                         q->error_status_flags|=
 497                                 QDIO_STATUS_MORE_THAN_ONE_QDIO_ERROR;
 498                 q->qdio_error=SLSB_P_OUTPUT_ERROR;
 499                 q->error_status_flags|=QDIO_STATUS_LOOK_FOR_ERROR;
 500
 501                 break;
 502
 503         /* no new buffers */
 504         default:
 505                 QDIO_DBF_TEXT5(0,trace,"outpni");
 506         }
 507 out:
 508         return (q->first_to_check=f_mod_no);
 509 }
 510
 511 /* all buffers are processed */
 512 inline static int
 513 qdio_is_outbound_q_done(struct qdio_q *q)
 514 {
 515         int no_used;
 516         char dbf_text[15];
 517
 518         no_used=atomic_read(&q->number_of_buffers_used);
 519
 520         if (no_used) {
 521                 sprintf(dbf_text,"oqisnt%02x",no_used);
 522                 QDIO_DBF_TEXT4(0,trace,dbf_text);
 523         } else {
 524                 QDIO_DBF_TEXT4(0,trace,"oqisdone");
 525         }
 526         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 527         return (no_used==0);
 528 }
 529
 530 inline static int
 531 qdio_has_outbound_q_moved(struct qdio_q *q)
 532 {
 533         int i;
 534
 535         i=qdio_get_outbound_buffer_frontier(q);
 536
 537         if ( (i!=GET_SAVED_FRONTIER(q)) ||
 538              (q->error_status_flags&QDIO_STATUS_LOOK_FOR_ERROR) ) {
 539                 SAVE_FRONTIER(q,i);
 540                 QDIO_DBF_TEXT4(0,trace,"oqhasmvd");
 541                 QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 542                 return 1;
 543         } else {
 544                 QDIO_DBF_TEXT4(0,trace,"oqhsntmv");
 545                 QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 546                 return 0;
 547         }
 548 }
 549
 550 inline static void
 551 qdio_kick_outbound_q(struct qdio_q *q)
 552 {
 553         int result;
 554         char dbf_text[15];
 555
 556         QDIO_DBF_TEXT4(0,trace,"kickoutq");
 557         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 558
 559         if (!q->siga_out)
 560                 return;
 561
 562         /* here's the story with cc=2 and busy bit set (thanks, Rick):
 563          * VM's CP could present us cc=2 and busy bit set on SIGA-write
 564          * during reconfiguration of their Guest LAN (only in HIPERS mode,
 565          * QDIO mode is asynchronous -- cc=2 and busy bit there will take
 566          * the queues down immediately; and not being under VM we have a
 567          * problem on cc=2 and busy bit set right away).
 568          *
 569          * Therefore qdio_siga_output will try for a short time constantly,
 570          * if such a condition occurs. If it doesn't change, it will
 571          * increase the busy_siga_counter and save the timestamp, and
 572          * schedule the queue for later processing (via mark_q, using the
 573          * queue tasklet). __qdio_outbound_processing will check out the
 574          * counter. If non-zero, it will call qdio_kick_outbound_q as often
 575          * as the value of the counter. This will attempt further SIGA
 576          * instructions. For each successful SIGA, the counter is
 577          * decreased, for failing SIGAs the counter remains the same, after
 578          * all.
 579          * After some time of no movement, qdio_kick_outbound_q will
 580          * finally fail and reflect corresponding error codes to call
 581          * the upper layer module and have it take the queues down.
 582          *
 583          * Note that this is a change from the original HiperSockets design
 584          * (saying cc=2 and busy bit means take the queues down), but in
 585          * these days Guest LAN didn't exist... excessive cc=2 with busy bit
 586          * conditions will still take the queues down, but the threshold is
 587          * higher due to the Guest LAN environment.
 588          */
 589
 590
 591         result=qdio_siga_output(q);
 592
 593                 switch (result) {
 594                 case 0:
 595                 /* went smooth this time, reset timestamp */
 596                         QDIO_DBF_TEXT3(0,trace,"cc2reslv");
 597                         sprintf(dbf_text,"%4x%2x%2x",q->irq,q->q_no,
 598                                 atomic_read(&q->busy_siga_counter));
 599                         QDIO_DBF_TEXT3(0,trace,dbf_text);
 600                         q->timing.busy_start=0;
 601                         break;
 602                 case (2|QDIO_SIGA_ERROR_B_BIT_SET):
 603                         /* cc=2 and busy bit: */
 604                 atomic_inc(&q->busy_siga_counter);
 605
 606                         /* if the last siga was successful, save
 607                          * timestamp here */
 608                         if (!q->timing.busy_start)
 609                                 q->timing.busy_start=NOW;
 610
 611                         /* if we're in time, don't touch error_status_flags
 612                          * and siga_error */
 613                         if (NOW-q->timing.busy_start<QDIO_BUSY_BIT_GIVE_UP) {
 614                                 qdio_mark_q(q);
 615                                 break;
 616                         }
 617                         QDIO_DBF_TEXT2(0,trace,"cc2REPRT");
 618                         sprintf(dbf_text,"%4x%2x%2x",q->irq,q->q_no,
 619                                 atomic_read(&q->busy_siga_counter));
 620                         QDIO_DBF_TEXT3(0,trace,dbf_text);
 621                         /* else fallthrough and report error */
 622                 default:
 623                         /* for plain cc=1, 2 or 3: */
 624                         if (q->siga_error)
 625                                 q->error_status_flags|=
 626                                         QDIO_STATUS_MORE_THAN_ONE_SIGA_ERROR;
 627                         q->error_status_flags|=
 628                                 QDIO_STATUS_LOOK_FOR_ERROR;
 629                         q->siga_error=result;
 630                 }
 631 }
 632
 633 inline static void
 634 qdio_kick_outbound_handler(struct qdio_q *q)
 635 {
 636         int start, end, real_end, count;
 637         char dbf_text[15];
 638
 639         start = q->first_element_to_kick;
 640         /* last_move_ftc was just updated */
 641         real_end = GET_SAVED_FRONTIER(q);
 642         end = (real_end+QDIO_MAX_BUFFERS_PER_Q-1)&
 643                 (QDIO_MAX_BUFFERS_PER_Q-1);
 644         count = (end+QDIO_MAX_BUFFERS_PER_Q+1-start)&
 645                 (QDIO_MAX_BUFFERS_PER_Q-1);
 646
 647         QDIO_DBF_TEXT4(0,trace,"kickouth");
 648         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 649
 650         sprintf(dbf_text,"s=%2xc=%2x",start,count);
 651         QDIO_DBF_TEXT4(0,trace,dbf_text);
 652
 653         if (q->state==QDIO_IRQ_STATE_ACTIVE)
 654                 q->handler(q->cdev,QDIO_STATUS_OUTBOUND_INT|
 655                            q->error_status_flags,
 656                            q->qdio_error,q->siga_error,q->q_no,start,count,
 657                            q->int_parm);
 658
 659         /* for the next time: */
 660         q->first_element_to_kick=real_end;
 661         q->qdio_error=0;
 662         q->siga_error=0;
 663         q->error_status_flags=0;
 664 }
 665
 666 static inline void
 667 __qdio_outbound_processing(struct qdio_q *q)
 668 {
 669         int siga_attempts;
 670
 671         QDIO_DBF_TEXT4(0,trace,"qoutproc");
 672         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 673
 674         if (unlikely(qdio_reserve_q(q))) {
 675                 qdio_release_q(q);
 676 #ifdef QDIO_PERFORMANCE_STATS
 677                 o_p_c++;
 678 #endif /* QDIO_PERFORMANCE_STATS */
 679                 /* as we're sissies, we'll check next time */
 680                 if (likely(!atomic_read(&q->is_in_shutdown))) {
 681                         qdio_mark_q(q);
 682                         QDIO_DBF_TEXT4(0,trace,"busy,agn");
 683                 }
 684                 return;
 685         }
 686 #ifdef QDIO_PERFORMANCE_STATS
 687         o_p_nc++;
 688         perf_stats.tl_runs++;
 689 #endif /* QDIO_PERFORMANCE_STATS */
 690
 691         /* see comment in qdio_kick_outbound_q */
 692         siga_attempts=atomic_read(&q->busy_siga_counter);
 693         while (siga_attempts) {
 694                 atomic_dec(&q->busy_siga_counter);
 695                 qdio_kick_outbound_q(q);
 696                 siga_attempts--;
 697         }
 698
 699         if (qdio_has_outbound_q_moved(q))
 700                 qdio_kick_outbound_handler(q);
 701
 702         if (q->is_iqdio_q) {
 703                 /*
 704                  * for asynchronous queues, we better check, if the fill
 705                  * level is too high. for synchronous queues, the fill
 706                  * level will never be that high.
 707                  */
 708                 if (atomic_read(&q->number_of_buffers_used)>
 709                     IQDIO_FILL_LEVEL_TO_POLL)
 710                         qdio_mark_q(q);
 711
 712         } else if (!q->hydra_gives_outbound_pcis)
 713                 if (!qdio_is_outbound_q_done(q))
 714                         qdio_mark_q(q);
 715
 716         qdio_release_q(q);
 717 }
 718
 719 static void
 720 qdio_outbound_processing(struct qdio_q *q)
 721 {
 722         __qdio_outbound_processing(q);
 723 }
 724
 725 /************************* INBOUND ROUTINES *******************************/
 726
 727
 728 inline static int
 729 qdio_get_inbound_buffer_frontier(struct qdio_q *q)
 730 {
 731         int f,f_mod_no;
 732         volatile char *slsb;
 733         int first_not_to_check;
 734         char dbf_text[15];
 735
 736         QDIO_DBF_TEXT4(0,trace,"getibfro");
 737         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 738
 739         slsb=&q->slsb.acc.val[0];
 740         f_mod_no=f=q->first_to_check;
 741         /*
 742          * we don't check 128 buffers, as otherwise qdio_has_inbound_q_moved
 743          * would return 0
 744          */
 745         first_not_to_check=f+qdio_min(atomic_read(&q->number_of_buffers_used),
 746                                       (QDIO_MAX_BUFFERS_PER_Q-1));
 747
 748         /*
 749          * we don't use this one, as a PCI or we after a thin interrupt
 750          * will sync the queues
 751          */
 752         /* SYNC_MEMORY;*/
 753
 754 check_next:
 755         f_mod_no=f&(QDIO_MAX_BUFFERS_PER_Q-1);
 756         if (f==first_not_to_check)
 757                 goto out;
 758         switch (slsb[f_mod_no]) {
 759
 760         /* CU_EMPTY means frontier is reached */
 761         case SLSB_CU_INPUT_EMPTY:
 762                 QDIO_DBF_TEXT5(0,trace,"inptempt");
 763                 break;
 764
 765         /* P_PRIMED means set slsb to P_PROCESSING and move on */
 766         case SLSB_P_INPUT_PRIMED:
 767                 QDIO_DBF_TEXT5(0,trace,"inptprim");
 768
 769 #ifdef QDIO_USE_PROCESSING_STATE
 770                 /*
 771                  * as soon as running under VM, polling the input queues will
 772                  * kill VM in terms of CP overhead
 773                  */
 774                 if (q->siga_sync) {
 775                         set_slsb(&slsb[f_mod_no],SLSB_P_INPUT_NOT_INIT);
 776                 } else {
 777                         set_slsb(&slsb[f_mod_no],SLSB_P_INPUT_PROCESSING);
 778                         atomic_set(&q->polling,1);
 779                 }
 780 #else /* QDIO_USE_PROCESSING_STATE */
 781                 set_slsb(&slsb[f_mod_no],SLSB_P_INPUT_NOT_INIT);
 782 #endif /* QDIO_USE_PROCESSING_STATE */
 783                 /*
 784                  * not needed, as the inbound queue will be synced on the next
 785                  * siga-r
 786                  */
 787                 /*SYNC_MEMORY;*/
 788                 f++;
 789                 atomic_dec(&q->number_of_buffers_used);
 790                 goto check_next;
 791
 792         case SLSB_P_INPUT_NOT_INIT:
 793         case SLSB_P_INPUT_PROCESSING:
 794                 QDIO_DBF_TEXT5(0,trace,"inpnipro");
 795                 break;
 796
 797         /* P_ERROR means frontier is reached, break and report error */
 798         case SLSB_P_INPUT_ERROR:
 799                 sprintf(dbf_text,"inperr%2x",f_mod_no);
 800                 QDIO_DBF_TEXT3(1,trace,dbf_text);
 801                 QDIO_DBF_HEX2(1,sbal,q->sbal[f_mod_no],256);
 802
 803                 /* kind of process the buffer */
 804                 set_slsb(&slsb[f_mod_no],SLSB_P_INPUT_NOT_INIT);
 805
 806                 if (q->qdio_error)
 807                         q->error_status_flags|=
 808                                 QDIO_STATUS_MORE_THAN_ONE_QDIO_ERROR;
 809                 q->qdio_error=SLSB_P_INPUT_ERROR;
 810                 q->error_status_flags|=QDIO_STATUS_LOOK_FOR_ERROR;
 811
 812                 /* we increment the frontier, as this buffer
 813                  * was processed obviously */
 814                 f_mod_no=(f_mod_no+1)&(QDIO_MAX_BUFFERS_PER_Q-1);
 815                 atomic_dec(&q->number_of_buffers_used);
 816
 817                 break;
 818
 819         /* everything else means frontier not changed (HALTED or so) */
 820         default:
 821                 break;
 822         }
 823 out:
 824         q->first_to_check=f_mod_no;
 825
 826         QDIO_DBF_HEX4(0,trace,&q->first_to_check,sizeof(int));
 827
 828         return q->first_to_check;
 829 }
 830
 831 inline static int
 832 qdio_has_inbound_q_moved(struct qdio_q *q)
 833 {
 834         int i;
 835
 836 #ifdef QDIO_PERFORMANCE_STATS
 837         static int old_pcis=0;
 838         static int old_thinints=0;
 839
 840         if ((old_pcis==perf_stats.pcis)&&(old_thinints==perf_stats.thinints))
 841                 perf_stats.start_time_inbound=NOW;
 842         else
 843                 old_pcis=perf_stats.pcis;
 844 #endif /* QDIO_PERFORMANCE_STATS */
 845
 846         i=qdio_get_inbound_buffer_frontier(q);
 847         if ( (i!=GET_SAVED_FRONTIER(q)) ||
 848              (q->error_status_flags&QDIO_STATUS_LOOK_FOR_ERROR) ) {
 849                 SAVE_FRONTIER(q,i);
 850                 if ((!q->siga_sync)&&(!q->hydra_gives_outbound_pcis))
 851                         SAVE_TIMESTAMP(q);
 852
 853                 QDIO_DBF_TEXT4(0,trace,"inhasmvd");
 854                 QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 855                 return 1;
 856         } else {
 857                 QDIO_DBF_TEXT4(0,trace,"inhsntmv");
 858                 QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 859                 return 0;
 860         }
 861 }
 862
 863 /* means, no more buffers to be filled */
 864 inline static int
 865 iqdio_is_inbound_q_done(struct qdio_q *q)
 866 {
 867         int no_used;
 868         char dbf_text[15];
 869
 870         no_used=atomic_read(&q->number_of_buffers_used);
 871
 872         /* propagate the change from 82 to 80 through VM */
 873         SYNC_MEMORY;
 874
 875         if (no_used) {
 876                 sprintf(dbf_text,"iqisnt%02x",no_used);
 877                 QDIO_DBF_TEXT4(0,trace,dbf_text);
 878         } else {
 879                 QDIO_DBF_TEXT4(0,trace,"iniqisdo");
 880         }
 881         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 882
 883         if (!no_used)
 884                 return 1;
 885
 886         if (!q->siga_sync)
 887                 /* we'll check for more primed buffers in qeth_stop_polling */
 888                 return 0;
 889
 890         if (q->slsb.acc.val[q->first_to_check]!=SLSB_P_INPUT_PRIMED)
 891                 /*
 892                  * nothing more to do, if next buffer is not PRIMED.
 893                  * note that we did a SYNC_MEMORY before, that there
 894                  * has been a sychnronization.
 895                  * we will return 0 below, as there is nothing to do
 896                  * (stop_polling not necessary, as we have not been
 897                  * using the PROCESSING state
 898                  */
 899                 return 0;
 900
 901         /*
 902          * ok, the next input buffer is primed. that means, that device state
 903          * change indicator and adapter local summary are set, so we will find
 904          * it next time.
 905          * we will return 0 below, as there is nothing to do, except scheduling
 906          * ourselves for the next time.
 907          */
 908         tiqdio_set_summary_bit((__u32*)q->dev_st_chg_ind);
 909         tiqdio_sched_tl();
 910         return 0;
 911 }
 912
 913 inline static int
 914 qdio_is_inbound_q_done(struct qdio_q *q)
 915 {
 916         int no_used;
 917         char dbf_text[15];
 918
 919         no_used=atomic_read(&q->number_of_buffers_used);
 920
 921         /*
 922          * we need that one for synchronization with the adapter, as it
 923          * does a kind of PCI avoidance
 924          */
 925         SYNC_MEMORY;
 926
 927         if (!no_used) {
 928                 QDIO_DBF_TEXT4(0,trace,"inqisdnA");
 929                 QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 930                 QDIO_DBF_TEXT4(0,trace,dbf_text);
 931                 return 1;
 932         }
 933
 934         if (q->slsb.acc.val[q->first_to_check]==SLSB_P_INPUT_PRIMED) {
 935                 /* we got something to do */
 936                 QDIO_DBF_TEXT4(0,trace,"inqisntA");
 937                 QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 938                 return 0;
 939         }
 940
 941         /* on VM, we don't poll, so the q is always done here */
 942         if (q->siga_sync)
 943                 return 1;
 944         if (q->hydra_gives_outbound_pcis)
 945                 return 1;
 946
 947         /*
 948          * at this point we know, that inbound first_to_check
 949          * has (probably) not moved (see qdio_inbound_processing)
 950          */
 951         if (NOW>GET_SAVED_TIMESTAMP(q)+q->timing.threshold) {
 952                 QDIO_DBF_TEXT4(0,trace,"inqisdon");
 953                 QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 954                 sprintf(dbf_text,"pf%02xcn%02x",q->first_to_check,no_used);
 955                 QDIO_DBF_TEXT4(0,trace,dbf_text);
 956                 return 1;
 957         } else {
 958                 QDIO_DBF_TEXT4(0,trace,"inqisntd");
 959                 QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 960                 sprintf(dbf_text,"pf%02xcn%02x",q->first_to_check,no_used);
 961                 QDIO_DBF_TEXT4(0,trace,dbf_text);
 962                 return 0;
 963         }
 964 }
 965
 966 inline static void
 967 qdio_kick_inbound_handler(struct qdio_q *q)
 968 {
 969         int count, start, end, real_end, i;
 970         char dbf_text[15];
 971
 972         QDIO_DBF_TEXT4(0,trace,"kickinh");
 973         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
 974
 975         start=q->first_element_to_kick;
 976         real_end=q->first_to_check;
 977         end=(real_end+QDIO_MAX_BUFFERS_PER_Q-1)&(QDIO_MAX_BUFFERS_PER_Q-1);
 978
 979         i=start;
 980         count=0;
 981         while (1) {
 982                 count++;
 983                 if (i==end)
 984                         break;
 985                 i=(i+1)&(QDIO_MAX_BUFFERS_PER_Q-1);
 986         }
 987
 988         sprintf(dbf_text,"s=%2xc=%2x",start,count);
 989         QDIO_DBF_TEXT4(0,trace,dbf_text);
 990
 991         if (likely(q->state==QDIO_IRQ_STATE_ACTIVE))
 992                 q->handler(q->cdev,
 993                            QDIO_STATUS_INBOUND_INT|q->error_status_flags,
 994                            q->qdio_error,q->siga_error,q->q_no,start,count,
 995                            q->int_parm);
 996
 997         /* for the next time: */
 998         q->first_element_to_kick=real_end;
 999         q->qdio_error=0;
1000         q->siga_error=0;
1001         q->error_status_flags=0;
1002
1003 #ifdef QDIO_PERFORMANCE_STATS
1004         perf_stats.inbound_time+=NOW-perf_stats.start_time_inbound;
1005         perf_stats.inbound_cnt++;
1006 #endif /* QDIO_PERFORMANCE_STATS */
1007 }
1008
1009 static inline void
1010 __tiqdio_inbound_processing(struct qdio_q *q, int spare_ind_was_set)
1011 {
1012         struct qdio_irq *irq_ptr;
1013         struct qdio_q *oq;
1014         int i;
1015
1016         QDIO_DBF_TEXT4(0,trace,"iqinproc");
1017         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
1018
1019         /*
1020          * we first want to reserve the q, so that we know, that we don't
1021          * interrupt ourselves and call qdio_unmark_q, as is_in_shutdown might
1022          * be set
1023          */
1024         if (unlikely(qdio_reserve_q(q))) {
1025                 qdio_release_q(q);
1026 #ifdef QDIO_PERFORMANCE_STATS
1027                 ii_p_c++;
1028 #endif /* QDIO_PERFORMANCE_STATS */
1029                 /*
1030                  * as we might just be about to stop polling, we make
1031                  * sure that we check again at least once more
1032                  */
1033                 tiqdio_sched_tl();
1034                 return;
1035         }
1036 #ifdef QDIO_PERFORMANCE_STATS
1037         ii_p_nc++;
1038 #endif /* QDIO_PERFORMANCE_STATS */
1039         if (unlikely(atomic_read(&q->is_in_shutdown))) {
1040                 qdio_unmark_q(q);
1041                 goto out;
1042         }
1043
1044         /*
1045          * we reset spare_ind_was_set, when the queue does not use the
1046          * spare indicator
1047          */
1048         if (spare_ind_was_set)
1049                 spare_ind_was_set = (q->dev_st_chg_ind == &spare_indicator);
1050
1051         if (!(*(q->dev_st_chg_ind)) && !spare_ind_was_set)
1052                 goto out;
1053         /*
1054          * q->dev_st_chg_ind is the indicator, be it shared or not.
1055          * only clear it, if indicator is non-shared
1056          */
1057         if (!spare_ind_was_set)
1058                 tiqdio_clear_summary_bit((__u32*)q->dev_st_chg_ind);
1059
1060         if (q->hydra_gives_outbound_pcis) {
1061                 if (!q->siga_sync_done_on_thinints) {
1062                         SYNC_MEMORY_ALL;
1063                 } else if ((!q->siga_sync_done_on_outb_tis)&&
1064                          (q->hydra_gives_outbound_pcis)) {
1065                         SYNC_MEMORY_ALL_OUTB;
1066                 }
1067         } else {
1068                 SYNC_MEMORY;
1069         }
1070         /*
1071          * maybe we have to do work on our outbound queues... at least
1072          * we have to check the outbound-int-capable thinint-capable
1073          * queues
1074          */
1075         if (q->hydra_gives_outbound_pcis) {
1076                 irq_ptr = (struct qdio_irq*)q->irq_ptr;
1077                 for (i=0;i<irq_ptr->no_output_qs;i++) {
1078                         oq = irq_ptr->output_qs[i];
1079 #ifdef QDIO_PERFORMANCE_STATS
1080                         perf_stats.tl_runs--;
1081 #endif /* QDIO_PERFORMANCE_STATS */
1082                         if (!qdio_is_outbound_q_done(oq))
1083                                 __qdio_outbound_processing(oq);
1084                 }
1085         }
1086
1087         if (!qdio_has_inbound_q_moved(q))
1088                 goto out;
1089
1090         qdio_kick_inbound_handler(q);
1091         if (iqdio_is_inbound_q_done(q))
1092                 if (!qdio_stop_polling(q)) {
1093                         /*
1094                          * we set the flags to get into the stuff next time,
1095                          * see also comment in qdio_stop_polling
1096                          */
1097                         tiqdio_set_summary_bit((__u32*)q->dev_st_chg_ind);
1098                         tiqdio_sched_tl();
1099                 }
1100 out:
1101         qdio_release_q(q);
1102 }
1103
1104 static void
1105 tiqdio_inbound_processing(struct qdio_q *q)
1106 {
1107         __tiqdio_inbound_processing(q, atomic_read(&spare_indicator_usecount));
1108 }
1109
1110 static inline void
1111 __qdio_inbound_processing(struct qdio_q *q)
1112 {
1113         int q_laps=0;
1114
1115         QDIO_DBF_TEXT4(0,trace,"qinproc");
1116         QDIO_DBF_HEX4(0,trace,&q,sizeof(void*));
1117
1118         if (unlikely(qdio_reserve_q(q))) {
1119                 qdio_release_q(q);
1120 #ifdef QDIO_PERFORMANCE_STATS
1121                 i_p_c++;
1122 #endif /* QDIO_PERFORMANCE_STATS */
1123                 /* as we're sissies, we'll check next time */
1124                 if (likely(!atomic_read(&q->is_in_shutdown))) {
1125                         qdio_mark_q(q);
1126                         QDIO_DBF_TEXT4(0,trace,"busy,agn");
1127                 }
1128                 return;
1129         }
1130 #ifdef QDIO_PERFORMANCE_STATS
1131         i_p_nc++;
1132         perf_stats.tl_runs++;
1133 #endif /* QDIO_PERFORMANCE_STATS */
1134
1135 again:
1136         if (qdio_has_inbound_q_moved(q)) {
1137                 qdio_kick_inbound_handler(q);
1138                 if (!qdio_stop_polling(q)) {
1139                         q_laps++;
1140                         if (q_laps<QDIO_Q_LAPS)
1141                                 goto again;
1142                 }
1143                 qdio_mark_q(q);
1144         } else {
1145                 if (!qdio_is_inbound_q_done(q))
1146                         /* means poll time is not yet over */
1147                         qdio_mark_q(q);
1148         }
1149
1150         qdio_release_q(q);
1151 }
1152
1153 static void
1154 qdio_inbound_processing(struct qdio_q *q)
1155 {
1156         __qdio_inbound_processing(q);
1157 }
1158
1159 /************************* MAIN ROUTINES *******************************/
1160
1161 #ifdef QDIO_USE_PROCESSING_STATE
1162 static inline int
1163 tiqdio_do_inbound_checks(struct qdio_q *q, int q_laps)
1164 {
1165         if (!q) {
1166                 tiqdio_sched_tl();
1167                 return 0;
1168         }
1169
1170         /*
1171          * under VM, we have not used the PROCESSING state, so no
1172          * need to stop polling
1173          */
1174         if (q->siga_sync)
1175                 return 2;
1176
1177         if (unlikely(qdio_reserve_q(q))) {
1178                 qdio_release_q(q);
1179 #ifdef QDIO_PERFORMANCE_STATS
1180                 ii_p_c++;
1181 #endif /* QDIO_PERFORMANCE_STATS */
1182                 /*
1183                  * as we might just be about to stop polling, we make
1184                  * sure that we check again at least once more
1185                  */
1186
1187                 /*
1188                  * sanity -- we'd get here without setting the
1189                  * dev st chg ind
1190                  */
1191                 tiqdio_set_summary_bit((__u32*)q->dev_st_chg_ind);
1192                 tiqdio_sched_tl();
1193                 return 0;
1194         }
1195         if (qdio_stop_polling(q)) {
1196                 qdio_release_q(q);
1197                 return 2;
1198         }
1199         if (q_laps<QDIO_Q_LAPS-1) {
1200                 qdio_release_q(q);
1201                 return 3;
1202         }
1203         /*
1204          * we set the flags to get into the stuff
1205          * next time, see also comment in qdio_stop_polling
1206          */
1207         tiqdio_set_summary_bit((__u32*)q->dev_st_chg_ind);
1208         tiqdio_sched_tl();
1209         qdio_release_q(q);
1210         return 1;
1211
1212 }
1213 #endif /* QDIO_USE_PROCESSING_STATE */
1214
1215 static inline void
1216 tiqdio_inbound_checks(void)
1217 {
1218         struct qdio_q *q;
1219         int spare_ind_was_set=0;
1220 #ifdef QDIO_USE_PROCESSING_STATE
1221         int q_laps=0;
1222 #endif /* QDIO_USE_PROCESSING_STATE */
1223
1224         QDIO_DBF_TEXT4(0,trace,"iqdinbck");
1225         QDIO_DBF_TEXT5(0,trace,"iqlocsum");
1226
1227 #ifdef QDIO_USE_PROCESSING_STATE
1228 again:
1229 #endif /* QDIO_USE_PROCESSING_STATE */
1230
1231         /* when the spare indicator is used and set, save that and clear it */
1232         if ((atomic_read(&spare_indicator_usecount)) && spare_indicator) {
1233                 spare_ind_was_set = 1;
1234                 tiqdio_clear_summary_bit((__u32*)&spare_indicator);
1235         }
1236
1237         q=(struct qdio_q*)tiq_list;
1238         do {
1239                 if (!q)
1240                         break;
1241                 __tiqdio_inbound_processing(q, spare_ind_was_set);
1242                 q=(struct qdio_q*)q->list_next;
1243         } while (q!=(struct qdio_q*)tiq_list);
1244
1245 #ifdef QDIO_USE_PROCESSING_STATE
1246         q=(struct qdio_q*)tiq_list;
1247         do {
1248                 int ret;
1249
1250                 ret = tiqdio_do_inbound_checks(q, q_laps);
1251                 switch (ret) {
1252                 case 0:
1253                         return;
1254                 case 1:
1255                         q_laps++;
1256                 case 2:
1257                         q = (struct qdio_q*)q->list_next;
1258                         break;
1259                 default:
1260                         q_laps++;
1261                         goto again;
1262                 }
1263         } while (q!=(struct qdio_q*)tiq_list);
1264 #endif /* QDIO_USE_PROCESSING_STATE */
1265 }
1266
1267 static void
1268 tiqdio_tl(unsigned long data)
1269 {
1270         QDIO_DBF_TEXT4(0,trace,"iqdio_tl");
1271
1272 #ifdef QDIO_PERFORMANCE_STATS
1273         perf_stats.tl_runs++;
1274 #endif /* QDIO_PERFORMANCE_STATS */
1275
1276         tiqdio_inbound_checks();
1277 }
1278
1279 /********************* GENERAL HELPER_ROUTINES ***********************/
1280
1281 static void
1282 qdio_release_irq_memory(struct qdio_irq *irq_ptr)
1283 {
1284         int i;
1285
1286         for (i=0;i<QDIO_MAX_QUEUES_PER_IRQ;i++) {
1287                 if (!irq_ptr->input_qs[i])
1288                         goto next;
1289
1290                 if (irq_ptr->input_qs[i]->slib)
1291                         kfree(irq_ptr->input_qs[i]->slib);
1292                 kfree(irq_ptr->input_qs[i]);
1293
1294 next:
1295                 if (!irq_ptr->output_qs[i])
1296                         continue;
1297
1298                 if (irq_ptr->output_qs[i]->slib)
1299                         kfree(irq_ptr->output_qs[i]->slib);
1300                 kfree(irq_ptr->output_qs[i]);
1301
1302         }
1303         kfree(irq_ptr->qdr);
1304         kfree(irq_ptr);
1305 }
1306
1307 static void
1308 qdio_set_impl_params(struct qdio_irq *irq_ptr,
1309                      unsigned int qib_param_field_format,
1310                      /* pointer to 128 bytes or NULL, if no param field */
1311                      unsigned char *qib_param_field,
1312                      /* pointer to no_queues*128 words of data or NULL */
1313                      unsigned int no_input_qs,
1314                      unsigned int no_output_qs,
1315                      unsigned long *input_slib_elements,
1316                      unsigned long *output_slib_elements)
1317 {
1318         int i,j;
1319
1320         if (!irq_ptr)
1321                 return;
1322
1323         irq_ptr->qib.pfmt=qib_param_field_format;
1324         if (qib_param_field)
1325                 memcpy(irq_ptr->qib.parm,qib_param_field,
1326                        QDIO_MAX_BUFFERS_PER_Q);
1327
1328         if (input_slib_elements)
1329                 for (i=0;i<no_input_qs;i++) {
1330                         for (j=0;j<QDIO_MAX_BUFFERS_PER_Q;j++)
1331                                 irq_ptr->input_qs[i]->slib->slibe[j].parms=
1332                                         input_slib_elements[
1333                                                 i*QDIO_MAX_BUFFERS_PER_Q+j];
1334                 }
1335         if (output_slib_elements)
1336                 for (i=0;i<no_output_qs;i++) {
1337                         for (j=0;j<QDIO_MAX_BUFFERS_PER_Q;j++)
1338                                 irq_ptr->output_qs[i]->slib->slibe[j].parms=
1339                                         output_slib_elements[
1340                                                 i*QDIO_MAX_BUFFERS_PER_Q+j];
1341                 }
1342 }
1343
1344 static int
1345 qdio_alloc_qs(struct qdio_irq *irq_ptr,
1346               int no_input_qs, int no_output_qs)
1347 {
1348         int i;
1349         struct qdio_q *q;
1350         int result=-ENOMEM;
1351
1352         for (i=0;i<no_input_qs;i++) {
1353                 q=kmalloc(sizeof(struct qdio_q),GFP_KERNEL);
1354
1355                 if (!q) {
1356                         QDIO_PRINT_ERR("kmalloc of q failed!\n");
1357                         goto out;
1358                 }
1359
1360                 memset(q,0,sizeof(struct qdio_q));
1361
1362                 q->slib=kmalloc(PAGE_SIZE,GFP_KERNEL);
1363                 if (!q->slib) {
1364                         QDIO_PRINT_ERR("kmalloc of slib failed!\n");
1365                         goto out;
1366                 }
1367
1368                 irq_ptr->input_qs[i]=q;
1369         }
1370
1371         for (i=0;i<no_output_qs;i++) {
1372                 q=kmalloc(sizeof(struct qdio_q),GFP_KERNEL);
1373
1374                 if (!q) {
1375                         goto out;
1376                 }
1377
1378                 memset(q,0,sizeof(struct qdio_q));
1379
1380                 q->slib=kmalloc(PAGE_SIZE,GFP_KERNEL);
1381                 if (!q->slib) {
1382                         QDIO_PRINT_ERR("kmalloc of slib failed!\n");
1383                         goto out;
1384                 }
1385
1386                 irq_ptr->output_qs[i]=q;
1387         }
1388
1389         result=0;
1390 out:
1391         return result;
1392 }
1393
1394 static void
1395 qdio_fill_qs(struct qdio_irq *irq_ptr, struct ccw_device *cdev,
1396              int no_input_qs, int no_output_qs,
1397              qdio_handler_t *input_handler,
1398              qdio_handler_t *output_handler,
1399              unsigned long int_parm,int q_format,
1400              unsigned long flags,
1401              void **inbound_sbals_array,
1402              void **outbound_sbals_array)
1403 {
1404         struct qdio_q *q;
1405         int i,j;
1406         char dbf_text[20]; /* see qdio_initialize */
1407         void *ptr;
1408         int available;
1409
1410         sprintf(dbf_text,"qfqs%4x",cdev->private->irq);
1411         QDIO_DBF_TEXT0(0,setup,dbf_text);
1412         for (i=0;i<no_input_qs;i++) {
1413                 q=irq_ptr->input_qs[i];
1414
1415                 memset(q,0,((char*)&q->slib)-((char*)q));
1416                 sprintf(dbf_text,"in-q%4x",i);
1417                 QDIO_DBF_TEXT0(0,setup,dbf_text);
1418                 QDIO_DBF_HEX0(0,setup,&q,sizeof(void*));
1419
1420                 memset(q->slib,0,PAGE_SIZE);
1421                 q->sl=(struct sl*)(((char*)q->slib)+PAGE_SIZE/2);
1422
1423                 available=0;
1424
1425                 for (j=0;j<QDIO_MAX_BUFFERS_PER_Q;j++)
1426                         q->sbal[j]=*(inbound_sbals_array++);
1427
1428                 q->queue_type=q_format;
1429                 q->int_parm=int_parm;
1430                 q->irq=irq_ptr->irq;
1431                 q->irq_ptr = irq_ptr;
1432                 q->cdev = cdev;
1433                 q->mask=1<<(31-i);
1434                 q->q_no=i;
1435                 q->is_input_q=1;
1436                 q->first_to_check=0;
1437                 q->last_move_ftc=0;
1438                 q->handler=input_handler;
1439                 q->dev_st_chg_ind=irq_ptr->dev_st_chg_ind;
1440
1441                 q->tasklet.data=(unsigned long)q;
1442                 /* q->is_thinint_q isn't valid at this time, but
1443                  * irq_ptr->is_thinint_irq is */
1444                 q->tasklet.func=(void(*)(unsigned long))
1445                         ((irq_ptr->is_thinint_irq)?&tiqdio_inbound_processing:
1446                          &qdio_inbound_processing);
1447
1448                 /* actually this is not used for inbound queues. yet. */
1449                 atomic_set(&q->busy_siga_counter,0);
1450                 q->timing.busy_start=0;
1451
1452 /*              for (j=0;j<QDIO_STATS_NUMBER;j++)
1453                         q->timing.last_transfer_times[j]=(qdio_get_micros()/
1454                                                           QDIO_STATS_NUMBER)*j;
1455                 q->timing.last_transfer_index=QDIO_STATS_NUMBER-1;
1456 */
1457
1458                 /* fill in slib */
1459                 if (i>0) irq_ptr->input_qs[i-1]->slib->nsliba=
1460                                  (unsigned long)(q->slib);
1461                 q->slib->sla=(unsigned long)(q->sl);
1462                 q->slib->slsba=(unsigned long)(&q->slsb.acc.val[0]);
1463
1464                 /* fill in sl */
1465                 for (j=0;j<QDIO_MAX_BUFFERS_PER_Q;j++)
1466                         q->sl->element[j].sbal=(unsigned long)(q->sbal[j]);
1467
1468                 QDIO_DBF_TEXT2(0,setup,"sl-sb-b0");
1469                 ptr=(void*)q->sl;
1470                 QDIO_DBF_HEX2(0,setup,&ptr,sizeof(void*));
1471                 ptr=(void*)&q->slsb;
1472                 QDIO_DBF_HEX2(0,setup,&ptr,sizeof(void*));
1473                 ptr=(void*)q->sbal[0];
1474                 QDIO_DBF_HEX2(0,setup,&ptr,sizeof(void*));
1475
1476                 /* fill in slsb */
1477                 for (j=0;j<QDIO_MAX_BUFFERS_PER_Q;j++) {
1478                         set_slsb(&q->slsb.acc.val[j],
1479                                  SLSB_P_INPUT_NOT_INIT);
1480 /*                      q->sbal[j]->element[1].sbalf.i1.key=QDIO_STORAGE_KEY;*/
1481                 }
1482         }
1483
1484         for (i=0;i<no_output_qs;i++) {
1485                 q=irq_ptr->output_qs[i];
1486                 memset(q,0,((char*)&q->slib)-((char*)q));
1487
1488                 sprintf(dbf_text,"outq%4x",i);
1489                 QDIO_DBF_TEXT0(0,setup,dbf_text);
1490                 QDIO_DBF_HEX0(0,setup,&q,sizeof(void*));
1491
1492                 memset(q->slib,0,PAGE_SIZE);
1493                 q->sl=(struct sl*)(((char*)q->slib)+PAGE_SIZE/2);
1494
1495                 available=0;
1496
1497                 for (j=0;j<QDIO_MAX_BUFFERS_PER_Q;j++)
1498                         q->sbal[j]=*(outbound_sbals_array++);
1499
1500                 q->queue_type=q_format;
1501                 q->int_parm=int_parm;
1502                 q->is_input_q=0;
1503                 q->irq=irq_ptr->irq;
1504                 q->cdev = cdev;
1505                 q->irq_ptr = irq_ptr;
1506                 q->mask=1<<(31-i);
1507                 q->q_no=i;
1508                 q->first_to_check=0;
1509                 q->last_move_ftc=0;
1510                 q->handler=output_handler;
1511
1512                 q->tasklet.data=(unsigned long)q;
1513                 q->tasklet.func=(void(*)(unsigned long))
1514                         &qdio_outbound_processing;
1515
1516                 atomic_set(&q->busy_siga_counter,0);
1517                 q->timing.busy_start=0;
1518
1519                 /* fill in slib */
1520                 if (i>0) irq_ptr->output_qs[i-1]->slib->nsliba=
1521                                  (unsigned long)(q->slib);
1522                 q->slib->sla=(unsigned long)(q->sl);
1523                 q->slib->slsba=(unsigned long)(&q->slsb.acc.val[0]);
1524
1525                 /* fill in sl */
1526                 for (j=0;j<QDIO_MAX_BUFFERS_PER_Q;j++)
1527                         q->sl->element[j].sbal=(unsigned long)(q->sbal[j]);
1528
1529                 QDIO_DBF_TEXT2(0,setup,"sl-sb-b0");
1530                 ptr=(void*)q->sl;
1531                 QDIO_DBF_HEX2(0,setup,&ptr,sizeof(void*));
1532                 ptr=(void*)&q->slsb;
1533                 QDIO_DBF_HEX2(0,setup,&ptr,sizeof(void*));
1534                 ptr=(void*)q->sbal[0];
1535                 QDIO_DBF_HEX2(0,setup,&ptr,sizeof(void*));
1536
1537                 /* fill in slsb */
1538                 for (j=0;j<QDIO_MAX_BUFFERS_PER_Q;j++) {
1539                         set_slsb(&q->slsb.acc.val[j],
1540                                  SLSB_P_OUTPUT_NOT_INIT);
1541 /*                      q->sbal[j]->element[1].sbalf.i1.key=QDIO_STORAGE_KEY;*/
1542                 }
1543         }
1544 }
1545
1546 static void
1547 qdio_fill_thresholds(struct qdio_irq *irq_ptr,
1548                      unsigned int no_input_qs,
1549                      unsigned int no_output_qs,
1550                      unsigned int min_input_threshold,
1551                      unsigned int max_input_threshold,
1552                      unsigned int min_output_threshold,
1553                      unsigned int max_output_threshold)
1554 {
1555         int i;
1556         struct qdio_q *q;
1557
1558         for (i=0;i<no_input_qs;i++) {
1559                 q=irq_ptr->input_qs[i];
1560                 q->timing.threshold=max_input_threshold;
1561 /*              for (j=0;j<QDIO_STATS_CLASSES;j++) {
1562                         q->threshold_classes[j].threshold=
1563                                 min_input_threshold+
1564                                 (max_input_threshold-min_input_threshold)/
1565                                 QDIO_STATS_CLASSES;
1566                 }
1567                 qdio_use_thresholds(q,QDIO_STATS_CLASSES/2);*/
1568         }
1569         for (i=0;i<no_output_qs;i++) {
1570                 q=irq_ptr->output_qs[i];
1571                 q->timing.threshold=max_output_threshold;
1572 /*              for (j=0;j<QDIO_STATS_CLASSES;j++) {
1573                         q->threshold_classes[j].threshold=
1574                                 min_output_threshold+
1575                                 (max_output_threshold-min_output_threshold)/
1576                                 QDIO_STATS_CLASSES;
1577                 }
1578                 qdio_use_thresholds(q,QDIO_STATS_CLASSES/2);*/
1579         }
1580 }
1581
1582 static int
1583 tiqdio_thinint_handler(void)
1584 {
1585         QDIO_DBF_TEXT4(0,trace,"thin_int");
1586
1587 #ifdef QDIO_PERFORMANCE_STATS
1588         perf_stats.thinints++;
1589         perf_stats.start_time_inbound=NOW;
1590 #endif /* QDIO_PERFORMANCE_STATS */
1591
1592         /* SVS only when needed:
1593          * issue SVS to benefit from iqdio interrupt avoidance
1594          * (SVS clears AISOI)*/
1595         if (!omit_svs)
1596                 tiqdio_clear_global_summary();
1597
1598         tiqdio_inbound_checks();
1599         return 0;
1600 }
1601
1602 static void
1603 qdio_set_state(struct qdio_irq *irq_ptr, enum qdio_irq_states state)
1604 {
1605         int i;
1606         char dbf_text[15];
1607
1608         QDIO_DBF_TEXT5(0,trace,"newstate");
1609         sprintf(dbf_text,"%4x%4x",irq_ptr->irq,state);
1610         QDIO_DBF_TEXT5(0,trace,dbf_text);
1611
1612         irq_ptr->state=state;
1613         for (i=0;i<irq_ptr->no_input_qs;i++)
1614                 irq_ptr->input_qs[i]->state=state;
1615         for (i=0;i<irq_ptr->no_output_qs;i++)
1616                 irq_ptr->output_qs[i]->state=state;
1617         mb();
1618 }
1619
1620 static inline void
1621 qdio_irq_check_sense(int irq, struct irb *irb)
1622 {
1623         char dbf_text[15];
1624
1625         if (irb->esw.esw0.erw.cons) {
1626                 sprintf(dbf_text,"sens%4x",irq);
1627                 QDIO_DBF_TEXT2(1,trace,dbf_text);
1628                 QDIO_DBF_HEX0(0,sense,irb,QDIO_DBF_SENSE_LEN);
1629
1630                 QDIO_PRINT_WARN("sense data available on qdio channel.\n");
1631                 HEXDUMP16(WARN,"irb: ",irb);
1632                 HEXDUMP16(WARN,"sense data: ",irb->ecw);
1633         }
1634
1635 }
1636
1637 static inline void
1638 qdio_handle_pci(struct qdio_irq *irq_ptr)
1639 {
1640         int i;
1641         struct qdio_q *q;
1642
1643 #ifdef QDIO_PERFORMANCE_STATS
1644         perf_stats.pcis++;
1645         perf_stats.start_time_inbound=NOW;
1646 #endif /* QDIO_PERFORMANCE_STATS */
1647         for (i=0;i<irq_ptr->no_input_qs;i++) {
1648                 q=irq_ptr->input_qs[i];
1649                 if (q->is_input_q&QDIO_FLAG_NO_INPUT_INTERRUPT_CONTEXT)
1650                         qdio_mark_q(q);
1651                 else {
1652 #ifdef QDIO_PERFORMANCE_STATS
1653                         perf_stats.tl_runs--;
1654 #endif /* QDIO_PERFORMANCE_STATS */
1655                         __qdio_inbound_processing(q);
1656                 }
1657         }
1658         if (!irq_ptr->hydra_gives_outbound_pcis)
1659                 return;
1660         for (i=0;i<irq_ptr->no_output_qs;i++) {
1661                 q=irq_ptr->output_qs[i];
1662 #ifdef QDIO_PERFORMANCE_STATS
1663                 perf_stats.tl_runs--;
1664 #endif /* QDIO_PERFORMANCE_STATS */
1665                 if (qdio_is_outbound_q_done(q))
1666                         continue;
1667                 if (!irq_ptr->sync_done_on_outb_pcis)
1668                         SYNC_MEMORY;
1669                 __qdio_outbound_processing(q);
1670         }
1671 }
1672
1673 static void qdio_establish_handle_irq(struct ccw_device*, int, int);
1674
1675 static inline void
1676 qdio_handle_activate_check(struct ccw_device *cdev, unsigned long intparm,
1677                            int cstat, int dstat)
1678 {
1679         struct qdio_irq *irq_ptr;
1680         struct qdio_q *q;
1681         char dbf_text[15];
1682
1683         irq_ptr = cdev->private->qdio_data;
1684
1685         QDIO_DBF_TEXT2(1, trace, "ick2");
1686         sprintf(dbf_text,"%s", cdev->dev.bus_id);
1687         QDIO_DBF_TEXT2(1,trace,dbf_text);
1688         QDIO_DBF_HEX2(0,trace,&intparm,sizeof(int));
1689         QDIO_DBF_HEX2(0,trace,&dstat,sizeof(int));
1690         QDIO_DBF_HEX2(0,trace,&cstat,sizeof(int));
1691         QDIO_PRINT_ERR("received check condition on activate " \
1692                        "queues on device %s (cs=x%x, ds=x%x).\n",
1693                        cdev->dev.bus_id, cstat, dstat);
1694         if (irq_ptr->no_input_qs) {
1695                 q=irq_ptr->input_qs[0];
1696         } else if (irq_ptr->no_output_qs) {
1697                 q=irq_ptr->output_qs[0];
1698         } else {
1699                 QDIO_PRINT_ERR("oops... no queue registered for device %s!?\n",
1700                                cdev->dev.bus_id);
1701                 goto omit_handler_call;
1702         }
1703         q->handler(q->cdev,QDIO_STATUS_ACTIVATE_CHECK_CONDITION|
1704                    QDIO_STATUS_LOOK_FOR_ERROR,
1705                    0,0,0,-1,-1,q->int_parm);
1706 omit_handler_call:
1707         qdio_set_state(irq_ptr,QDIO_IRQ_STATE_STOPPED);
1708
1709 }
1710
1711 static void
1712 qdio_call_shutdown(void *data)
1713 {
1714         struct ccw_device *cdev;
1715
1716         cdev = (struct ccw_device *)data;
1717         qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1718         put_device(&cdev->dev);
1719 }
1720
1721 static void
1722 qdio_timeout_handler(struct ccw_device *cdev)
1723 {
1724         struct qdio_irq *irq_ptr;
1725         char dbf_text[15];
1726
1727         QDIO_DBF_TEXT2(0, trace, "qtoh");
1728         sprintf(dbf_text, "%s", cdev->dev.bus_id);
1729         QDIO_DBF_TEXT2(0, trace, dbf_text);
1730
1731         irq_ptr = cdev->private->qdio_data;
1732         sprintf(dbf_text, "state:%d", irq_ptr->state);
1733         QDIO_DBF_TEXT2(0, trace, dbf_text);
1734
1735         switch (irq_ptr->state) {
1736         case QDIO_IRQ_STATE_INACTIVE:
1737                 QDIO_PRINT_ERR("establish queues on irq %04x: timed out\n",
1738                                irq_ptr->irq);
1739                 QDIO_DBF_TEXT2(1,setup,"eq:timeo");
1740                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
1741                 break;
1742         case QDIO_IRQ_STATE_CLEANUP:
1743                 QDIO_PRINT_INFO("Did not get interrupt on cleanup, irq=0x%x.\n",
1744                                 irq_ptr->irq);
1745                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
1746                 break;
1747         case QDIO_IRQ_STATE_ESTABLISHED:
1748         case QDIO_IRQ_STATE_ACTIVE:
1749                 /* I/O has been terminated by common I/O layer. */
1750                 QDIO_PRINT_INFO("Queues on irq %04x killed by cio.\n",
1751                                 irq_ptr->irq);
1752                 QDIO_DBF_TEXT2(1, trace, "cio:term");
1753                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
1754                 if (get_device(&cdev->dev)) {
1755                         /* Can't call shutdown from interrupt context. */
1756                         PREPARE_WORK(&cdev->private->kick_work,
1757                                      qdio_call_shutdown, (void *)cdev);
1758                         queue_work(ccw_device_work, &cdev->private->kick_work);
1759                 }
1760                 break;
1761         default:
1762                 BUG();
1763         }
1764         ccw_device_set_timeout(cdev, 0);
1765         wake_up(&cdev->private->wait_q);
1766 }
1767
1768 static void
1769 qdio_handler(struct ccw_device *cdev, unsigned long intparm, struct irb *irb)
1770 {
1771         struct qdio_irq *irq_ptr;
1772         int cstat,dstat;
1773         char dbf_text[15];
1774
1775         QDIO_DBF_TEXT4(0, trace, "qint");
1776         sprintf(dbf_text, "%s", cdev->dev.bus_id);
1777         QDIO_DBF_TEXT4(0, trace, dbf_text);
1778
1779         if (!intparm) {
1780                 QDIO_PRINT_ERR("got unsolicited interrupt in qdio " \
1781                                   "handler, device %s\n", cdev->dev.bus_id);
1782                 return;
1783         }
1784
1785         irq_ptr = cdev->private->qdio_data;
1786         if (!irq_ptr) {
1787                 QDIO_DBF_TEXT2(1, trace, "uint");
1788                 sprintf(dbf_text,"%s", cdev->dev.bus_id);
1789                 QDIO_DBF_TEXT2(1,trace,dbf_text);
1790                 QDIO_PRINT_ERR("received interrupt on unused device %s!\n",
1791                                cdev->dev.bus_id);
1792                 return;
1793         }
1794
1795         if (IS_ERR(irb)) {
1796                 /* Currently running i/o is in error. */
1797                 switch (PTR_ERR(irb)) {
1798                 case -EIO:
1799                         QDIO_PRINT_ERR("i/o error on device %s\n",
1800                                        cdev->dev.bus_id);
1801                         return;
1802                 case -ETIMEDOUT:
1803                         qdio_timeout_handler(cdev);
1804                         return;
1805                 default:
1806                         QDIO_PRINT_ERR("unknown error state %ld on device %s\n",
1807                                        PTR_ERR(irb), cdev->dev.bus_id);
1808                         return;
1809                 }
1810         }
1811
1812         qdio_irq_check_sense(irq_ptr->irq, irb);
1813
1814         sprintf(dbf_text, "state:%d", irq_ptr->state);
1815         QDIO_DBF_TEXT4(0, trace, dbf_text);
1816
1817         cstat = irb->scsw.cstat;
1818         dstat = irb->scsw.dstat;
1819
1820         switch (irq_ptr->state) {
1821         case QDIO_IRQ_STATE_INACTIVE:
1822                 qdio_establish_handle_irq(cdev, cstat, dstat);
1823                 break;
1824
1825         case QDIO_IRQ_STATE_CLEANUP:
1826                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1827                 break;
1828
1829         case QDIO_IRQ_STATE_ESTABLISHED:
1830         case QDIO_IRQ_STATE_ACTIVE:
1831                 if (cstat & SCHN_STAT_PCI) {
1832                         qdio_handle_pci(irq_ptr);
1833                         break;
1834                 }
1835
1836                 if ((cstat&~SCHN_STAT_PCI)||dstat) {
1837                         qdio_handle_activate_check(cdev, intparm, cstat, dstat);
1838                         break;
1839                 }
1840         default:
1841                 QDIO_PRINT_ERR("got interrupt for queues in state %d on " \
1842                                "device %s?!\n",
1843                                irq_ptr->state, cdev->dev.bus_id);
1844         }
1845         wake_up(&cdev->private->wait_q);
1846
1847 }
1848
1849 int
1850 qdio_synchronize(struct ccw_device *cdev, unsigned int flags,
1851                  unsigned int queue_number)
1852 {
1853         int cc;
1854         struct qdio_q *q;
1855         struct qdio_irq *irq_ptr;
1856         char dbf_text[15]="SyncXXXX";
1857         void *ptr;
1858
1859         irq_ptr = cdev->private->qdio_data;
1860         if (!irq_ptr)
1861                 return -ENODEV;
1862
1863         *((int*)(&dbf_text[4])) = irq_ptr->irq;
1864         QDIO_DBF_HEX4(0,trace,dbf_text,QDIO_DBF_TRACE_LEN);
1865         *((int*)(&dbf_text[0]))=flags;
1866         *((int*)(&dbf_text[4]))=queue_number;
1867         QDIO_DBF_HEX4(0,trace,dbf_text,QDIO_DBF_TRACE_LEN);
1868
1869         if (flags&QDIO_FLAG_SYNC_INPUT) {
1870                 q=irq_ptr->input_qs[queue_number];
1871                 if (!q)
1872                         return -EINVAL;
1873                 cc = do_siga_sync(q->irq, 0, q->mask);
1874         } else if (flags&QDIO_FLAG_SYNC_OUTPUT) {
1875                 q=irq_ptr->output_qs[queue_number];
1876                 if (!q)
1877                         return -EINVAL;
1878                 cc = do_siga_sync(q->irq, q->mask, 0);
1879         } else
1880                 return -EINVAL;
1881
1882         ptr=&cc;
1883         if (cc)
1884                 QDIO_DBF_HEX3(0,trace,&ptr,sizeof(int));
1885
1886         return cc;
1887 }
1888
1889 static unsigned char
1890 qdio_check_siga_needs(int sch)
1891 {
1892         int result;
1893         unsigned char qdioac;
1894
1895         struct {
1896                 struct chsc_header request;
1897                 u16 reserved1;
1898                 u16 first_sch;
1899                 u16 reserved2;
1900                 u16 last_sch;
1901                 u32 reserved3;
1902                 struct chsc_header response;
1903                 u32 reserved4;
1904                 u8  flags;
1905                 u8  reserved5;
1906                 u16 sch;
1907                 u8  qfmt;
1908                 u8  reserved6;
1909                 u8  qdioac;
1910                 u8  sch_class;
1911                 u8  reserved7;
1912                 u8  icnt;
1913                 u8  reserved8;
1914                 u8  ocnt;
1915         } *ssqd_area;
1916
1917         ssqd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
1918         if (!ssqd_area) {
1919                 QDIO_PRINT_WARN("Could not get memory for chsc. Using all " \
1920                                 "SIGAs for sch x%x.\n", sch);
1921                 return CHSC_FLAG_SIGA_INPUT_NECESSARY ||
1922                         CHSC_FLAG_SIGA_OUTPUT_NECESSARY ||
1923                         CHSC_FLAG_SIGA_SYNC_NECESSARY; /* all flags set */
1924         }
1925         ssqd_area->request = (struct chsc_header) {
1926                 .length = 0x0010,
1927                 .code   = 0x0024,
1928         };
1929
1930         ssqd_area->first_sch = sch;
1931         ssqd_area->last_sch = sch;
1932
1933         result=chsc(ssqd_area);
1934
1935         if (result) {
1936                 QDIO_PRINT_WARN("CHSC returned cc %i. Using all " \
1937                                 "SIGAs for sch x%x.\n",
1938                                 result,sch);
1939                 qdioac = CHSC_FLAG_SIGA_INPUT_NECESSARY ||
1940                         CHSC_FLAG_SIGA_OUTPUT_NECESSARY ||
1941                         CHSC_FLAG_SIGA_SYNC_NECESSARY; /* all flags set */
1942                 goto out;
1943         }
1944
1945         if (ssqd_area->response.code != QDIO_CHSC_RESPONSE_CODE_OK) {
1946                 QDIO_PRINT_WARN("response upon checking SIGA needs " \
1947                                 "is 0x%x. Using all SIGAs for sch x%x.\n",
1948                                 ssqd_area->response.code, sch);
1949                 qdioac = CHSC_FLAG_SIGA_INPUT_NECESSARY ||
1950                         CHSC_FLAG_SIGA_OUTPUT_NECESSARY ||
1951                         CHSC_FLAG_SIGA_SYNC_NECESSARY; /* all flags set */
1952                 goto out;
1953         }
1954         if (!(ssqd_area->flags & CHSC_FLAG_QDIO_CAPABILITY) ||
1955             !(ssqd_area->flags & CHSC_FLAG_VALIDITY) ||
1956             (ssqd_area->sch != sch)) {
1957                 QDIO_PRINT_WARN("huh? problems checking out sch x%x... " \
1958                                 "using all SIGAs.\n",sch);
1959                 qdioac = CHSC_FLAG_SIGA_INPUT_NECESSARY |
1960                         CHSC_FLAG_SIGA_OUTPUT_NECESSARY |
1961                         CHSC_FLAG_SIGA_SYNC_NECESSARY; /* worst case */
1962                 goto out;
1963         }
1964
1965         qdioac = ssqd_area->qdioac;
1966 out:
1967         free_page ((unsigned long) ssqd_area);
1968         return qdioac;
1969 }
1970
1971 static unsigned int
1972 tiqdio_check_chsc_availability(void)
1973 {
1974         int result;
1975         char dbf_text[15];
1976
1977         struct {
1978                 struct chsc_header request;
1979                 u32 reserved1;
1980                 u32 reserved2;
1981                 u32 reserved3;
1982                 struct chsc_header response;
1983                 u32 reserved4;
1984                 u32 general_char[510];
1985                 u32 chsc_char[518];
1986         } *scsc_area;
1987
1988         scsc_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
1989         if (!scsc_area) {
1990                 QDIO_PRINT_WARN("Was not able to determine available" \
1991                                 "CHSCs due to no memory.\n");
1992                 return -ENOMEM;
1993         }
1994
1995         scsc_area->request = (struct chsc_header) {
1996                 .length = 0x0010,
1997                 .code   = 0x0010,
1998         };
1999
2000         result=chsc(scsc_area);
2001         if (result) {
2002                 QDIO_PRINT_WARN("Was not able to determine " \
2003                                 "available CHSCs, cc=%i.\n",
2004                                 result);
2005                 result=-EIO;
2006                 goto exit;
2007         }
2008
2009         if (scsc_area->response.code != 1) {
2010                 QDIO_PRINT_WARN("Was not able to determine " \
2011                                 "available CHSCs.\n");
2012                 result=-EIO;
2013                 goto exit;
2014         }
2015         /* Check for bit 41. */
2016         if ((scsc_area->general_char[1] & 0x00400000) != 0x00400000) {
2017                 QDIO_PRINT_WARN("Adapter interruption facility not " \
2018                                 "installed.\n");
2019                 result=-ENOENT;
2020                 goto exit;
2021         }
2022         /* Check for bits 107 and 108. */
2023         if ((scsc_area->chsc_char[3] & 0x00180000) != 0x00180000) {
2024                 QDIO_PRINT_WARN("Set Chan Subsys. Char. & Fast-CHSCs " \
2025                                 "not available.\n");
2026                 result=-ENOENT;
2027                 goto exit;
2028         }
2029
2030         /* Check for OSA/FCP thin interrupts (bit 67). */
2031         hydra_thinints = ((scsc_area->general_char[2] & 0x10000000)
2032                 == 0x10000000);
2033         sprintf(dbf_text,"hydrati%1x", hydra_thinints);
2034         QDIO_DBF_TEXT0(0,setup,dbf_text);
2035
2036         /* Check for aif time delay disablement fac (bit 56). If installed,
2037          * omit svs even under lpar (good point by rick again) */
2038         omit_svs = ((scsc_area->general_char[1] & 0x00000080)
2039                 == 0x00000080);
2040         sprintf(dbf_text,"omitsvs%1x", omit_svs);
2041         QDIO_DBF_TEXT0(0,setup,dbf_text);
2042 exit:
2043         free_page ((unsigned long) scsc_area);
2044         return result;
2045 }
2046
2047
2048 static unsigned int
2049 tiqdio_set_subchannel_ind(struct qdio_irq *irq_ptr, int reset_to_zero)
2050 {
2051         unsigned long real_addr_local_summary_bit;
2052         unsigned long real_addr_dev_st_chg_ind;
2053         void *ptr;
2054         char dbf_text[15];
2055
2056         unsigned int resp_code;
2057         int result;
2058
2059         struct {
2060                 struct chsc_header request;
2061                 u16 operation_code;
2062                 u16 reserved1;
2063                 u32 reserved2;
2064                 u32 reserved3;
2065                 u64 summary_indicator_addr;
2066                 u64 subchannel_indicator_addr;
2067                 u32 ks:4;
2068                 u32 kc:4;
2069                 u32 reserved4:21;
2070                 u32 isc:3;
2071                 u32 reserved5[2];
2072                 u32 subsystem_id;
2073                 u32 reserved6[1004];
2074                 struct chsc_header response;
2075                 u32 reserved7;
2076         } *scssc_area;
2077
2078         if (!irq_ptr->is_thinint_irq)
2079                 return -ENODEV;
2080
2081         if (reset_to_zero) {
2082                 real_addr_local_summary_bit=0;
2083                 real_addr_dev_st_chg_ind=0;
2084         } else {
2085                 real_addr_local_summary_bit=
2086                         virt_to_phys((volatile void *)indicators);
2087                 real_addr_dev_st_chg_ind=
2088                         virt_to_phys((volatile void *)irq_ptr->dev_st_chg_ind);
2089         }
2090
2091         scssc_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
2092         if (!scssc_area) {
2093                 QDIO_PRINT_WARN("No memory for setting indicators on " \
2094                                 "subchannel x%x.\n", irq_ptr->irq);
2095                 return -ENOMEM;
2096         }
2097         scssc_area->request = (struct chsc_header) {
2098                 .length = 0x0fe0,
2099                 .code   = 0x0021,
2100         };
2101         scssc_area->operation_code = 0;
2102
2103         scssc_area->summary_indicator_addr = real_addr_local_summary_bit;
2104         scssc_area->subchannel_indicator_addr = real_addr_dev_st_chg_ind;
2105         scssc_area->ks = QDIO_STORAGE_KEY;
2106         scssc_area->kc = QDIO_STORAGE_KEY;
2107         scssc_area->isc = TIQDIO_THININT_ISC;
2108         scssc_area->subsystem_id = (1<<16) + irq_ptr->irq;
2109
2110         result = chsc(scssc_area);
2111         if (result) {
2112                 QDIO_PRINT_WARN("could not set indicators on irq x%x, " \
2113                                 "cc=%i.\n",irq_ptr->irq,result);
2114                 result = -EIO;
2115                 goto out;
2116         }
2117
2118         resp_code = scssc_area->response.code;
2119         if (resp_code!=QDIO_CHSC_RESPONSE_CODE_OK) {
2120                 QDIO_PRINT_WARN("response upon setting indicators " \
2121                                 "is 0x%x.\n",resp_code);
2122                 sprintf(dbf_text,"sidR%4x",resp_code);
2123                 QDIO_DBF_TEXT1(0,trace,dbf_text);
2124                 QDIO_DBF_TEXT1(0,setup,dbf_text);
2125                 ptr=&scssc_area->response;
2126                 QDIO_DBF_HEX2(1,setup,&ptr,QDIO_DBF_SETUP_LEN);
2127                 result = -EIO;
2128                 goto out;
2129         }
2130
2131         QDIO_DBF_TEXT2(0,setup,"setscind");
2132         QDIO_DBF_HEX2(0,setup,&real_addr_local_summary_bit,
2133                       sizeof(unsigned long));
2134         QDIO_DBF_HEX2(0,setup,&real_addr_dev_st_chg_ind,sizeof(unsigned long));
2135         result = 0;
2136 out:
2137         free_page ((unsigned long) scssc_area);
2138         return result;
2139
2140 }
2141
2142 static unsigned int
2143 tiqdio_set_delay_target(struct qdio_irq *irq_ptr, unsigned long delay_target)
2144 {
2145         unsigned int resp_code;
2146         int result;
2147         void *ptr;
2148         char dbf_text[15];
2149
2150         struct {
2151                 struct chsc_header request;
2152                 u16 operation_code;
2153                 u16 reserved1;
2154                 u32 reserved2;
2155                 u32 reserved3;
2156                 u32 reserved4[2];
2157                 u32 delay_target;
2158                 u32 reserved5[1009];
2159                 struct chsc_header response;
2160                 u32 reserved6;
2161         } *scsscf_area;
2162
2163         if (!irq_ptr->is_thinint_irq)
2164                 return -ENODEV;
2165
2166         scsscf_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
2167         if (!scsscf_area) {
2168                 QDIO_PRINT_WARN("No memory for setting delay target on " \
2169                                 "subchannel x%x.\n", irq_ptr->irq);
2170                 return -ENOMEM;
2171         }
2172         scsscf_area->request = (struct chsc_header) {
2173                 .length = 0x0fe0,
2174                 .code   = 0x1027,
2175         };
2176
2177         scsscf_area->delay_target = delay_target<<16;
2178
2179         result=chsc(scsscf_area);
2180         if (result) {
2181                 QDIO_PRINT_WARN("could not set delay target on irq x%x, " \
2182                                 "cc=%i. Continuing.\n",irq_ptr->irq,result);
2183                 result = -EIO;
2184                 goto out;
2185         }
2186
2187         resp_code = scsscf_area->response.code;
2188         if (resp_code!=QDIO_CHSC_RESPONSE_CODE_OK) {
2189                 QDIO_PRINT_WARN("response upon setting delay target " \
2190                                 "is 0x%x. Continuing.\n",resp_code);
2191                 sprintf(dbf_text,"sdtR%4x",resp_code);
2192                 QDIO_DBF_TEXT1(0,trace,dbf_text);
2193                 QDIO_DBF_TEXT1(0,setup,dbf_text);
2194                 ptr=&scsscf_area->response;
2195                 QDIO_DBF_HEX2(1,trace,&ptr,QDIO_DBF_TRACE_LEN);
2196         }
2197         QDIO_DBF_TEXT2(0,trace,"delytrgt");
2198         QDIO_DBF_HEX2(0,trace,&delay_target,sizeof(unsigned long));
2199         result = 0; /* not critical */
2200 out:
2201         free_page ((unsigned long) scsscf_area);
2202         return result;
2203 }
2204
2205 int
2206 qdio_cleanup(struct ccw_device *cdev, int how)
2207 {
2208         struct qdio_irq *irq_ptr;
2209         char dbf_text[15];
2210         int rc;
2211
2212         irq_ptr = cdev->private->qdio_data;
2213         if (!irq_ptr)
2214                 return -ENODEV;
2215
2216         sprintf(dbf_text,"qcln%4x",irq_ptr->irq);
2217         QDIO_DBF_TEXT1(0,trace,dbf_text);
2218         QDIO_DBF_TEXT0(0,setup,dbf_text);
2219
2220         rc = qdio_shutdown(cdev, how);
2221         if ((rc == 0) || (rc == -EINPROGRESS))
2222                 rc = qdio_free(cdev);
2223         return rc;
2224 }
2225
2226 int
2227 qdio_shutdown(struct ccw_device *cdev, int how)
2228 {
2229         struct qdio_irq *irq_ptr;
2230         int i;
2231         int result = 0;
2232         int rc;
2233         unsigned long flags;
2234         int timeout;
2235         char dbf_text[15];
2236
2237         irq_ptr = cdev->private->qdio_data;
2238         if (!irq_ptr)
2239                 return -ENODEV;
2240
2241         down(&irq_ptr->setting_up_sema);
2242
2243         sprintf(dbf_text,"qsqs%4x",irq_ptr->irq);
2244         QDIO_DBF_TEXT1(0,trace,dbf_text);
2245         QDIO_DBF_TEXT0(0,setup,dbf_text);
2246
2247         /* mark all qs as uninteresting */
2248         for (i=0;i<irq_ptr->no_input_qs;i++)
2249                 atomic_set(&irq_ptr->input_qs[i]->is_in_shutdown,1);
2250
2251         for (i=0;i<irq_ptr->no_output_qs;i++)
2252                 atomic_set(&irq_ptr->output_qs[i]->is_in_shutdown,1);
2253
2254         tasklet_kill(&tiqdio_tasklet);
2255
2256         for (i=0;i<irq_ptr->no_input_qs;i++) {
2257                 qdio_unmark_q(irq_ptr->input_qs[i]);
2258                 tasklet_kill(&irq_ptr->input_qs[i]->tasklet);
2259                 wait_event_interruptible_timeout(cdev->private->wait_q,
2260                                                  !atomic_read(&irq_ptr->
2261                                                               input_qs[i]->
2262                                                               use_count),
2263                                                  QDIO_NO_USE_COUNT_TIMEOUT);
2264                 if (atomic_read(&irq_ptr->input_qs[i]->use_count))
2265                         result=-EINPROGRESS;
2266         }
2267
2268         for (i=0;i<irq_ptr->no_output_qs;i++) {
2269                 tasklet_kill(&irq_ptr->output_qs[i]->tasklet);
2270                 wait_event_interruptible_timeout(cdev->private->wait_q,
2271                                                  !atomic_read(&irq_ptr->
2272                                                               output_qs[i]->
2273                                                               use_count),
2274                                                  QDIO_NO_USE_COUNT_TIMEOUT);
2275                 if (atomic_read(&irq_ptr->output_qs[i]->use_count))
2276                         result=-EINPROGRESS;
2277         }
2278
2279         /* cleanup subchannel */
2280         spin_lock_irqsave(get_ccwdev_lock(cdev),flags);
2281         if (how&QDIO_FLAG_CLEANUP_USING_CLEAR) {
2282                 rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
2283                 timeout=QDIO_CLEANUP_CLEAR_TIMEOUT;
2284         } else if (how&QDIO_FLAG_CLEANUP_USING_HALT) {
2285                 rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
2286                 timeout=QDIO_CLEANUP_HALT_TIMEOUT;
2287         } else { /* default behaviour */
2288                 rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
2289                 timeout=QDIO_CLEANUP_HALT_TIMEOUT;
2290         }
2291         if (rc == -ENODEV) {
2292                 /* No need to wait for device no longer present. */
2293                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
2294                 spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
2295         } else if (rc == 0) {
2296                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_CLEANUP);
2297                 ccw_device_set_timeout(cdev, timeout);
2298                 spin_unlock_irqrestore(get_ccwdev_lock(cdev),flags);
2299
2300                 wait_event(cdev->private->wait_q,
2301                            irq_ptr->state == QDIO_IRQ_STATE_INACTIVE ||
2302                            irq_ptr->state == QDIO_IRQ_STATE_ERR);
2303         } else {
2304                 QDIO_PRINT_INFO("ccw_device_{halt,clear} returned %d for "
2305                                 "device %s\n", result, cdev->dev.bus_id);
2306                 spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
2307                 result = rc;
2308                 goto out;
2309         }
2310         if (irq_ptr->is_thinint_irq) {
2311                 qdio_put_indicator((__u32*)irq_ptr->dev_st_chg_ind);
2312                 tiqdio_set_subchannel_ind(irq_ptr,1);
2313                 /* reset adapter interrupt indicators */
2314         }
2315
2316         /* exchange int handlers, if necessary */
2317         if ((void*)cdev->handler == (void*)qdio_handler)
2318                 cdev->handler=irq_ptr->original_int_handler;
2319
2320         /* Ignore errors. */
2321         qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
2322         ccw_device_set_timeout(cdev, 0);
2323 out:
2324         up(&irq_ptr->setting_up_sema);
2325         return result;
2326 }
2327
2328 int
2329 qdio_free(struct ccw_device *cdev)
2330 {
2331         struct qdio_irq *irq_ptr;
2332         char dbf_text[15];
2333
2334         irq_ptr = cdev->private->qdio_data;
2335         if (!irq_ptr)
2336                 return -ENODEV;
2337
2338         down(&irq_ptr->setting_up_sema);
2339
2340         sprintf(dbf_text,"qfqs%4x",irq_ptr->irq);
2341         QDIO_DBF_TEXT1(0,trace,dbf_text);
2342         QDIO_DBF_TEXT0(0,setup,dbf_text);
2343
2344         cdev->private->qdio_data = 0;
2345
2346         up(&irq_ptr->setting_up_sema);
2347
2348         qdio_release_irq_memory(irq_ptr);
2349         module_put(THIS_MODULE);
2350         return 0;
2351 }
2352
2353 static inline void
2354 qdio_allocate_do_dbf(struct qdio_initialize *init_data)
2355 {
2356         char dbf_text[20]; /* if a printf printed out more than 8 chars */
2357
2358         sprintf(dbf_text,"qfmt:%x",init_data->q_format);
2359         QDIO_DBF_TEXT0(0,setup,dbf_text);
2360         QDIO_DBF_HEX0(0,setup,init_data->adapter_name,8);
2361         sprintf(dbf_text,"qpff%4x",init_data->qib_param_field_format);
2362         QDIO_DBF_TEXT0(0,setup,dbf_text);
2363         QDIO_DBF_HEX0(0,setup,&init_data->qib_param_field,sizeof(char*));
2364         QDIO_DBF_HEX0(0,setup,&init_data->input_slib_elements,sizeof(long*));
2365         QDIO_DBF_HEX0(0,setup,&init_data->output_slib_elements,sizeof(long*));
2366         sprintf(dbf_text,"miit%4x",init_data->min_input_threshold);
2367         QDIO_DBF_TEXT0(0,setup,dbf_text);
2368         sprintf(dbf_text,"mait%4x",init_data->max_input_threshold);
2369         QDIO_DBF_TEXT0(0,setup,dbf_text);
2370         sprintf(dbf_text,"miot%4x",init_data->min_output_threshold);
2371         QDIO_DBF_TEXT0(0,setup,dbf_text);
2372         sprintf(dbf_text,"maot%4x",init_data->max_output_threshold);
2373         QDIO_DBF_TEXT0(0,setup,dbf_text);
2374         sprintf(dbf_text,"niq:%4x",init_data->no_input_qs);
2375         QDIO_DBF_TEXT0(0,setup,dbf_text);
2376         sprintf(dbf_text,"noq:%4x",init_data->no_output_qs);
2377         QDIO_DBF_TEXT0(0,setup,dbf_text);
2378         QDIO_DBF_HEX0(0,setup,&init_data->input_handler,sizeof(void*));
2379         QDIO_DBF_HEX0(0,setup,&init_data->output_handler,sizeof(void*));
2380         QDIO_DBF_HEX0(0,setup,&init_data->int_parm,sizeof(long));
2381         QDIO_DBF_HEX0(0,setup,&init_data->flags,sizeof(long));
2382         QDIO_DBF_HEX0(0,setup,&init_data->input_sbal_addr_array,sizeof(void*));
2383         QDIO_DBF_HEX0(0,setup,&init_data->output_sbal_addr_array,sizeof(void*));
2384 }
2385
2386 static inline void
2387 qdio_allocate_fill_input_desc(struct qdio_irq *irq_ptr, int i, int iqfmt)
2388 {
2389         irq_ptr->input_qs[i]->is_iqdio_q = iqfmt;
2390         irq_ptr->input_qs[i]->is_thinint_q = irq_ptr->is_thinint_irq;
2391
2392         irq_ptr->qdr->qdf0[i].sliba=(unsigned long)(irq_ptr->input_qs[i]->slib);
2393
2394         irq_ptr->qdr->qdf0[i].sla=(unsigned long)(irq_ptr->input_qs[i]->sl);
2395
2396         irq_ptr->qdr->qdf0[i].slsba=
2397                 (unsigned long)(&irq_ptr->input_qs[i]->slsb.acc.val[0]);
2398
2399         irq_ptr->qdr->qdf0[i].akey=QDIO_STORAGE_KEY;
2400         irq_ptr->qdr->qdf0[i].bkey=QDIO_STORAGE_KEY;
2401         irq_ptr->qdr->qdf0[i].ckey=QDIO_STORAGE_KEY;
2402         irq_ptr->qdr->qdf0[i].dkey=QDIO_STORAGE_KEY;
2403 }
2404
2405 static inline void
2406 qdio_allocate_fill_output_desc(struct qdio_irq *irq_ptr, int i,
2407                                int j, int iqfmt)
2408 {
2409         irq_ptr->output_qs[i]->is_iqdio_q = iqfmt;
2410         irq_ptr->output_qs[i]->is_thinint_q = irq_ptr->is_thinint_irq;
2411
2412         irq_ptr->qdr->qdf0[i+j].sliba=(unsigned long)(irq_ptr->output_qs[i]->slib);
2413
2414         irq_ptr->qdr->qdf0[i+j].sla=(unsigned long)(irq_ptr->output_qs[i]->sl);
2415
2416         irq_ptr->qdr->qdf0[i+j].slsba=
2417                 (unsigned long)(&irq_ptr->output_qs[i]->slsb.acc.val[0]);
2418
2419         irq_ptr->qdr->qdf0[i+j].akey=QDIO_STORAGE_KEY;
2420         irq_ptr->qdr->qdf0[i+j].bkey=QDIO_STORAGE_KEY;
2421         irq_ptr->qdr->qdf0[i+j].ckey=QDIO_STORAGE_KEY;
2422         irq_ptr->qdr->qdf0[i+j].dkey=QDIO_STORAGE_KEY;
2423 }
2424
2425
2426 static inline void
2427 qdio_initialize_set_siga_flags_input(struct qdio_irq *irq_ptr)
2428 {
2429         int i;
2430
2431         for (i=0;i<irq_ptr->no_input_qs;i++) {
2432                 irq_ptr->input_qs[i]->siga_sync=
2433                         irq_ptr->qdioac&CHSC_FLAG_SIGA_SYNC_NECESSARY;
2434                 irq_ptr->input_qs[i]->siga_in=
2435                         irq_ptr->qdioac&CHSC_FLAG_SIGA_INPUT_NECESSARY;
2436                 irq_ptr->input_qs[i]->siga_out=
2437                         irq_ptr->qdioac&CHSC_FLAG_SIGA_OUTPUT_NECESSARY;
2438                 irq_ptr->input_qs[i]->siga_sync_done_on_thinints=
2439                         irq_ptr->qdioac&CHSC_FLAG_SIGA_SYNC_DONE_ON_THININTS;
2440                 irq_ptr->input_qs[i]->hydra_gives_outbound_pcis=
2441                         irq_ptr->hydra_gives_outbound_pcis;
2442                 irq_ptr->input_qs[i]->siga_sync_done_on_outb_tis=
2443                         ((irq_ptr->qdioac&
2444                           (CHSC_FLAG_SIGA_SYNC_DONE_ON_OUTB_PCIS|
2445                            CHSC_FLAG_SIGA_SYNC_DONE_ON_THININTS))==
2446                          (CHSC_FLAG_SIGA_SYNC_DONE_ON_OUTB_PCIS|
2447                           CHSC_FLAG_SIGA_SYNC_DONE_ON_THININTS));
2448
2449         }
2450 }
2451
2452 static inline void
2453 qdio_initialize_set_siga_flags_output(struct qdio_irq *irq_ptr)
2454 {
2455         int i;
2456
2457         for (i=0;i<irq_ptr->no_output_qs;i++) {
2458                 irq_ptr->output_qs[i]->siga_sync=
2459                         irq_ptr->qdioac&CHSC_FLAG_SIGA_SYNC_NECESSARY;
2460                 irq_ptr->output_qs[i]->siga_in=
2461                         irq_ptr->qdioac&CHSC_FLAG_SIGA_INPUT_NECESSARY;
2462                 irq_ptr->output_qs[i]->siga_out=
2463                         irq_ptr->qdioac&CHSC_FLAG_SIGA_OUTPUT_NECESSARY;
2464                 irq_ptr->output_qs[i]->siga_sync_done_on_thinints=
2465                         irq_ptr->qdioac&CHSC_FLAG_SIGA_SYNC_DONE_ON_THININTS;
2466                 irq_ptr->output_qs[i]->hydra_gives_outbound_pcis=
2467                         irq_ptr->hydra_gives_outbound_pcis;
2468                 irq_ptr->output_qs[i]->siga_sync_done_on_outb_tis=
2469                         ((irq_ptr->qdioac&
2470                           (CHSC_FLAG_SIGA_SYNC_DONE_ON_OUTB_PCIS|
2471                            CHSC_FLAG_SIGA_SYNC_DONE_ON_THININTS))==
2472                          (CHSC_FLAG_SIGA_SYNC_DONE_ON_OUTB_PCIS|
2473                           CHSC_FLAG_SIGA_SYNC_DONE_ON_THININTS));
2474
2475         }
2476 }
2477
2478 static inline int
2479 qdio_establish_irq_check_for_errors(struct ccw_device *cdev, int cstat,
2480                                     int dstat)
2481 {
2482         char dbf_text[15];
2483         struct qdio_irq *irq_ptr;
2484
2485         irq_ptr = cdev->private->qdio_data;
2486
2487         if (cstat || (dstat & ~(DEV_STAT_CHN_END|DEV_STAT_DEV_END))) {
2488                 sprintf(dbf_text,"ick1%4x",irq_ptr->irq);
2489                 QDIO_DBF_TEXT2(1,trace,dbf_text);
2490                 QDIO_DBF_HEX2(0,trace,&dstat,sizeof(int));
2491                 QDIO_DBF_HEX2(0,trace,&cstat,sizeof(int));
2492                 QDIO_PRINT_ERR("received check condition on establish " \
2493                                "queues on irq 0x%x (cs=x%x, ds=x%x).\n",
2494                                irq_ptr->irq,cstat,dstat);
2495                 qdio_set_state(irq_ptr,QDIO_IRQ_STATE_ERR);
2496         }
2497
2498         if (!(dstat & DEV_STAT_DEV_END)) {
2499                 QDIO_DBF_TEXT2(1,setup,"eq:no de");
2500                 QDIO_DBF_HEX2(0,setup,&dstat, sizeof(dstat));
2501                 QDIO_DBF_HEX2(0,setup,&cstat, sizeof(cstat));
2502                 QDIO_PRINT_ERR("establish queues on irq %04x: didn't get "
2503                                "device end: dstat=%02x, cstat=%02x\n",
2504                                irq_ptr->irq, dstat, cstat);
2505                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
2506                 return 1;
2507         }
2508
2509         if (dstat & ~(DEV_STAT_CHN_END|DEV_STAT_DEV_END)) {
2510                 QDIO_DBF_TEXT2(1,setup,"eq:badio");
2511                 QDIO_DBF_HEX2(0,setup,&dstat, sizeof(dstat));
2512                 QDIO_DBF_HEX2(0,setup,&cstat, sizeof(cstat));
2513                 QDIO_PRINT_ERR("establish queues on irq %04x: got "
2514                                "the following devstat: dstat=%02x, "
2515                                "cstat=%02x\n",
2516                                irq_ptr->irq, dstat, cstat);
2517                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
2518                 return 1;
2519         }
2520         return 0;
2521 }
2522
2523 static void
2524 qdio_establish_handle_irq(struct ccw_device *cdev, int cstat, int dstat)
2525 {
2526         struct qdio_irq *irq_ptr;
2527         char dbf_text[15];
2528
2529         irq_ptr = cdev->private->qdio_data;
2530
2531         sprintf(dbf_text,"qehi%4x",cdev->private->irq);
2532         QDIO_DBF_TEXT0(0,setup,dbf_text);
2533         QDIO_DBF_TEXT0(0,trace,dbf_text);
2534
2535         if (qdio_establish_irq_check_for_errors(cdev, cstat, dstat)) {
2536                 ccw_device_set_timeout(cdev, 0);
2537                 return;
2538         }
2539
2540         qdio_set_state(irq_ptr,QDIO_IRQ_STATE_ESTABLISHED);
2541         ccw_device_set_timeout(cdev, 0);
2542 }
2543
2544 int
2545 qdio_initialize(struct qdio_initialize *init_data)
2546 {
2547         int rc;
2548         char dbf_text[15];
2549
2550         sprintf(dbf_text,"qini%4x",init_data->cdev->private->irq);
2551         QDIO_DBF_TEXT0(0,setup,dbf_text);
2552         QDIO_DBF_TEXT0(0,trace,dbf_text);
2553
2554         rc = qdio_allocate(init_data);
2555         if (rc == 0) {
2556                 rc = qdio_establish(init_data);
2557                 if (rc != 0)
2558                         qdio_free(init_data->cdev);
2559         }
2560
2561         return rc;
2562 }
2563
2564
2565 int
2566 qdio_allocate(struct qdio_initialize *init_data)
2567 {
2568         struct qdio_irq *irq_ptr;
2569         char dbf_text[15];
2570
2571         sprintf(dbf_text,"qalc%4x",init_data->cdev->private->irq);
2572         QDIO_DBF_TEXT0(0,setup,dbf_text);
2573         QDIO_DBF_TEXT0(0,trace,dbf_text);
2574         if ( (init_data->no_input_qs>QDIO_MAX_QUEUES_PER_IRQ) ||
2575              (init_data->no_output_qs>QDIO_MAX_QUEUES_PER_IRQ) ||
2576              ((init_data->no_input_qs) && (!init_data->input_handler)) ||
2577              ((init_data->no_output_qs) && (!init_data->output_handler)) )
2578                 return -EINVAL;
2579
2580         if (!init_data->input_sbal_addr_array)
2581                 return -EINVAL;
2582
2583         if (!init_data->output_sbal_addr_array)
2584                 return -EINVAL;
2585
2586         qdio_allocate_do_dbf(init_data);
2587
2588         /* create irq */
2589         irq_ptr=kmalloc(sizeof(struct qdio_irq), GFP_KERNEL | GFP_DMA);
2590
2591         QDIO_DBF_TEXT0(0,setup,"irq_ptr:");
2592         QDIO_DBF_HEX0(0,setup,&irq_ptr,sizeof(void*));
2593
2594         if (!irq_ptr) {
2595                 QDIO_PRINT_ERR("kmalloc of irq_ptr failed!\n");
2596                 return -ENOMEM;
2597         }
2598
2599         memset(irq_ptr,0,sizeof(struct qdio_irq));
2600
2601         init_MUTEX(&irq_ptr->setting_up_sema);
2602
2603         irq_ptr->qdr=kmalloc(sizeof(struct qdr), GFP_KERNEL | GFP_DMA);
2604         if (!(irq_ptr->qdr)) {
2605                 kfree(irq_ptr);
2606                 QDIO_PRINT_ERR("kmalloc of irq_ptr->qdr failed!\n");
2607                 return -ENOMEM;
2608         }
2609         QDIO_DBF_TEXT0(0,setup,"qdr:");
2610         QDIO_DBF_HEX0(0,setup,&irq_ptr->qdr,sizeof(void*));
2611
2612         if (qdio_alloc_qs(irq_ptr,
2613                           init_data->no_input_qs,
2614                           init_data->no_output_qs)) {
2615                 qdio_release_irq_memory(irq_ptr);
2616                 return -ENOMEM;
2617         }
2618
2619         init_data->cdev->private->qdio_data = irq_ptr;
2620
2621         qdio_set_state(irq_ptr,QDIO_IRQ_STATE_INACTIVE);
2622
2623         return 0;
2624 }
2625
2626 int qdio_fill_irq(struct qdio_initialize *init_data)
2627 {
2628         int i;
2629         char dbf_text[15];
2630         struct ciw *ciw;
2631         int is_iqdio;
2632         struct qdio_irq *irq_ptr;
2633
2634         irq_ptr = init_data->cdev->private->qdio_data;
2635
2636         memset(irq_ptr,0,((char*)&irq_ptr->qdr)-((char*)irq_ptr));
2637
2638         /* wipes qib.ac, required by ar7063 */
2639         memset(irq_ptr->qdr,0,sizeof(struct qdr));
2640
2641         irq_ptr->int_parm=init_data->int_parm;
2642
2643         irq_ptr->irq = init_data->cdev->private->irq;
2644         irq_ptr->no_input_qs=init_data->no_input_qs;
2645         irq_ptr->no_output_qs=init_data->no_output_qs;
2646
2647         if (init_data->q_format==QDIO_IQDIO_QFMT) {
2648                 irq_ptr->is_iqdio_irq=1;
2649                 irq_ptr->is_thinint_irq=1;
2650         } else {
2651                 irq_ptr->is_iqdio_irq=0;
2652                 irq_ptr->is_thinint_irq=hydra_thinints;
2653         }
2654         sprintf(dbf_text,"is_i_t%1x%1x",
2655                 irq_ptr->is_iqdio_irq,irq_ptr->is_thinint_irq);
2656         QDIO_DBF_TEXT2(0,setup,dbf_text);
2657
2658         if (irq_ptr->is_thinint_irq) {
2659                 irq_ptr->dev_st_chg_ind=qdio_get_indicator();
2660                 QDIO_DBF_HEX1(0,setup,&irq_ptr->dev_st_chg_ind,sizeof(void*));
2661                 if (!irq_ptr->dev_st_chg_ind) {
2662                         QDIO_PRINT_WARN("no indicator location available " \
2663                                         "for irq 0x%x\n",irq_ptr->irq);
2664                         qdio_release_irq_memory(irq_ptr);
2665                         return -ENOBUFS;
2666                 }
2667         }
2668
2669         /* defaults */
2670         irq_ptr->equeue.cmd=DEFAULT_ESTABLISH_QS_CMD;
2671         irq_ptr->equeue.count=DEFAULT_ESTABLISH_QS_COUNT;
2672         irq_ptr->aqueue.cmd=DEFAULT_ACTIVATE_QS_CMD;
2673         irq_ptr->aqueue.count=DEFAULT_ACTIVATE_QS_COUNT;
2674
2675         qdio_fill_qs(irq_ptr, init_data->cdev,
2676                      init_data->no_input_qs,
2677                      init_data->no_output_qs,
2678                      init_data->input_handler,
2679                      init_data->output_handler,init_data->int_parm,
2680                      init_data->q_format,init_data->flags,
2681                      init_data->input_sbal_addr_array,
2682                      init_data->output_sbal_addr_array);
2683
2684         if (!try_module_get(THIS_MODULE)) {
2685                 QDIO_PRINT_CRIT("try_module_get() failed!\n");
2686                 qdio_release_irq_memory(irq_ptr);
2687                 return -EINVAL;
2688         }
2689
2690         qdio_fill_thresholds(irq_ptr,init_data->no_input_qs,
2691                              init_data->no_output_qs,
2692                              init_data->min_input_threshold,
2693                              init_data->max_input_threshold,
2694                              init_data->min_output_threshold,
2695                              init_data->max_output_threshold);
2696
2697         /* fill in qdr */
2698         irq_ptr->qdr->qfmt=init_data->q_format;
2699         irq_ptr->qdr->iqdcnt=init_data->no_input_qs;
2700         irq_ptr->qdr->oqdcnt=init_data->no_output_qs;
2701         irq_ptr->qdr->iqdsz=sizeof(struct qdesfmt0)/4; /* size in words */
2702         irq_ptr->qdr->oqdsz=sizeof(struct qdesfmt0)/4;
2703
2704         irq_ptr->qdr->qiba=(unsigned long)&irq_ptr->qib;
2705         irq_ptr->qdr->qkey=QDIO_STORAGE_KEY;
2706
2707         /* fill in qib */
2708         irq_ptr->qib.qfmt=init_data->q_format;
2709         if (init_data->no_input_qs)
2710                 irq_ptr->qib.isliba=(unsigned long)(irq_ptr->input_qs[0]->slib);
2711         if (init_data->no_output_qs)
2712                 irq_ptr->qib.osliba=(unsigned long)(irq_ptr->output_qs[0]->slib);
2713         memcpy(irq_ptr->qib.ebcnam,init_data->adapter_name,8);
2714
2715         qdio_set_impl_params(irq_ptr,init_data->qib_param_field_format,
2716                              init_data->qib_param_field,
2717                              init_data->no_input_qs,
2718                              init_data->no_output_qs,
2719                              init_data->input_slib_elements,
2720                              init_data->output_slib_elements);
2721
2722         /* first input descriptors, then output descriptors */
2723         is_iqdio = (init_data->q_format == QDIO_IQDIO_QFMT) ? 1 : 0;
2724         for (i=0;i<init_data->no_input_qs;i++)
2725                 qdio_allocate_fill_input_desc(irq_ptr, i, is_iqdio);
2726
2727         for (i=0;i<init_data->no_output_qs;i++)
2728                 qdio_allocate_fill_output_desc(irq_ptr, i,
2729                                                init_data->no_input_qs,
2730                                                is_iqdio);
2731
2732         /* qdr, qib, sls, slsbs, slibs, sbales filled. */
2733
2734         /* get qdio commands */
2735         ciw = ccw_device_get_ciw(init_data->cdev, CIW_TYPE_EQUEUE);
2736         if (!ciw) {
2737                 QDIO_DBF_TEXT2(1,setup,"no eq");
2738                 QDIO_PRINT_INFO("No equeue CIW found for QDIO commands. "
2739                                 "Trying to use default.\n");
2740         } else
2741                 irq_ptr->equeue = *ciw;
2742         ciw = ccw_device_get_ciw(init_data->cdev, CIW_TYPE_AQUEUE);
2743         if (!ciw) {
2744                 QDIO_DBF_TEXT2(1,setup,"no aq");
2745                 QDIO_PRINT_INFO("No aqueue CIW found for QDIO commands. "
2746                                 "Trying to use default.\n");
2747         } else
2748                 irq_ptr->aqueue = *ciw;
2749
2750         /* Set new interrupt handler. */
2751         irq_ptr->original_int_handler = init_data->cdev->handler;
2752         init_data->cdev->handler = qdio_handler;
2753
2754         return 0;
2755 }
2756
2757 int
2758 qdio_establish(struct qdio_initialize *init_data)
2759 {
2760         struct qdio_irq *irq_ptr;
2761         unsigned long saveflags;
2762         int result, result2;
2763         struct ccw_device *cdev;
2764         char dbf_text[20];
2765
2766         cdev=init_data->cdev;
2767         irq_ptr = cdev->private->qdio_data;
2768         if (!irq_ptr)
2769                 return -EINVAL;
2770
2771         if (cdev->private->state != DEV_STATE_ONLINE)
2772                 return -EINVAL;
2773
2774         down(&irq_ptr->setting_up_sema);
2775
2776         qdio_fill_irq(init_data);
2777
2778         /* the thinint CHSC stuff */
2779         if (irq_ptr->is_thinint_irq) {
2780
2781                 result = tiqdio_set_subchannel_ind(irq_ptr,0);
2782                 if (result) {
2783                         up(&irq_ptr->setting_up_sema);
2784                         qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
2785                         return result;
2786                 }
2787                 tiqdio_set_delay_target(irq_ptr,TIQDIO_DELAY_TARGET);
2788         }
2789
2790         sprintf(dbf_text,"qest%4x",cdev->private->irq);
2791         QDIO_DBF_TEXT0(0,setup,dbf_text);
2792         QDIO_DBF_TEXT0(0,trace,dbf_text);
2793
2794         /* establish q */
2795         irq_ptr->ccw.cmd_code=irq_ptr->equeue.cmd;
2796         irq_ptr->ccw.flags=CCW_FLAG_SLI;
2797         irq_ptr->ccw.count=irq_ptr->equeue.count;
2798         irq_ptr->ccw.cda=QDIO_GET_ADDR(irq_ptr->qdr);
2799
2800         spin_lock_irqsave(get_ccwdev_lock(cdev),saveflags);
2801
2802         ccw_device_set_options(cdev, 0);
2803         result=ccw_device_start_timeout(cdev,&irq_ptr->ccw,
2804                                         QDIO_DOING_ESTABLISH,0, 0,
2805                                         QDIO_ESTABLISH_TIMEOUT);
2806         if (result) {
2807                 result2=ccw_device_start_timeout(cdev,&irq_ptr->ccw,
2808                                                  QDIO_DOING_ESTABLISH,0,0,
2809                                                  QDIO_ESTABLISH_TIMEOUT);
2810                 sprintf(dbf_text,"eq:io%4x",result);
2811                 QDIO_DBF_TEXT2(1,setup,dbf_text);
2812                 if (result2) {
2813                         sprintf(dbf_text,"eq:io%4x",result);
2814                         QDIO_DBF_TEXT2(1,setup,dbf_text);
2815                 }
2816                 QDIO_PRINT_WARN("establish queues on irq %04x: do_IO " \
2817                            "returned %i, next try returned %i\n",
2818                            irq_ptr->irq,result,result2);
2819                 result=result2;
2820                 if (result)
2821                         ccw_device_set_timeout(cdev, 0);
2822         }
2823
2824         spin_unlock_irqrestore(get_ccwdev_lock(cdev),saveflags);
2825
2826         if (result) {
2827                 up(&irq_ptr->setting_up_sema);
2828                 qdio_shutdown(cdev,QDIO_FLAG_CLEANUP_USING_CLEAR);
2829                 return result;
2830         }
2831
2832         wait_event_interruptible_timeout(cdev->private->wait_q,
2833                  irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED ||
2834                  irq_ptr->state == QDIO_IRQ_STATE_ERR,
2835                  QDIO_ESTABLISH_TIMEOUT);
2836
2837         if (irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED)
2838                 result = 0;
2839         else {
2840                 up(&irq_ptr->setting_up_sema);
2841                 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
2842                 return -EIO;
2843         }
2844
2845         irq_ptr->qdioac=qdio_check_siga_needs(irq_ptr->irq);
2846         /* if this gets set once, we're running under VM and can omit SVSes */
2847         if (irq_ptr->qdioac&CHSC_FLAG_SIGA_SYNC_NECESSARY)
2848                 omit_svs=1;
2849
2850         sprintf(dbf_text,"qdioac%2x",irq_ptr->qdioac);
2851         QDIO_DBF_TEXT2(0,setup,dbf_text);
2852
2853         sprintf(dbf_text,"qib ac%2x",irq_ptr->qib.ac);
2854         QDIO_DBF_TEXT2(0,setup,dbf_text);
2855
2856         irq_ptr->hydra_gives_outbound_pcis=
2857                 irq_ptr->qib.ac&QIB_AC_OUTBOUND_PCI_SUPPORTED;
2858         irq_ptr->sync_done_on_outb_pcis=
2859                 irq_ptr->qdioac&CHSC_FLAG_SIGA_SYNC_DONE_ON_OUTB_PCIS;
2860
2861         qdio_initialize_set_siga_flags_input(irq_ptr);
2862         qdio_initialize_set_siga_flags_output(irq_ptr);
2863
2864         up(&irq_ptr->setting_up_sema);
2865
2866         return result;
2867
2868 }
2869
2870 int
2871 qdio_activate(struct ccw_device *cdev, int flags)
2872 {
2873         struct qdio_irq *irq_ptr;
2874         int i,result=0,result2;
2875         unsigned long saveflags;
2876         char dbf_text[20]; /* see qdio_initialize */
2877
2878         irq_ptr = cdev->private->qdio_data;
2879         if (!irq_ptr)
2880                 return -ENODEV;
2881
2882         if (cdev->private->state != DEV_STATE_ONLINE)
2883                 return -EINVAL;
2884
2885         down(&irq_ptr->setting_up_sema);
2886         if (irq_ptr->state==QDIO_IRQ_STATE_INACTIVE) {
2887                 result=-EBUSY;
2888                 goto out;
2889         }
2890
2891         sprintf(dbf_text,"qact%4x", irq_ptr->irq);
2892         QDIO_DBF_TEXT2(0,setup,dbf_text);
2893         QDIO_DBF_TEXT2(0,trace,dbf_text);
2894
2895         /* activate q */
2896         irq_ptr->ccw.cmd_code=irq_ptr->aqueue.cmd;
2897         irq_ptr->ccw.flags=CCW_FLAG_SLI;
2898         irq_ptr->ccw.count=irq_ptr->aqueue.count;
2899         irq_ptr->ccw.cda=QDIO_GET_ADDR(0);
2900
2901         spin_lock_irqsave(get_ccwdev_lock(cdev),saveflags);
2902
2903         ccw_device_set_timeout(cdev, 0);
2904         ccw_device_set_options(cdev, CCWDEV_REPORT_ALL);
2905         result=ccw_device_start(cdev,&irq_ptr->ccw,QDIO_DOING_ACTIVATE,
2906                                 0, DOIO_DENY_PREFETCH);
2907         if (result) {
2908                 result2=ccw_device_start(cdev,&irq_ptr->ccw,
2909                                          QDIO_DOING_ACTIVATE,0,0);
2910                 sprintf(dbf_text,"aq:io%4x",result);
2911                 QDIO_DBF_TEXT2(1,setup,dbf_text);
2912                 if (result2) {
2913                         sprintf(dbf_text,"aq:io%4x",result);
2914                         QDIO_DBF_TEXT2(1,setup,dbf_text);
2915                 }
2916                 QDIO_PRINT_WARN("activate queues on irq %04x: do_IO " \
2917                            "returned %i, next try returned %i\n",
2918                            irq_ptr->irq,result,result2);
2919                 result=result2;
2920         }
2921
2922         spin_unlock_irqrestore(get_ccwdev_lock(cdev),saveflags);
2923         if (result)
2924                 goto out;
2925
2926         for (i=0;i<irq_ptr->no_input_qs;i++) {
2927                 if (irq_ptr->is_thinint_irq) {
2928                         /*
2929                          * that way we know, that, if we will get interrupted
2930                          * by tiqdio_inbound_processing, qdio_unmark_q will
2931                          * not be called
2932                          */
2933                         qdio_reserve_q(irq_ptr->input_qs[i]);
2934                         qdio_mark_tiq(irq_ptr->input_qs[i]);
2935                         qdio_release_q(irq_ptr->input_qs[i]);
2936                 }
2937         }
2938
2939         if (flags&QDIO_FLAG_NO_INPUT_INTERRUPT_CONTEXT) {
2940                 for (i=0;i<irq_ptr->no_input_qs;i++) {
2941                         irq_ptr->input_qs[i]->is_input_q|=
2942                                 QDIO_FLAG_NO_INPUT_INTERRUPT_CONTEXT;
2943                 }
2944         }
2945
2946         wait_event_interruptible_timeout(cdev->private->wait_q,
2947                                          ((irq_ptr->state ==
2948                                           QDIO_IRQ_STATE_STOPPED) ||
2949                                           (irq_ptr->state ==
2950                                            QDIO_IRQ_STATE_ERR)),
2951                                          QDIO_ACTIVATE_TIMEOUT);
2952
2953         switch (irq_ptr->state) {
2954         case QDIO_IRQ_STATE_STOPPED:
2955         case QDIO_IRQ_STATE_ERR:
2956                 up(&irq_ptr->setting_up_sema);
2957                 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
2958                 down(&irq_ptr->setting_up_sema);
2959                 result = -EIO;
2960                 break;
2961         default:
2962                 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ACTIVE);
2963                 result = 0;
2964         }
2965  out:
2966         up(&irq_ptr->setting_up_sema);
2967
2968         return result;
2969 }
2970
2971 /* buffers filled forwards again to make Rick happy */
2972 static inline void
2973 qdio_do_qdio_fill_input(struct qdio_q *q, unsigned int qidx,
2974                         unsigned int count, struct qdio_buffer *buffers)
2975 {
2976         for (;;) {
2977                 set_slsb(&q->slsb.acc.val[qidx],SLSB_CU_INPUT_EMPTY);
2978                 count--;
2979                 if (!count) break;
2980                 qidx=(qidx+1)&(QDIO_MAX_BUFFERS_PER_Q-1);
2981         }
2982
2983         /* not necessary, as the queues are synced during the SIGA read */
2984         /*SYNC_MEMORY;*/
2985 }
2986
2987 static inline void
2988 qdio_do_qdio_fill_output(struct qdio_q *q, unsigned int qidx,
2989                          unsigned int count, struct qdio_buffer *buffers)
2990 {
2991         for (;;) {
2992                 set_slsb(&q->slsb.acc.val[qidx],SLSB_CU_OUTPUT_PRIMED);
2993                 count--;
2994                 if (!count) break;
2995                 qidx=(qidx+1)&(QDIO_MAX_BUFFERS_PER_Q-1);
2996         }
2997
2998         /* SIGA write will sync the queues */
2999         /*SYNC_MEMORY;*/
3000 }
3001
3002 static inline void
3003 do_qdio_handle_inbound(struct qdio_q *q, unsigned int callflags,
3004                        unsigned int qidx, unsigned int count,
3005                        struct qdio_buffer *buffers)
3006 {
3007         int used_elements;
3008
3009         /* This is the inbound handling of queues */
3010         used_elements=atomic_add_return(count, &q->number_of_buffers_used) - count;
3011
3012         qdio_do_qdio_fill_input(q,qidx,count,buffers);
3013
3014         if ((used_elements+count==QDIO_MAX_BUFFERS_PER_Q)&&
3015             (callflags&QDIO_FLAG_UNDER_INTERRUPT))
3016                 atomic_swap(&q->polling,0);
3017
3018         if (used_elements)
3019                 return;
3020         if (callflags&QDIO_FLAG_DONT_SIGA)
3021                 return;
3022         if (q->siga_in) {
3023                 int result;
3024
3025                 result=qdio_siga_input(q);
3026                 if (result) {
3027                         if (q->siga_error)
3028                                 q->error_status_flags|=
3029                                         QDIO_STATUS_MORE_THAN_ONE_SIGA_ERROR;
3030                         q->error_status_flags|=QDIO_STATUS_LOOK_FOR_ERROR;
3031                         q->siga_error=result;
3032                 }
3033         }
3034
3035         qdio_mark_q(q);
3036 }
3037
3038 static inline void
3039 do_qdio_handle_outbound(struct qdio_q *q, unsigned int callflags,
3040                         unsigned int qidx, unsigned int count,
3041                         struct qdio_buffer *buffers)
3042 {
3043         int used_elements;
3044
3045         /* This is the outbound handling of queues */
3046 #ifdef QDIO_PERFORMANCE_STATS
3047         perf_stats.start_time_outbound=NOW;
3048 #endif /* QDIO_PERFORMANCE_STATS */
3049
3050         qdio_do_qdio_fill_output(q,qidx,count,buffers);
3051
3052         used_elements=atomic_add_return(count, &q->number_of_buffers_used) - count;
3053
3054         if (callflags&QDIO_FLAG_DONT_SIGA) {
3055 #ifdef QDIO_PERFORMANCE_STATS
3056                 perf_stats.outbound_time+=NOW-perf_stats.start_time_outbound;
3057                 perf_stats.outbound_cnt++;
3058 #endif /* QDIO_PERFORMANCE_STATS */
3059                 return;
3060         }
3061         if (q->is_iqdio_q) {
3062                 /* one siga for every sbal */
3063                 while (count--)
3064                         qdio_kick_outbound_q(q);
3065
3066                 __qdio_outbound_processing(q);
3067         } else {
3068                 /* under VM, we do a SIGA sync unconditionally */
3069                 SYNC_MEMORY;
3070                 else {
3071                         /*
3072                          * w/o shadow queues (else branch of
3073                          * SYNC_MEMORY :-/ ), we try to
3074                          * fast-requeue buffers
3075                          */
3076                         if (q->slsb.acc.val[(qidx+QDIO_MAX_BUFFERS_PER_Q-1)
3077                                             &(QDIO_MAX_BUFFERS_PER_Q-1)]!=
3078                             SLSB_CU_OUTPUT_PRIMED) {
3079                                 qdio_kick_outbound_q(q);
3080                         } else {
3081                                 QDIO_DBF_TEXT3(0,trace, "fast-req");
3082 #ifdef QDIO_PERFORMANCE_STATS
3083                                 perf_stats.fast_reqs++;
3084 #endif /* QDIO_PERFORMANCE_STATS */
3085                         }
3086                 }
3087                 /*
3088                  * only marking the q could take too long,
3089                  * the upper layer module could do a lot of
3090                  * traffic in that time
3091                  */
3092                 __qdio_outbound_processing(q);
3093         }
3094
3095 #ifdef QDIO_PERFORMANCE_STATS
3096         perf_stats.outbound_time+=NOW-perf_stats.start_time_outbound;
3097         perf_stats.outbound_cnt++;
3098 #endif /* QDIO_PERFORMANCE_STATS */
3099 }
3100
3101 /* count must be 1 in iqdio */
3102 int
3103 do_QDIO(struct ccw_device *cdev,unsigned int callflags,
3104         unsigned int queue_number, unsigned int qidx,
3105         unsigned int count,struct qdio_buffer *buffers)
3106 {
3107         struct qdio_irq *irq_ptr;
3108
3109         char dbf_text[20];
3110
3111         sprintf(dbf_text,"doQD%04x",cdev->private->irq);
3112         QDIO_DBF_TEXT3(0,trace,dbf_text);
3113
3114         if ( (qidx>QDIO_MAX_BUFFERS_PER_Q) ||
3115              (count>QDIO_MAX_BUFFERS_PER_Q) ||
3116              (queue_number>QDIO_MAX_QUEUES_PER_IRQ) )
3117                 return -EINVAL;
3118
3119         if (count==0)
3120                 return 0;
3121
3122         irq_ptr = cdev->private->qdio_data;
3123         if (!irq_ptr)
3124                 return -ENODEV;
3125
3126         if (callflags&QDIO_FLAG_SYNC_INPUT)
3127                 QDIO_DBF_HEX3(0,trace,&irq_ptr->input_qs[queue_number],
3128                               sizeof(void*));
3129         else
3130                 QDIO_DBF_HEX3(0,trace,&irq_ptr->output_qs[queue_number],
3131                               sizeof(void*));
3132         sprintf(dbf_text,"flag%04x",callflags);
3133         QDIO_DBF_TEXT3(0,trace,dbf_text);
3134         sprintf(dbf_text,"qi%02xct%02x",qidx,count);
3135         QDIO_DBF_TEXT3(0,trace,dbf_text);
3136
3137         if (irq_ptr->state!=QDIO_IRQ_STATE_ACTIVE)
3138                 return -EBUSY;
3139
3140         if (callflags&QDIO_FLAG_SYNC_INPUT)
3141                 do_qdio_handle_inbound(irq_ptr->input_qs[queue_number],
3142                                        callflags, qidx, count, buffers);
3143         else if (callflags&QDIO_FLAG_SYNC_OUTPUT)
3144                 do_qdio_handle_outbound(irq_ptr->output_qs[queue_number],
3145                                         callflags, qidx, count, buffers);
3146         else {
3147                 QDIO_DBF_TEXT3(1,trace,"doQD:inv");
3148                 return -EINVAL;
3149         }
3150         return 0;
3151 }
3152
3153 #ifdef QDIO_PERFORMANCE_STATS
3154 static int
3155 qdio_perf_procfile_read(char *buffer, char **buffer_location, off_t offset,
3156                         int buffer_length, int *eof, void *data)
3157 {
3158         int c=0;
3159
3160         /* we are always called with buffer_length=4k, so we all
3161            deliver on the first read */
3162         if (offset>0)
3163                 return 0;
3164
3165 #define _OUTP_IT(x...) c+=sprintf(buffer+c,x)
3166         _OUTP_IT("i_p_nc/c=%lu/%lu\n",i_p_nc,i_p_c);
3167         _OUTP_IT("ii_p_nc/c=%lu/%lu\n",ii_p_nc,ii_p_c);
3168         _OUTP_IT("o_p_nc/c=%lu/%lu\n",o_p_nc,o_p_c);
3169         _OUTP_IT("Number of tasklet runs (total)                  : %u\n",
3170                  perf_stats.tl_runs);
3171         _OUTP_IT("\n");
3172         _OUTP_IT("Number of SIGA sync's issued                    : %u\n",
3173                  perf_stats.siga_syncs);
3174         _OUTP_IT("Number of SIGA in's issued                      : %u\n",
3175                  perf_stats.siga_ins);
3176         _OUTP_IT("Number of SIGA out's issued                     : %u\n",
3177                  perf_stats.siga_outs);
3178         _OUTP_IT("Number of PCIs caught                           : %u\n",
3179                  perf_stats.pcis);
3180         _OUTP_IT("Number of adapter interrupts caught             : %u\n",
3181                  perf_stats.thinints);
3182         _OUTP_IT("Number of fast requeues (outg. SBALs w/o SIGA)  : %u\n",
3183                  perf_stats.fast_reqs);
3184         _OUTP_IT("\n");
3185         _OUTP_IT("Total time of all inbound actions (us) incl. UL : %u\n",
3186                  perf_stats.inbound_time);
3187         _OUTP_IT("Number of inbound transfers                     : %u\n",
3188                  perf_stats.inbound_cnt);
3189         _OUTP_IT("Total time of all outbound do_QDIOs (us)        : %u\n",
3190                  perf_stats.outbound_time);
3191         _OUTP_IT("Number of do_QDIOs outbound                     : %u\n",
3192                  perf_stats.outbound_cnt);
3193         _OUTP_IT("\n");
3194
3195         return c;
3196 }
3197
3198 static struct proc_dir_entry *qdio_perf_proc_file;
3199 #endif /* QDIO_PERFORMANCE_STATS */
3200
3201 static void
3202 qdio_add_procfs_entry(void)
3203 {
3204 #ifdef QDIO_PERFORMANCE_STATS
3205         proc_perf_file_registration=0;
3206         qdio_perf_proc_file=create_proc_entry(QDIO_PERF,
3207                                               S_IFREG|0444,&proc_root);
3208         if (qdio_perf_proc_file) {
3209                 qdio_perf_proc_file->read_proc=&qdio_perf_procfile_read;
3210         } else proc_perf_file_registration=-1;
3211
3212         if (proc_perf_file_registration)
3213                 QDIO_PRINT_WARN("was not able to register perf. " \
3214                                 "proc-file (%i).\n",
3215                                 proc_perf_file_registration);
3216 #endif /* QDIO_PERFORMANCE_STATS */
3217 }
3218
3219 static void
3220 qdio_remove_procfs_entry(void)
3221 {
3222 #ifdef QDIO_PERFORMANCE_STATS
3223         perf_stats.tl_runs=0;
3224
3225         if (!proc_perf_file_registration) /* means if it went ok earlier */
3226                 remove_proc_entry(QDIO_PERF,&proc_root);
3227 #endif /* QDIO_PERFORMANCE_STATS */
3228 }
3229
3230 static void
3231 tiqdio_register_thinints(void)
3232 {
3233         char dbf_text[20];
3234         register_thinint_result=
3235                 s390_register_adapter_interrupt(&tiqdio_thinint_handler);
3236         if (register_thinint_result) {
3237                 sprintf(dbf_text,"regthn%x",(register_thinint_result&0xff));
3238                 QDIO_DBF_TEXT0(0,setup,dbf_text);
3239                 QDIO_PRINT_ERR("failed to register adapter handler " \
3240                                "(rc=%i).\nAdapter interrupts might " \
3241                                "not work. Continuing.\n",
3242                                register_thinint_result);
3243         }
3244 }
3245
3246 static void
3247 tiqdio_unregister_thinints(void)
3248 {
3249         if (!register_thinint_result)
3250                 s390_unregister_adapter_interrupt(&tiqdio_thinint_handler);
3251 }
3252
3253 static int
3254 qdio_get_qdio_memory(void)
3255 {
3256         int i;
3257         indicator_used[0]=1;
3258
3259         for (i=1;i<INDICATORS_PER_CACHELINE;i++)
3260                 indicator_used[i]=0;
3261         indicators=(__u32*)kmalloc(sizeof(__u32)*(INDICATORS_PER_CACHELINE),
3262                                    GFP_KERNEL);
3263         if (!indicators) return -ENOMEM;
3264         memset(indicators,0,sizeof(__u32)*(INDICATORS_PER_CACHELINE));
3265         return 0;
3266 }
3267
3268 static void
3269 qdio_release_qdio_memory(void)
3270 {
3271         if (indicators)
3272                 kfree(indicators);
3273 }
3274
3275 static void
3276 qdio_unregister_dbf_views(void)
3277 {
3278         if (qdio_dbf_setup)
3279                 debug_unregister(qdio_dbf_setup);
3280         if (qdio_dbf_sbal)
3281                 debug_unregister(qdio_dbf_sbal);
3282         if (qdio_dbf_sense)
3283                 debug_unregister(qdio_dbf_sense);
3284         if (qdio_dbf_trace)
3285                 debug_unregister(qdio_dbf_trace);
3286 #ifdef QDIO_DBF_LIKE_HELL
3287         if (qdio_dbf_slsb_out)
3288                 debug_unregister(qdio_dbf_slsb_out);
3289         if (qdio_dbf_slsb_in)
3290                 debug_unregister(qdio_dbf_slsb_in);
3291 #endif /* QDIO_DBF_LIKE_HELL */
3292 }
3293
3294 static int
3295 qdio_register_dbf_views(void)
3296 {
3297         qdio_dbf_setup=debug_register(QDIO_DBF_SETUP_NAME,
3298                                       QDIO_DBF_SETUP_INDEX,
3299                                       QDIO_DBF_SETUP_NR_AREAS,
3300                                       QDIO_DBF_SETUP_LEN);
3301         if (!qdio_dbf_setup)
3302                 goto oom;
3303         debug_register_view(qdio_dbf_setup,&debug_hex_ascii_view);
3304         debug_set_level(qdio_dbf_setup,QDIO_DBF_SETUP_LEVEL);
3305
3306         qdio_dbf_sbal=debug_register(QDIO_DBF_SBAL_NAME,
3307                                      QDIO_DBF_SBAL_INDEX,
3308                                      QDIO_DBF_SBAL_NR_AREAS,
3309                                      QDIO_DBF_SBAL_LEN);
3310         if (!qdio_dbf_sbal)
3311                 goto oom;
3312
3313         debug_register_view(qdio_dbf_sbal,&debug_hex_ascii_view);
3314         debug_set_level(qdio_dbf_sbal,QDIO_DBF_SBAL_LEVEL);
3315
3316         qdio_dbf_sense=debug_register(QDIO_DBF_SENSE_NAME,
3317                                       QDIO_DBF_SENSE_INDEX,
3318                                       QDIO_DBF_SENSE_NR_AREAS,
3319                                       QDIO_DBF_SENSE_LEN);
3320         if (!qdio_dbf_sense)
3321                 goto oom;
3322
3323         debug_register_view(qdio_dbf_sense,&debug_hex_ascii_view);
3324         debug_set_level(qdio_dbf_sense,QDIO_DBF_SENSE_LEVEL);
3325
3326         qdio_dbf_trace=debug_register(QDIO_DBF_TRACE_NAME,
3327                                       QDIO_DBF_TRACE_INDEX,
3328                                       QDIO_DBF_TRACE_NR_AREAS,
3329                                       QDIO_DBF_TRACE_LEN);
3330         if (!qdio_dbf_trace)
3331                 goto oom;
3332
3333         debug_register_view(qdio_dbf_trace,&debug_hex_ascii_view);
3334         debug_set_level(qdio_dbf_trace,QDIO_DBF_TRACE_LEVEL);
3335
3336 #ifdef QDIO_DBF_LIKE_HELL
3337         qdio_dbf_slsb_out=debug_register(QDIO_DBF_SLSB_OUT_NAME,
3338                                          QDIO_DBF_SLSB_OUT_INDEX,
3339                                          QDIO_DBF_SLSB_OUT_NR_AREAS,
3340                                          QDIO_DBF_SLSB_OUT_LEN);
3341         if (!qdio_dbf_slsb_out)
3342                 goto oom;
3343         debug_register_view(qdio_dbf_slsb_out,&debug_hex_ascii_view);
3344         debug_set_level(qdio_dbf_slsb_out,QDIO_DBF_SLSB_OUT_LEVEL);
3345
3346         qdio_dbf_slsb_in=debug_register(QDIO_DBF_SLSB_IN_NAME,
3347                                         QDIO_DBF_SLSB_IN_INDEX,
3348                                         QDIO_DBF_SLSB_IN_NR_AREAS,
3349                                         QDIO_DBF_SLSB_IN_LEN);
3350         if (!qdio_dbf_slsb_in)
3351                 goto oom;
3352         debug_register_view(qdio_dbf_slsb_in,&debug_hex_ascii_view);
3353         debug_set_level(qdio_dbf_slsb_in,QDIO_DBF_SLSB_IN_LEVEL);
3354 #endif /* QDIO_DBF_LIKE_HELL */
3355         return 0;
3356 oom:
3357         QDIO_PRINT_ERR("not enough memory for dbf.\n");
3358         qdio_unregister_dbf_views();
3359         return -ENOMEM;
3360 }
3361
3362 static int __init
3363 init_QDIO(void)
3364 {
3365         int res;
3366 #ifdef QDIO_PERFORMANCE_STATS
3367         void *ptr;
3368 #endif /* QDIO_PERFORMANCE_STATS */
3369
3370         printk("qdio: loading %s\n",version);
3371
3372         res=qdio_get_qdio_memory();
3373         if (res)
3374                 return res;
3375
3376         res = qdio_register_dbf_views();
3377         if (res)
3378                 return res;
3379
3380         QDIO_DBF_TEXT0(0,setup,"initQDIO");
3381
3382 #ifdef QDIO_PERFORMANCE_STATS
3383         memset((void*)&perf_stats,0,sizeof(perf_stats));
3384         QDIO_DBF_TEXT0(0,setup,"perfstat");
3385         ptr=&perf_stats;
3386         QDIO_DBF_HEX0(0,setup,&ptr,sizeof(void*));
3387 #endif /* QDIO_PERFORMANCE_STATS */
3388
3389         qdio_add_procfs_entry();
3390
3391         if (tiqdio_check_chsc_availability())
3392                 QDIO_PRINT_ERR("Not all CHSCs supported. Continuing.\n");
3393
3394         tiqdio_register_thinints();
3395
3396         return 0;
3397  }
3398
3399 static void __exit
3400 cleanup_QDIO(void)
3401 {
3402         tiqdio_unregister_thinints();
3403         qdio_remove_procfs_entry();
3404         qdio_release_qdio_memory();
3405         qdio_unregister_dbf_views();
3406
3407         printk("qdio: %s: module removed\n",version);
3408 }
3409
3410 module_init(init_QDIO);
3411 module_exit(cleanup_QDIO);
3412
3413 EXPORT_SYMBOL(qdio_allocate);
3414 EXPORT_SYMBOL(qdio_establish);
3415 EXPORT_SYMBOL(qdio_initialize);
3416 EXPORT_SYMBOL(qdio_activate);
3417 EXPORT_SYMBOL(do_QDIO);
3418 EXPORT_SYMBOL(qdio_shutdown);
3419 EXPORT_SYMBOL(qdio_free);
3420 EXPORT_SYMBOL(qdio_cleanup);
3421 EXPORT_SYMBOL(qdio_synchronize);