1/** 2 * IBM Accelerator Family 'GenWQE' 3 * 4 * (C) Copyright IBM Corp. 2013 5 * 6 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com> 7 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com> 8 * Author: Michael Jung <mijung@gmx.net> 9 * Author: Michael Ruettger <michael@ibmra.de> 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License (version 2 only) 13 * as published by the Free Software Foundation. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 */ 20 21/* 22 * Device Driver Control Block (DDCB) queue support. Definition of 23 * interrupt handlers for queue support as well as triggering the 24 * health monitor code in case of problems. The current hardware uses 25 * an MSI interrupt which is shared between error handling and 26 * functional code. 27 */ 28 29#include <linux/types.h> 30#include <linux/module.h> 31#include <linux/sched.h> 32#include <linux/wait.h> 33#include <linux/pci.h> 34#include <linux/string.h> 35#include <linux/dma-mapping.h> 36#include <linux/delay.h> 37#include <linux/module.h> 38#include <linux/interrupt.h> 39#include <linux/crc-itu-t.h> 40 41#include "card_base.h" 42#include "card_ddcb.h" 43 44/* 45 * N: next DDCB, this is where the next DDCB will be put. 46 * A: active DDCB, this is where the code will look for the next completion. 47 * x: DDCB is enqueued, we are waiting for its completion. 48 49 * Situation (1): Empty queue 50 * +---+---+---+---+---+---+---+---+ 51 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 52 * | | | | | | | | | 53 * +---+---+---+---+---+---+---+---+ 54 * A/N 55 * enqueued_ddcbs = A - N = 2 - 2 = 0 56 * 57 * Situation (2): Wrapped, N > A 58 * +---+---+---+---+---+---+---+---+ 59 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 60 * | | | x | x | | | | | 61 * +---+---+---+---+---+---+---+---+ 62 * A N 63 * enqueued_ddcbs = N - A = 4 - 2 = 2 64 * 65 * Situation (3): Queue wrapped, A > N 66 * +---+---+---+---+---+---+---+---+ 67 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 68 * | x | x | | | x | x | x | x | 69 * +---+---+---+---+---+---+---+---+ 70 * N A 71 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 2) = 6 72 * 73 * Situation (4a): Queue full N > A 74 * +---+---+---+---+---+---+---+---+ 75 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 76 * | x | x | x | x | x | x | x | | 77 * +---+---+---+---+---+---+---+---+ 78 * A N 79 * 80 * enqueued_ddcbs = N - A = 7 - 0 = 7 81 * 82 * Situation (4a): Queue full A > N 83 * +---+---+---+---+---+---+---+---+ 84 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 85 * | x | x | x | | x | x | x | x | 86 * +---+---+---+---+---+---+---+---+ 87 * N A 88 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7 89 */ 90 91static int queue_empty(struct ddcb_queue *queue) 92{ 93 return queue->ddcb_next == queue->ddcb_act; 94} 95 96static int queue_enqueued_ddcbs(struct ddcb_queue *queue) 97{ 98 if (queue->ddcb_next >= queue->ddcb_act) 99 return queue->ddcb_next - queue->ddcb_act; 100 101 return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next); 102} 103 104static int queue_free_ddcbs(struct ddcb_queue *queue) 105{ 106 int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1; 107 108 if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */ 109 return 0; 110 } 111 return free_ddcbs; 112} 113 114/* 115 * Use of the PRIV field in the DDCB for queue debugging: 116 * 117 * (1) Trying to get rid of a DDCB which saw a timeout: 118 * pddcb->priv[6] = 0xcc; # cleared 119 * 120 * (2) Append a DDCB via NEXT bit: 121 * pddcb->priv[7] = 0xaa; # appended 122 * 123 * (3) DDCB needed tapping: 124 * pddcb->priv[7] = 0xbb; # tapped 125 * 126 * (4) DDCB marked as correctly finished: 127 * pddcb->priv[6] = 0xff; # finished 128 */ 129 130static inline void ddcb_mark_tapped(struct ddcb *pddcb) 131{ 132 pddcb->priv[7] = 0xbb; /* tapped */ 133} 134 135static inline void ddcb_mark_appended(struct ddcb *pddcb) 136{ 137 pddcb->priv[7] = 0xaa; /* appended */ 138} 139 140static inline void ddcb_mark_cleared(struct ddcb *pddcb) 141{ 142 pddcb->priv[6] = 0xcc; /* cleared */ 143} 144 145static inline void ddcb_mark_finished(struct ddcb *pddcb) 146{ 147 pddcb->priv[6] = 0xff; /* finished */ 148} 149 150static inline void ddcb_mark_unused(struct ddcb *pddcb) 151{ 152 pddcb->priv_64 = cpu_to_be64(0); /* not tapped */ 153} 154 155/** 156 * genwqe_crc16() - Generate 16-bit crc as required for DDCBs 157 * @buff: pointer to data buffer 158 * @len: length of data for calculation 159 * @init: initial crc (0xffff at start) 160 * 161 * Polynomial = x^16 + x^12 + x^5 + 1 (0x1021) 162 * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff 163 * should result in a crc16 of 0x89c3 164 * 165 * Return: crc16 checksum in big endian format ! 166 */ 167static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init) 168{ 169 return crc_itu_t(init, buff, len); 170} 171 172static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue) 173{ 174 int i; 175 struct ddcb *pddcb; 176 unsigned long flags; 177 struct pci_dev *pci_dev = cd->pci_dev; 178 179 spin_lock_irqsave(&cd->print_lock, flags); 180 181 dev_info(&pci_dev->dev, 182 "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n", 183 cd->card_idx, queue->ddcb_act, queue->ddcb_next); 184 185 pddcb = queue->ddcb_vaddr; 186 for (i = 0; i < queue->ddcb_max; i++) { 187 dev_err(&pci_dev->dev, 188 " %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n", 189 i == queue->ddcb_act ? '>' : ' ', 190 i, 191 be16_to_cpu(pddcb->retc_16), 192 be16_to_cpu(pddcb->seqnum_16), 193 pddcb->hsi, 194 pddcb->shi, 195 be64_to_cpu(pddcb->priv_64), 196 pddcb->cmd); 197 pddcb++; 198 } 199 spin_unlock_irqrestore(&cd->print_lock, flags); 200} 201 202struct genwqe_ddcb_cmd *ddcb_requ_alloc(void) 203{ 204 struct ddcb_requ *req; 205 206 req = kzalloc(sizeof(*req), GFP_ATOMIC); 207 if (!req) 208 return NULL; 209 210 return &req->cmd; 211} 212 213void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd) 214{ 215 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd); 216 217 kfree(req); 218} 219 220static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req) 221{ 222 return req->req_state; 223} 224 225static inline void ddcb_requ_set_state(struct ddcb_requ *req, 226 enum genwqe_requ_state new_state) 227{ 228 req->req_state = new_state; 229} 230 231static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req) 232{ 233 return req->cmd.ddata_addr != 0x0; 234} 235 236/** 237 * ddcb_requ_finished() - Returns the hardware state of the associated DDCB 238 * @cd: pointer to genwqe device descriptor 239 * @req: DDCB work request 240 * 241 * Status of ddcb_requ mirrors this hardware state, but is copied in 242 * the ddcb_requ on interrupt/polling function. The lowlevel code 243 * should check the hardware state directly, the higher level code 244 * should check the copy. 245 * 246 * This function will also return true if the state of the queue is 247 * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the 248 * shutdown case. 249 */ 250static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req) 251{ 252 return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) || 253 (cd->card_state != GENWQE_CARD_USED); 254} 255 256/** 257 * enqueue_ddcb() - Enqueue a DDCB 258 * @cd: pointer to genwqe device descriptor 259 * @queue: queue this operation should be done on 260 * @ddcb_no: pointer to ddcb number being tapped 261 * 262 * Start execution of DDCB by tapping or append to queue via NEXT 263 * bit. This is done by an atomic 'compare and swap' instruction and 264 * checking SHI and HSI of the previous DDCB. 265 * 266 * This function must only be called with ddcb_lock held. 267 * 268 * Return: 1 if new DDCB is appended to previous 269 * 2 if DDCB queue is tapped via register/simulation 270 */ 271#define RET_DDCB_APPENDED 1 272#define RET_DDCB_TAPPED 2 273 274static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue, 275 struct ddcb *pddcb, int ddcb_no) 276{ 277 unsigned int try; 278 int prev_no; 279 struct ddcb *prev_ddcb; 280 __be32 old, new, icrc_hsi_shi; 281 u64 num; 282 283 /* 284 * For performance checks a Dispatch Timestamp can be put into 285 * DDCB It is supposed to use the SLU's free running counter, 286 * but this requires PCIe cycles. 287 */ 288 ddcb_mark_unused(pddcb); 289 290 /* check previous DDCB if already fetched */ 291 prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1; 292 prev_ddcb = &queue->ddcb_vaddr[prev_no]; 293 294 /* 295 * It might have happened that the HSI.FETCHED bit is 296 * set. Retry in this case. Therefore I expect maximum 2 times 297 * trying. 298 */ 299 ddcb_mark_appended(pddcb); 300 for (try = 0; try < 2; try++) { 301 old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ 302 303 /* try to append via NEXT bit if prev DDCB is not completed */ 304 if ((old & DDCB_COMPLETED_BE32) != 0x00000000) 305 break; 306 307 new = (old | DDCB_NEXT_BE32); 308 309 wmb(); /* need to ensure write ordering */ 310 icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new); 311 312 if (icrc_hsi_shi == old) 313 return RET_DDCB_APPENDED; /* appended to queue */ 314 } 315 316 /* Queue must be re-started by updating QUEUE_OFFSET */ 317 ddcb_mark_tapped(pddcb); 318 num = (u64)ddcb_no << 8; 319 320 wmb(); /* need to ensure write ordering */ 321 __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */ 322 323 return RET_DDCB_TAPPED; 324} 325 326/** 327 * copy_ddcb_results() - Copy output state from real DDCB to request 328 * 329 * Copy DDCB ASV to request struct. There is no endian 330 * conversion made, since data structure in ASV is still 331 * unknown here. 332 * 333 * This is needed by: 334 * - genwqe_purge_ddcb() 335 * - genwqe_check_ddcb_queue() 336 */ 337static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no) 338{ 339 struct ddcb_queue *queue = req->queue; 340 struct ddcb *pddcb = &queue->ddcb_vaddr[req->num]; 341 342 memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH); 343 344 /* copy status flags of the variant part */ 345 req->cmd.vcrc = be16_to_cpu(pddcb->vcrc_16); 346 req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64); 347 req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64); 348 349 req->cmd.attn = be16_to_cpu(pddcb->attn_16); 350 req->cmd.progress = be32_to_cpu(pddcb->progress_32); 351 req->cmd.retc = be16_to_cpu(pddcb->retc_16); 352 353 if (ddcb_requ_collect_debug_data(req)) { 354 int prev_no = (ddcb_no == 0) ? 355 queue->ddcb_max - 1 : ddcb_no - 1; 356 struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no]; 357 358 memcpy(&req->debug_data.ddcb_finished, pddcb, 359 sizeof(req->debug_data.ddcb_finished)); 360 memcpy(&req->debug_data.ddcb_prev, prev_pddcb, 361 sizeof(req->debug_data.ddcb_prev)); 362 } 363} 364 365/** 366 * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work equests. 367 * @cd: pointer to genwqe device descriptor 368 * 369 * Return: Number of DDCBs which were finished 370 */ 371static int genwqe_check_ddcb_queue(struct genwqe_dev *cd, 372 struct ddcb_queue *queue) 373{ 374 unsigned long flags; 375 int ddcbs_finished = 0; 376 struct pci_dev *pci_dev = cd->pci_dev; 377 378 spin_lock_irqsave(&queue->ddcb_lock, flags); 379 380 /* FIXME avoid soft locking CPU */ 381 while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) { 382 383 struct ddcb *pddcb; 384 struct ddcb_requ *req; 385 u16 vcrc, vcrc_16, retc_16; 386 387 pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; 388 389 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 390 0x00000000) 391 goto go_home; /* not completed, continue waiting */ 392 393 wmb(); /* Add sync to decouple prev. read operations */ 394 395 /* Note: DDCB could be purged */ 396 req = queue->ddcb_req[queue->ddcb_act]; 397 if (req == NULL) { 398 /* this occurs if DDCB is purged, not an error */ 399 /* Move active DDCB further; Nothing to do anymore. */ 400 goto pick_next_one; 401 } 402 403 /* 404 * HSI=0x44 (fetched and completed), but RETC is 405 * 0x101, or even worse 0x000. 406 * 407 * In case of seeing the queue in inconsistent state 408 * we read the errcnts and the queue status to provide 409 * a trigger for our PCIe analyzer stop capturing. 410 */ 411 retc_16 = be16_to_cpu(pddcb->retc_16); 412 if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) { 413 u64 errcnts, status; 414 u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr; 415 416 errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS); 417 status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS); 418 419 dev_err(&pci_dev->dev, 420 "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n", 421 __func__, be16_to_cpu(pddcb->seqnum_16), 422 pddcb->hsi, retc_16, errcnts, status, 423 queue->ddcb_daddr + ddcb_offs); 424 } 425 426 copy_ddcb_results(req, queue->ddcb_act); 427 queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */ 428 429 dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num); 430 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 431 432 ddcb_mark_finished(pddcb); 433 434 /* calculate CRC_16 to see if VCRC is correct */ 435 vcrc = genwqe_crc16(pddcb->asv, 436 VCRC_LENGTH(req->cmd.asv_length), 437 0xffff); 438 vcrc_16 = be16_to_cpu(pddcb->vcrc_16); 439 if (vcrc != vcrc_16) { 440 printk_ratelimited(KERN_ERR 441 "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n", 442 GENWQE_DEVNAME, dev_name(&pci_dev->dev), 443 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length), 444 vcrc, vcrc_16); 445 } 446 447 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED); 448 queue->ddcbs_completed++; 449 queue->ddcbs_in_flight--; 450 451 /* wake up process waiting for this DDCB, and 452 processes on the busy queue */ 453 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); 454 wake_up_interruptible(&queue->busy_waitq); 455 456pick_next_one: 457 queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max; 458 ddcbs_finished++; 459 } 460 461 go_home: 462 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 463 return ddcbs_finished; 464} 465 466/** 467 * __genwqe_wait_ddcb(): Waits until DDCB is completed 468 * @cd: pointer to genwqe device descriptor 469 * @req: pointer to requsted DDCB parameters 470 * 471 * The Service Layer will update the RETC in DDCB when processing is 472 * pending or done. 473 * 474 * Return: > 0 remaining jiffies, DDCB completed 475 * -ETIMEDOUT when timeout 476 * -ERESTARTSYS when ^C 477 * -EINVAL when unknown error condition 478 * 479 * When an error is returned the called needs to ensure that 480 * purge_ddcb() is being called to get the &req removed from the 481 * queue. 482 */ 483int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) 484{ 485 int rc; 486 unsigned int ddcb_no; 487 struct ddcb_queue *queue; 488 struct pci_dev *pci_dev = cd->pci_dev; 489 490 if (req == NULL) 491 return -EINVAL; 492 493 queue = req->queue; 494 if (queue == NULL) 495 return -EINVAL; 496 497 ddcb_no = req->num; 498 if (ddcb_no >= queue->ddcb_max) 499 return -EINVAL; 500 501 rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no], 502 ddcb_requ_finished(cd, req), 503 genwqe_ddcb_software_timeout * HZ); 504 505 /* 506 * We need to distinguish 3 cases here: 507 * 1. rc == 0 timeout occured 508 * 2. rc == -ERESTARTSYS signal received 509 * 3. rc > 0 remaining jiffies condition is true 510 */ 511 if (rc == 0) { 512 struct ddcb_queue *queue = req->queue; 513 struct ddcb *pddcb; 514 515 /* 516 * Timeout may be caused by long task switching time. 517 * When timeout happens, check if the request has 518 * meanwhile completed. 519 */ 520 genwqe_check_ddcb_queue(cd, req->queue); 521 if (ddcb_requ_finished(cd, req)) 522 return rc; 523 524 dev_err(&pci_dev->dev, 525 "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n", 526 __func__, req->num, rc, ddcb_requ_get_state(req), 527 req); 528 dev_err(&pci_dev->dev, 529 "[%s] IO_QUEUE_STATUS=0x%016llx\n", __func__, 530 __genwqe_readq(cd, queue->IO_QUEUE_STATUS)); 531 532 pddcb = &queue->ddcb_vaddr[req->num]; 533 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 534 535 print_ddcb_info(cd, req->queue); 536 return -ETIMEDOUT; 537 538 } else if (rc == -ERESTARTSYS) { 539 return rc; 540 /* 541 * EINTR: Stops the application 542 * ERESTARTSYS: Restartable systemcall; called again 543 */ 544 545 } else if (rc < 0) { 546 dev_err(&pci_dev->dev, 547 "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n", 548 __func__, req->num, rc, ddcb_requ_get_state(req)); 549 return -EINVAL; 550 } 551 552 /* Severe error occured. Driver is forced to stop operation */ 553 if (cd->card_state != GENWQE_CARD_USED) { 554 dev_err(&pci_dev->dev, 555 "[%s] err: DDCB#%d forced to stop (rc=%d)\n", 556 __func__, req->num, rc); 557 return -EIO; 558 } 559 return rc; 560} 561 562/** 563 * get_next_ddcb() - Get next available DDCB 564 * @cd: pointer to genwqe device descriptor 565 * 566 * DDCB's content is completely cleared but presets for PRE and 567 * SEQNUM. This function must only be called when ddcb_lock is held. 568 * 569 * Return: NULL if no empty DDCB available otherwise ptr to next DDCB. 570 */ 571static struct ddcb *get_next_ddcb(struct genwqe_dev *cd, 572 struct ddcb_queue *queue, 573 int *num) 574{ 575 u64 *pu64; 576 struct ddcb *pddcb; 577 578 if (queue_free_ddcbs(queue) == 0) /* queue is full */ 579 return NULL; 580 581 /* find new ddcb */ 582 pddcb = &queue->ddcb_vaddr[queue->ddcb_next]; 583 584 /* if it is not completed, we are not allowed to use it */ 585 /* barrier(); */ 586 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000) 587 return NULL; 588 589 *num = queue->ddcb_next; /* internal DDCB number */ 590 queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max; 591 592 /* clear important DDCB fields */ 593 pu64 = (u64 *)pddcb; 594 pu64[0] = 0ULL; /* offs 0x00 (ICRC,HSI,SHI,...) */ 595 pu64[1] = 0ULL; /* offs 0x01 (ACFUNC,CMD...) */ 596 597 /* destroy previous results in ASV */ 598 pu64[0x80/8] = 0ULL; /* offs 0x80 (ASV + 0) */ 599 pu64[0x88/8] = 0ULL; /* offs 0x88 (ASV + 0x08) */ 600 pu64[0x90/8] = 0ULL; /* offs 0x90 (ASV + 0x10) */ 601 pu64[0x98/8] = 0ULL; /* offs 0x98 (ASV + 0x18) */ 602 pu64[0xd0/8] = 0ULL; /* offs 0xd0 (RETC,ATTN...) */ 603 604 pddcb->pre = DDCB_PRESET_PRE; /* 128 */ 605 pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++); 606 return pddcb; 607} 608 609/** 610 * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue 611 * @cd: genwqe device descriptor 612 * @req: DDCB request 613 * 614 * This will fail when the request was already FETCHED. In this case 615 * we need to wait until it is finished. Else the DDCB can be 616 * reused. This function also ensures that the request data structure 617 * is removed from ddcb_req[]. 618 * 619 * Do not forget to call this function when genwqe_wait_ddcb() fails, 620 * such that the request gets really removed from ddcb_req[]. 621 * 622 * Return: 0 success 623 */ 624int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) 625{ 626 struct ddcb *pddcb = NULL; 627 unsigned int t; 628 unsigned long flags; 629 struct ddcb_queue *queue = req->queue; 630 struct pci_dev *pci_dev = cd->pci_dev; 631 u64 queue_status; 632 __be32 icrc_hsi_shi = 0x0000; 633 __be32 old, new; 634 635 /* unsigned long flags; */ 636 if (genwqe_ddcb_software_timeout <= 0) { 637 dev_err(&pci_dev->dev, 638 "[%s] err: software timeout is not set!\n", __func__); 639 return -EFAULT; 640 } 641 642 pddcb = &queue->ddcb_vaddr[req->num]; 643 644 for (t = 0; t < genwqe_ddcb_software_timeout * 10; t++) { 645 646 spin_lock_irqsave(&queue->ddcb_lock, flags); 647 648 /* Check if req was meanwhile finished */ 649 if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) 650 goto go_home; 651 652 /* try to set PURGE bit if FETCHED/COMPLETED are not set */ 653 old = pddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ 654 if ((old & DDCB_FETCHED_BE32) == 0x00000000) { 655 656 new = (old | DDCB_PURGE_BE32); 657 icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32, 658 old, new); 659 if (icrc_hsi_shi == old) 660 goto finish_ddcb; 661 } 662 663 /* normal finish with HSI bit */ 664 barrier(); 665 icrc_hsi_shi = pddcb->icrc_hsi_shi_32; 666 if (icrc_hsi_shi & DDCB_COMPLETED_BE32) 667 goto finish_ddcb; 668 669 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 670 671 /* 672 * Here the check_ddcb() function will most likely 673 * discover this DDCB to be finished some point in 674 * time. It will mark the req finished and free it up 675 * in the list. 676 */ 677 678 copy_ddcb_results(req, req->num); /* for the failing case */ 679 msleep(100); /* sleep for 1/10 second and try again */ 680 continue; 681 682finish_ddcb: 683 copy_ddcb_results(req, req->num); 684 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED); 685 queue->ddcbs_in_flight--; 686 queue->ddcb_req[req->num] = NULL; /* delete from array */ 687 ddcb_mark_cleared(pddcb); 688 689 /* Move active DDCB further; Nothing to do here anymore. */ 690 691 /* 692 * We need to ensure that there is at least one free 693 * DDCB in the queue. To do that, we must update 694 * ddcb_act only if the COMPLETED bit is set for the 695 * DDCB we are working on else we treat that DDCB even 696 * if we PURGED it as occupied (hardware is supposed 697 * to set the COMPLETED bit yet!). 698 */ 699 icrc_hsi_shi = pddcb->icrc_hsi_shi_32; 700 if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) && 701 (queue->ddcb_act == req->num)) { 702 queue->ddcb_act = ((queue->ddcb_act + 1) % 703 queue->ddcb_max); 704 } 705go_home: 706 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 707 return 0; 708 } 709 710 /* 711 * If the card is dead and the queue is forced to stop, we 712 * might see this in the queue status register. 713 */ 714 queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS); 715 716 dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num); 717 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 718 719 dev_err(&pci_dev->dev, 720 "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n", 721 __func__, req->num, genwqe_ddcb_software_timeout, 722 queue_status); 723 724 print_ddcb_info(cd, req->queue); 725 726 return -EFAULT; 727} 728 729int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d) 730{ 731 int len; 732 struct pci_dev *pci_dev = cd->pci_dev; 733 734 if (d == NULL) { 735 dev_err(&pci_dev->dev, 736 "[%s] err: invalid memory for debug data!\n", 737 __func__); 738 return -EFAULT; 739 } 740 741 len = sizeof(d->driver_version); 742 snprintf(d->driver_version, len, "%s", DRV_VERSION); 743 d->slu_unitcfg = cd->slu_unitcfg; 744 d->app_unitcfg = cd->app_unitcfg; 745 return 0; 746} 747 748/** 749 * __genwqe_enqueue_ddcb() - Enqueue a DDCB 750 * @cd: pointer to genwqe device descriptor 751 * @req: pointer to DDCB execution request 752 * @f_flags: file mode: blocking, non-blocking 753 * 754 * Return: 0 if enqueuing succeeded 755 * -EIO if card is unusable/PCIe problems 756 * -EBUSY if enqueuing failed 757 */ 758int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req, 759 unsigned int f_flags) 760{ 761 struct ddcb *pddcb; 762 unsigned long flags; 763 struct ddcb_queue *queue; 764 struct pci_dev *pci_dev = cd->pci_dev; 765 u16 icrc; 766 767 retry: 768 if (cd->card_state != GENWQE_CARD_USED) { 769 printk_ratelimited(KERN_ERR 770 "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n", 771 GENWQE_DEVNAME, dev_name(&pci_dev->dev), 772 __func__, req->num); 773 return -EIO; 774 } 775 776 queue = req->queue = &cd->queue; 777 778 /* FIXME circumvention to improve performance when no irq is 779 * there. 780 */ 781 if (genwqe_polling_enabled) 782 genwqe_check_ddcb_queue(cd, queue); 783 784 /* 785 * It must be ensured to process all DDCBs in successive 786 * order. Use a lock here in order to prevent nested DDCB 787 * enqueuing. 788 */ 789 spin_lock_irqsave(&queue->ddcb_lock, flags); 790 791 pddcb = get_next_ddcb(cd, queue, &req->num); /* get ptr and num */ 792 if (pddcb == NULL) { 793 int rc; 794 795 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 796 797 if (f_flags & O_NONBLOCK) { 798 queue->return_on_busy++; 799 return -EBUSY; 800 } 801 802 queue->wait_on_busy++; 803 rc = wait_event_interruptible(queue->busy_waitq, 804 queue_free_ddcbs(queue) != 0); 805 dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n", 806 __func__, rc); 807 if (rc == -ERESTARTSYS) 808 return rc; /* interrupted by a signal */ 809 810 goto retry; 811 } 812 813 if (queue->ddcb_req[req->num] != NULL) { 814 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 815 816 dev_err(&pci_dev->dev, 817 "[%s] picked DDCB %d with req=%p still in use!!\n", 818 __func__, req->num, req); 819 return -EFAULT; 820 } 821 ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED); 822 queue->ddcb_req[req->num] = req; 823 824 pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts); 825 pddcb->cmd = req->cmd.cmd; 826 pddcb->acfunc = req->cmd.acfunc; /* functional unit */ 827 828 /* 829 * We know that we can get retc 0x104 with CRC error, do not 830 * stop the queue in those cases for this command. XDIR = 1 831 * does not work for old SLU versions. 832 * 833 * Last bitstream with the old XDIR behavior had SLU_ID 834 * 0x34199. 835 */ 836 if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull) 837 pddcb->xdir = 0x1; 838 else 839 pddcb->xdir = 0x0; 840 841 842 pddcb->psp = (((req->cmd.asiv_length / 8) << 4) | 843 ((req->cmd.asv_length / 8))); 844 pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts); 845 846 /* 847 * If copying the whole DDCB_ASIV_LENGTH is impacting 848 * performance we need to change it to 849 * req->cmd.asiv_length. But simulation benefits from some 850 * non-architectured bits behind the architectured content. 851 * 852 * How much data is copied depends on the availability of the 853 * ATS field, which was introduced late. If the ATS field is 854 * supported ASIV is 8 bytes shorter than it used to be. Since 855 * the ATS field is copied too, the code should do exactly 856 * what it did before, but I wanted to make copying of the ATS 857 * field very explicit. 858 */ 859 if (genwqe_get_slu_id(cd) <= 0x2) { 860 memcpy(&pddcb->__asiv[0], /* destination */ 861 &req->cmd.__asiv[0], /* source */ 862 DDCB_ASIV_LENGTH); /* req->cmd.asiv_length */ 863 } else { 864 pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats); 865 memcpy(&pddcb->n.asiv[0], /* destination */ 866 &req->cmd.asiv[0], /* source */ 867 DDCB_ASIV_LENGTH_ATS); /* req->cmd.asiv_length */ 868 } 869 870 pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */ 871 872 /* 873 * Calculate CRC_16 for corresponding range PSP(7:4). Include 874 * empty 4 bytes prior to the data. 875 */ 876 icrc = genwqe_crc16((const u8 *)pddcb, 877 ICRC_LENGTH(req->cmd.asiv_length), 0xffff); 878 pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16); 879 880 /* enable DDCB completion irq */ 881 if (!genwqe_polling_enabled) 882 pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32; 883 884 dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num); 885 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb)); 886 887 if (ddcb_requ_collect_debug_data(req)) { 888 /* use the kernel copy of debug data. copying back to 889 user buffer happens later */ 890 891 genwqe_init_debug_data(cd, &req->debug_data); 892 memcpy(&req->debug_data.ddcb_before, pddcb, 893 sizeof(req->debug_data.ddcb_before)); 894 } 895 896 enqueue_ddcb(cd, queue, pddcb, req->num); 897 queue->ddcbs_in_flight++; 898 899 if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight) 900 queue->ddcbs_max_in_flight = queue->ddcbs_in_flight; 901 902 ddcb_requ_set_state(req, GENWQE_REQU_TAPPED); 903 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 904 wake_up_interruptible(&cd->queue_waitq); 905 906 return 0; 907} 908 909/** 910 * __genwqe_execute_raw_ddcb() - Setup and execute DDCB 911 * @cd: pointer to genwqe device descriptor 912 * @req: user provided DDCB request 913 * @f_flags: file mode: blocking, non-blocking 914 */ 915int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd, 916 struct genwqe_ddcb_cmd *cmd, 917 unsigned int f_flags) 918{ 919 int rc = 0; 920 struct pci_dev *pci_dev = cd->pci_dev; 921 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd); 922 923 if (cmd->asiv_length > DDCB_ASIV_LENGTH) { 924 dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n", 925 __func__, cmd->asiv_length); 926 return -EINVAL; 927 } 928 if (cmd->asv_length > DDCB_ASV_LENGTH) { 929 dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n", 930 __func__, cmd->asiv_length); 931 return -EINVAL; 932 } 933 rc = __genwqe_enqueue_ddcb(cd, req, f_flags); 934 if (rc != 0) 935 return rc; 936 937 rc = __genwqe_wait_ddcb(cd, req); 938 if (rc < 0) /* error or signal interrupt */ 939 goto err_exit; 940 941 if (ddcb_requ_collect_debug_data(req)) { 942 if (copy_to_user((struct genwqe_debug_data __user *) 943 (unsigned long)cmd->ddata_addr, 944 &req->debug_data, 945 sizeof(struct genwqe_debug_data))) 946 return -EFAULT; 947 } 948 949 /* 950 * Higher values than 0x102 indicate completion with faults, 951 * lower values than 0x102 indicate processing faults. Note 952 * that DDCB might have been purged. E.g. Cntl+C. 953 */ 954 if (cmd->retc != DDCB_RETC_COMPLETE) { 955 /* This might happen e.g. flash read, and needs to be 956 handled by the upper layer code. */ 957 rc = -EBADMSG; /* not processed/error retc */ 958 } 959 960 return rc; 961 962 err_exit: 963 __genwqe_purge_ddcb(cd, req); 964 965 if (ddcb_requ_collect_debug_data(req)) { 966 if (copy_to_user((struct genwqe_debug_data __user *) 967 (unsigned long)cmd->ddata_addr, 968 &req->debug_data, 969 sizeof(struct genwqe_debug_data))) 970 return -EFAULT; 971 } 972 return rc; 973} 974 975/** 976 * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished 977 * 978 * We use this as condition for our wait-queue code. 979 */ 980static int genwqe_next_ddcb_ready(struct genwqe_dev *cd) 981{ 982 unsigned long flags; 983 struct ddcb *pddcb; 984 struct ddcb_queue *queue = &cd->queue; 985 986 spin_lock_irqsave(&queue->ddcb_lock, flags); 987 988 if (queue_empty(queue)) { /* emtpy queue */ 989 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 990 return 0; 991 } 992 993 pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; 994 if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */ 995 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 996 return 1; 997 } 998 999 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 1000 return 0; 1001} 1002 1003/** 1004 * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight 1005 * 1006 * Keep track on the number of DDCBs which ware currently in the 1007 * queue. This is needed for statistics as well as conditon if we want 1008 * to wait or better do polling in case of no interrupts available. 1009 */ 1010int genwqe_ddcbs_in_flight(struct genwqe_dev *cd) 1011{ 1012 unsigned long flags; 1013 int ddcbs_in_flight = 0; 1014 struct ddcb_queue *queue = &cd->queue; 1015 1016 spin_lock_irqsave(&queue->ddcb_lock, flags); 1017 ddcbs_in_flight += queue->ddcbs_in_flight; 1018 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 1019 1020 return ddcbs_in_flight; 1021} 1022 1023static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) 1024{ 1025 int rc, i; 1026 struct ddcb *pddcb; 1027 u64 val64; 1028 unsigned int queue_size; 1029 struct pci_dev *pci_dev = cd->pci_dev; 1030 1031 if (genwqe_ddcb_max < 2) 1032 return -EINVAL; 1033 1034 queue_size = roundup(genwqe_ddcb_max * sizeof(struct ddcb), PAGE_SIZE); 1035 1036 queue->ddcbs_in_flight = 0; /* statistics */ 1037 queue->ddcbs_max_in_flight = 0; 1038 queue->ddcbs_completed = 0; 1039 queue->return_on_busy = 0; 1040 queue->wait_on_busy = 0; 1041 1042 queue->ddcb_seq = 0x100; /* start sequence number */ 1043 queue->ddcb_max = genwqe_ddcb_max; /* module parameter */ 1044 queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size, 1045 &queue->ddcb_daddr); 1046 if (queue->ddcb_vaddr == NULL) { 1047 dev_err(&pci_dev->dev, 1048 "[%s] **err: could not allocate DDCB **\n", __func__); 1049 return -ENOMEM; 1050 } 1051 memset(queue->ddcb_vaddr, 0, queue_size); 1052 1053 queue->ddcb_req = kzalloc(sizeof(struct ddcb_requ *) * 1054 queue->ddcb_max, GFP_KERNEL); 1055 if (!queue->ddcb_req) { 1056 rc = -ENOMEM; 1057 goto free_ddcbs; 1058 } 1059 1060 queue->ddcb_waitqs = kzalloc(sizeof(wait_queue_head_t) * 1061 queue->ddcb_max, GFP_KERNEL); 1062 if (!queue->ddcb_waitqs) { 1063 rc = -ENOMEM; 1064 goto free_requs; 1065 } 1066 1067 for (i = 0; i < queue->ddcb_max; i++) { 1068 pddcb = &queue->ddcb_vaddr[i]; /* DDCBs */ 1069 pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32; 1070 pddcb->retc_16 = cpu_to_be16(0xfff); 1071 1072 queue->ddcb_req[i] = NULL; /* requests */ 1073 init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */ 1074 } 1075 1076 queue->ddcb_act = 0; 1077 queue->ddcb_next = 0; /* queue is empty */ 1078 1079 spin_lock_init(&queue->ddcb_lock); 1080 init_waitqueue_head(&queue->busy_waitq); 1081 1082 val64 = ((u64)(queue->ddcb_max - 1) << 8); /* lastptr */ 1083 __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG, 0x07); /* iCRC/vCRC */ 1084 __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr); 1085 __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq); 1086 __genwqe_writeq(cd, queue->IO_QUEUE_WRAP, val64); 1087 return 0; 1088 1089 free_requs: 1090 kfree(queue->ddcb_req); 1091 queue->ddcb_req = NULL; 1092 free_ddcbs: 1093 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr, 1094 queue->ddcb_daddr); 1095 queue->ddcb_vaddr = NULL; 1096 queue->ddcb_daddr = 0ull; 1097 return -ENODEV; 1098 1099} 1100 1101static int ddcb_queue_initialized(struct ddcb_queue *queue) 1102{ 1103 return queue->ddcb_vaddr != NULL; 1104} 1105 1106static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) 1107{ 1108 unsigned int queue_size; 1109 1110 queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE); 1111 1112 kfree(queue->ddcb_req); 1113 queue->ddcb_req = NULL; 1114 1115 if (queue->ddcb_vaddr) { 1116 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr, 1117 queue->ddcb_daddr); 1118 queue->ddcb_vaddr = NULL; 1119 queue->ddcb_daddr = 0ull; 1120 } 1121} 1122 1123static irqreturn_t genwqe_pf_isr(int irq, void *dev_id) 1124{ 1125 u64 gfir; 1126 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id; 1127 struct pci_dev *pci_dev = cd->pci_dev; 1128 1129 /* 1130 * In case of fatal FIR error the queue is stopped, such that 1131 * we can safely check it without risking anything. 1132 */ 1133 cd->irqs_processed++; 1134 wake_up_interruptible(&cd->queue_waitq); 1135 1136 /* 1137 * Checking for errors before kicking the queue might be 1138 * safer, but slower for the good-case ... See above. 1139 */ 1140 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR); 1141 if (((gfir & GFIR_ERR_TRIGGER) != 0x0) && 1142 !pci_channel_offline(pci_dev)) { 1143 1144 if (cd->use_platform_recovery) { 1145 /* 1146 * Since we use raw accessors, EEH errors won't be 1147 * detected by the platform until we do a non-raw 1148 * MMIO or config space read 1149 */ 1150 readq(cd->mmio + IO_SLC_CFGREG_GFIR); 1151 1152 /* Don't do anything if the PCI channel is frozen */ 1153 if (pci_channel_offline(pci_dev)) 1154 goto exit; 1155 } 1156 1157 wake_up_interruptible(&cd->health_waitq); 1158 1159 /* 1160 * By default GFIRs causes recovery actions. This 1161 * count is just for debug when recovery is masked. 1162 */ 1163 dev_err_ratelimited(&pci_dev->dev, 1164 "[%s] GFIR=%016llx\n", 1165 __func__, gfir); 1166 } 1167 1168 exit: 1169 return IRQ_HANDLED; 1170} 1171 1172static irqreturn_t genwqe_vf_isr(int irq, void *dev_id) 1173{ 1174 struct genwqe_dev *cd = (struct genwqe_dev *)dev_id; 1175 1176 cd->irqs_processed++; 1177 wake_up_interruptible(&cd->queue_waitq); 1178 1179 return IRQ_HANDLED; 1180} 1181 1182/** 1183 * genwqe_card_thread() - Work thread for the DDCB queue 1184 * 1185 * The idea is to check if there are DDCBs in processing. If there are 1186 * some finished DDCBs, we process them and wakeup the 1187 * requestors. Otherwise we give other processes time using 1188 * cond_resched(). 1189 */ 1190static int genwqe_card_thread(void *data) 1191{ 1192 int should_stop = 0, rc = 0; 1193 struct genwqe_dev *cd = (struct genwqe_dev *)data; 1194 1195 while (!kthread_should_stop()) { 1196 1197 genwqe_check_ddcb_queue(cd, &cd->queue); 1198 1199 if (genwqe_polling_enabled) { 1200 rc = wait_event_interruptible_timeout( 1201 cd->queue_waitq, 1202 genwqe_ddcbs_in_flight(cd) || 1203 (should_stop = kthread_should_stop()), 1); 1204 } else { 1205 rc = wait_event_interruptible_timeout( 1206 cd->queue_waitq, 1207 genwqe_next_ddcb_ready(cd) || 1208 (should_stop = kthread_should_stop()), HZ); 1209 } 1210 if (should_stop) 1211 break; 1212 1213 /* 1214 * Avoid soft lockups on heavy loads; we do not want 1215 * to disable our interrupts. 1216 */ 1217 cond_resched(); 1218 } 1219 return 0; 1220} 1221 1222/** 1223 * genwqe_setup_service_layer() - Setup DDCB queue 1224 * @cd: pointer to genwqe device descriptor 1225 * 1226 * Allocate DDCBs. Configure Service Layer Controller (SLC). 1227 * 1228 * Return: 0 success 1229 */ 1230int genwqe_setup_service_layer(struct genwqe_dev *cd) 1231{ 1232 int rc; 1233 struct ddcb_queue *queue; 1234 struct pci_dev *pci_dev = cd->pci_dev; 1235 1236 if (genwqe_is_privileged(cd)) { 1237 rc = genwqe_card_reset(cd); 1238 if (rc < 0) { 1239 dev_err(&pci_dev->dev, 1240 "[%s] err: reset failed.\n", __func__); 1241 return rc; 1242 } 1243 genwqe_read_softreset(cd); 1244 } 1245 1246 queue = &cd->queue; 1247 queue->IO_QUEUE_CONFIG = IO_SLC_QUEUE_CONFIG; 1248 queue->IO_QUEUE_STATUS = IO_SLC_QUEUE_STATUS; 1249 queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT; 1250 queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN; 1251 queue->IO_QUEUE_OFFSET = IO_SLC_QUEUE_OFFSET; 1252 queue->IO_QUEUE_WRAP = IO_SLC_QUEUE_WRAP; 1253 queue->IO_QUEUE_WTIME = IO_SLC_QUEUE_WTIME; 1254 queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS; 1255 queue->IO_QUEUE_LRW = IO_SLC_QUEUE_LRW; 1256 1257 rc = setup_ddcb_queue(cd, queue); 1258 if (rc != 0) { 1259 rc = -ENODEV; 1260 goto err_out; 1261 } 1262 1263 init_waitqueue_head(&cd->queue_waitq); 1264 cd->card_thread = kthread_run(genwqe_card_thread, cd, 1265 GENWQE_DEVNAME "%d_thread", 1266 cd->card_idx); 1267 if (IS_ERR(cd->card_thread)) { 1268 rc = PTR_ERR(cd->card_thread); 1269 cd->card_thread = NULL; 1270 goto stop_free_queue; 1271 } 1272 1273 rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS); 1274 if (rc) 1275 goto stop_kthread; 1276 1277 /* 1278 * We must have all wait-queues initialized when we enable the 1279 * interrupts. Otherwise we might crash if we get an early 1280 * irq. 1281 */ 1282 init_waitqueue_head(&cd->health_waitq); 1283 1284 if (genwqe_is_privileged(cd)) { 1285 rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED, 1286 GENWQE_DEVNAME, cd); 1287 } else { 1288 rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED, 1289 GENWQE_DEVNAME, cd); 1290 } 1291 if (rc < 0) { 1292 dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq); 1293 goto stop_irq_cap; 1294 } 1295 1296 cd->card_state = GENWQE_CARD_USED; 1297 return 0; 1298 1299 stop_irq_cap: 1300 genwqe_reset_interrupt_capability(cd); 1301 stop_kthread: 1302 kthread_stop(cd->card_thread); 1303 cd->card_thread = NULL; 1304 stop_free_queue: 1305 free_ddcb_queue(cd, queue); 1306 err_out: 1307 return rc; 1308} 1309 1310/** 1311 * queue_wake_up_all() - Handles fatal error case 1312 * 1313 * The PCI device got unusable and we have to stop all pending 1314 * requests as fast as we can. The code after this must purge the 1315 * DDCBs in question and ensure that all mappings are freed. 1316 */ 1317static int queue_wake_up_all(struct genwqe_dev *cd) 1318{ 1319 unsigned int i; 1320 unsigned long flags; 1321 struct ddcb_queue *queue = &cd->queue; 1322 1323 spin_lock_irqsave(&queue->ddcb_lock, flags); 1324 1325 for (i = 0; i < queue->ddcb_max; i++) 1326 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); 1327 1328 wake_up_interruptible(&queue->busy_waitq); 1329 spin_unlock_irqrestore(&queue->ddcb_lock, flags); 1330 1331 return 0; 1332} 1333 1334/** 1335 * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces 1336 * 1337 * Relies on the pre-condition that there are no users of the card 1338 * device anymore e.g. with open file-descriptors. 1339 * 1340 * This function must be robust enough to be called twice. 1341 */ 1342int genwqe_finish_queue(struct genwqe_dev *cd) 1343{ 1344 int i, rc = 0, in_flight; 1345 int waitmax = genwqe_ddcb_software_timeout; 1346 struct pci_dev *pci_dev = cd->pci_dev; 1347 struct ddcb_queue *queue = &cd->queue; 1348 1349 if (!ddcb_queue_initialized(queue)) 1350 return 0; 1351 1352 /* Do not wipe out the error state. */ 1353 if (cd->card_state == GENWQE_CARD_USED) 1354 cd->card_state = GENWQE_CARD_UNUSED; 1355 1356 /* Wake up all requests in the DDCB queue such that they 1357 should be removed nicely. */ 1358 queue_wake_up_all(cd); 1359 1360 /* We must wait to get rid of the DDCBs in flight */ 1361 for (i = 0; i < waitmax; i++) { 1362 in_flight = genwqe_ddcbs_in_flight(cd); 1363 1364 if (in_flight == 0) 1365 break; 1366 1367 dev_dbg(&pci_dev->dev, 1368 " DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n", 1369 i, waitmax, in_flight); 1370 1371 /* 1372 * Severe severe error situation: The card itself has 1373 * 16 DDCB queues, each queue has e.g. 32 entries, 1374 * each DDBC has a hardware timeout of currently 250 1375 * msec but the PFs have a hardware timeout of 8 sec 1376 * ... so I take something large. 1377 */ 1378 msleep(1000); 1379 } 1380 if (i == waitmax) { 1381 dev_err(&pci_dev->dev, " [%s] err: queue is not empty!!\n", 1382 __func__); 1383 rc = -EIO; 1384 } 1385 return rc; 1386} 1387 1388/** 1389 * genwqe_release_service_layer() - Shutdown DDCB queue 1390 * @cd: genwqe device descriptor 1391 * 1392 * This function must be robust enough to be called twice. 1393 */ 1394int genwqe_release_service_layer(struct genwqe_dev *cd) 1395{ 1396 struct pci_dev *pci_dev = cd->pci_dev; 1397 1398 if (!ddcb_queue_initialized(&cd->queue)) 1399 return 1; 1400 1401 free_irq(pci_dev->irq, cd); 1402 genwqe_reset_interrupt_capability(cd); 1403 1404 if (cd->card_thread != NULL) { 1405 kthread_stop(cd->card_thread); 1406 cd->card_thread = NULL; 1407 } 1408 1409 free_ddcb_queue(cd, &cd->queue); 1410 return 0; 1411} 1412