root/drivers/infiniband/hw/qib/qib.h

INCLUDED FROM

DEFINITIONS

1. dd_from_ppd
2. dd_from_dev
3. dd_from_ibdev
4. ppd_from_ibp
5. to_iport
6. qib_sdma_empty
7. qib_sdma_descq_freecnt
8. __qib_sdma_running
9. qib_clear_rcvhdrtail
10. qib_get_rcvhdrtail
11. qib_get_hdrqtail
12. qib_flush_wc

   1 #ifndef _QIB_KERNEL_H
   2 #define _QIB_KERNEL_H
   3 /*
   4  * Copyright (c) 2012 - 2017 Intel Corporation.  All rights reserved.
   5  * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
   6  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
   7  *
   8  * This software is available to you under a choice of one of two
   9  * licenses.  You may choose to be licensed under the terms of the GNU
  10  * General Public License (GPL) Version 2, available from the file
  11  * COPYING in the main directory of this source tree, or the
  12  * OpenIB.org BSD license below:
  13  *
  14  *     Redistribution and use in source and binary forms, with or
  15  *     without modification, are permitted provided that the following
  16  *     conditions are met:
  17  *
  18  *      - Redistributions of source code must retain the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer.
  21  *
  22  *      - Redistributions in binary form must reproduce the above
  23  *        copyright notice, this list of conditions and the following
  24  *        disclaimer in the documentation and/or other materials
  25  *        provided with the distribution.
  26  *
  27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34  * SOFTWARE.
  35  */
  36 
  37 /*
  38  * This header file is the base header file for qlogic_ib kernel code
  39  * qib_user.h serves a similar purpose for user code.
  40  */
  41 
  42 #include <linux/interrupt.h>
  43 #include <linux/pci.h>
  44 #include <linux/dma-mapping.h>
  45 #include <linux/mutex.h>
  46 #include <linux/list.h>
  47 #include <linux/scatterlist.h>
  48 #include <linux/slab.h>
  49 #include <linux/io.h>
  50 #include <linux/fs.h>
  51 #include <linux/completion.h>
  52 #include <linux/kref.h>
  53 #include <linux/sched.h>
  54 #include <linux/kthread.h>
  55 #include <linux/xarray.h>
  56 #include <rdma/ib_hdrs.h>
  57 #include <rdma/rdma_vt.h>
  58 
  59 #include "qib_common.h"
  60 #include "qib_verbs.h"
  61 
  62 /* only s/w major version of QLogic_IB we can handle */
  63 #define QIB_CHIP_VERS_MAJ 2U
  64 
  65 /* don't care about this except printing */
  66 #define QIB_CHIP_VERS_MIN 0U
  67 
  68 /* The Organization Unique Identifier (Mfg code), and its position in GUID */
  69 #define QIB_OUI 0x001175
  70 #define QIB_OUI_LSB 40
  71 
  72 /*
  73  * per driver stats, either not device nor port-specific, or
  74  * summed over all of the devices and ports.
  75  * They are described by name via ipathfs filesystem, so layout
  76  * and number of elements can change without breaking compatibility.
  77  * If members are added or deleted qib_statnames[] in qib_fs.c must
  78  * change to match.
  79  */
  80 struct qlogic_ib_stats {
  81         __u64 sps_ints; /* number of interrupts handled */
  82         __u64 sps_errints; /* number of error interrupts */
  83         __u64 sps_txerrs; /* tx-related packet errors */
  84         __u64 sps_rcverrs; /* non-crc rcv packet errors */
  85         __u64 sps_hwerrs; /* hardware errors reported (parity, etc.) */
  86         __u64 sps_nopiobufs; /* no pio bufs avail from kernel */
  87         __u64 sps_ctxts; /* number of contexts currently open */
  88         __u64 sps_lenerrs; /* number of kernel packets where RHF != LRH len */
  89         __u64 sps_buffull;
  90         __u64 sps_hdrfull;
  91 };
  92 
  93 extern struct qlogic_ib_stats qib_stats;
  94 extern const struct pci_error_handlers qib_pci_err_handler;
  95 
  96 #define QIB_CHIP_SWVERSION QIB_CHIP_VERS_MAJ
  97 /*
  98  * First-cut critierion for "device is active" is
  99  * two thousand dwords combined Tx, Rx traffic per
 100  * 5-second interval. SMA packets are 64 dwords,
 101  * and occur "a few per second", presumably each way.
 102  */
 103 #define QIB_TRAFFIC_ACTIVE_THRESHOLD (2000)
 104 
 105 /*
 106  * Below contains all data related to a single context (formerly called port).
 107  */
 108 
 109 #ifdef CONFIG_DEBUG_FS
 110 struct qib_opcode_stats_perctx;
 111 #endif
 112 
 113 struct qib_ctxtdata {
 114         void **rcvegrbuf;
 115         dma_addr_t *rcvegrbuf_phys;
 116         /* rcvhdrq base, needs mmap before useful */
 117         void *rcvhdrq;
 118         /* kernel virtual address where hdrqtail is updated */
 119         void *rcvhdrtail_kvaddr;
 120         /*
 121          * temp buffer for expected send setup, allocated at open, instead
 122          * of each setup call
 123          */
 124         void *tid_pg_list;
 125         /*
 126          * Shared page for kernel to signal user processes that send buffers
 127          * need disarming.  The process should call QIB_CMD_DISARM_BUFS
 128          * or QIB_CMD_ACK_EVENT with IPATH_EVENT_DISARM_BUFS set.
 129          */
 130         unsigned long *user_event_mask;
 131         /* when waiting for rcv or pioavail */
 132         wait_queue_head_t wait;
 133         /*
 134          * rcvegr bufs base, physical, must fit
 135          * in 44 bits so 32 bit programs mmap64 44 bit works)
 136          */
 137         dma_addr_t rcvegr_phys;
 138         /* mmap of hdrq, must fit in 44 bits */
 139         dma_addr_t rcvhdrq_phys;
 140         dma_addr_t rcvhdrqtailaddr_phys;
 141 
 142         /*
 143          * number of opens (including slave sub-contexts) on this instance
 144          * (ignoring forks, dup, etc. for now)
 145          */
 146         int cnt;
 147         /*
 148          * how much space to leave at start of eager TID entries for
 149          * protocol use, on each TID
 150          */
 151         /* instead of calculating it */
 152         unsigned ctxt;
 153         /* local node of context */
 154         int node_id;
 155         /* non-zero if ctxt is being shared. */
 156         u16 subctxt_cnt;
 157         /* non-zero if ctxt is being shared. */
 158         u16 subctxt_id;
 159         /* number of eager TID entries. */
 160         u16 rcvegrcnt;
 161         /* index of first eager TID entry. */
 162         u16 rcvegr_tid_base;
 163         /* number of pio bufs for this ctxt (all procs, if shared) */
 164         u32 piocnt;
 165         /* first pio buffer for this ctxt */
 166         u32 pio_base;
 167         /* chip offset of PIO buffers for this ctxt */
 168         u32 piobufs;
 169         /* how many alloc_pages() chunks in rcvegrbuf_pages */
 170         u32 rcvegrbuf_chunks;
 171         /* how many egrbufs per chunk */
 172         u16 rcvegrbufs_perchunk;
 173         /* ilog2 of above */
 174         u16 rcvegrbufs_perchunk_shift;
 175         /* order for rcvegrbuf_pages */
 176         size_t rcvegrbuf_size;
 177         /* rcvhdrq size (for freeing) */
 178         size_t rcvhdrq_size;
 179         /* per-context flags for fileops/intr communication */
 180         unsigned long flag;
 181         /* next expected TID to check when looking for free */
 182         u32 tidcursor;
 183         /* WAIT_RCV that timed out, no interrupt */
 184         u32 rcvwait_to;
 185         /* WAIT_PIO that timed out, no interrupt */
 186         u32 piowait_to;
 187         /* WAIT_RCV already happened, no wait */
 188         u32 rcvnowait;
 189         /* WAIT_PIO already happened, no wait */
 190         u32 pionowait;
 191         /* total number of polled urgent packets */
 192         u32 urgent;
 193         /* saved total number of polled urgent packets for poll edge trigger */
 194         u32 urgent_poll;
 195         /* pid of process using this ctxt */
 196         pid_t pid;
 197         pid_t subpid[QLOGIC_IB_MAX_SUBCTXT];
 198         /* same size as task_struct .comm[], command that opened context */
 199         char comm[16];
 200         /* pkeys set by this use of this ctxt */
 201         u16 pkeys[4];
 202         /* so file ops can get at unit */
 203         struct qib_devdata *dd;
 204         /* so funcs that need physical port can get it easily */
 205         struct qib_pportdata *ppd;
 206         /* A page of memory for rcvhdrhead, rcvegrhead, rcvegrtail * N */
 207         void *subctxt_uregbase;
 208         /* An array of pages for the eager receive buffers * N */
 209         void *subctxt_rcvegrbuf;
 210         /* An array of pages for the eager header queue entries * N */
 211         void *subctxt_rcvhdr_base;
 212         /* The version of the library which opened this ctxt */
 213         u32 userversion;
 214         /* Bitmask of active slaves */
 215         u32 active_slaves;
 216         /* Type of packets or conditions we want to poll for */
 217         u16 poll_type;
 218         /* receive packet sequence counter */
 219         u8 seq_cnt;
 220         u8 redirect_seq_cnt;
 221         /* ctxt rcvhdrq head offset */
 222         u32 head;
 223         /* QPs waiting for context processing */
 224         struct list_head qp_wait_list;
 225 #ifdef CONFIG_DEBUG_FS
 226         /* verbs stats per CTX */
 227         struct qib_opcode_stats_perctx *opstats;
 228 #endif
 229 };
 230 
 231 struct rvt_sge_state;
 232 
 233 struct qib_sdma_txreq {
 234         int                 flags;
 235         int                 sg_count;
 236         dma_addr_t          addr;
 237         void              (*callback)(struct qib_sdma_txreq *, int);
 238         u16                 start_idx;  /* sdma private */
 239         u16                 next_descq_idx;  /* sdma private */
 240         struct list_head    list;       /* sdma private */
 241 };
 242 
 243 struct qib_sdma_desc {
 244         __le64 qw[2];
 245 };
 246 
 247 struct qib_verbs_txreq {
 248         struct qib_sdma_txreq   txreq;
 249         struct rvt_qp           *qp;
 250         struct rvt_swqe         *wqe;
 251         u32                     dwords;
 252         u16                     hdr_dwords;
 253         u16                     hdr_inx;
 254         struct qib_pio_header   *align_buf;
 255         struct rvt_mregion      *mr;
 256         struct rvt_sge_state    *ss;
 257 };
 258 
 259 #define QIB_SDMA_TXREQ_F_USELARGEBUF  0x1
 260 #define QIB_SDMA_TXREQ_F_HEADTOHOST   0x2
 261 #define QIB_SDMA_TXREQ_F_INTREQ       0x4
 262 #define QIB_SDMA_TXREQ_F_FREEBUF      0x8
 263 #define QIB_SDMA_TXREQ_F_FREEDESC     0x10
 264 
 265 #define QIB_SDMA_TXREQ_S_OK        0
 266 #define QIB_SDMA_TXREQ_S_SENDERROR 1
 267 #define QIB_SDMA_TXREQ_S_ABORTED   2
 268 #define QIB_SDMA_TXREQ_S_SHUTDOWN  3
 269 
 270 /*
 271  * Get/Set IB link-level config parameters for f_get/set_ib_cfg()
 272  * Mostly for MADs that set or query link parameters, also ipath
 273  * config interfaces
 274  */
 275 #define QIB_IB_CFG_LIDLMC 0 /* LID (LS16b) and Mask (MS16b) */
 276 #define QIB_IB_CFG_LWID_ENB 2 /* allowed Link-width */
 277 #define QIB_IB_CFG_LWID 3 /* currently active Link-width */
 278 #define QIB_IB_CFG_SPD_ENB 4 /* allowed Link speeds */
 279 #define QIB_IB_CFG_SPD 5 /* current Link spd */
 280 #define QIB_IB_CFG_RXPOL_ENB 6 /* Auto-RX-polarity enable */
 281 #define QIB_IB_CFG_LREV_ENB 7 /* Auto-Lane-reversal enable */
 282 #define QIB_IB_CFG_LINKLATENCY 8 /* Link Latency (IB1.2 only) */
 283 #define QIB_IB_CFG_HRTBT 9 /* IB heartbeat off/enable/auto; DDR/QDR only */
 284 #define QIB_IB_CFG_OP_VLS 10 /* operational VLs */
 285 #define QIB_IB_CFG_VL_HIGH_CAP 11 /* num of VL high priority weights */
 286 #define QIB_IB_CFG_VL_LOW_CAP 12 /* num of VL low priority weights */
 287 #define QIB_IB_CFG_OVERRUN_THRESH 13 /* IB overrun threshold */
 288 #define QIB_IB_CFG_PHYERR_THRESH 14 /* IB PHY error threshold */
 289 #define QIB_IB_CFG_LINKDEFAULT 15 /* IB link default (sleep/poll) */
 290 #define QIB_IB_CFG_PKEYS 16 /* update partition keys */
 291 #define QIB_IB_CFG_MTU 17 /* update MTU in IBC */
 292 #define QIB_IB_CFG_LSTATE 18 /* update linkcmd and linkinitcmd in IBC */
 293 #define QIB_IB_CFG_VL_HIGH_LIMIT 19
 294 #define QIB_IB_CFG_PMA_TICKS 20 /* PMA sample tick resolution */
 295 #define QIB_IB_CFG_PORT 21 /* switch port we are connected to */
 296 
 297 /*
 298  * for CFG_LSTATE: LINKCMD in upper 16 bits, LINKINITCMD in lower 16
 299  * IB_LINKINITCMD_POLL and SLEEP are also used as set/get values for
 300  * QIB_IB_CFG_LINKDEFAULT cmd
 301  */
 302 #define   IB_LINKCMD_DOWN   (0 << 16)
 303 #define   IB_LINKCMD_ARMED  (1 << 16)
 304 #define   IB_LINKCMD_ACTIVE (2 << 16)
 305 #define   IB_LINKINITCMD_NOP     0
 306 #define   IB_LINKINITCMD_POLL    1
 307 #define   IB_LINKINITCMD_SLEEP   2
 308 #define   IB_LINKINITCMD_DISABLE 3
 309 
 310 /*
 311  * valid states passed to qib_set_linkstate() user call
 312  */
 313 #define QIB_IB_LINKDOWN         0
 314 #define QIB_IB_LINKARM          1
 315 #define QIB_IB_LINKACTIVE       2
 316 #define QIB_IB_LINKDOWN_ONLY    3
 317 #define QIB_IB_LINKDOWN_SLEEP   4
 318 #define QIB_IB_LINKDOWN_DISABLE 5
 319 
 320 /*
 321  * These 7 values (SDR, DDR, and QDR may be ORed for auto-speed
 322  * negotiation) are used for the 3rd argument to path_f_set_ib_cfg
 323  * with cmd QIB_IB_CFG_SPD_ENB, by direct calls or via sysfs.  They
 324  * are also the the possible values for qib_link_speed_enabled and active
 325  * The values were chosen to match values used within the IB spec.
 326  */
 327 #define QIB_IB_SDR 1
 328 #define QIB_IB_DDR 2
 329 #define QIB_IB_QDR 4
 330 
 331 #define QIB_DEFAULT_MTU 4096
 332 
 333 /* max number of IB ports supported per HCA */
 334 #define QIB_MAX_IB_PORTS 2
 335 
 336 /*
 337  * Possible IB config parameters for f_get/set_ib_table()
 338  */
 339 #define QIB_IB_TBL_VL_HIGH_ARB 1 /* Get/set VL high priority weights */
 340 #define QIB_IB_TBL_VL_LOW_ARB 2 /* Get/set VL low priority weights */
 341 
 342 /*
 343  * Possible "operations" for f_rcvctrl(ppd, op, ctxt)
 344  * these are bits so they can be combined, e.g.
 345  * QIB_RCVCTRL_INTRAVAIL_ENB | QIB_RCVCTRL_CTXT_ENB
 346  */
 347 #define QIB_RCVCTRL_TAILUPD_ENB 0x01
 348 #define QIB_RCVCTRL_TAILUPD_DIS 0x02
 349 #define QIB_RCVCTRL_CTXT_ENB 0x04
 350 #define QIB_RCVCTRL_CTXT_DIS 0x08
 351 #define QIB_RCVCTRL_INTRAVAIL_ENB 0x10
 352 #define QIB_RCVCTRL_INTRAVAIL_DIS 0x20
 353 #define QIB_RCVCTRL_PKEY_ENB 0x40  /* Note, default is enabled */
 354 #define QIB_RCVCTRL_PKEY_DIS 0x80
 355 #define QIB_RCVCTRL_BP_ENB 0x0100
 356 #define QIB_RCVCTRL_BP_DIS 0x0200
 357 #define QIB_RCVCTRL_TIDFLOW_ENB 0x0400
 358 #define QIB_RCVCTRL_TIDFLOW_DIS 0x0800
 359 
 360 /*
 361  * Possible "operations" for f_sendctrl(ppd, op, var)
 362  * these are bits so they can be combined, e.g.
 363  * QIB_SENDCTRL_BUFAVAIL_ENB | QIB_SENDCTRL_ENB
 364  * Some operations (e.g. DISARM, ABORT) are known to
 365  * be "one-shot", so do not modify shadow.
 366  */
 367 #define QIB_SENDCTRL_DISARM       (0x1000)
 368 #define QIB_SENDCTRL_DISARM_BUF(bufn) ((bufn) | QIB_SENDCTRL_DISARM)
 369         /* available (0x2000) */
 370 #define QIB_SENDCTRL_AVAIL_DIS    (0x4000)
 371 #define QIB_SENDCTRL_AVAIL_ENB    (0x8000)
 372 #define QIB_SENDCTRL_AVAIL_BLIP  (0x10000)
 373 #define QIB_SENDCTRL_SEND_DIS    (0x20000)
 374 #define QIB_SENDCTRL_SEND_ENB    (0x40000)
 375 #define QIB_SENDCTRL_FLUSH       (0x80000)
 376 #define QIB_SENDCTRL_CLEAR      (0x100000)
 377 #define QIB_SENDCTRL_DISARM_ALL (0x200000)
 378 
 379 /*
 380  * These are the generic indices for requesting per-port
 381  * counter values via the f_portcntr function.  They
 382  * are always returned as 64 bit values, although most
 383  * are 32 bit counters.
 384  */
 385 /* send-related counters */
 386 #define QIBPORTCNTR_PKTSEND         0U
 387 #define QIBPORTCNTR_WORDSEND        1U
 388 #define QIBPORTCNTR_PSXMITDATA      2U
 389 #define QIBPORTCNTR_PSXMITPKTS      3U
 390 #define QIBPORTCNTR_PSXMITWAIT      4U
 391 #define QIBPORTCNTR_SENDSTALL       5U
 392 /* receive-related counters */
 393 #define QIBPORTCNTR_PKTRCV          6U
 394 #define QIBPORTCNTR_PSRCVDATA       7U
 395 #define QIBPORTCNTR_PSRCVPKTS       8U
 396 #define QIBPORTCNTR_RCVEBP          9U
 397 #define QIBPORTCNTR_RCVOVFL         10U
 398 #define QIBPORTCNTR_WORDRCV         11U
 399 /* IB link related error counters */
 400 #define QIBPORTCNTR_RXLOCALPHYERR   12U
 401 #define QIBPORTCNTR_RXVLERR         13U
 402 #define QIBPORTCNTR_ERRICRC         14U
 403 #define QIBPORTCNTR_ERRVCRC         15U
 404 #define QIBPORTCNTR_ERRLPCRC        16U
 405 #define QIBPORTCNTR_BADFORMAT       17U
 406 #define QIBPORTCNTR_ERR_RLEN        18U
 407 #define QIBPORTCNTR_IBSYMBOLERR     19U
 408 #define QIBPORTCNTR_INVALIDRLEN     20U
 409 #define QIBPORTCNTR_UNSUPVL         21U
 410 #define QIBPORTCNTR_EXCESSBUFOVFL   22U
 411 #define QIBPORTCNTR_ERRLINK         23U
 412 #define QIBPORTCNTR_IBLINKDOWN      24U
 413 #define QIBPORTCNTR_IBLINKERRRECOV  25U
 414 #define QIBPORTCNTR_LLI             26U
 415 /* other error counters */
 416 #define QIBPORTCNTR_RXDROPPKT       27U
 417 #define QIBPORTCNTR_VL15PKTDROP     28U
 418 #define QIBPORTCNTR_ERRPKEY         29U
 419 #define QIBPORTCNTR_KHDROVFL        30U
 420 /* sampling counters (these are actually control registers) */
 421 #define QIBPORTCNTR_PSINTERVAL      31U
 422 #define QIBPORTCNTR_PSSTART         32U
 423 #define QIBPORTCNTR_PSSTAT          33U
 424 
 425 /* how often we check for packet activity for "power on hours (in seconds) */
 426 #define ACTIVITY_TIMER 5
 427 
 428 #define MAX_NAME_SIZE 64
 429 
 430 #ifdef CONFIG_INFINIBAND_QIB_DCA
 431 struct qib_irq_notify;
 432 #endif
 433 
 434 struct qib_msix_entry {
 435         void *arg;
 436 #ifdef CONFIG_INFINIBAND_QIB_DCA
 437         int dca;
 438         int rcv;
 439         struct qib_irq_notify *notifier;
 440 #endif
 441         cpumask_var_t mask;
 442 };
 443 
 444 /* Below is an opaque struct. Each chip (device) can maintain
 445  * private data needed for its operation, but not germane to the
 446  * rest of the driver.  For convenience, we define another that
 447  * is chip-specific, per-port
 448  */
 449 struct qib_chip_specific;
 450 struct qib_chipport_specific;
 451 
 452 enum qib_sdma_states {
 453         qib_sdma_state_s00_hw_down,
 454         qib_sdma_state_s10_hw_start_up_wait,
 455         qib_sdma_state_s20_idle,
 456         qib_sdma_state_s30_sw_clean_up_wait,
 457         qib_sdma_state_s40_hw_clean_up_wait,
 458         qib_sdma_state_s50_hw_halt_wait,
 459         qib_sdma_state_s99_running,
 460 };
 461 
 462 enum qib_sdma_events {
 463         qib_sdma_event_e00_go_hw_down,
 464         qib_sdma_event_e10_go_hw_start,
 465         qib_sdma_event_e20_hw_started,
 466         qib_sdma_event_e30_go_running,
 467         qib_sdma_event_e40_sw_cleaned,
 468         qib_sdma_event_e50_hw_cleaned,
 469         qib_sdma_event_e60_hw_halted,
 470         qib_sdma_event_e70_go_idle,
 471         qib_sdma_event_e7220_err_halted,
 472         qib_sdma_event_e7322_err_halted,
 473         qib_sdma_event_e90_timer_tick,
 474 };
 475 
 476 struct sdma_set_state_action {
 477         unsigned op_enable:1;
 478         unsigned op_intenable:1;
 479         unsigned op_halt:1;
 480         unsigned op_drain:1;
 481         unsigned go_s99_running_tofalse:1;
 482         unsigned go_s99_running_totrue:1;
 483 };
 484 
 485 struct qib_sdma_state {
 486         struct kref          kref;
 487         struct completion    comp;
 488         enum qib_sdma_states current_state;
 489         struct sdma_set_state_action *set_state_action;
 490         unsigned             current_op;
 491         unsigned             go_s99_running;
 492         unsigned             first_sendbuf;
 493         unsigned             last_sendbuf; /* really last +1 */
 494         /* debugging/devel */
 495         enum qib_sdma_states previous_state;
 496         unsigned             previous_op;
 497         enum qib_sdma_events last_event;
 498 };
 499 
 500 struct xmit_wait {
 501         struct timer_list timer;
 502         u64 counter;
 503         u8 flags;
 504         struct cache {
 505                 u64 psxmitdata;
 506                 u64 psrcvdata;
 507                 u64 psxmitpkts;
 508                 u64 psrcvpkts;
 509                 u64 psxmitwait;
 510         } counter_cache;
 511 };
 512 
 513 /*
 514  * The structure below encapsulates data relevant to a physical IB Port.
 515  * Current chips support only one such port, but the separation
 516  * clarifies things a bit. Note that to conform to IB conventions,
 517  * port-numbers are one-based. The first or only port is port1.
 518  */
 519 struct qib_pportdata {
 520         struct qib_ibport ibport_data;
 521 
 522         struct qib_devdata *dd;
 523         struct qib_chippport_specific *cpspec; /* chip-specific per-port */
 524         struct kobject pport_kobj;
 525         struct kobject pport_cc_kobj;
 526         struct kobject sl2vl_kobj;
 527         struct kobject diagc_kobj;
 528 
 529         /* GUID for this interface, in network order */
 530         __be64 guid;
 531 
 532         /* QIB_POLL, etc. link-state specific flags, per port */
 533         u32 lflags;
 534         /* qib_lflags driver is waiting for */
 535         u32 state_wanted;
 536         spinlock_t lflags_lock;
 537 
 538         /* ref count for each pkey */
 539         atomic_t pkeyrefs[4];
 540 
 541         /*
 542          * this address is mapped readonly into user processes so they can
 543          * get status cheaply, whenever they want.  One qword of status per port
 544          */
 545         u64 *statusp;
 546 
 547         /* SendDMA related entries */
 548 
 549         /* read mostly */
 550         struct qib_sdma_desc *sdma_descq;
 551         struct workqueue_struct *qib_wq;
 552         struct qib_sdma_state sdma_state;
 553         dma_addr_t       sdma_descq_phys;
 554         volatile __le64 *sdma_head_dma; /* DMA'ed by chip */
 555         dma_addr_t       sdma_head_phys;
 556         u16                   sdma_descq_cnt;
 557 
 558         /* read/write using lock */
 559         spinlock_t            sdma_lock ____cacheline_aligned_in_smp;
 560         struct list_head      sdma_activelist;
 561         struct list_head      sdma_userpending;
 562         u64                   sdma_descq_added;
 563         u64                   sdma_descq_removed;
 564         u16                   sdma_descq_tail;
 565         u16                   sdma_descq_head;
 566         u8                    sdma_generation;
 567         u8                    sdma_intrequest;
 568 
 569         struct tasklet_struct sdma_sw_clean_up_task
 570                 ____cacheline_aligned_in_smp;
 571 
 572         wait_queue_head_t state_wait; /* for state_wanted */
 573 
 574         /* HoL blocking for SMP replies */
 575         unsigned          hol_state;
 576         struct timer_list hol_timer;
 577 
 578         /*
 579          * Shadow copies of registers; size indicates read access size.
 580          * Most of them are readonly, but some are write-only register,
 581          * where we manipulate the bits in the shadow copy, and then write
 582          * the shadow copy to qlogic_ib.
 583          *
 584          * We deliberately make most of these 32 bits, since they have
 585          * restricted range.  For any that we read, we won't to generate 32
 586          * bit accesses, since Opteron will generate 2 separate 32 bit HT
 587          * transactions for a 64 bit read, and we want to avoid unnecessary
 588          * bus transactions.
 589          */
 590 
 591         /* This is the 64 bit group */
 592         /* last ibcstatus.  opaque outside chip-specific code */
 593         u64 lastibcstat;
 594 
 595         /* these are the "32 bit" regs */
 596 
 597         /*
 598          * the following two are 32-bit bitmasks, but {test,clear,set}_bit
 599          * all expect bit fields to be "unsigned long"
 600          */
 601         unsigned long p_rcvctrl; /* shadow per-port rcvctrl */
 602         unsigned long p_sendctrl; /* shadow per-port sendctrl */
 603 
 604         u32 ibmtu; /* The MTU programmed for this unit */
 605         /*
 606          * Current max size IB packet (in bytes) including IB headers, that
 607          * we can send. Changes when ibmtu changes.
 608          */
 609         u32 ibmaxlen;
 610         /*
 611          * ibmaxlen at init time, limited by chip and by receive buffer
 612          * size.  Not changed after init.
 613          */
 614         u32 init_ibmaxlen;
 615         /* LID programmed for this instance */
 616         u16 lid;
 617         /* list of pkeys programmed; 0 if not set */
 618         u16 pkeys[4];
 619         /* LID mask control */
 620         u8 lmc;
 621         u8 link_width_supported;
 622         u8 link_speed_supported;
 623         u8 link_width_enabled;
 624         u8 link_speed_enabled;
 625         u8 link_width_active;
 626         u8 link_speed_active;
 627         u8 vls_supported;
 628         u8 vls_operational;
 629         /* Rx Polarity inversion (compensate for ~tx on partner) */
 630         u8 rx_pol_inv;
 631 
 632         u8 hw_pidx;     /* physical port index */
 633         u8 port;        /* IB port number and index into dd->pports - 1 */
 634 
 635         u8 delay_mult;
 636 
 637         /* used to override LED behavior */
 638         u8 led_override;  /* Substituted for normal value, if non-zero */
 639         u16 led_override_timeoff; /* delta to next timer event */
 640         u8 led_override_vals[2]; /* Alternates per blink-frame */
 641         u8 led_override_phase; /* Just counts, LSB picks from vals[] */
 642         atomic_t led_override_timer_active;
 643         /* Used to flash LEDs in override mode */
 644         struct timer_list led_override_timer;
 645         struct xmit_wait cong_stats;
 646         struct timer_list symerr_clear_timer;
 647 
 648         /* Synchronize access between driver writes and sysfs reads */
 649         spinlock_t cc_shadow_lock
 650                 ____cacheline_aligned_in_smp;
 651 
 652         /* Shadow copy of the congestion control table */
 653         struct cc_table_shadow *ccti_entries_shadow;
 654 
 655         /* Shadow copy of the congestion control entries */
 656         struct ib_cc_congestion_setting_attr_shadow *congestion_entries_shadow;
 657 
 658         /* List of congestion control table entries */
 659         struct ib_cc_table_entry_shadow *ccti_entries;
 660 
 661         /* 16 congestion entries with each entry corresponding to a SL */
 662         struct ib_cc_congestion_entry_shadow *congestion_entries;
 663 
 664         /* Maximum number of congestion control entries that the agent expects
 665          * the manager to send.
 666          */
 667         u16 cc_supported_table_entries;
 668 
 669         /* Total number of congestion control table entries */
 670         u16 total_cct_entry;
 671 
 672         /* Bit map identifying service level */
 673         u16 cc_sl_control_map;
 674 
 675         /* maximum congestion control table index */
 676         u16 ccti_limit;
 677 
 678         /* CA's max number of 64 entry units in the congestion control table */
 679         u8 cc_max_table_entries;
 680 };
 681 
 682 /* Observers. Not to be taken lightly, possibly not to ship. */
 683 /*
 684  * If a diag read or write is to (bottom <= offset <= top),
 685  * the "hoook" is called, allowing, e.g. shadows to be
 686  * updated in sync with the driver. struct diag_observer
 687  * is the "visible" part.
 688  */
 689 struct diag_observer;
 690 
 691 typedef int (*diag_hook) (struct qib_devdata *dd,
 692         const struct diag_observer *op,
 693         u32 offs, u64 *data, u64 mask, int only_32);
 694 
 695 struct diag_observer {
 696         diag_hook hook;
 697         u32 bottom;
 698         u32 top;
 699 };
 700 
 701 extern int qib_register_observer(struct qib_devdata *dd,
 702         const struct diag_observer *op);
 703 
 704 /* Only declared here, not defined. Private to diags */
 705 struct diag_observer_list_elt;
 706 
 707 /* device data struct now contains only "general per-device" info.
 708  * fields related to a physical IB port are in a qib_pportdata struct,
 709  * described above) while fields only used by a particular chip-type are in
 710  * a qib_chipdata struct, whose contents are opaque to this file.
 711  */
 712 struct qib_devdata {
 713         struct qib_ibdev verbs_dev;     /* must be first */
 714         struct list_head list;
 715         /* pointers to related structs for this device */
 716         /* pci access data structure */
 717         struct pci_dev *pcidev;
 718         struct cdev *user_cdev;
 719         struct cdev *diag_cdev;
 720         struct device *user_device;
 721         struct device *diag_device;
 722 
 723         /* mem-mapped pointer to base of chip regs */
 724         u64 __iomem *kregbase;
 725         /* end of mem-mapped chip space excluding sendbuf and user regs */
 726         u64 __iomem *kregend;
 727         /* physical address of chip for io_remap, etc. */
 728         resource_size_t physaddr;
 729         /* qib_cfgctxts pointers */
 730         struct qib_ctxtdata **rcd; /* Receive Context Data */
 731 
 732         /* qib_pportdata, points to array of (physical) port-specific
 733          * data structs, indexed by pidx (0..n-1)
 734          */
 735         struct qib_pportdata *pport;
 736         struct qib_chip_specific *cspec; /* chip-specific */
 737 
 738         /* kvirt address of 1st 2k pio buffer */
 739         void __iomem *pio2kbase;
 740         /* kvirt address of 1st 4k pio buffer */
 741         void __iomem *pio4kbase;
 742         /* mem-mapped pointer to base of PIO buffers (if using WC PAT) */
 743         void __iomem *piobase;
 744         /* mem-mapped pointer to base of user chip regs (if using WC PAT) */
 745         u64 __iomem *userbase;
 746         void __iomem *piovl15base; /* base of VL15 buffers, if not WC */
 747         /*
 748          * points to area where PIOavail registers will be DMA'ed.
 749          * Has to be on a page of it's own, because the page will be
 750          * mapped into user program space.  This copy is *ONLY* ever
 751          * written by DMA, not by the driver!  Need a copy per device
 752          * when we get to multiple devices
 753          */
 754         volatile __le64 *pioavailregs_dma; /* DMA'ed by chip */
 755         /* physical address where updates occur */
 756         dma_addr_t pioavailregs_phys;
 757 
 758         /* device-specific implementations of functions needed by
 759          * common code. Contrary to previous consensus, we can't
 760          * really just point to a device-specific table, because we
 761          * may need to "bend", e.g. *_f_put_tid
 762          */
 763         /* fallback to alternate interrupt type if possible */
 764         int (*f_intr_fallback)(struct qib_devdata *);
 765         /* hard reset chip */
 766         int (*f_reset)(struct qib_devdata *);
 767         void (*f_quiet_serdes)(struct qib_pportdata *);
 768         int (*f_bringup_serdes)(struct qib_pportdata *);
 769         int (*f_early_init)(struct qib_devdata *);
 770         void (*f_clear_tids)(struct qib_devdata *, struct qib_ctxtdata *);
 771         void (*f_put_tid)(struct qib_devdata *, u64 __iomem*,
 772                                 u32, unsigned long);
 773         void (*f_cleanup)(struct qib_devdata *);
 774         void (*f_setextled)(struct qib_pportdata *, u32);
 775         /* fill out chip-specific fields */
 776         int (*f_get_base_info)(struct qib_ctxtdata *, struct qib_base_info *);
 777         /* free irq */
 778         void (*f_free_irq)(struct qib_devdata *);
 779         struct qib_message_header *(*f_get_msgheader)
 780                                         (struct qib_devdata *, __le32 *);
 781         void (*f_config_ctxts)(struct qib_devdata *);
 782         int (*f_get_ib_cfg)(struct qib_pportdata *, int);
 783         int (*f_set_ib_cfg)(struct qib_pportdata *, int, u32);
 784         int (*f_set_ib_loopback)(struct qib_pportdata *, const char *);
 785         int (*f_get_ib_table)(struct qib_pportdata *, int, void *);
 786         int (*f_set_ib_table)(struct qib_pportdata *, int, void *);
 787         u32 (*f_iblink_state)(u64);
 788         u8 (*f_ibphys_portstate)(u64);
 789         void (*f_xgxs_reset)(struct qib_pportdata *);
 790         /* per chip actions needed for IB Link up/down changes */
 791         int (*f_ib_updown)(struct qib_pportdata *, int, u64);
 792         u32 __iomem *(*f_getsendbuf)(struct qib_pportdata *, u64, u32 *);
 793         /* Read/modify/write of GPIO pins (potentially chip-specific */
 794         int (*f_gpio_mod)(struct qib_devdata *dd, u32 out, u32 dir,
 795                 u32 mask);
 796         /* Enable writes to config EEPROM (if supported) */
 797         int (*f_eeprom_wen)(struct qib_devdata *dd, int wen);
 798         /*
 799          * modify rcvctrl shadow[s] and write to appropriate chip-regs.
 800          * see above QIB_RCVCTRL_xxx_ENB/DIS for operations.
 801          * (ctxt == -1) means "all contexts", only meaningful for
 802          * clearing. Could remove if chip_spec shutdown properly done.
 803          */
 804         void (*f_rcvctrl)(struct qib_pportdata *, unsigned int op,
 805                 int ctxt);
 806         /* Read/modify/write sendctrl appropriately for op and port. */
 807         void (*f_sendctrl)(struct qib_pportdata *, u32 op);
 808         void (*f_set_intr_state)(struct qib_devdata *, u32);
 809         void (*f_set_armlaunch)(struct qib_devdata *, u32);
 810         void (*f_wantpiobuf_intr)(struct qib_devdata *, u32);
 811         int (*f_late_initreg)(struct qib_devdata *);
 812         int (*f_init_sdma_regs)(struct qib_pportdata *);
 813         u16 (*f_sdma_gethead)(struct qib_pportdata *);
 814         int (*f_sdma_busy)(struct qib_pportdata *);
 815         void (*f_sdma_update_tail)(struct qib_pportdata *, u16);
 816         void (*f_sdma_set_desc_cnt)(struct qib_pportdata *, unsigned);
 817         void (*f_sdma_sendctrl)(struct qib_pportdata *, unsigned);
 818         void (*f_sdma_hw_clean_up)(struct qib_pportdata *);
 819         void (*f_sdma_hw_start_up)(struct qib_pportdata *);
 820         void (*f_sdma_init_early)(struct qib_pportdata *);
 821         void (*f_set_cntr_sample)(struct qib_pportdata *, u32, u32);
 822         void (*f_update_usrhead)(struct qib_ctxtdata *, u64, u32, u32, u32);
 823         u32 (*f_hdrqempty)(struct qib_ctxtdata *);
 824         u64 (*f_portcntr)(struct qib_pportdata *, u32);
 825         u32 (*f_read_cntrs)(struct qib_devdata *, loff_t, char **,
 826                 u64 **);
 827         u32 (*f_read_portcntrs)(struct qib_devdata *, loff_t, u32,
 828                 char **, u64 **);
 829         u32 (*f_setpbc_control)(struct qib_pportdata *, u32, u8, u8);
 830         void (*f_initvl15_bufs)(struct qib_devdata *);
 831         void (*f_init_ctxt)(struct qib_ctxtdata *);
 832         void (*f_txchk_change)(struct qib_devdata *, u32, u32, u32,
 833                 struct qib_ctxtdata *);
 834         void (*f_writescratch)(struct qib_devdata *, u32);
 835         int (*f_tempsense_rd)(struct qib_devdata *, int regnum);
 836 #ifdef CONFIG_INFINIBAND_QIB_DCA
 837         int (*f_notify_dca)(struct qib_devdata *, unsigned long event);
 838 #endif
 839 
 840         char *boardname; /* human readable board info */
 841 
 842         /* template for writing TIDs  */
 843         u64 tidtemplate;
 844         /* value to write to free TIDs */
 845         u64 tidinvalid;
 846 
 847         /* number of registers used for pioavail */
 848         u32 pioavregs;
 849         /* device (not port) flags, basically device capabilities */
 850         u32 flags;
 851         /* last buffer for user use */
 852         u32 lastctxt_piobuf;
 853 
 854         /* reset value */
 855         u64 z_int_counter;
 856         /* percpu intcounter */
 857         u64 __percpu *int_counter;
 858 
 859         /* pio bufs allocated per ctxt */
 860         u32 pbufsctxt;
 861         /* if remainder on bufs/ctxt, ctxts < extrabuf get 1 extra */
 862         u32 ctxts_extrabuf;
 863         /*
 864          * number of ctxts configured as max; zero is set to number chip
 865          * supports, less gives more pio bufs/ctxt, etc.
 866          */
 867         u32 cfgctxts;
 868         /*
 869          * number of ctxts available for PSM open
 870          */
 871         u32 freectxts;
 872 
 873         /*
 874          * hint that we should update pioavailshadow before
 875          * looking for a PIO buffer
 876          */
 877         u32 upd_pio_shadow;
 878 
 879         /* internal debugging stats */
 880         u32 maxpkts_call;
 881         u32 avgpkts_call;
 882         u64 nopiobufs;
 883 
 884         /* PCI Vendor ID (here for NodeInfo) */
 885         u16 vendorid;
 886         /* PCI Device ID (here for NodeInfo) */
 887         u16 deviceid;
 888         /* for write combining settings */
 889         int wc_cookie;
 890         unsigned long wc_base;
 891         unsigned long wc_len;
 892 
 893         /* shadow copy of struct page *'s for exp tid pages */
 894         struct page **pageshadow;
 895         /* shadow copy of dma handles for exp tid pages */
 896         dma_addr_t *physshadow;
 897         u64 __iomem *egrtidbase;
 898         spinlock_t sendctrl_lock; /* protect changes to sendctrl shadow */
 899         /* around rcd and (user ctxts) ctxt_cnt use (intr vs free) */
 900         spinlock_t uctxt_lock; /* rcd and user context changes */
 901         /*
 902          * per unit status, see also portdata statusp
 903          * mapped readonly into user processes so they can get unit and
 904          * IB link status cheaply
 905          */
 906         u64 *devstatusp;
 907         char *freezemsg; /* freeze msg if hw error put chip in freeze */
 908         u32 freezelen; /* max length of freezemsg */
 909         /* timer used to prevent stats overflow, error throttling, etc. */
 910         struct timer_list stats_timer;
 911 
 912         /* timer to verify interrupts work, and fallback if possible */
 913         struct timer_list intrchk_timer;
 914         unsigned long ureg_align; /* user register alignment */
 915 
 916         /*
 917          * Protects pioavailshadow, pioavailkernel, pio_need_disarm, and
 918          * pio_writing.
 919          */
 920         spinlock_t pioavail_lock;
 921         /*
 922          * index of last buffer to optimize search for next
 923          */
 924         u32 last_pio;
 925         /*
 926          * min kernel pio buffer to optimize search
 927          */
 928         u32 min_kernel_pio;
 929         /*
 930          * Shadow copies of registers; size indicates read access size.
 931          * Most of them are readonly, but some are write-only register,
 932          * where we manipulate the bits in the shadow copy, and then write
 933          * the shadow copy to qlogic_ib.
 934          *
 935          * We deliberately make most of these 32 bits, since they have
 936          * restricted range.  For any that we read, we won't to generate 32
 937          * bit accesses, since Opteron will generate 2 separate 32 bit HT
 938          * transactions for a 64 bit read, and we want to avoid unnecessary
 939          * bus transactions.
 940          */
 941 
 942         /* This is the 64 bit group */
 943 
 944         unsigned long pioavailshadow[6];
 945         /* bitmap of send buffers available for the kernel to use with PIO. */
 946         unsigned long pioavailkernel[6];
 947         /* bitmap of send buffers which need to be disarmed. */
 948         unsigned long pio_need_disarm[3];
 949         /* bitmap of send buffers which are being written to. */
 950         unsigned long pio_writing[3];
 951         /* kr_revision shadow */
 952         u64 revision;
 953         /* Base GUID for device (from eeprom, network order) */
 954         __be64 base_guid;
 955 
 956         /*
 957          * kr_sendpiobufbase value (chip offset of pio buffers), and the
 958          * base of the 2KB buffer s(user processes only use 2K)
 959          */
 960         u64 piobufbase;
 961         u32 pio2k_bufbase;
 962 
 963         /* these are the "32 bit" regs */
 964 
 965         /* number of GUIDs in the flash for this interface */
 966         u32 nguid;
 967         /*
 968          * the following two are 32-bit bitmasks, but {test,clear,set}_bit
 969          * all expect bit fields to be "unsigned long"
 970          */
 971         unsigned long rcvctrl; /* shadow per device rcvctrl */
 972         unsigned long sendctrl; /* shadow per device sendctrl */
 973 
 974         /* value we put in kr_rcvhdrcnt */
 975         u32 rcvhdrcnt;
 976         /* value we put in kr_rcvhdrsize */
 977         u32 rcvhdrsize;
 978         /* value we put in kr_rcvhdrentsize */
 979         u32 rcvhdrentsize;
 980         /* kr_ctxtcnt value */
 981         u32 ctxtcnt;
 982         /* kr_pagealign value */
 983         u32 palign;
 984         /* number of "2KB" PIO buffers */
 985         u32 piobcnt2k;
 986         /* size in bytes of "2KB" PIO buffers */
 987         u32 piosize2k;
 988         /* max usable size in dwords of a "2KB" PIO buffer before going "4KB" */
 989         u32 piosize2kmax_dwords;
 990         /* number of "4KB" PIO buffers */
 991         u32 piobcnt4k;
 992         /* size in bytes of "4KB" PIO buffers */
 993         u32 piosize4k;
 994         /* kr_rcvegrbase value */
 995         u32 rcvegrbase;
 996         /* kr_rcvtidbase value */
 997         u32 rcvtidbase;
 998         /* kr_rcvtidcnt value */
 999         u32 rcvtidcnt;
1000         /* kr_userregbase */
1001         u32 uregbase;
1002         /* shadow the control register contents */
1003         u32 control;
1004 
1005         /* chip address space used by 4k pio buffers */
1006         u32 align4k;
1007         /* size of each rcvegrbuffer */
1008         u16 rcvegrbufsize;
1009         /* log2 of above */
1010         u16 rcvegrbufsize_shift;
1011         /* localbus width (1, 2,4,8,16,32) from config space  */
1012         u32 lbus_width;
1013         /* localbus speed in MHz */
1014         u32 lbus_speed;
1015         int unit; /* unit # of this chip */
1016 
1017         /* start of CHIP_SPEC move to chipspec, but need code changes */
1018         /* low and high portions of MSI capability/vector */
1019         u32 msi_lo;
1020         /* saved after PCIe init for restore after reset */
1021         u32 msi_hi;
1022         /* MSI data (vector) saved for restore */
1023         u16 msi_data;
1024         /* so we can rewrite it after a chip reset */
1025         u32 pcibar0;
1026         /* so we can rewrite it after a chip reset */
1027         u32 pcibar1;
1028         u64 rhdrhead_intr_off;
1029 
1030         /*
1031          * ASCII serial number, from flash, large enough for original
1032          * all digit strings, and longer QLogic serial number format
1033          */
1034         u8 serial[16];
1035         /* human readable board version */
1036         u8 boardversion[96];
1037         u8 lbus_info[32]; /* human readable localbus info */
1038         /* chip major rev, from qib_revision */
1039         u8 majrev;
1040         /* chip minor rev, from qib_revision */
1041         u8 minrev;
1042 
1043         /* Misc small ints */
1044         /* Number of physical ports available */
1045         u8 num_pports;
1046         /* Lowest context number which can be used by user processes */
1047         u8 first_user_ctxt;
1048         u8 n_krcv_queues;
1049         u8 qpn_mask;
1050         u8 skip_kctxt_mask;
1051 
1052         u16 rhf_offset; /* offset of RHF within receive header entry */
1053 
1054         /*
1055          * GPIO pins for twsi-connected devices, and device code for eeprom
1056          */
1057         u8 gpio_sda_num;
1058         u8 gpio_scl_num;
1059         u8 twsi_eeprom_dev;
1060         u8 board_atten;
1061 
1062         /* Support (including locks) for EEPROM logging of errors and time */
1063         /* control access to actual counters, timer */
1064         spinlock_t eep_st_lock;
1065         /* control high-level access to EEPROM */
1066         struct mutex eep_lock;
1067         uint64_t traffic_wds;
1068         struct qib_diag_client *diag_client;
1069         spinlock_t qib_diag_trans_lock; /* protect diag observer ops */
1070         struct diag_observer_list_elt *diag_observer_list;
1071 
1072         u8 psxmitwait_supported;
1073         /* cycle length of PS* counters in HW (in picoseconds) */
1074         u16 psxmitwait_check_rate;
1075         /* high volume overflow errors defered to tasklet */
1076         struct tasklet_struct error_tasklet;
1077 
1078         int assigned_node_id; /* NUMA node closest to HCA */
1079 };
1080 
1081 /* hol_state values */
1082 #define QIB_HOL_UP       0
1083 #define QIB_HOL_INIT     1
1084 
1085 #define QIB_SDMA_SENDCTRL_OP_ENABLE    (1U << 0)
1086 #define QIB_SDMA_SENDCTRL_OP_INTENABLE (1U << 1)
1087 #define QIB_SDMA_SENDCTRL_OP_HALT      (1U << 2)
1088 #define QIB_SDMA_SENDCTRL_OP_CLEANUP   (1U << 3)
1089 #define QIB_SDMA_SENDCTRL_OP_DRAIN     (1U << 4)
1090 
1091 /* operation types for f_txchk_change() */
1092 #define TXCHK_CHG_TYPE_DIS1  3
1093 #define TXCHK_CHG_TYPE_ENAB1 2
1094 #define TXCHK_CHG_TYPE_KERN  1
1095 #define TXCHK_CHG_TYPE_USER  0
1096 
1097 #define QIB_CHASE_TIME msecs_to_jiffies(145)
1098 #define QIB_CHASE_DIS_TIME msecs_to_jiffies(160)
1099 
1100 /* Private data for file operations */
1101 struct qib_filedata {
1102         struct qib_ctxtdata *rcd;
1103         unsigned subctxt;
1104         unsigned tidcursor;
1105         struct qib_user_sdma_queue *pq;
1106         int rec_cpu_num; /* for cpu affinity; -1 if none */
1107 };
1108 
1109 extern struct xarray qib_dev_table;
1110 extern struct qib_devdata *qib_lookup(int unit);
1111 extern u32 qib_cpulist_count;
1112 extern unsigned long *qib_cpulist;
1113 extern unsigned qib_cc_table_size;
1114 
1115 int qib_init(struct qib_devdata *, int);
1116 int init_chip_wc_pat(struct qib_devdata *dd, u32);
1117 int qib_enable_wc(struct qib_devdata *dd);
1118 void qib_disable_wc(struct qib_devdata *dd);
1119 int qib_count_units(int *npresentp, int *nupp);
1120 int qib_count_active_units(void);
1121 
1122 int qib_cdev_init(int minor, const char *name,
1123                   const struct file_operations *fops,
1124                   struct cdev **cdevp, struct device **devp);
1125 void qib_cdev_cleanup(struct cdev **cdevp, struct device **devp);
1126 int qib_dev_init(void);
1127 void qib_dev_cleanup(void);
1128 
1129 int qib_diag_add(struct qib_devdata *);
1130 void qib_diag_remove(struct qib_devdata *);
1131 void qib_handle_e_ibstatuschanged(struct qib_pportdata *, u64);
1132 void qib_sdma_update_tail(struct qib_pportdata *, u16); /* hold sdma_lock */
1133 
1134 int qib_decode_err(struct qib_devdata *dd, char *buf, size_t blen, u64 err);
1135 void qib_bad_intrstatus(struct qib_devdata *);
1136 void qib_handle_urcv(struct qib_devdata *, u64);
1137 
1138 /* clean up any per-chip chip-specific stuff */
1139 void qib_chip_cleanup(struct qib_devdata *);
1140 /* clean up any chip type-specific stuff */
1141 void qib_chip_done(void);
1142 
1143 /* check to see if we have to force ordering for write combining */
1144 int qib_unordered_wc(void);
1145 void qib_pio_copy(void __iomem *to, const void *from, size_t count);
1146 
1147 void qib_disarm_piobufs(struct qib_devdata *, unsigned, unsigned);
1148 int qib_disarm_piobufs_ifneeded(struct qib_ctxtdata *);
1149 void qib_disarm_piobufs_set(struct qib_devdata *, unsigned long *, unsigned);
1150 void qib_cancel_sends(struct qib_pportdata *);
1151 
1152 int qib_create_rcvhdrq(struct qib_devdata *, struct qib_ctxtdata *);
1153 int qib_setup_eagerbufs(struct qib_ctxtdata *);
1154 void qib_set_ctxtcnt(struct qib_devdata *);
1155 int qib_create_ctxts(struct qib_devdata *dd);
1156 struct qib_ctxtdata *qib_create_ctxtdata(struct qib_pportdata *, u32, int);
1157 int qib_init_pportdata(struct qib_pportdata *, struct qib_devdata *, u8, u8);
1158 void qib_free_ctxtdata(struct qib_devdata *, struct qib_ctxtdata *);
1159 
1160 u32 qib_kreceive(struct qib_ctxtdata *, u32 *, u32 *);
1161 int qib_reset_device(int);
1162 int qib_wait_linkstate(struct qib_pportdata *, u32, int);
1163 int qib_set_linkstate(struct qib_pportdata *, u8);
1164 int qib_set_mtu(struct qib_pportdata *, u16);
1165 int qib_set_lid(struct qib_pportdata *, u32, u8);
1166 void qib_hol_down(struct qib_pportdata *);
1167 void qib_hol_init(struct qib_pportdata *);
1168 void qib_hol_up(struct qib_pportdata *);
1169 void qib_hol_event(struct timer_list *);
1170 void qib_disable_after_error(struct qib_devdata *);
1171 int qib_set_uevent_bits(struct qib_pportdata *, const int);
1172 
1173 /* for use in system calls, where we want to know device type, etc. */
1174 #define ctxt_fp(fp) \
1175         (((struct qib_filedata *)(fp)->private_data)->rcd)
1176 #define subctxt_fp(fp) \
1177         (((struct qib_filedata *)(fp)->private_data)->subctxt)
1178 #define tidcursor_fp(fp) \
1179         (((struct qib_filedata *)(fp)->private_data)->tidcursor)
1180 #define user_sdma_queue_fp(fp) \
1181         (((struct qib_filedata *)(fp)->private_data)->pq)
1182 
1183 static inline struct qib_devdata *dd_from_ppd(struct qib_pportdata *ppd)
1184 {
1185         return ppd->dd;
1186 }
1187 
1188 static inline struct qib_devdata *dd_from_dev(struct qib_ibdev *dev)
1189 {
1190         return container_of(dev, struct qib_devdata, verbs_dev);
1191 }
1192 
1193 static inline struct qib_devdata *dd_from_ibdev(struct ib_device *ibdev)
1194 {
1195         return dd_from_dev(to_idev(ibdev));
1196 }
1197 
1198 static inline struct qib_pportdata *ppd_from_ibp(struct qib_ibport *ibp)
1199 {
1200         return container_of(ibp, struct qib_pportdata, ibport_data);
1201 }
1202 
1203 static inline struct qib_ibport *to_iport(struct ib_device *ibdev, u8 port)
1204 {
1205         struct qib_devdata *dd = dd_from_ibdev(ibdev);
1206         unsigned pidx = port - 1; /* IB number port from 1, hdw from 0 */
1207 
1208         WARN_ON(pidx >= dd->num_pports);
1209         return &dd->pport[pidx].ibport_data;
1210 }
1211 
1212 /*
1213  * values for dd->flags (_device_ related flags) and
1214  */
1215 #define QIB_HAS_LINK_LATENCY  0x1 /* supports link latency (IB 1.2) */
1216 #define QIB_INITTED           0x2 /* chip and driver up and initted */
1217 #define QIB_DOING_RESET       0x4  /* in the middle of doing chip reset */
1218 #define QIB_PRESENT           0x8  /* chip accesses can be done */
1219 #define QIB_PIO_FLUSH_WC      0x10 /* Needs Write combining flush for PIO */
1220 #define QIB_HAS_THRESH_UPDATE 0x40
1221 #define QIB_HAS_SDMA_TIMEOUT  0x80
1222 #define QIB_USE_SPCL_TRIG     0x100 /* SpecialTrigger launch enabled */
1223 #define QIB_NODMA_RTAIL       0x200 /* rcvhdrtail register DMA enabled */
1224 #define QIB_HAS_INTX          0x800 /* Supports INTx interrupts */
1225 #define QIB_HAS_SEND_DMA      0x1000 /* Supports Send DMA */
1226 #define QIB_HAS_VLSUPP        0x2000 /* Supports multiple VLs; PBC different */
1227 #define QIB_HAS_HDRSUPP       0x4000 /* Supports header suppression */
1228 #define QIB_BADINTR           0x8000 /* severe interrupt problems */
1229 #define QIB_DCA_ENABLED       0x10000 /* Direct Cache Access enabled */
1230 #define QIB_HAS_QSFP          0x20000 /* device (card instance) has QSFP */
1231 #define QIB_SHUTDOWN          0x40000 /* device is shutting down */
1232 
1233 /*
1234  * values for ppd->lflags (_ib_port_ related flags)
1235  */
1236 #define QIBL_LINKV             0x1 /* IB link state valid */
1237 #define QIBL_LINKDOWN          0x8 /* IB link is down */
1238 #define QIBL_LINKINIT          0x10 /* IB link level is up */
1239 #define QIBL_LINKARMED         0x20 /* IB link is ARMED */
1240 #define QIBL_LINKACTIVE        0x40 /* IB link is ACTIVE */
1241 /* leave a gap for more IB-link state */
1242 #define QIBL_IB_AUTONEG_INPROG 0x1000 /* non-IBTA DDR/QDR neg active */
1243 #define QIBL_IB_AUTONEG_FAILED 0x2000 /* non-IBTA DDR/QDR neg failed */
1244 #define QIBL_IB_LINK_DISABLED  0x4000 /* Linkdown-disable forced,
1245                                        * Do not try to bring up */
1246 #define QIBL_IB_FORCE_NOTIFY   0x8000 /* force notify on next ib change */
1247 
1248 /* IB dword length mask in PBC (lower 11 bits); same for all chips */
1249 #define QIB_PBC_LENGTH_MASK                     ((1 << 11) - 1)
1250 
1251 
1252 /* ctxt_flag bit offsets */
1253                 /* waiting for a packet to arrive */
1254 #define QIB_CTXT_WAITING_RCV   2
1255                 /* master has not finished initializing */
1256 #define QIB_CTXT_MASTER_UNINIT 4
1257                 /* waiting for an urgent packet to arrive */
1258 #define QIB_CTXT_WAITING_URG 5
1259 
1260 /* free up any allocated data at closes */
1261 void qib_free_data(struct qib_ctxtdata *dd);
1262 void qib_chg_pioavailkernel(struct qib_devdata *, unsigned, unsigned,
1263                             u32, struct qib_ctxtdata *);
1264 struct qib_devdata *qib_init_iba7322_funcs(struct pci_dev *,
1265                                            const struct pci_device_id *);
1266 struct qib_devdata *qib_init_iba7220_funcs(struct pci_dev *,
1267                                            const struct pci_device_id *);
1268 struct qib_devdata *qib_init_iba6120_funcs(struct pci_dev *,
1269                                            const struct pci_device_id *);
1270 void qib_free_devdata(struct qib_devdata *);
1271 struct qib_devdata *qib_alloc_devdata(struct pci_dev *pdev, size_t extra);
1272 
1273 #define QIB_TWSI_NO_DEV 0xFF
1274 /* Below qib_twsi_ functions must be called with eep_lock held */
1275 int qib_twsi_reset(struct qib_devdata *dd);
1276 int qib_twsi_blk_rd(struct qib_devdata *dd, int dev, int addr, void *buffer,
1277                     int len);
1278 int qib_twsi_blk_wr(struct qib_devdata *dd, int dev, int addr,
1279                     const void *buffer, int len);
1280 void qib_get_eeprom_info(struct qib_devdata *);
1281 void qib_dump_lookup_output_queue(struct qib_devdata *);
1282 void qib_force_pio_avail_update(struct qib_devdata *);
1283 void qib_clear_symerror_on_linkup(struct timer_list *t);
1284 
1285 /*
1286  * Set LED override, only the two LSBs have "public" meaning, but
1287  * any non-zero value substitutes them for the Link and LinkTrain
1288  * LED states.
1289  */
1290 #define QIB_LED_PHYS 1 /* Physical (linktraining) GREEN LED */
1291 #define QIB_LED_LOG 2  /* Logical (link) YELLOW LED */
1292 void qib_set_led_override(struct qib_pportdata *ppd, unsigned int val);
1293 
1294 /* send dma routines */
1295 int qib_setup_sdma(struct qib_pportdata *);
1296 void qib_teardown_sdma(struct qib_pportdata *);
1297 void __qib_sdma_intr(struct qib_pportdata *);
1298 void qib_sdma_intr(struct qib_pportdata *);
1299 void qib_user_sdma_send_desc(struct qib_pportdata *dd,
1300                         struct list_head *pktlist);
1301 int qib_sdma_verbs_send(struct qib_pportdata *, struct rvt_sge_state *,
1302                         u32, struct qib_verbs_txreq *);
1303 /* ppd->sdma_lock should be locked before calling this. */
1304 int qib_sdma_make_progress(struct qib_pportdata *dd);
1305 
1306 static inline int qib_sdma_empty(const struct qib_pportdata *ppd)
1307 {
1308         return ppd->sdma_descq_added == ppd->sdma_descq_removed;
1309 }
1310 
1311 /* must be called under qib_sdma_lock */
1312 static inline u16 qib_sdma_descq_freecnt(const struct qib_pportdata *ppd)
1313 {
1314         return ppd->sdma_descq_cnt -
1315                 (ppd->sdma_descq_added - ppd->sdma_descq_removed) - 1;
1316 }
1317 
1318 static inline int __qib_sdma_running(struct qib_pportdata *ppd)
1319 {
1320         return ppd->sdma_state.current_state == qib_sdma_state_s99_running;
1321 }
1322 int qib_sdma_running(struct qib_pportdata *);
1323 void dump_sdma_state(struct qib_pportdata *ppd);
1324 void __qib_sdma_process_event(struct qib_pportdata *, enum qib_sdma_events);
1325 void qib_sdma_process_event(struct qib_pportdata *, enum qib_sdma_events);
1326 
1327 /*
1328  * number of words used for protocol header if not set by qib_userinit();
1329  */
1330 #define QIB_DFLT_RCVHDRSIZE 9
1331 
1332 /*
1333  * We need to be able to handle an IB header of at least 24 dwords.
1334  * We need the rcvhdrq large enough to handle largest IB header, but
1335  * still have room for a 2KB MTU standard IB packet.
1336  * Additionally, some processor/memory controller combinations
1337  * benefit quite strongly from having the DMA'ed data be cacheline
1338  * aligned and a cacheline multiple, so we set the size to 32 dwords
1339  * (2 64-byte primary cachelines for pretty much all processors of
1340  * interest).  The alignment hurts nothing, other than using somewhat
1341  * more memory.
1342  */
1343 #define QIB_RCVHDR_ENTSIZE 32
1344 
1345 int qib_get_user_pages(unsigned long, size_t, struct page **);
1346 void qib_release_user_pages(struct page **, size_t);
1347 int qib_eeprom_read(struct qib_devdata *, u8, void *, int);
1348 int qib_eeprom_write(struct qib_devdata *, u8, const void *, int);
1349 u32 __iomem *qib_getsendbuf_range(struct qib_devdata *, u32 *, u32, u32);
1350 void qib_sendbuf_done(struct qib_devdata *, unsigned);
1351 
1352 static inline void qib_clear_rcvhdrtail(const struct qib_ctxtdata *rcd)
1353 {
1354         *((u64 *) rcd->rcvhdrtail_kvaddr) = 0ULL;
1355 }
1356 
1357 static inline u32 qib_get_rcvhdrtail(const struct qib_ctxtdata *rcd)
1358 {
1359         /*
1360          * volatile because it's a DMA target from the chip, routine is
1361          * inlined, and don't want register caching or reordering.
1362          */
1363         return (u32) le64_to_cpu(
1364                 *((volatile __le64 *)rcd->rcvhdrtail_kvaddr)); /* DMA'ed */
1365 }
1366 
1367 static inline u32 qib_get_hdrqtail(const struct qib_ctxtdata *rcd)
1368 {
1369         const struct qib_devdata *dd = rcd->dd;
1370         u32 hdrqtail;
1371 
1372         if (dd->flags & QIB_NODMA_RTAIL) {
1373                 __le32 *rhf_addr;
1374                 u32 seq;
1375 
1376                 rhf_addr = (__le32 *) rcd->rcvhdrq +
1377                         rcd->head + dd->rhf_offset;
1378                 seq = qib_hdrget_seq(rhf_addr);
1379                 hdrqtail = rcd->head;
1380                 if (seq == rcd->seq_cnt)
1381                         hdrqtail++;
1382         } else
1383                 hdrqtail = qib_get_rcvhdrtail(rcd);
1384 
1385         return hdrqtail;
1386 }
1387 
1388 /*
1389  * sysfs interface.
1390  */
1391 
1392 extern const char ib_qib_version[];
1393 extern const struct attribute_group qib_attr_group;
1394 
1395 int qib_device_create(struct qib_devdata *);
1396 void qib_device_remove(struct qib_devdata *);
1397 
1398 int qib_create_port_files(struct ib_device *ibdev, u8 port_num,
1399                           struct kobject *kobj);
1400 void qib_verbs_unregister_sysfs(struct qib_devdata *);
1401 /* Hook for sysfs read of QSFP */
1402 extern int qib_qsfp_dump(struct qib_pportdata *ppd, char *buf, int len);
1403 
1404 int __init qib_init_qibfs(void);
1405 int __exit qib_exit_qibfs(void);
1406 
1407 int qibfs_add(struct qib_devdata *);
1408 int qibfs_remove(struct qib_devdata *);
1409 
1410 int qib_pcie_init(struct pci_dev *, const struct pci_device_id *);
1411 int qib_pcie_ddinit(struct qib_devdata *, struct pci_dev *,
1412                     const struct pci_device_id *);
1413 void qib_pcie_ddcleanup(struct qib_devdata *);
1414 int qib_pcie_params(struct qib_devdata *dd, u32 minw, u32 *nent);
1415 void qib_free_irq(struct qib_devdata *dd);
1416 int qib_reinit_intr(struct qib_devdata *dd);
1417 void qib_pcie_getcmd(struct qib_devdata *, u16 *, u8 *, u8 *);
1418 void qib_pcie_reenable(struct qib_devdata *, u16, u8, u8);
1419 /* interrupts for device */
1420 u64 qib_int_counter(struct qib_devdata *);
1421 /* interrupt for all devices */
1422 u64 qib_sps_ints(void);
1423 
1424 /*
1425  * dma_addr wrappers - all 0's invalid for hw
1426  */
1427 int qib_map_page(struct pci_dev *d, struct page *p, dma_addr_t *daddr);
1428 struct pci_dev *qib_get_pci_dev(struct rvt_dev_info *rdi);
1429 
1430 /*
1431  * Flush write combining store buffers (if present) and perform a write
1432  * barrier.
1433  */
1434 static inline void qib_flush_wc(void)
1435 {
1436 #if defined(CONFIG_X86_64)
1437         asm volatile("sfence" : : : "memory");
1438 #else
1439         wmb(); /* no reorder around wc flush */
1440 #endif
1441 }
1442 
1443 /* global module parameter variables */
1444 extern unsigned qib_ibmtu;
1445 extern ushort qib_cfgctxts;
1446 extern ushort qib_num_cfg_vls;
1447 extern ushort qib_mini_init; /* If set, do few (ideally 0) writes to chip */
1448 extern unsigned qib_n_krcv_queues;
1449 extern unsigned qib_sdma_fetch_arb;
1450 extern unsigned qib_compat_ddr_negotiate;
1451 extern int qib_special_trigger;
1452 extern unsigned qib_numa_aware;
1453 
1454 extern struct mutex qib_mutex;
1455 
1456 /* Number of seconds before our card status check...  */
1457 #define STATUS_TIMEOUT 60
1458 
1459 #define QIB_DRV_NAME            "ib_qib"
1460 #define QIB_USER_MINOR_BASE     0
1461 #define QIB_TRACE_MINOR         127
1462 #define QIB_DIAGPKT_MINOR       128
1463 #define QIB_DIAG_MINOR_BASE     129
1464 #define QIB_NMINORS             255
1465 
1466 #define PCI_VENDOR_ID_PATHSCALE 0x1fc1
1467 #define PCI_VENDOR_ID_QLOGIC 0x1077
1468 #define PCI_DEVICE_ID_QLOGIC_IB_6120 0x10
1469 #define PCI_DEVICE_ID_QLOGIC_IB_7220 0x7220
1470 #define PCI_DEVICE_ID_QLOGIC_IB_7322 0x7322
1471 
1472 /*
1473  * qib_early_err is used (only!) to print early errors before devdata is
1474  * allocated, or when dd->pcidev may not be valid, and at the tail end of
1475  * cleanup when devdata may have been freed, etc.  qib_dev_porterr is
1476  * the same as qib_dev_err, but is used when the message really needs
1477  * the IB port# to be definitive as to what's happening..
1478  * All of these go to the trace log, and the trace log entry is done
1479  * first to avoid possible serial port delays from printk.
1480  */
1481 #define qib_early_err(dev, fmt, ...) \
1482         dev_err(dev, fmt, ##__VA_ARGS__)
1483 
1484 #define qib_dev_err(dd, fmt, ...) \
1485         dev_err(&(dd)->pcidev->dev, "%s: " fmt, \
1486                 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
1487 
1488 #define qib_dev_warn(dd, fmt, ...) \
1489         dev_warn(&(dd)->pcidev->dev, "%s: " fmt, \
1490                  rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
1491 
1492 #define qib_dev_porterr(dd, port, fmt, ...) \
1493         dev_err(&(dd)->pcidev->dev, "%s: IB%u:%u " fmt, \
1494                 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), (dd)->unit, (port), \
1495                 ##__VA_ARGS__)
1496 
1497 #define qib_devinfo(pcidev, fmt, ...) \
1498         dev_info(&(pcidev)->dev, fmt, ##__VA_ARGS__)
1499 
1500 /*
1501  * this is used for formatting hw error messages...
1502  */
1503 struct qib_hwerror_msgs {
1504         u64 mask;
1505         const char *msg;
1506         size_t sz;
1507 };
1508 
1509 #define QLOGIC_IB_HWE_MSG(a, b) { .mask = a, .msg = b }
1510 
1511 /* in qib_intr.c... */
1512 void qib_format_hwerrors(u64 hwerrs,
1513                          const struct qib_hwerror_msgs *hwerrmsgs,
1514                          size_t nhwerrmsgs, char *msg, size_t lmsg);
1515 
1516 void qib_stop_send_queue(struct rvt_qp *qp);
1517 void qib_quiesce_qp(struct rvt_qp *qp);
1518 void qib_flush_qp_waiters(struct rvt_qp *qp);
1519 int qib_mtu_to_path_mtu(u32 mtu);
1520 u32 qib_mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu);
1521 void qib_notify_error_qp(struct rvt_qp *qp);
1522 int qib_get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp,
1523                            struct ib_qp_attr *attr);
1524 
1525 #endif                          /* _QIB_KERNEL_H */