1/*
2 * Copyright (c) 2013 Intel Corporation. All rights reserved.
3 * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.
4 * All rights reserved.
5 * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses.  You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
12 *
13 *     Redistribution and use in source and binary forms, with or
14 *     without modification, are permitted provided that the following
15 *     conditions are met:
16 *
17 *      - Redistributions of source code must retain the above
18 *        copyright notice, this list of conditions and the following
19 *        disclaimer.
20 *
21 *      - Redistributions in binary form must reproduce the above
22 *        copyright notice, this list of conditions and the following
23 *        disclaimer in the documentation and/or other materials
24 *        provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
34 */
35/*
36 * This file contains all of the code that is specific to the
37 * QLogic_IB 6120 PCIe chip.
38 */
39
40#include <linux/interrupt.h>
41#include <linux/pci.h>
42#include <linux/delay.h>
43#include <rdma/ib_verbs.h>
44
45#include "qib.h"
46#include "qib_6120_regs.h"
47
48static void qib_6120_setup_setextled(struct qib_pportdata *, u32);
49static void sendctrl_6120_mod(struct qib_pportdata *ppd, u32 op);
50static u8 qib_6120_phys_portstate(u64);
51static u32 qib_6120_iblink_state(u64);
52
53/*
54 * This file contains all the chip-specific register information and
55 * access functions for the Intel Intel_IB PCI-Express chip.
56 *
57 */
58
59/* KREG_IDX uses machine-generated #defines */
60#define KREG_IDX(regname) (QIB_6120_##regname##_OFFS / sizeof(u64))
61
62/* Use defines to tie machine-generated names to lower-case names */
63#define kr_extctrl KREG_IDX(EXTCtrl)
64#define kr_extstatus KREG_IDX(EXTStatus)
65#define kr_gpio_clear KREG_IDX(GPIOClear)
66#define kr_gpio_mask KREG_IDX(GPIOMask)
67#define kr_gpio_out KREG_IDX(GPIOOut)
68#define kr_gpio_status KREG_IDX(GPIOStatus)
69#define kr_rcvctrl KREG_IDX(RcvCtrl)
70#define kr_sendctrl KREG_IDX(SendCtrl)
71#define kr_partitionkey KREG_IDX(RcvPartitionKey)
72#define kr_hwdiagctrl KREG_IDX(HwDiagCtrl)
73#define kr_ibcstatus KREG_IDX(IBCStatus)
74#define kr_ibcctrl KREG_IDX(IBCCtrl)
75#define kr_sendbuffererror KREG_IDX(SendBufErr0)
76#define kr_rcvbthqp KREG_IDX(RcvBTHQP)
77#define kr_counterregbase KREG_IDX(CntrRegBase)
78#define kr_palign KREG_IDX(PageAlign)
79#define kr_rcvegrbase KREG_IDX(RcvEgrBase)
80#define kr_rcvegrcnt KREG_IDX(RcvEgrCnt)
81#define kr_rcvhdrcnt KREG_IDX(RcvHdrCnt)
82#define kr_rcvhdrentsize KREG_IDX(RcvHdrEntSize)
83#define kr_rcvhdrsize KREG_IDX(RcvHdrSize)
84#define kr_rcvtidbase KREG_IDX(RcvTIDBase)
85#define kr_rcvtidcnt KREG_IDX(RcvTIDCnt)
86#define kr_scratch KREG_IDX(Scratch)
87#define kr_sendctrl KREG_IDX(SendCtrl)
88#define kr_sendpioavailaddr KREG_IDX(SendPIOAvailAddr)
89#define kr_sendpiobufbase KREG_IDX(SendPIOBufBase)
90#define kr_sendpiobufcnt KREG_IDX(SendPIOBufCnt)
91#define kr_sendpiosize KREG_IDX(SendPIOSize)
92#define kr_sendregbase KREG_IDX(SendRegBase)
93#define kr_userregbase KREG_IDX(UserRegBase)
94#define kr_control KREG_IDX(Control)
95#define kr_intclear KREG_IDX(IntClear)
96#define kr_intmask KREG_IDX(IntMask)
97#define kr_intstatus KREG_IDX(IntStatus)
98#define kr_errclear KREG_IDX(ErrClear)
99#define kr_errmask KREG_IDX(ErrMask)
100#define kr_errstatus KREG_IDX(ErrStatus)
101#define kr_hwerrclear KREG_IDX(HwErrClear)
102#define kr_hwerrmask KREG_IDX(HwErrMask)
103#define kr_hwerrstatus KREG_IDX(HwErrStatus)
104#define kr_revision KREG_IDX(Revision)
105#define kr_portcnt KREG_IDX(PortCnt)
106#define kr_serdes_cfg0 KREG_IDX(SerdesCfg0)
107#define kr_serdes_cfg1 (kr_serdes_cfg0 + 1)
108#define kr_serdes_stat KREG_IDX(SerdesStat)
109#define kr_xgxs_cfg KREG_IDX(XGXSCfg)
110
111/* These must only be written via qib_write_kreg_ctxt() */
112#define kr_rcvhdraddr KREG_IDX(RcvHdrAddr0)
113#define kr_rcvhdrtailaddr KREG_IDX(RcvHdrTailAddr0)
114
115#define CREG_IDX(regname) ((QIB_6120_##regname##_OFFS - \
116			QIB_6120_LBIntCnt_OFFS) / sizeof(u64))
117
118#define cr_badformat CREG_IDX(RxBadFormatCnt)
119#define cr_erricrc CREG_IDX(RxICRCErrCnt)
120#define cr_errlink CREG_IDX(RxLinkProblemCnt)
121#define cr_errlpcrc CREG_IDX(RxLPCRCErrCnt)
122#define cr_errpkey CREG_IDX(RxPKeyMismatchCnt)
123#define cr_rcvflowctrl_err CREG_IDX(RxFlowCtrlErrCnt)
124#define cr_err_rlen CREG_IDX(RxLenErrCnt)
125#define cr_errslen CREG_IDX(TxLenErrCnt)
126#define cr_errtidfull CREG_IDX(RxTIDFullErrCnt)
127#define cr_errtidvalid CREG_IDX(RxTIDValidErrCnt)
128#define cr_errvcrc CREG_IDX(RxVCRCErrCnt)
129#define cr_ibstatuschange CREG_IDX(IBStatusChangeCnt)
130#define cr_lbint CREG_IDX(LBIntCnt)
131#define cr_invalidrlen CREG_IDX(RxMaxMinLenErrCnt)
132#define cr_invalidslen CREG_IDX(TxMaxMinLenErrCnt)
133#define cr_lbflowstall CREG_IDX(LBFlowStallCnt)
134#define cr_pktrcv CREG_IDX(RxDataPktCnt)
135#define cr_pktrcvflowctrl CREG_IDX(RxFlowPktCnt)
136#define cr_pktsend CREG_IDX(TxDataPktCnt)
137#define cr_pktsendflow CREG_IDX(TxFlowPktCnt)
138#define cr_portovfl CREG_IDX(RxP0HdrEgrOvflCnt)
139#define cr_rcvebp CREG_IDX(RxEBPCnt)
140#define cr_rcvovfl CREG_IDX(RxBufOvflCnt)
141#define cr_senddropped CREG_IDX(TxDroppedPktCnt)
142#define cr_sendstall CREG_IDX(TxFlowStallCnt)
143#define cr_sendunderrun CREG_IDX(TxUnderrunCnt)
144#define cr_wordrcv CREG_IDX(RxDwordCnt)
145#define cr_wordsend CREG_IDX(TxDwordCnt)
146#define cr_txunsupvl CREG_IDX(TxUnsupVLErrCnt)
147#define cr_rxdroppkt CREG_IDX(RxDroppedPktCnt)
148#define cr_iblinkerrrecov CREG_IDX(IBLinkErrRecoveryCnt)
149#define cr_iblinkdown CREG_IDX(IBLinkDownedCnt)
150#define cr_ibsymbolerr CREG_IDX(IBSymbolErrCnt)
151
152#define SYM_RMASK(regname, fldname) ((u64)              \
153	QIB_6120_##regname##_##fldname##_RMASK)
154#define SYM_MASK(regname, fldname) ((u64)               \
155	QIB_6120_##regname##_##fldname##_RMASK <<       \
156	 QIB_6120_##regname##_##fldname##_LSB)
157#define SYM_LSB(regname, fldname) (QIB_6120_##regname##_##fldname##_LSB)
158
159#define SYM_FIELD(value, regname, fldname) ((u64) \
160	(((value) >> SYM_LSB(regname, fldname)) & \
161	 SYM_RMASK(regname, fldname)))
162#define ERR_MASK(fldname) SYM_MASK(ErrMask, fldname##Mask)
163#define HWE_MASK(fldname) SYM_MASK(HwErrMask, fldname##Mask)
164
165/* link training states, from IBC */
166#define IB_6120_LT_STATE_DISABLED        0x00
167#define IB_6120_LT_STATE_LINKUP          0x01
168#define IB_6120_LT_STATE_POLLACTIVE      0x02
169#define IB_6120_LT_STATE_POLLQUIET       0x03
170#define IB_6120_LT_STATE_SLEEPDELAY      0x04
171#define IB_6120_LT_STATE_SLEEPQUIET      0x05
172#define IB_6120_LT_STATE_CFGDEBOUNCE     0x08
173#define IB_6120_LT_STATE_CFGRCVFCFG      0x09
174#define IB_6120_LT_STATE_CFGWAITRMT      0x0a
175#define IB_6120_LT_STATE_CFGIDLE 0x0b
176#define IB_6120_LT_STATE_RECOVERRETRAIN  0x0c
177#define IB_6120_LT_STATE_RECOVERWAITRMT  0x0e
178#define IB_6120_LT_STATE_RECOVERIDLE     0x0f
179
180/* link state machine states from IBC */
181#define IB_6120_L_STATE_DOWN             0x0
182#define IB_6120_L_STATE_INIT             0x1
183#define IB_6120_L_STATE_ARM              0x2
184#define IB_6120_L_STATE_ACTIVE           0x3
185#define IB_6120_L_STATE_ACT_DEFER        0x4
186
187static const u8 qib_6120_physportstate[0x20] = {
188	[IB_6120_LT_STATE_DISABLED] = IB_PHYSPORTSTATE_DISABLED,
189	[IB_6120_LT_STATE_LINKUP] = IB_PHYSPORTSTATE_LINKUP,
190	[IB_6120_LT_STATE_POLLACTIVE] = IB_PHYSPORTSTATE_POLL,
191	[IB_6120_LT_STATE_POLLQUIET] = IB_PHYSPORTSTATE_POLL,
192	[IB_6120_LT_STATE_SLEEPDELAY] = IB_PHYSPORTSTATE_SLEEP,
193	[IB_6120_LT_STATE_SLEEPQUIET] = IB_PHYSPORTSTATE_SLEEP,
194	[IB_6120_LT_STATE_CFGDEBOUNCE] =
195		IB_PHYSPORTSTATE_CFG_TRAIN,
196	[IB_6120_LT_STATE_CFGRCVFCFG] =
197		IB_PHYSPORTSTATE_CFG_TRAIN,
198	[IB_6120_LT_STATE_CFGWAITRMT] =
199		IB_PHYSPORTSTATE_CFG_TRAIN,
200	[IB_6120_LT_STATE_CFGIDLE] = IB_PHYSPORTSTATE_CFG_TRAIN,
201	[IB_6120_LT_STATE_RECOVERRETRAIN] =
202		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
203	[IB_6120_LT_STATE_RECOVERWAITRMT] =
204		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
205	[IB_6120_LT_STATE_RECOVERIDLE] =
206		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
207	[0x10] = IB_PHYSPORTSTATE_CFG_TRAIN,
208	[0x11] = IB_PHYSPORTSTATE_CFG_TRAIN,
209	[0x12] = IB_PHYSPORTSTATE_CFG_TRAIN,
210	[0x13] = IB_PHYSPORTSTATE_CFG_TRAIN,
211	[0x14] = IB_PHYSPORTSTATE_CFG_TRAIN,
212	[0x15] = IB_PHYSPORTSTATE_CFG_TRAIN,
213	[0x16] = IB_PHYSPORTSTATE_CFG_TRAIN,
214	[0x17] = IB_PHYSPORTSTATE_CFG_TRAIN
215};
216
217
218struct qib_chip_specific {
219	u64 __iomem *cregbase;
220	u64 *cntrs;
221	u64 *portcntrs;
222	void *dummy_hdrq;   /* used after ctxt close */
223	dma_addr_t dummy_hdrq_phys;
224	spinlock_t kernel_tid_lock; /* no back to back kernel TID writes */
225	spinlock_t user_tid_lock; /* no back to back user TID writes */
226	spinlock_t rcvmod_lock; /* protect rcvctrl shadow changes */
227	spinlock_t gpio_lock; /* RMW of shadows/regs for ExtCtrl and GPIO */
228	u64 hwerrmask;
229	u64 errormask;
230	u64 gpio_out; /* shadow of kr_gpio_out, for rmw ops */
231	u64 gpio_mask; /* shadow the gpio mask register */
232	u64 extctrl; /* shadow the gpio output enable, etc... */
233	/*
234	 * these 5 fields are used to establish deltas for IB symbol
235	 * errors and linkrecovery errors.  They can be reported on
236	 * some chips during link negotiation prior to INIT, and with
237	 * DDR when faking DDR negotiations with non-IBTA switches.
238	 * The chip counters are adjusted at driver unload if there is
239	 * a non-zero delta.
240	 */
241	u64 ibdeltainprog;
242	u64 ibsymdelta;
243	u64 ibsymsnap;
244	u64 iblnkerrdelta;
245	u64 iblnkerrsnap;
246	u64 ibcctrl; /* shadow for kr_ibcctrl */
247	u32 lastlinkrecov; /* link recovery issue */
248	int irq;
249	u32 cntrnamelen;
250	u32 portcntrnamelen;
251	u32 ncntrs;
252	u32 nportcntrs;
253	/* used with gpio interrupts to implement IB counters */
254	u32 rxfc_unsupvl_errs;
255	u32 overrun_thresh_errs;
256	/*
257	 * these count only cases where _successive_ LocalLinkIntegrity
258	 * errors were seen in the receive headers of IB standard packets
259	 */
260	u32 lli_errs;
261	u32 lli_counter;
262	u64 lli_thresh;
263	u64 sword; /* total dwords sent (sample result) */
264	u64 rword; /* total dwords received (sample result) */
265	u64 spkts; /* total packets sent (sample result) */
266	u64 rpkts; /* total packets received (sample result) */
267	u64 xmit_wait; /* # of ticks no data sent (sample result) */
268	struct timer_list pma_timer;
269	char emsgbuf[128];
270	char bitsmsgbuf[64];
271	u8 pma_sample_status;
272};
273
274/* ibcctrl bits */
275#define QLOGIC_IB_IBCC_LINKINITCMD_DISABLE 1
276/* cycle through TS1/TS2 till OK */
277#define QLOGIC_IB_IBCC_LINKINITCMD_POLL 2
278/* wait for TS1, then go on */
279#define QLOGIC_IB_IBCC_LINKINITCMD_SLEEP 3
280#define QLOGIC_IB_IBCC_LINKINITCMD_SHIFT 16
281
282#define QLOGIC_IB_IBCC_LINKCMD_DOWN 1           /* move to 0x11 */
283#define QLOGIC_IB_IBCC_LINKCMD_ARMED 2          /* move to 0x21 */
284#define QLOGIC_IB_IBCC_LINKCMD_ACTIVE 3 /* move to 0x31 */
285#define QLOGIC_IB_IBCC_LINKCMD_SHIFT 18
286
287/*
288 * We could have a single register get/put routine, that takes a group type,
289 * but this is somewhat clearer and cleaner.  It also gives us some error
290 * checking.  64 bit register reads should always work, but are inefficient
291 * on opteron (the northbridge always generates 2 separate HT 32 bit reads),
292 * so we use kreg32 wherever possible.  User register and counter register
293 * reads are always 32 bit reads, so only one form of those routines.
294 */
295
296/**
297 * qib_read_ureg32 - read 32-bit virtualized per-context register
298 * @dd: device
299 * @regno: register number
300 * @ctxt: context number
301 *
302 * Return the contents of a register that is virtualized to be per context.
303 * Returns -1 on errors (not distinguishable from valid contents at
304 * runtime; we may add a separate error variable at some point).
305 */
306static inline u32 qib_read_ureg32(const struct qib_devdata *dd,
307				  enum qib_ureg regno, int ctxt)
308{
309	if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
310		return 0;
311
312	if (dd->userbase)
313		return readl(regno + (u64 __iomem *)
314			     ((char __iomem *)dd->userbase +
315			      dd->ureg_align * ctxt));
316	else
317		return readl(regno + (u64 __iomem *)
318			     (dd->uregbase +
319			      (char __iomem *)dd->kregbase +
320			      dd->ureg_align * ctxt));
321}
322
323/**
324 * qib_write_ureg - write 32-bit virtualized per-context register
325 * @dd: device
326 * @regno: register number
327 * @value: value
328 * @ctxt: context
329 *
330 * Write the contents of a register that is virtualized to be per context.
331 */
332static inline void qib_write_ureg(const struct qib_devdata *dd,
333				  enum qib_ureg regno, u64 value, int ctxt)
334{
335	u64 __iomem *ubase;
336
337	if (dd->userbase)
338		ubase = (u64 __iomem *)
339			((char __iomem *) dd->userbase +
340			 dd->ureg_align * ctxt);
341	else
342		ubase = (u64 __iomem *)
343			(dd->uregbase +
344			 (char __iomem *) dd->kregbase +
345			 dd->ureg_align * ctxt);
346
347	if (dd->kregbase && (dd->flags & QIB_PRESENT))
348		writeq(value, &ubase[regno]);
349}
350
351static inline u32 qib_read_kreg32(const struct qib_devdata *dd,
352				  const u16 regno)
353{
354	if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
355		return -1;
356	return readl((u32 __iomem *)&dd->kregbase[regno]);
357}
358
359static inline u64 qib_read_kreg64(const struct qib_devdata *dd,
360				  const u16 regno)
361{
362	if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
363		return -1;
364
365	return readq(&dd->kregbase[regno]);
366}
367
368static inline void qib_write_kreg(const struct qib_devdata *dd,
369				  const u16 regno, u64 value)
370{
371	if (dd->kregbase && (dd->flags & QIB_PRESENT))
372		writeq(value, &dd->kregbase[regno]);
373}
374
375/**
376 * qib_write_kreg_ctxt - write a device's per-ctxt 64-bit kernel register
377 * @dd: the qlogic_ib device
378 * @regno: the register number to write
379 * @ctxt: the context containing the register
380 * @value: the value to write
381 */
382static inline void qib_write_kreg_ctxt(const struct qib_devdata *dd,
383				       const u16 regno, unsigned ctxt,
384				       u64 value)
385{
386	qib_write_kreg(dd, regno + ctxt, value);
387}
388
389static inline void write_6120_creg(const struct qib_devdata *dd,
390				   u16 regno, u64 value)
391{
392	if (dd->cspec->cregbase && (dd->flags & QIB_PRESENT))
393		writeq(value, &dd->cspec->cregbase[regno]);
394}
395
396static inline u64 read_6120_creg(const struct qib_devdata *dd, u16 regno)
397{
398	if (!dd->cspec->cregbase || !(dd->flags & QIB_PRESENT))
399		return 0;
400	return readq(&dd->cspec->cregbase[regno]);
401}
402
403static inline u32 read_6120_creg32(const struct qib_devdata *dd, u16 regno)
404{
405	if (!dd->cspec->cregbase || !(dd->flags & QIB_PRESENT))
406		return 0;
407	return readl(&dd->cspec->cregbase[regno]);
408}
409
410/* kr_control bits */
411#define QLOGIC_IB_C_RESET 1U
412
413/* kr_intstatus, kr_intclear, kr_intmask bits */
414#define QLOGIC_IB_I_RCVURG_MASK ((1U << 5) - 1)
415#define QLOGIC_IB_I_RCVURG_SHIFT 0
416#define QLOGIC_IB_I_RCVAVAIL_MASK ((1U << 5) - 1)
417#define QLOGIC_IB_I_RCVAVAIL_SHIFT 12
418
419#define QLOGIC_IB_C_FREEZEMODE 0x00000002
420#define QLOGIC_IB_C_LINKENABLE 0x00000004
421#define QLOGIC_IB_I_ERROR               0x0000000080000000ULL
422#define QLOGIC_IB_I_SPIOSENT            0x0000000040000000ULL
423#define QLOGIC_IB_I_SPIOBUFAVAIL        0x0000000020000000ULL
424#define QLOGIC_IB_I_GPIO                0x0000000010000000ULL
425#define QLOGIC_IB_I_BITSEXTANT \
426		((QLOGIC_IB_I_RCVURG_MASK << QLOGIC_IB_I_RCVURG_SHIFT) | \
427		(QLOGIC_IB_I_RCVAVAIL_MASK << \
428		 QLOGIC_IB_I_RCVAVAIL_SHIFT) | \
429		QLOGIC_IB_I_ERROR | QLOGIC_IB_I_SPIOSENT | \
430		QLOGIC_IB_I_SPIOBUFAVAIL | QLOGIC_IB_I_GPIO)
431
432/* kr_hwerrclear, kr_hwerrmask, kr_hwerrstatus, bits */
433#define QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK  0x000000000000003fULL
434#define QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT 0
435#define QLOGIC_IB_HWE_PCIEPOISONEDTLP      0x0000000010000000ULL
436#define QLOGIC_IB_HWE_PCIECPLTIMEOUT       0x0000000020000000ULL
437#define QLOGIC_IB_HWE_PCIEBUSPARITYXTLH    0x0000000040000000ULL
438#define QLOGIC_IB_HWE_PCIEBUSPARITYXADM    0x0000000080000000ULL
439#define QLOGIC_IB_HWE_PCIEBUSPARITYRADM    0x0000000100000000ULL
440#define QLOGIC_IB_HWE_COREPLL_FBSLIP       0x0080000000000000ULL
441#define QLOGIC_IB_HWE_COREPLL_RFSLIP       0x0100000000000000ULL
442#define QLOGIC_IB_HWE_PCIE1PLLFAILED       0x0400000000000000ULL
443#define QLOGIC_IB_HWE_PCIE0PLLFAILED       0x0800000000000000ULL
444#define QLOGIC_IB_HWE_SERDESPLLFAILED      0x1000000000000000ULL
445
446
447/* kr_extstatus bits */
448#define QLOGIC_IB_EXTS_FREQSEL 0x2
449#define QLOGIC_IB_EXTS_SERDESSEL 0x4
450#define QLOGIC_IB_EXTS_MEMBIST_ENDTEST     0x0000000000004000
451#define QLOGIC_IB_EXTS_MEMBIST_FOUND       0x0000000000008000
452
453/* kr_xgxsconfig bits */
454#define QLOGIC_IB_XGXS_RESET          0x5ULL
455
456#define _QIB_GPIO_SDA_NUM 1
457#define _QIB_GPIO_SCL_NUM 0
458
459/* Bits in GPIO for the added IB link interrupts */
460#define GPIO_RXUVL_BIT 3
461#define GPIO_OVRUN_BIT 4
462#define GPIO_LLI_BIT 5
463#define GPIO_ERRINTR_MASK 0x38
464
465
466#define QLOGIC_IB_RT_BUFSIZE_MASK 0xe0000000ULL
467#define QLOGIC_IB_RT_BUFSIZE_SHIFTVAL(tid) \
468	((((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) >> 29) + 11 - 1)
469#define QLOGIC_IB_RT_BUFSIZE(tid) (1 << QLOGIC_IB_RT_BUFSIZE_SHIFTVAL(tid))
470#define QLOGIC_IB_RT_IS_VALID(tid) \
471	(((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) && \
472	 ((((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) != QLOGIC_IB_RT_BUFSIZE_MASK)))
473#define QLOGIC_IB_RT_ADDR_MASK 0x1FFFFFFFULL /* 29 bits valid */
474#define QLOGIC_IB_RT_ADDR_SHIFT 10
475
476#define QLOGIC_IB_R_INTRAVAIL_SHIFT 16
477#define QLOGIC_IB_R_TAILUPD_SHIFT 31
478#define IBA6120_R_PKEY_DIS_SHIFT 30
479
480#define PBC_6120_VL15_SEND_CTRL (1ULL << 31) /* pbc; VL15; link_buf only */
481
482#define IBCBUSFRSPCPARITYERR HWE_MASK(IBCBusFromSPCParityErr)
483#define IBCBUSTOSPCPARITYERR HWE_MASK(IBCBusToSPCParityErr)
484
485#define SYM_MASK_BIT(regname, fldname, bit) ((u64) \
486	((1ULL << (SYM_LSB(regname, fldname) + (bit)))))
487
488#define TXEMEMPARITYERR_PIOBUF \
489	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 0)
490#define TXEMEMPARITYERR_PIOPBC \
491	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 1)
492#define TXEMEMPARITYERR_PIOLAUNCHFIFO \
493	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 2)
494
495#define RXEMEMPARITYERR_RCVBUF \
496	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 0)
497#define RXEMEMPARITYERR_LOOKUPQ \
498	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 1)
499#define RXEMEMPARITYERR_EXPTID \
500	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 2)
501#define RXEMEMPARITYERR_EAGERTID \
502	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 3)
503#define RXEMEMPARITYERR_FLAGBUF \
504	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 4)
505#define RXEMEMPARITYERR_DATAINFO \
506	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 5)
507#define RXEMEMPARITYERR_HDRINFO \
508	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 6)
509
510/* 6120 specific hardware errors... */
511static const struct qib_hwerror_msgs qib_6120_hwerror_msgs[] = {
512	/* generic hardware errors */
513	QLOGIC_IB_HWE_MSG(IBCBUSFRSPCPARITYERR, "QIB2IB Parity"),
514	QLOGIC_IB_HWE_MSG(IBCBUSTOSPCPARITYERR, "IB2QIB Parity"),
515
516	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOBUF,
517			  "TXE PIOBUF Memory Parity"),
518	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOPBC,
519			  "TXE PIOPBC Memory Parity"),
520	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOLAUNCHFIFO,
521			  "TXE PIOLAUNCHFIFO Memory Parity"),
522
523	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_RCVBUF,
524			  "RXE RCVBUF Memory Parity"),
525	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_LOOKUPQ,
526			  "RXE LOOKUPQ Memory Parity"),
527	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_EAGERTID,
528			  "RXE EAGERTID Memory Parity"),
529	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_EXPTID,
530			  "RXE EXPTID Memory Parity"),
531	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_FLAGBUF,
532			  "RXE FLAGBUF Memory Parity"),
533	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_DATAINFO,
534			  "RXE DATAINFO Memory Parity"),
535	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_HDRINFO,
536			  "RXE HDRINFO Memory Parity"),
537
538	/* chip-specific hardware errors */
539	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEPOISONEDTLP,
540			  "PCIe Poisoned TLP"),
541	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIECPLTIMEOUT,
542			  "PCIe completion timeout"),
543	/*
544	 * In practice, it's unlikely wthat we'll see PCIe PLL, or bus
545	 * parity or memory parity error failures, because most likely we
546	 * won't be able to talk to the core of the chip.  Nonetheless, we
547	 * might see them, if they are in parts of the PCIe core that aren't
548	 * essential.
549	 */
550	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIE1PLLFAILED,
551			  "PCIePLL1"),
552	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIE0PLLFAILED,
553			  "PCIePLL0"),
554	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYXTLH,
555			  "PCIe XTLH core parity"),
556	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYXADM,
557			  "PCIe ADM TX core parity"),
558	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYRADM,
559			  "PCIe ADM RX core parity"),
560	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_SERDESPLLFAILED,
561			  "SerDes PLL"),
562};
563
564#define TXE_PIO_PARITY (TXEMEMPARITYERR_PIOBUF | TXEMEMPARITYERR_PIOPBC)
565#define _QIB_PLL_FAIL (QLOGIC_IB_HWE_COREPLL_FBSLIP |   \
566		QLOGIC_IB_HWE_COREPLL_RFSLIP)
567
568	/* variables for sanity checking interrupt and errors */
569#define IB_HWE_BITSEXTANT \
570	(HWE_MASK(RXEMemParityErr) |					\
571	 HWE_MASK(TXEMemParityErr) |					\
572	 (QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK <<			\
573	  QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT) |			\
574	 QLOGIC_IB_HWE_PCIE1PLLFAILED |					\
575	 QLOGIC_IB_HWE_PCIE0PLLFAILED |					\
576	 QLOGIC_IB_HWE_PCIEPOISONEDTLP |				\
577	 QLOGIC_IB_HWE_PCIECPLTIMEOUT |					\
578	 QLOGIC_IB_HWE_PCIEBUSPARITYXTLH |				\
579	 QLOGIC_IB_HWE_PCIEBUSPARITYXADM |				\
580	 QLOGIC_IB_HWE_PCIEBUSPARITYRADM |				\
581	 HWE_MASK(PowerOnBISTFailed) |					\
582	 QLOGIC_IB_HWE_COREPLL_FBSLIP |					\
583	 QLOGIC_IB_HWE_COREPLL_RFSLIP |					\
584	 QLOGIC_IB_HWE_SERDESPLLFAILED |				\
585	 HWE_MASK(IBCBusToSPCParityErr) |				\
586	 HWE_MASK(IBCBusFromSPCParityErr))
587
588#define IB_E_BITSEXTANT \
589	(ERR_MASK(RcvFormatErr) | ERR_MASK(RcvVCRCErr) |		\
590	 ERR_MASK(RcvICRCErr) | ERR_MASK(RcvMinPktLenErr) |		\
591	 ERR_MASK(RcvMaxPktLenErr) | ERR_MASK(RcvLongPktLenErr) |	\
592	 ERR_MASK(RcvShortPktLenErr) | ERR_MASK(RcvUnexpectedCharErr) | \
593	 ERR_MASK(RcvUnsupportedVLErr) | ERR_MASK(RcvEBPErr) |		\
594	 ERR_MASK(RcvIBFlowErr) | ERR_MASK(RcvBadVersionErr) |		\
595	 ERR_MASK(RcvEgrFullErr) | ERR_MASK(RcvHdrFullErr) |		\
596	 ERR_MASK(RcvBadTidErr) | ERR_MASK(RcvHdrLenErr) |		\
597	 ERR_MASK(RcvHdrErr) | ERR_MASK(RcvIBLostLinkErr) |		\
598	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendMaxPktLenErr) |	\
599	 ERR_MASK(SendUnderRunErr) | ERR_MASK(SendPktLenErr) |		\
600	 ERR_MASK(SendDroppedSmpPktErr) |				\
601	 ERR_MASK(SendDroppedDataPktErr) |				\
602	 ERR_MASK(SendPioArmLaunchErr) |				\
603	 ERR_MASK(SendUnexpectedPktNumErr) |				\
604	 ERR_MASK(SendUnsupportedVLErr) | ERR_MASK(IBStatusChanged) |	\
605	 ERR_MASK(InvalidAddrErr) | ERR_MASK(ResetNegated) |		\
606	 ERR_MASK(HardwareErr))
607
608#define QLOGIC_IB_E_PKTERRS ( \
609		ERR_MASK(SendPktLenErr) |				\
610		ERR_MASK(SendDroppedDataPktErr) |			\
611		ERR_MASK(RcvVCRCErr) |					\
612		ERR_MASK(RcvICRCErr) |					\
613		ERR_MASK(RcvShortPktLenErr) |				\
614		ERR_MASK(RcvEBPErr))
615
616/* These are all rcv-related errors which we want to count for stats */
617#define E_SUM_PKTERRS						\
618	(ERR_MASK(RcvHdrLenErr) | ERR_MASK(RcvBadTidErr) |		\
619	 ERR_MASK(RcvBadVersionErr) | ERR_MASK(RcvHdrErr) |		\
620	 ERR_MASK(RcvLongPktLenErr) | ERR_MASK(RcvShortPktLenErr) |	\
621	 ERR_MASK(RcvMaxPktLenErr) | ERR_MASK(RcvMinPktLenErr) |	\
622	 ERR_MASK(RcvFormatErr) | ERR_MASK(RcvUnsupportedVLErr) |	\
623	 ERR_MASK(RcvUnexpectedCharErr) | ERR_MASK(RcvEBPErr))
624
625/* These are all send-related errors which we want to count for stats */
626#define E_SUM_ERRS							\
627	(ERR_MASK(SendPioArmLaunchErr) |				\
628	 ERR_MASK(SendUnexpectedPktNumErr) |				\
629	 ERR_MASK(SendDroppedDataPktErr) |				\
630	 ERR_MASK(SendDroppedSmpPktErr) |				\
631	 ERR_MASK(SendMaxPktLenErr) | ERR_MASK(SendUnsupportedVLErr) |	\
632	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendPktLenErr) |		\
633	 ERR_MASK(InvalidAddrErr))
634
635/*
636 * this is similar to E_SUM_ERRS, but can't ignore armlaunch, don't ignore
637 * errors not related to freeze and cancelling buffers.  Can't ignore
638 * armlaunch because could get more while still cleaning up, and need
639 * to cancel those as they happen.
640 */
641#define E_SPKT_ERRS_IGNORE \
642	(ERR_MASK(SendDroppedDataPktErr) |				\
643	 ERR_MASK(SendDroppedSmpPktErr) |				\
644	 ERR_MASK(SendMaxPktLenErr) | ERR_MASK(SendMinPktLenErr) |	\
645	 ERR_MASK(SendPktLenErr))
646
647/*
648 * these are errors that can occur when the link changes state while
649 * a packet is being sent or received.  This doesn't cover things
650 * like EBP or VCRC that can be the result of a sending having the
651 * link change state, so we receive a "known bad" packet.
652 */
653#define E_SUM_LINK_PKTERRS		\
654	(ERR_MASK(SendDroppedDataPktErr) |				\
655	 ERR_MASK(SendDroppedSmpPktErr) |				\
656	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendPktLenErr) |		\
657	 ERR_MASK(RcvShortPktLenErr) | ERR_MASK(RcvMinPktLenErr) |	\
658	 ERR_MASK(RcvUnexpectedCharErr))
659
660static void qib_6120_put_tid_2(struct qib_devdata *, u64 __iomem *,
661			       u32, unsigned long);
662
663/*
664 * On platforms using this chip, and not having ordered WC stores, we
665 * can get TXE parity errors due to speculative reads to the PIO buffers,
666 * and this, due to a chip issue can result in (many) false parity error
667 * reports.  So it's a debug print on those, and an info print on systems
668 * where the speculative reads don't occur.
669 */
670static void qib_6120_txe_recover(struct qib_devdata *dd)
671{
672	if (!qib_unordered_wc())
673		qib_devinfo(dd->pcidev,
674			    "Recovering from TXE PIO parity error\n");
675}
676
677/* enable/disable chip from delivering interrupts */
678static void qib_6120_set_intr_state(struct qib_devdata *dd, u32 enable)
679{
680	if (enable) {
681		if (dd->flags & QIB_BADINTR)
682			return;
683		qib_write_kreg(dd, kr_intmask, ~0ULL);
684		/* force re-interrupt of any pending interrupts. */
685		qib_write_kreg(dd, kr_intclear, 0ULL);
686	} else
687		qib_write_kreg(dd, kr_intmask, 0ULL);
688}
689
690/*
691 * Try to cleanup as much as possible for anything that might have gone
692 * wrong while in freeze mode, such as pio buffers being written by user
693 * processes (causing armlaunch), send errors due to going into freeze mode,
694 * etc., and try to avoid causing extra interrupts while doing so.
695 * Forcibly update the in-memory pioavail register copies after cleanup
696 * because the chip won't do it while in freeze mode (the register values
697 * themselves are kept correct).
698 * Make sure that we don't lose any important interrupts by using the chip
699 * feature that says that writing 0 to a bit in *clear that is set in
700 * *status will cause an interrupt to be generated again (if allowed by
701 * the *mask value).
702 * This is in chip-specific code because of all of the register accesses,
703 * even though the details are similar on most chips
704 */
705static void qib_6120_clear_freeze(struct qib_devdata *dd)
706{
707	/* disable error interrupts, to avoid confusion */
708	qib_write_kreg(dd, kr_errmask, 0ULL);
709
710	/* also disable interrupts; errormask is sometimes overwriten */
711	qib_6120_set_intr_state(dd, 0);
712
713	qib_cancel_sends(dd->pport);
714
715	/* clear the freeze, and be sure chip saw it */
716	qib_write_kreg(dd, kr_control, dd->control);
717	qib_read_kreg32(dd, kr_scratch);
718
719	/* force in-memory update now we are out of freeze */
720	qib_force_pio_avail_update(dd);
721
722	/*
723	 * force new interrupt if any hwerr, error or interrupt bits are
724	 * still set, and clear "safe" send packet errors related to freeze
725	 * and cancelling sends.  Re-enable error interrupts before possible
726	 * force of re-interrupt on pending interrupts.
727	 */
728	qib_write_kreg(dd, kr_hwerrclear, 0ULL);
729	qib_write_kreg(dd, kr_errclear, E_SPKT_ERRS_IGNORE);
730	qib_write_kreg(dd, kr_errmask, dd->cspec->errormask);
731	qib_6120_set_intr_state(dd, 1);
732}
733
734/**
735 * qib_handle_6120_hwerrors - display hardware errors.
736 * @dd: the qlogic_ib device
737 * @msg: the output buffer
738 * @msgl: the size of the output buffer
739 *
740 * Use same msg buffer as regular errors to avoid excessive stack
741 * use.  Most hardware errors are catastrophic, but for right now,
742 * we'll print them and continue.  Reuse the same message buffer as
743 * handle_6120_errors() to avoid excessive stack usage.
744 */
745static void qib_handle_6120_hwerrors(struct qib_devdata *dd, char *msg,
746				     size_t msgl)
747{
748	u64 hwerrs;
749	u32 bits, ctrl;
750	int isfatal = 0;
751	char *bitsmsg;
752	int log_idx;
753
754	hwerrs = qib_read_kreg64(dd, kr_hwerrstatus);
755	if (!hwerrs)
756		return;
757	if (hwerrs == ~0ULL) {
758		qib_dev_err(dd,
759			"Read of hardware error status failed (all bits set); ignoring\n");
760		return;
761	}
762	qib_stats.sps_hwerrs++;
763
764	/* Always clear the error status register, except MEMBISTFAIL,
765	 * regardless of whether we continue or stop using the chip.
766	 * We want that set so we know it failed, even across driver reload.
767	 * We'll still ignore it in the hwerrmask.  We do this partly for
768	 * diagnostics, but also for support */
769	qib_write_kreg(dd, kr_hwerrclear,
770		       hwerrs & ~HWE_MASK(PowerOnBISTFailed));
771
772	hwerrs &= dd->cspec->hwerrmask;
773
774	/* We log some errors to EEPROM, check if we have any of those. */
775	for (log_idx = 0; log_idx < QIB_EEP_LOG_CNT; ++log_idx)
776		if (hwerrs & dd->eep_st_masks[log_idx].hwerrs_to_log)
777			qib_inc_eeprom_err(dd, log_idx, 1);
778
779	/*
780	 * Make sure we get this much out, unless told to be quiet,
781	 * or it's occurred within the last 5 seconds.
782	 */
783	if (hwerrs & ~(TXE_PIO_PARITY | RXEMEMPARITYERR_EAGERTID))
784		qib_devinfo(dd->pcidev,
785			"Hardware error: hwerr=0x%llx (cleared)\n",
786			(unsigned long long) hwerrs);
787
788	if (hwerrs & ~IB_HWE_BITSEXTANT)
789		qib_dev_err(dd,
790			"hwerror interrupt with unknown errors %llx set\n",
791			(unsigned long long)(hwerrs & ~IB_HWE_BITSEXTANT));
792
793	ctrl = qib_read_kreg32(dd, kr_control);
794	if ((ctrl & QLOGIC_IB_C_FREEZEMODE) && !dd->diag_client) {
795		/*
796		 * Parity errors in send memory are recoverable,
797		 * just cancel the send (if indicated in * sendbuffererror),
798		 * count the occurrence, unfreeze (if no other handled
799		 * hardware error bits are set), and continue. They can
800		 * occur if a processor speculative read is done to the PIO
801		 * buffer while we are sending a packet, for example.
802		 */
803		if (hwerrs & TXE_PIO_PARITY) {
804			qib_6120_txe_recover(dd);
805			hwerrs &= ~TXE_PIO_PARITY;
806		}
807
808		if (!hwerrs) {
809			static u32 freeze_cnt;
810
811			freeze_cnt++;
812			qib_6120_clear_freeze(dd);
813		} else
814			isfatal = 1;
815	}
816
817	*msg = '\0';
818
819	if (hwerrs & HWE_MASK(PowerOnBISTFailed)) {
820		isfatal = 1;
821		strlcat(msg,
822			"[Memory BIST test failed, InfiniPath hardware unusable]",
823			msgl);
824		/* ignore from now on, so disable until driver reloaded */
825		dd->cspec->hwerrmask &= ~HWE_MASK(PowerOnBISTFailed);
826		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
827	}
828
829	qib_format_hwerrors(hwerrs, qib_6120_hwerror_msgs,
830			    ARRAY_SIZE(qib_6120_hwerror_msgs), msg, msgl);
831
832	bitsmsg = dd->cspec->bitsmsgbuf;
833	if (hwerrs & (QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK <<
834		      QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT)) {
835		bits = (u32) ((hwerrs >>
836			       QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT) &
837			      QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK);
838		snprintf(bitsmsg, sizeof(dd->cspec->bitsmsgbuf),
839			 "[PCIe Mem Parity Errs %x] ", bits);
840		strlcat(msg, bitsmsg, msgl);
841	}
842
843	if (hwerrs & _QIB_PLL_FAIL) {
844		isfatal = 1;
845		snprintf(bitsmsg, sizeof(dd->cspec->bitsmsgbuf),
846			 "[PLL failed (%llx), InfiniPath hardware unusable]",
847			 (unsigned long long) hwerrs & _QIB_PLL_FAIL);
848		strlcat(msg, bitsmsg, msgl);
849		/* ignore from now on, so disable until driver reloaded */
850		dd->cspec->hwerrmask &= ~(hwerrs & _QIB_PLL_FAIL);
851		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
852	}
853
854	if (hwerrs & QLOGIC_IB_HWE_SERDESPLLFAILED) {
855		/*
856		 * If it occurs, it is left masked since the external
857		 * interface is unused
858		 */
859		dd->cspec->hwerrmask &= ~QLOGIC_IB_HWE_SERDESPLLFAILED;
860		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
861	}
862
863	if (hwerrs)
864		/*
865		 * if any set that we aren't ignoring; only
866		 * make the complaint once, in case it's stuck
867		 * or recurring, and we get here multiple
868		 * times.
869		 */
870		qib_dev_err(dd, "%s hardware error\n", msg);
871	else
872		*msg = 0; /* recovered from all of them */
873
874	if (isfatal && !dd->diag_client) {
875		qib_dev_err(dd,
876			"Fatal Hardware Error, no longer usable, SN %.16s\n",
877			dd->serial);
878		/*
879		 * for /sys status file and user programs to print; if no
880		 * trailing brace is copied, we'll know it was truncated.
881		 */
882		if (dd->freezemsg)
883			snprintf(dd->freezemsg, dd->freezelen,
884				 "{%s}", msg);
885		qib_disable_after_error(dd);
886	}
887}
888
889/*
890 * Decode the error status into strings, deciding whether to always
891 * print * it or not depending on "normal packet errors" vs everything
892 * else.   Return 1 if "real" errors, otherwise 0 if only packet
893 * errors, so caller can decide what to print with the string.
894 */
895static int qib_decode_6120_err(struct qib_devdata *dd, char *buf, size_t blen,
896			       u64 err)
897{
898	int iserr = 1;
899
900	*buf = '\0';
901	if (err & QLOGIC_IB_E_PKTERRS) {
902		if (!(err & ~QLOGIC_IB_E_PKTERRS))
903			iserr = 0;
904		if ((err & ERR_MASK(RcvICRCErr)) &&
905		    !(err&(ERR_MASK(RcvVCRCErr)|ERR_MASK(RcvEBPErr))))
906			strlcat(buf, "CRC ", blen);
907		if (!iserr)
908			goto done;
909	}
910	if (err & ERR_MASK(RcvHdrLenErr))
911		strlcat(buf, "rhdrlen ", blen);
912	if (err & ERR_MASK(RcvBadTidErr))
913		strlcat(buf, "rbadtid ", blen);
914	if (err & ERR_MASK(RcvBadVersionErr))
915		strlcat(buf, "rbadversion ", blen);
916	if (err & ERR_MASK(RcvHdrErr))
917		strlcat(buf, "rhdr ", blen);
918	if (err & ERR_MASK(RcvLongPktLenErr))
919		strlcat(buf, "rlongpktlen ", blen);
920	if (err & ERR_MASK(RcvMaxPktLenErr))
921		strlcat(buf, "rmaxpktlen ", blen);
922	if (err & ERR_MASK(RcvMinPktLenErr))
923		strlcat(buf, "rminpktlen ", blen);
924	if (err & ERR_MASK(SendMinPktLenErr))
925		strlcat(buf, "sminpktlen ", blen);
926	if (err & ERR_MASK(RcvFormatErr))
927		strlcat(buf, "rformaterr ", blen);
928	if (err & ERR_MASK(RcvUnsupportedVLErr))
929		strlcat(buf, "runsupvl ", blen);
930	if (err & ERR_MASK(RcvUnexpectedCharErr))
931		strlcat(buf, "runexpchar ", blen);
932	if (err & ERR_MASK(RcvIBFlowErr))
933		strlcat(buf, "ribflow ", blen);
934	if (err & ERR_MASK(SendUnderRunErr))
935		strlcat(buf, "sunderrun ", blen);
936	if (err & ERR_MASK(SendPioArmLaunchErr))
937		strlcat(buf, "spioarmlaunch ", blen);
938	if (err & ERR_MASK(SendUnexpectedPktNumErr))
939		strlcat(buf, "sunexperrpktnum ", blen);
940	if (err & ERR_MASK(SendDroppedSmpPktErr))
941		strlcat(buf, "sdroppedsmppkt ", blen);
942	if (err & ERR_MASK(SendMaxPktLenErr))
943		strlcat(buf, "smaxpktlen ", blen);
944	if (err & ERR_MASK(SendUnsupportedVLErr))
945		strlcat(buf, "sunsupVL ", blen);
946	if (err & ERR_MASK(InvalidAddrErr))
947		strlcat(buf, "invalidaddr ", blen);
948	if (err & ERR_MASK(RcvEgrFullErr))
949		strlcat(buf, "rcvegrfull ", blen);
950	if (err & ERR_MASK(RcvHdrFullErr))
951		strlcat(buf, "rcvhdrfull ", blen);
952	if (err & ERR_MASK(IBStatusChanged))
953		strlcat(buf, "ibcstatuschg ", blen);
954	if (err & ERR_MASK(RcvIBLostLinkErr))
955		strlcat(buf, "riblostlink ", blen);
956	if (err & ERR_MASK(HardwareErr))
957		strlcat(buf, "hardware ", blen);
958	if (err & ERR_MASK(ResetNegated))
959		strlcat(buf, "reset ", blen);
960done:
961	return iserr;
962}
963
964/*
965 * Called when we might have an error that is specific to a particular
966 * PIO buffer, and may need to cancel that buffer, so it can be re-used.
967 */
968static void qib_disarm_6120_senderrbufs(struct qib_pportdata *ppd)
969{
970	unsigned long sbuf[2];
971	struct qib_devdata *dd = ppd->dd;
972
973	/*
974	 * It's possible that sendbuffererror could have bits set; might
975	 * have already done this as a result of hardware error handling.
976	 */
977	sbuf[0] = qib_read_kreg64(dd, kr_sendbuffererror);
978	sbuf[1] = qib_read_kreg64(dd, kr_sendbuffererror + 1);
979
980	if (sbuf[0] || sbuf[1])
981		qib_disarm_piobufs_set(dd, sbuf,
982				       dd->piobcnt2k + dd->piobcnt4k);
983}
984
985static int chk_6120_linkrecovery(struct qib_devdata *dd, u64 ibcs)
986{
987	int ret = 1;
988	u32 ibstate = qib_6120_iblink_state(ibcs);
989	u32 linkrecov = read_6120_creg32(dd, cr_iblinkerrrecov);
990
991	if (linkrecov != dd->cspec->lastlinkrecov) {
992		/* and no more until active again */
993		dd->cspec->lastlinkrecov = 0;
994		qib_set_linkstate(dd->pport, QIB_IB_LINKDOWN);
995		ret = 0;
996	}
997	if (ibstate == IB_PORT_ACTIVE)
998		dd->cspec->lastlinkrecov =
999			read_6120_creg32(dd, cr_iblinkerrrecov);
1000	return ret;
1001}
1002
1003static void handle_6120_errors(struct qib_devdata *dd, u64 errs)
1004{
1005	char *msg;
1006	u64 ignore_this_time = 0;
1007	u64 iserr = 0;
1008	int log_idx;
1009	struct qib_pportdata *ppd = dd->pport;
1010	u64 mask;
1011
1012	/* don't report errors that are masked */
1013	errs &= dd->cspec->errormask;
1014	msg = dd->cspec->emsgbuf;
1015
1016	/* do these first, they are most important */
1017	if (errs & ERR_MASK(HardwareErr))
1018		qib_handle_6120_hwerrors(dd, msg, sizeof(dd->cspec->emsgbuf));
1019	else
1020		for (log_idx = 0; log_idx < QIB_EEP_LOG_CNT; ++log_idx)
1021			if (errs & dd->eep_st_masks[log_idx].errs_to_log)
1022				qib_inc_eeprom_err(dd, log_idx, 1);
1023
1024	if (errs & ~IB_E_BITSEXTANT)
1025		qib_dev_err(dd,
1026			"error interrupt with unknown errors %llx set\n",
1027			(unsigned long long) (errs & ~IB_E_BITSEXTANT));
1028
1029	if (errs & E_SUM_ERRS) {
1030		qib_disarm_6120_senderrbufs(ppd);
1031		if ((errs & E_SUM_LINK_PKTERRS) &&
1032		    !(ppd->lflags & QIBL_LINKACTIVE)) {
1033			/*
1034			 * This can happen when trying to bring the link
1035			 * up, but the IB link changes state at the "wrong"
1036			 * time. The IB logic then complains that the packet
1037			 * isn't valid.  We don't want to confuse people, so
1038			 * we just don't print them, except at debug
1039			 */
1040			ignore_this_time = errs & E_SUM_LINK_PKTERRS;
1041		}
1042	} else if ((errs & E_SUM_LINK_PKTERRS) &&
1043		   !(ppd->lflags & QIBL_LINKACTIVE)) {
1044		/*
1045		 * This can happen when SMA is trying to bring the link
1046		 * up, but the IB link changes state at the "wrong" time.
1047		 * The IB logic then complains that the packet isn't
1048		 * valid.  We don't want to confuse people, so we just
1049		 * don't print them, except at debug
1050		 */
1051		ignore_this_time = errs & E_SUM_LINK_PKTERRS;
1052	}
1053
1054	qib_write_kreg(dd, kr_errclear, errs);
1055
1056	errs &= ~ignore_this_time;
1057	if (!errs)
1058		goto done;
1059
1060	/*
1061	 * The ones we mask off are handled specially below
1062	 * or above.
1063	 */
1064	mask = ERR_MASK(IBStatusChanged) | ERR_MASK(RcvEgrFullErr) |
1065		ERR_MASK(RcvHdrFullErr) | ERR_MASK(HardwareErr);
1066	qib_decode_6120_err(dd, msg, sizeof(dd->cspec->emsgbuf), errs & ~mask);
1067
1068	if (errs & E_SUM_PKTERRS)
1069		qib_stats.sps_rcverrs++;
1070	if (errs & E_SUM_ERRS)
1071		qib_stats.sps_txerrs++;
1072
1073	iserr = errs & ~(E_SUM_PKTERRS | QLOGIC_IB_E_PKTERRS);
1074
1075	if (errs & ERR_MASK(IBStatusChanged)) {
1076		u64 ibcs = qib_read_kreg64(dd, kr_ibcstatus);
1077		u32 ibstate = qib_6120_iblink_state(ibcs);
1078		int handle = 1;
1079
1080		if (ibstate != IB_PORT_INIT && dd->cspec->lastlinkrecov)
1081			handle = chk_6120_linkrecovery(dd, ibcs);
1082		/*
1083		 * Since going into a recovery state causes the link state
1084		 * to go down and since recovery is transitory, it is better
1085		 * if we "miss" ever seeing the link training state go into
1086		 * recovery (i.e., ignore this transition for link state
1087		 * special handling purposes) without updating lastibcstat.
1088		 */
1089		if (handle && qib_6120_phys_portstate(ibcs) ==
1090					    IB_PHYSPORTSTATE_LINK_ERR_RECOVER)
1091			handle = 0;
1092		if (handle)
1093			qib_handle_e_ibstatuschanged(ppd, ibcs);
1094	}
1095
1096	if (errs & ERR_MASK(ResetNegated)) {
1097		qib_dev_err(dd,
1098			"Got reset, requires re-init (unload and reload driver)\n");
1099		dd->flags &= ~QIB_INITTED;  /* needs re-init */
1100		/* mark as having had error */
1101		*dd->devstatusp |= QIB_STATUS_HWERROR;
1102		*dd->pport->statusp &= ~QIB_STATUS_IB_CONF;
1103	}
1104
1105	if (*msg && iserr)
1106		qib_dev_porterr(dd, ppd->port, "%s error\n", msg);
1107
1108	if (ppd->state_wanted & ppd->lflags)
1109		wake_up_interruptible(&ppd->state_wait);
1110
1111	/*
1112	 * If there were hdrq or egrfull errors, wake up any processes
1113	 * waiting in poll.  We used to try to check which contexts had
1114	 * the overflow, but given the cost of that and the chip reads
1115	 * to support it, it's better to just wake everybody up if we
1116	 * get an overflow; waiters can poll again if it's not them.
1117	 */
1118	if (errs & (ERR_MASK(RcvEgrFullErr) | ERR_MASK(RcvHdrFullErr))) {
1119		qib_handle_urcv(dd, ~0U);
1120		if (errs & ERR_MASK(RcvEgrFullErr))
1121			qib_stats.sps_buffull++;
1122		else
1123			qib_stats.sps_hdrfull++;
1124	}
1125done:
1126	return;
1127}
1128
1129/**
1130 * qib_6120_init_hwerrors - enable hardware errors
1131 * @dd: the qlogic_ib device
1132 *
1133 * now that we have finished initializing everything that might reasonably
1134 * cause a hardware error, and cleared those errors bits as they occur,
1135 * we can enable hardware errors in the mask (potentially enabling
1136 * freeze mode), and enable hardware errors as errors (along with
1137 * everything else) in errormask
1138 */
1139static void qib_6120_init_hwerrors(struct qib_devdata *dd)
1140{
1141	u64 val;
1142	u64 extsval;
1143
1144	extsval = qib_read_kreg64(dd, kr_extstatus);
1145
1146	if (!(extsval & QLOGIC_IB_EXTS_MEMBIST_ENDTEST))
1147		qib_dev_err(dd, "MemBIST did not complete!\n");
1148
1149	/* init so all hwerrors interrupt, and enter freeze, ajdust below */
1150	val = ~0ULL;
1151	if (dd->minrev < 2) {
1152		/*
1153		 * Avoid problem with internal interface bus parity
1154		 * checking. Fixed in Rev2.
1155		 */
1156		val &= ~QLOGIC_IB_HWE_PCIEBUSPARITYRADM;
1157	}
1158	/* avoid some intel cpu's speculative read freeze mode issue */
1159	val &= ~TXEMEMPARITYERR_PIOBUF;
1160
1161	dd->cspec->hwerrmask = val;
1162
1163	qib_write_kreg(dd, kr_hwerrclear, ~HWE_MASK(PowerOnBISTFailed));
1164	qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
1165
1166	/* clear all */
1167	qib_write_kreg(dd, kr_errclear, ~0ULL);
1168	/* enable errors that are masked, at least this first time. */
1169	qib_write_kreg(dd, kr_errmask, ~0ULL);
1170	dd->cspec->errormask = qib_read_kreg64(dd, kr_errmask);
1171	/* clear any interrupts up to this point (ints still not enabled) */
1172	qib_write_kreg(dd, kr_intclear, ~0ULL);
1173
1174	qib_write_kreg(dd, kr_rcvbthqp,
1175		       dd->qpn_mask << (QIB_6120_RcvBTHQP_BTHQP_Mask_LSB - 1) |
1176		       QIB_KD_QP);
1177}
1178
1179/*
1180 * Disable and enable the armlaunch error.  Used for PIO bandwidth testing
1181 * on chips that are count-based, rather than trigger-based.  There is no
1182 * reference counting, but that's also fine, given the intended use.
1183 * Only chip-specific because it's all register accesses
1184 */
1185static void qib_set_6120_armlaunch(struct qib_devdata *dd, u32 enable)
1186{
1187	if (enable) {
1188		qib_write_kreg(dd, kr_errclear,
1189			       ERR_MASK(SendPioArmLaunchErr));
1190		dd->cspec->errormask |= ERR_MASK(SendPioArmLaunchErr);
1191	} else
1192		dd->cspec->errormask &= ~ERR_MASK(SendPioArmLaunchErr);
1193	qib_write_kreg(dd, kr_errmask, dd->cspec->errormask);
1194}
1195
1196/*
1197 * Formerly took parameter <which> in pre-shifted,
1198 * pre-merged form with LinkCmd and LinkInitCmd
1199 * together, and assuming the zero was NOP.
1200 */
1201static void qib_set_ib_6120_lstate(struct qib_pportdata *ppd, u16 linkcmd,
1202				   u16 linitcmd)
1203{
1204	u64 mod_wd;
1205	struct qib_devdata *dd = ppd->dd;
1206	unsigned long flags;
1207
1208	if (linitcmd == QLOGIC_IB_IBCC_LINKINITCMD_DISABLE) {
1209		/*
1210		 * If we are told to disable, note that so link-recovery
1211		 * code does not attempt to bring us back up.
1212		 */
1213		spin_lock_irqsave(&ppd->lflags_lock, flags);
1214		ppd->lflags |= QIBL_IB_LINK_DISABLED;
1215		spin_unlock_irqrestore(&ppd->lflags_lock, flags);
1216	} else if (linitcmd || linkcmd == QLOGIC_IB_IBCC_LINKCMD_DOWN) {
1217		/*
1218		 * Any other linkinitcmd will lead to LINKDOWN and then
1219		 * to INIT (if all is well), so clear flag to let
1220		 * link-recovery code attempt to bring us back up.
1221		 */
1222		spin_lock_irqsave(&ppd->lflags_lock, flags);
1223		ppd->lflags &= ~QIBL_IB_LINK_DISABLED;
1224		spin_unlock_irqrestore(&ppd->lflags_lock, flags);
1225	}
1226
1227	mod_wd = (linkcmd << QLOGIC_IB_IBCC_LINKCMD_SHIFT) |
1228		(linitcmd << QLOGIC_IB_IBCC_LINKINITCMD_SHIFT);
1229
1230	qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl | mod_wd);
1231	/* write to chip to prevent back-to-back writes of control reg */
1232	qib_write_kreg(dd, kr_scratch, 0);
1233}
1234
1235/**
1236 * qib_6120_bringup_serdes - bring up the serdes
1237 * @dd: the qlogic_ib device
1238 */
1239static int qib_6120_bringup_serdes(struct qib_pportdata *ppd)
1240{
1241	struct qib_devdata *dd = ppd->dd;
1242	u64 val, config1, prev_val, hwstat, ibc;
1243
1244	/* Put IBC in reset, sends disabled */
1245	dd->control &= ~QLOGIC_IB_C_LINKENABLE;
1246	qib_write_kreg(dd, kr_control, 0ULL);
1247
1248	dd->cspec->ibdeltainprog = 1;
1249	dd->cspec->ibsymsnap = read_6120_creg32(dd, cr_ibsymbolerr);
1250	dd->cspec->iblnkerrsnap = read_6120_creg32(dd, cr_iblinkerrrecov);
1251
1252	/* flowcontrolwatermark is in units of KBytes */
1253	ibc = 0x5ULL << SYM_LSB(IBCCtrl, FlowCtrlWaterMark);
1254	/*
1255	 * How often flowctrl sent.  More or less in usecs; balance against
1256	 * watermark value, so that in theory senders always get a flow
1257	 * control update in time to not let the IB link go idle.
1258	 */
1259	ibc |= 0x3ULL << SYM_LSB(IBCCtrl, FlowCtrlPeriod);
1260	/* max error tolerance */
1261	dd->cspec->lli_thresh = 0xf;
1262	ibc |= (u64) dd->cspec->lli_thresh << SYM_LSB(IBCCtrl, PhyerrThreshold);
1263	/* use "real" buffer space for */
1264	ibc |= 4ULL << SYM_LSB(IBCCtrl, CreditScale);
1265	/* IB credit flow control. */
1266	ibc |= 0xfULL << SYM_LSB(IBCCtrl, OverrunThreshold);
1267	/*
1268	 * set initial max size pkt IBC will send, including ICRC; it's the
1269	 * PIO buffer size in dwords, less 1; also see qib_set_mtu()
1270	 */
1271	ibc |= ((u64)(ppd->ibmaxlen >> 2) + 1) << SYM_LSB(IBCCtrl, MaxPktLen);
1272	dd->cspec->ibcctrl = ibc; /* without linkcmd or linkinitcmd! */
1273
1274	/* initially come up waiting for TS1, without sending anything. */
1275	val = dd->cspec->ibcctrl | (QLOGIC_IB_IBCC_LINKINITCMD_DISABLE <<
1276		QLOGIC_IB_IBCC_LINKINITCMD_SHIFT);
1277	qib_write_kreg(dd, kr_ibcctrl, val);
1278
1279	val = qib_read_kreg64(dd, kr_serdes_cfg0);
1280	config1 = qib_read_kreg64(dd, kr_serdes_cfg1);
1281
1282	/*
1283	 * Force reset on, also set rxdetect enable.  Must do before reading
1284	 * serdesstatus at least for simulation, or some of the bits in
1285	 * serdes status will come back as undefined and cause simulation
1286	 * failures
1287	 */
1288	val |= SYM_MASK(SerdesCfg0, ResetPLL) |
1289		SYM_MASK(SerdesCfg0, RxDetEnX) |
1290		(SYM_MASK(SerdesCfg0, L1PwrDnA) |
1291		 SYM_MASK(SerdesCfg0, L1PwrDnB) |
1292		 SYM_MASK(SerdesCfg0, L1PwrDnC) |
1293		 SYM_MASK(SerdesCfg0, L1PwrDnD));
1294	qib_write_kreg(dd, kr_serdes_cfg0, val);
1295	/* be sure chip saw it */
1296	qib_read_kreg64(dd, kr_scratch);
1297	udelay(5);              /* need pll reset set at least for a bit */
1298	/*
1299	 * after PLL is reset, set the per-lane Resets and TxIdle and
1300	 * clear the PLL reset and rxdetect (to get falling edge).
1301	 * Leave L1PWR bits set (permanently)
1302	 */
1303	val &= ~(SYM_MASK(SerdesCfg0, RxDetEnX) |
1304		 SYM_MASK(SerdesCfg0, ResetPLL) |
1305		 (SYM_MASK(SerdesCfg0, L1PwrDnA) |
1306		  SYM_MASK(SerdesCfg0, L1PwrDnB) |
1307		  SYM_MASK(SerdesCfg0, L1PwrDnC) |
1308		  SYM_MASK(SerdesCfg0, L1PwrDnD)));
1309	val |= (SYM_MASK(SerdesCfg0, ResetA) |
1310		SYM_MASK(SerdesCfg0, ResetB) |
1311		SYM_MASK(SerdesCfg0, ResetC) |
1312		SYM_MASK(SerdesCfg0, ResetD)) |
1313		SYM_MASK(SerdesCfg0, TxIdeEnX);
1314	qib_write_kreg(dd, kr_serdes_cfg0, val);
1315	/* be sure chip saw it */
1316	(void) qib_read_kreg64(dd, kr_scratch);
1317	/* need PLL reset clear for at least 11 usec before lane
1318	 * resets cleared; give it a few more to be sure */
1319	udelay(15);
1320	val &= ~((SYM_MASK(SerdesCfg0, ResetA) |
1321		  SYM_MASK(SerdesCfg0, ResetB) |
1322		  SYM_MASK(SerdesCfg0, ResetC) |
1323		  SYM_MASK(SerdesCfg0, ResetD)) |
1324		 SYM_MASK(SerdesCfg0, TxIdeEnX));
1325
1326	qib_write_kreg(dd, kr_serdes_cfg0, val);
1327	/* be sure chip saw it */
1328	(void) qib_read_kreg64(dd, kr_scratch);
1329
1330	val = qib_read_kreg64(dd, kr_xgxs_cfg);
1331	prev_val = val;
1332	if (val & QLOGIC_IB_XGXS_RESET)
1333		val &= ~QLOGIC_IB_XGXS_RESET;
1334	if (SYM_FIELD(val, XGXSCfg, polarity_inv) != ppd->rx_pol_inv) {
1335		/* need to compensate for Tx inversion in partner */
1336		val &= ~SYM_MASK(XGXSCfg, polarity_inv);
1337		val |= (u64)ppd->rx_pol_inv << SYM_LSB(XGXSCfg, polarity_inv);
1338	}
1339	if (val != prev_val)
1340		qib_write_kreg(dd, kr_xgxs_cfg, val);
1341
1342	val = qib_read_kreg64(dd, kr_serdes_cfg0);
1343
1344	/* clear current and de-emphasis bits */
1345	config1 &= ~0x0ffffffff00ULL;
1346	/* set current to 20ma */
1347	config1 |= 0x00000000000ULL;
1348	/* set de-emphasis to -5.68dB */
1349	config1 |= 0x0cccc000000ULL;
1350	qib_write_kreg(dd, kr_serdes_cfg1, config1);
1351
1352	/* base and port guid same for single port */
1353	ppd->guid = dd->base_guid;
1354
1355	/*
1356	 * the process of setting and un-resetting the serdes normally
1357	 * causes a serdes PLL error, so check for that and clear it
1358	 * here.  Also clearr hwerr bit in errstatus, but not others.
1359	 */
1360	hwstat = qib_read_kreg64(dd, kr_hwerrstatus);
1361	if (hwstat) {
1362		/* should just have PLL, clear all set, in an case */
1363		qib_write_kreg(dd, kr_hwerrclear, hwstat);
1364		qib_write_kreg(dd, kr_errclear, ERR_MASK(HardwareErr));
1365	}
1366
1367	dd->control |= QLOGIC_IB_C_LINKENABLE;
1368	dd->control &= ~QLOGIC_IB_C_FREEZEMODE;
1369	qib_write_kreg(dd, kr_control, dd->control);
1370
1371	return 0;
1372}
1373
1374/**
1375 * qib_6120_quiet_serdes - set serdes to txidle
1376 * @ppd: physical port of the qlogic_ib device
1377 * Called when driver is being unloaded
1378 */
1379static void qib_6120_quiet_serdes(struct qib_pportdata *ppd)
1380{
1381	struct qib_devdata *dd = ppd->dd;
1382	u64 val;
1383
1384	qib_set_ib_6120_lstate(ppd, 0, QLOGIC_IB_IBCC_LINKINITCMD_DISABLE);
1385
1386	/* disable IBC */
1387	dd->control &= ~QLOGIC_IB_C_LINKENABLE;
1388	qib_write_kreg(dd, kr_control,
1389		       dd->control | QLOGIC_IB_C_FREEZEMODE);
1390
1391	if (dd->cspec->ibsymdelta || dd->cspec->iblnkerrdelta ||
1392	    dd->cspec->ibdeltainprog) {
1393		u64 diagc;
1394
1395		/* enable counter writes */
1396		diagc = qib_read_kreg64(dd, kr_hwdiagctrl);
1397		qib_write_kreg(dd, kr_hwdiagctrl,
1398			       diagc | SYM_MASK(HwDiagCtrl, CounterWrEnable));
1399
1400		if (dd->cspec->ibsymdelta || dd->cspec->ibdeltainprog) {
1401			val = read_6120_creg32(dd, cr_ibsymbolerr);
1402			if (dd->cspec->ibdeltainprog)
1403				val -= val - dd->cspec->ibsymsnap;
1404			val -= dd->cspec->ibsymdelta;
1405			write_6120_creg(dd, cr_ibsymbolerr, val);
1406		}
1407		if (dd->cspec->iblnkerrdelta || dd->cspec->ibdeltainprog) {
1408			val = read_6120_creg32(dd, cr_iblinkerrrecov);
1409			if (dd->cspec->ibdeltainprog)
1410				val -= val - dd->cspec->iblnkerrsnap;
1411			val -= dd->cspec->iblnkerrdelta;
1412			write_6120_creg(dd, cr_iblinkerrrecov, val);
1413		}
1414
1415		/* and disable counter writes */
1416		qib_write_kreg(dd, kr_hwdiagctrl, diagc);
1417	}
1418
1419	val = qib_read_kreg64(dd, kr_serdes_cfg0);
1420	val |= SYM_MASK(SerdesCfg0, TxIdeEnX);
1421	qib_write_kreg(dd, kr_serdes_cfg0, val);
1422}
1423
1424/**
1425 * qib_6120_setup_setextled - set the state of the two external LEDs
1426 * @dd: the qlogic_ib device
1427 * @on: whether the link is up or not
1428 *
1429 * The exact combo of LEDs if on is true is determined by looking
1430 * at the ibcstatus.
1431
1432 * These LEDs indicate the physical and logical state of IB link.
1433 * For this chip (at least with recommended board pinouts), LED1
1434 * is Yellow (logical state) and LED2 is Green (physical state),
1435 *
1436 * Note:  We try to match the Mellanox HCA LED behavior as best
1437 * we can.  Green indicates physical link state is OK (something is
1438 * plugged in, and we can train).
1439 * Amber indicates the link is logically up (ACTIVE).
1440 * Mellanox further blinks the amber LED to indicate data packet
1441 * activity, but we have no hardware support for that, so it would
1442 * require waking up every 10-20 msecs and checking the counters
1443 * on the chip, and then turning the LED off if appropriate.  That's
1444 * visible overhead, so not something we will do.
1445 *
1446 */
1447static void qib_6120_setup_setextled(struct qib_pportdata *ppd, u32 on)
1448{
1449	u64 extctl, val, lst, ltst;
1450	unsigned long flags;
1451	struct qib_devdata *dd = ppd->dd;
1452
1453	/*
1454	 * The diags use the LED to indicate diag info, so we leave
1455	 * the external LED alone when the diags are running.
1456	 */
1457	if (dd->diag_client)
1458		return;
1459
1460	/* Allow override of LED display for, e.g. Locating system in rack */
1461	if (ppd->led_override) {
1462		ltst = (ppd->led_override & QIB_LED_PHYS) ?
1463			IB_PHYSPORTSTATE_LINKUP : IB_PHYSPORTSTATE_DISABLED,
1464		lst = (ppd->led_override & QIB_LED_LOG) ?
1465			IB_PORT_ACTIVE : IB_PORT_DOWN;
1466	} else if (on) {
1467		val = qib_read_kreg64(dd, kr_ibcstatus);
1468		ltst = qib_6120_phys_portstate(val);
1469		lst = qib_6120_iblink_state(val);
1470	} else {
1471		ltst = 0;
1472		lst = 0;
1473	}
1474
1475	spin_lock_irqsave(&dd->cspec->gpio_lock, flags);
1476	extctl = dd->cspec->extctrl & ~(SYM_MASK(EXTCtrl, LEDPriPortGreenOn) |
1477				 SYM_MASK(EXTCtrl, LEDPriPortYellowOn));
1478
1479	if (ltst == IB_PHYSPORTSTATE_LINKUP)
1480		extctl |= SYM_MASK(EXTCtrl, LEDPriPortYellowOn);
1481	if (lst == IB_PORT_ACTIVE)
1482		extctl |= SYM_MASK(EXTCtrl, LEDPriPortGreenOn);
1483	dd->cspec->extctrl = extctl;
1484	qib_write_kreg(dd, kr_extctrl, extctl);
1485	spin_unlock_irqrestore(&dd->cspec->gpio_lock, flags);
1486}
1487
1488static void qib_6120_free_irq(struct qib_devdata *dd)
1489{
1490	if (dd->cspec->irq) {
1491		free_irq(dd->cspec->irq, dd);
1492		dd->cspec->irq = 0;
1493	}
1494	qib_nomsi(dd);
1495}
1496
1497/**
1498 * qib_6120_setup_cleanup - clean up any per-chip chip-specific stuff
1499 * @dd: the qlogic_ib device
1500 *
1501 * This is called during driver unload.
1502*/
1503static void qib_6120_setup_cleanup(struct qib_devdata *dd)
1504{
1505	qib_6120_free_irq(dd);
1506	kfree(dd->cspec->cntrs);
1507	kfree(dd->cspec->portcntrs);
1508	if (dd->cspec->dummy_hdrq) {
1509		dma_free_coherent(&dd->pcidev->dev,
1510				  ALIGN(dd->rcvhdrcnt *
1511					dd->rcvhdrentsize *
1512					sizeof(u32), PAGE_SIZE),
1513				  dd->cspec->dummy_hdrq,
1514				  dd->cspec->dummy_hdrq_phys);
1515		dd->cspec->dummy_hdrq = NULL;
1516	}
1517}
1518
1519static void qib_wantpiobuf_6120_intr(struct qib_devdata *dd, u32 needint)
1520{
1521	unsigned long flags;
1522
1523	spin_lock_irqsave(&dd->sendctrl_lock, flags);
1524	if (needint)
1525		dd->sendctrl |= SYM_MASK(SendCtrl, PIOIntBufAvail);
1526	else
1527		dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOIntBufAvail);
1528	qib_write_kreg(dd, kr_sendctrl, dd->sendctrl);
1529	qib_write_kreg(dd, kr_scratch, 0ULL);
1530	spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
1531}
1532
1533/*
1534 * handle errors and unusual events first, separate function
1535 * to improve cache hits for fast path interrupt handling
1536 */
1537static noinline void unlikely_6120_intr(struct qib_devdata *dd, u64 istat)
1538{
1539	if (unlikely(istat & ~QLOGIC_IB_I_BITSEXTANT))
1540		qib_dev_err(dd, "interrupt with unknown interrupts %Lx set\n",
1541			    istat & ~QLOGIC_IB_I_BITSEXTANT);
1542
1543	if (istat & QLOGIC_IB_I_ERROR) {
1544		u64 estat = 0;
1545
1546		qib_stats.sps_errints++;
1547		estat = qib_read_kreg64(dd, kr_errstatus);
1548		if (!estat)
1549			qib_devinfo(dd->pcidev,
1550				"error interrupt (%Lx), but no error bits set!\n",
1551				istat);
1552		handle_6120_errors(dd, estat);
1553	}
1554
1555	if (istat & QLOGIC_IB_I_GPIO) {
1556		u32 gpiostatus;
1557		u32 to_clear = 0;
1558
1559		/*
1560		 * GPIO_3..5 on IBA6120 Rev2 chips indicate
1561		 * errors that we need to count.
1562		 */
1563		gpiostatus = qib_read_kreg32(dd, kr_gpio_status);
1564		/* First the error-counter case. */
1565		if (gpiostatus & GPIO_ERRINTR_MASK) {
1566			/* want to clear the bits we see asserted. */
1567			to_clear |= (gpiostatus & GPIO_ERRINTR_MASK);
1568
1569			/*
1570			 * Count appropriately, clear bits out of our copy,
1571			 * as they have been "handled".
1572			 */
1573			if (gpiostatus & (1 << GPIO_RXUVL_BIT))
1574				dd->cspec->rxfc_unsupvl_errs++;
1575			if (gpiostatus & (1 << GPIO_OVRUN_BIT))
1576				dd->cspec->overrun_thresh_errs++;
1577			if (gpiostatus & (1 << GPIO_LLI_BIT))
1578				dd->cspec->lli_errs++;
1579			gpiostatus &= ~GPIO_ERRINTR_MASK;
1580		}
1581		if (gpiostatus) {
1582			/*
1583			 * Some unexpected bits remain. If they could have
1584			 * caused the interrupt, complain and clear.
1585			 * To avoid repetition of this condition, also clear
1586			 * the mask. It is almost certainly due to error.
1587			 */
1588			const u32 mask = qib_read_kreg32(dd, kr_gpio_mask);
1589
1590			/*
1591			 * Also check that the chip reflects our shadow,
1592			 * and report issues, If they caused the interrupt.
1593			 * we will suppress by refreshing from the shadow.
1594			 */
1595			if (mask & gpiostatus) {
1596				to_clear |= (gpiostatus & mask);
1597				dd->cspec->gpio_mask &= ~(gpiostatus & mask);
1598				qib_write_kreg(dd, kr_gpio_mask,
1599					       dd->cspec->gpio_mask);
1600			}
1601		}
1602		if (to_clear)
1603			qib_write_kreg(dd, kr_gpio_clear, (u64) to_clear);
1604	}
1605}
1606
1607static irqreturn_t qib_6120intr(int irq, void *data)
1608{
1609	struct qib_devdata *dd = data;
1610	irqreturn_t ret;
1611	u32 istat, ctxtrbits, rmask, crcs = 0;
1612	unsigned i;
1613
1614	if ((dd->flags & (QIB_PRESENT | QIB_BADINTR)) != QIB_PRESENT) {
1615		/*
1616		 * This return value is not great, but we do not want the
1617		 * interrupt core code to remove our interrupt handler
1618		 * because we don't appear to be handling an interrupt
1619		 * during a chip reset.
1620		 */
1621		ret = IRQ_HANDLED;
1622		goto bail;
1623	}
1624
1625	istat = qib_read_kreg32(dd, kr_intstatus);
1626
1627	if (unlikely(!istat)) {
1628		ret = IRQ_NONE; /* not our interrupt, or already handled */
1629		goto bail;
1630	}
1631	if (unlikely(istat == -1)) {
1632		qib_bad_intrstatus(dd);
1633		/* don't know if it was our interrupt or not */
1634		ret = IRQ_NONE;
1635		goto bail;
1636	}
1637
1638	this_cpu_inc(*dd->int_counter);
1639
1640	if (unlikely(istat & (~QLOGIC_IB_I_BITSEXTANT |
1641			      QLOGIC_IB_I_GPIO | QLOGIC_IB_I_ERROR)))
1642		unlikely_6120_intr(dd, istat);
1643
1644	/*
1645	 * Clear the interrupt bits we found set, relatively early, so we
1646	 * "know" know the chip will have seen this by the time we process
1647	 * the queue, and will re-interrupt if necessary.  The processor
1648	 * itself won't take the interrupt again until we return.
1649	 */
1650	qib_write_kreg(dd, kr_intclear, istat);
1651
1652	/*
1653	 * Handle kernel receive queues before checking for pio buffers
1654	 * available since receives can overflow; piobuf waiters can afford
1655	 * a few extra cycles, since they were waiting anyway.
1656	 */
1657	ctxtrbits = istat &
1658		((QLOGIC_IB_I_RCVAVAIL_MASK << QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1659		 (QLOGIC_IB_I_RCVURG_MASK << QLOGIC_IB_I_RCVURG_SHIFT));
1660	if (ctxtrbits) {
1661		rmask = (1U << QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1662			(1U << QLOGIC_IB_I_RCVURG_SHIFT);
1663		for (i = 0; i < dd->first_user_ctxt; i++) {
1664			if (ctxtrbits & rmask) {
1665				ctxtrbits &= ~rmask;
1666				crcs += qib_kreceive(dd->rcd[i],
1667						     &dd->cspec->lli_counter,
1668						     NULL);
1669			}
1670			rmask <<= 1;
1671		}
1672		if (crcs) {
1673			u32 cntr = dd->cspec->lli_counter;
1674
1675			cntr += crcs;
1676			if (cntr) {
1677				if (cntr > dd->cspec->lli_thresh) {
1678					dd->cspec->lli_counter = 0;
1679					dd->cspec->lli_errs++;
1680				} else
1681					dd->cspec->lli_counter += cntr;
1682			}
1683		}
1684
1685
1686		if (ctxtrbits) {
1687			ctxtrbits =
1688				(ctxtrbits >> QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1689				(ctxtrbits >> QLOGIC_IB_I_RCVURG_SHIFT);
1690			qib_handle_urcv(dd, ctxtrbits);
1691		}
1692	}
1693
1694	if ((istat & QLOGIC_IB_I_SPIOBUFAVAIL) && (dd->flags & QIB_INITTED))
1695		qib_ib_piobufavail(dd);
1696
1697	ret = IRQ_HANDLED;
1698bail:
1699	return ret;
1700}
1701
1702/*
1703 * Set up our chip-specific interrupt handler
1704 * The interrupt type has already been setup, so
1705 * we just need to do the registration and error checking.
1706 */
1707static void qib_setup_6120_interrupt(struct qib_devdata *dd)
1708{
1709	/*
1710	 * If the chip supports added error indication via GPIO pins,
1711	 * enable interrupts on those bits so the interrupt routine
1712	 * can count the events. Also set flag so interrupt routine
1713	 * can know they are expected.
1714	 */
1715	if (SYM_FIELD(dd->revision, Revision_R,
1716		      ChipRevMinor) > 1) {
1717		/* Rev2+ reports extra errors via internal GPIO pins */
1718		dd->cspec->gpio_mask |= GPIO_ERRINTR_MASK;
1719		qib_write_kreg(dd, kr_gpio_mask, dd->cspec->gpio_mask);
1720	}
1721
1722	if (!dd->cspec->irq)
1723		qib_dev_err(dd,
1724			"irq is 0, BIOS error?  Interrupts won't work\n");
1725	else {
1726		int ret;
1727
1728		ret = request_irq(dd->cspec->irq, qib_6120intr, 0,
1729				  QIB_DRV_NAME, dd);
1730		if (ret)
1731			qib_dev_err(dd,
1732				"Couldn't setup interrupt (irq=%d): %d\n",
1733				dd->cspec->irq, ret);
1734	}
1735}
1736
1737/**
1738 * pe_boardname - fill in the board name
1739 * @dd: the qlogic_ib device
1740 *
1741 * info is based on the board revision register
1742 */
1743static void pe_boardname(struct qib_devdata *dd)
1744{
1745	char *n;
1746	u32 boardid, namelen;
1747
1748	boardid = SYM_FIELD(dd->revision, Revision,
1749			    BoardID);
1750
1751	switch (boardid) {
1752	case 2:
1753		n = "InfiniPath_QLE7140";
1754		break;
1755	default:
1756		qib_dev_err(dd, "Unknown 6120 board with ID %u\n", boardid);
1757		n = "Unknown_InfiniPath_6120";
1758		break;
1759	}
1760	namelen = strlen(n) + 1;
1761	dd->boardname = kmalloc(namelen, GFP_KERNEL);
1762	if (!dd->boardname)
1763		qib_dev_err(dd, "Failed allocation for board name: %s\n", n);
1764	else
1765		snprintf(dd->boardname, namelen, "%s", n);
1766
1767	if (dd->majrev != 4 || !dd->minrev || dd->minrev > 2)
1768		qib_dev_err(dd,
1769			"Unsupported InfiniPath hardware revision %u.%u!\n",
1770			dd->majrev, dd->minrev);
1771
1772	snprintf(dd->boardversion, sizeof(dd->boardversion),
1773		 "ChipABI %u.%u, %s, InfiniPath%u %u.%u, SW Compat %u\n",
1774		 QIB_CHIP_VERS_MAJ, QIB_CHIP_VERS_MIN, dd->boardname,
1775		 (unsigned)SYM_FIELD(dd->revision, Revision_R, Arch),
1776		 dd->majrev, dd->minrev,
1777		 (unsigned)SYM_FIELD(dd->revision, Revision_R, SW));
1778
1779}
1780
1781/*
1782 * This routine sleeps, so it can only be called from user context, not
1783 * from interrupt context.  If we need interrupt context, we can split
1784 * it into two routines.
1785 */
1786static int qib_6120_setup_reset(struct qib_devdata *dd)
1787{
1788	u64 val;
1789	int i;
1790	int ret;
1791	u16 cmdval;
1792	u8 int_line, clinesz;
1793
1794	qib_pcie_getcmd(dd, &cmdval, &int_line, &clinesz);
1795
1796	/* Use ERROR so it shows up in logs, etc. */
1797	qib_dev_err(dd, "Resetting InfiniPath unit %u\n", dd->unit);
1798
1799	/* no interrupts till re-initted */
1800	qib_6120_set_intr_state(dd, 0);
1801
1802	dd->cspec->ibdeltainprog = 0;
1803	dd->cspec->ibsymdelta = 0;
1804	dd->cspec->iblnkerrdelta = 0;
1805
1806	/*
1807	 * Keep chip from being accessed until we are ready.  Use
1808	 * writeq() directly, to allow the write even though QIB_PRESENT
1809	 * isn't set.
1810	 */
1811	dd->flags &= ~(QIB_INITTED | QIB_PRESENT);
1812	/* so we check interrupts work again */
1813	dd->z_int_counter = qib_int_counter(dd);
1814	val = dd->control | QLOGIC_IB_C_RESET;
1815	writeq(val, &dd->kregbase[kr_control]);
1816	mb(); /* prevent compiler re-ordering around actual reset */
1817
1818	for (i = 1; i <= 5; i++) {
1819		/*
1820		 * Allow MBIST, etc. to complete; longer on each retry.
1821		 * We sometimes get machine checks from bus timeout if no
1822		 * response, so for now, make it *really* long.
1823		 */
1824		msleep(1000 + (1 + i) * 2000);
1825
1826		qib_pcie_reenable(dd, cmdval, int_line, clinesz);
1827
1828		/*
1829		 * Use readq directly, so we don't need to mark it as PRESENT
1830		 * until we get a successful indication that all is well.
1831		 */
1832		val = readq(&dd->kregbase[kr_revision]);
1833		if (val == dd->revision) {
1834			dd->flags |= QIB_PRESENT; /* it's back */
1835			ret = qib_reinit_intr(dd);
1836			goto bail;
1837		}
1838	}
1839	ret = 0; /* failed */
1840
1841bail:
1842	if (ret) {
1843		if (qib_pcie_params(dd, dd->lbus_width, NULL, NULL))
1844			qib_dev_err(dd,
1845				"Reset failed to setup PCIe or interrupts; continuing anyway\n");
1846		/* clear the reset error, init error/hwerror mask */
1847		qib_6120_init_hwerrors(dd);
1848		/* for Rev2 error interrupts; nop for rev 1 */
1849		qib_write_kreg(dd, kr_gpio_mask, dd->cspec->gpio_mask);
1850		/* clear the reset error, init error/hwerror mask */
1851		qib_6120_init_hwerrors(dd);
1852	}
1853	return ret;
1854}
1855
1856/**
1857 * qib_6120_put_tid - write a TID in chip
1858 * @dd: the qlogic_ib device
1859 * @tidptr: pointer to the expected TID (in chip) to update
1860 * @tidtype: RCVHQ_RCV_TYPE_EAGER (1) for eager, RCVHQ_RCV_TYPE_EXPECTED (0)
1861 * for expected
1862 * @pa: physical address of in memory buffer; tidinvalid if freeing
1863 *
1864 * This exists as a separate routine to allow for special locking etc.
1865 * It's used for both the full cleanup on exit, as well as the normal
1866 * setup and teardown.
1867 */
1868static void qib_6120_put_tid(struct qib_devdata *dd, u64 __iomem *tidptr,
1869			     u32 type, unsigned long pa)
1870{
1871	u32 __iomem *tidp32 = (u32 __iomem *)tidptr;
1872	unsigned long flags;
1873	int tidx;
1874	spinlock_t *tidlockp; /* select appropriate spinlock */
1875
1876	if (!dd->kregbase)
1877		return;
1878
1879	if (pa != dd->tidinvalid) {
1880		if (pa & ((1U << 11) - 1)) {
1881			qib_dev_err(dd, "Physaddr %lx not 2KB aligned!\n",
1882				    pa);
1883			return;
1884		}
1885		pa >>= 11;
1886		if (pa & ~QLOGIC_IB_RT_ADDR_MASK) {
1887			qib_dev_err(dd,
1888				"Physical page address 0x%lx larger than supported\n",
1889				pa);
1890			return;
1891		}
1892
1893		if (type == RCVHQ_RCV_TYPE_EAGER)
1894			pa |= dd->tidtemplate;
1895		else /* for now, always full 4KB page */
1896			pa |= 2 << 29;
1897	}
1898
1899	/*
1900	 * Avoid chip issue by writing the scratch register
1901	 * before and after the TID, and with an io write barrier.
1902	 * We use a spinlock around the writes, so they can't intermix
1903	 * with other TID (eager or expected) writes (the chip problem
1904	 * is triggered by back to back TID writes). Unfortunately, this
1905	 * call can be done from interrupt level for the ctxt 0 eager TIDs,
1906	 * so we have to use irqsave locks.
1907	 */
1908	/*
1909	 * Assumes tidptr always > egrtidbase
1910	 * if type == RCVHQ_RCV_TYPE_EAGER.
1911	 */
1912	tidx = tidptr - dd->egrtidbase;
1913
1914	tidlockp = (type == RCVHQ_RCV_TYPE_EAGER && tidx < dd->rcvhdrcnt)
1915		? &dd->cspec->kernel_tid_lock : &dd->cspec->user_tid_lock;
1916	spin_lock_irqsave(tidlockp, flags);
1917	qib_write_kreg(dd, kr_scratch, 0xfeeddeaf);
1918	writel(pa, tidp32);
1919	qib_write_kreg(dd, kr_scratch, 0xdeadbeef);
1920	mmiowb();
1921	spin_unlock_irqrestore(tidlockp, flags);
1922}
1923
1924/**
1925 * qib_6120_put_tid_2 - write a TID in chip, Revision 2 or higher
1926 * @dd: the qlogic_ib device
1927 * @tidptr: pointer to the expected TID (in chip) to update
1928 * @tidtype: RCVHQ_RCV_TYPE_EAGER (1) for eager, RCVHQ_RCV_TYPE_EXPECTED (0)
1929 * for expected
1930 * @pa: physical address of in memory buffer; tidinvalid if freeing
1931 *
1932 * This exists as a separate routine to allow for selection of the
1933 * appropriate "flavor". The static calls in cleanup just use the
1934 * revision-agnostic form, as they are not performance critical.
1935 */
1936static void qib_6120_put_tid_2(struct qib_devdata *dd, u64 __iomem *tidptr,
1937			       u32 type, unsigned long pa)
1938{
1939	u32 __iomem *tidp32 = (u32 __iomem *)tidptr;
1940	u32 tidx;
1941
1942	if (!dd->kregbase)
1943		return;
1944
1945	if (pa != dd->tidinvalid) {
1946		if (pa & ((1U << 11) - 1)) {
1947			qib_dev_err(dd, "Physaddr %lx not 2KB aligned!\n",
1948				    pa);
1949			return;
1950		}
1951		pa >>= 11;
1952		if (pa & ~QLOGIC_IB_RT_ADDR_MASK) {
1953			qib_dev_err(dd,
1954				"Physical page address 0x%lx larger than supported\n",
1955				pa);
1956			return;
1957		}
1958
1959		if (type == RCVHQ_RCV_TYPE_EAGER)
1960			pa |= dd->tidtemplate;
1961		else /* for now, always full 4KB page */
1962			pa |= 2 << 29;
1963	}
1964	tidx = tidptr - dd->egrtidbase;
1965	writel(pa, tidp32);
1966	mmiowb();
1967}
1968
1969
1970/**
1971 * qib_6120_clear_tids - clear all TID entries for a context, expected and eager
1972 * @dd: the qlogic_ib device
1973 * @ctxt: the context
1974 *
1975 * clear all TID entries for a context, expected and eager.
1976 * Used from qib_close().  On this chip, TIDs are only 32 bits,
1977 * not 64, but they are still on 64 bit boundaries, so tidbase
1978 * is declared as u64 * for the pointer math, even though we write 32 bits
1979 */
1980static void qib_6120_clear_tids(struct qib_devdata *dd,
1981				struct qib_ctxtdata *rcd)
1982{
1983	u64 __iomem *tidbase;
1984	unsigned long tidinv;
1985	u32 ctxt;
1986	int i;
1987
1988	if (!dd->kregbase || !rcd)
1989		return;
1990
1991	ctxt = rcd->ctxt;
1992
1993	tidinv = dd->tidinvalid;
1994	tidbase = (u64 __iomem *)
1995		((char __iomem *)(dd->kregbase) +
1996		 dd->rcvtidbase +
1997		 ctxt * dd->rcvtidcnt * sizeof(*tidbase));
1998
1999	for (i = 0; i < dd->rcvtidcnt; i++)
2000		/* use func pointer because could be one of two funcs */
2001		dd->f_put_tid(dd, &tidbase[i], RCVHQ_RCV_TYPE_EXPECTED,
2002				  tidinv);
2003
2004	tidbase = (u64 __iomem *)
2005		((char __iomem *)(dd->kregbase) +
2006		 dd->rcvegrbase +
2007		 rcd->rcvegr_tid_base * sizeof(*tidbase));
2008
2009	for (i = 0; i < rcd->rcvegrcnt; i++)
2010		/* use func pointer because could be one of two funcs */
2011		dd->f_put_tid(dd, &tidbase[i], RCVHQ_RCV_TYPE_EAGER,
2012				  tidinv);
2013}
2014
2015/**
2016 * qib_6120_tidtemplate - setup constants for TID updates
2017 * @dd: the qlogic_ib device
2018 *
2019 * We setup stuff that we use a lot, to avoid calculating each time
2020 */
2021static void qib_6120_tidtemplate(struct qib_devdata *dd)
2022{
2023	u32 egrsize = dd->rcvegrbufsize;
2024
2025	/*
2026	 * For now, we always allocate 4KB buffers (at init) so we can
2027	 * receive max size packets.  We may want a module parameter to
2028	 * specify 2KB or 4KB and/or make be per ctxt instead of per device
2029	 * for those who want to reduce memory footprint.  Note that the
2030	 * rcvhdrentsize size must be large enough to hold the largest
2031	 * IB header (currently 96 bytes) that we expect to handle (plus of
2032	 * course the 2 dwords of RHF).
2033	 */
2034	if (egrsize == 2048)
2035		dd->tidtemplate = 1U << 29;
2036	else if (egrsize == 4096)
2037		dd->tidtemplate = 2U << 29;
2038	dd->tidinvalid = 0;
2039}
2040
2041int __attribute__((weak)) qib_unordered_wc(void)
2042{
2043	return 0;
2044}
2045
2046/**
2047 * qib_6120_get_base_info - set chip-specific flags for user code
2048 * @rcd: the qlogic_ib ctxt
2049 * @kbase: qib_base_info pointer
2050 *
2051 * We set the PCIE flag because the lower bandwidth on PCIe vs
2052 * HyperTransport can affect some user packet algorithms.
2053 */
2054static int qib_6120_get_base_info(struct qib_ctxtdata *rcd,
2055				  struct qib_base_info *kinfo)
2056{
2057	if (qib_unordered_wc())
2058		kinfo->spi_runtime_flags |= QIB_RUNTIME_FORCE_WC_ORDER;
2059
2060	kinfo->spi_runtime_flags |= QIB_RUNTIME_PCIE |
2061		QIB_RUNTIME_FORCE_PIOAVAIL | QIB_RUNTIME_PIO_REGSWAPPED;
2062	return 0;
2063}
2064
2065
2066static struct qib_message_header *
2067qib_6120_get_msgheader(struct qib_devdata *dd, __le32 *rhf_addr)
2068{
2069	return (struct qib_message_header *)
2070		&rhf_addr[sizeof(u64) / sizeof(u32)];
2071}
2072
2073static void qib_6120_config_ctxts(struct qib_devdata *dd)
2074{
2075	dd->ctxtcnt = qib_read_kreg32(dd, kr_portcnt);
2076	if (qib_n_krcv_queues > 1) {
2077		dd->first_user_ctxt = qib_n_krcv_queues * dd->num_pports;
2078		if (dd->first_user_ctxt > dd->ctxtcnt)
2079			dd->first_user_ctxt = dd->ctxtcnt;
2080		dd->qpn_mask = dd->first_user_ctxt <= 2 ? 2 : 6;
2081	} else
2082		dd->first_user_ctxt = dd->num_pports;
2083	dd->n_krcv_queues = dd->first_user_ctxt;
2084}
2085
2086static void qib_update_6120_usrhead(struct qib_ctxtdata *rcd, u64 hd,
2087				    u32 updegr, u32 egrhd, u32 npkts)
2088{
2089	if (updegr)
2090		qib_write_ureg(rcd->dd, ur_rcvegrindexhead, egrhd, rcd->ctxt);
2091	mmiowb();
2092	qib_write_ureg(rcd->dd, ur_rcvhdrhead, hd, rcd->ctxt);
2093	mmiowb();
2094}
2095
2096static u32 qib_6120_hdrqempty(struct qib_ctxtdata *rcd)
2097{
2098	u32 head, tail;
2099
2100	head = qib_read_ureg32(rcd->dd, ur_rcvhdrhead, rcd->ctxt);
2101	if (rcd->rcvhdrtail_kvaddr)
2102		tail = qib_get_rcvhdrtail(rcd);
2103	else
2104		tail = qib_read_ureg32(rcd->dd, ur_rcvhdrtail, rcd->ctxt);
2105	return head == tail;
2106}
2107
2108/*
2109 * Used when we close any ctxt, for DMA already in flight
2110 * at close.  Can't be done until we know hdrq size, so not
2111 * early in chip init.
2112 */
2113static void alloc_dummy_hdrq(struct qib_devdata *dd)
2114{
2115	dd->cspec->dummy_hdrq = dma_alloc_coherent(&dd->pcidev->dev,
2116					dd->rcd[0]->rcvhdrq_size,
2117					&dd->cspec->dummy_hdrq_phys,
2118					GFP_ATOMIC | __GFP_COMP);
2119	if (!dd->cspec->dummy_hdrq) {
2120		qib_devinfo(dd->pcidev, "Couldn't allocate dummy hdrq\n");
2121		/* fallback to just 0'ing */
2122		dd->cspec->dummy_hdrq_phys = 0UL;
2123	}
2124}
2125
2126/*
2127 * Modify the RCVCTRL register in chip-specific way. This
2128 * is a function because bit positions and (future) register
2129 * location is chip-specific, but the needed operations are
2130 * generic. <op> is a bit-mask because we often want to
2131 * do multiple modifications.
2132 */
2133static void rcvctrl_6120_mod(struct qib_pportdata *ppd, unsigned int op,
2134			     int ctxt)
2135{
2136	struct qib_devdata *dd = ppd->dd;
2137	u64 mask, val;
2138	unsigned long flags;
2139
2140	spin_lock_irqsave(&dd->cspec->rcvmod_lock, flags);
2141
2142	if (op & QIB_RCVCTRL_TAILUPD_ENB)
2143		dd->rcvctrl |= (1ULL << QLOGIC_IB_R_TAILUPD_SHIFT);
2144	if (op & QIB_RCVCTRL_TAILUPD_DIS)
2145		dd->rcvctrl &= ~(1ULL << QLOGIC_IB_R_TAILUPD_SHIFT);
2146	if (op & QIB_RCVCTRL_PKEY_ENB)
2147		dd->rcvctrl &= ~(1ULL << IBA6120_R_PKEY_DIS_SHIFT);
2148	if (op & QIB_RCVCTRL_PKEY_DIS)
2149		dd->rcvctrl |= (1ULL << IBA6120_R_PKEY_DIS_SHIFT);
2150	if (ctxt < 0)
2151		mask = (1ULL << dd->ctxtcnt) - 1;
2152	else
2153		mask = (1ULL << ctxt);
2154	if (op & QIB_RCVCTRL_CTXT_ENB) {
2155		/* always done for specific ctxt */
2156		dd->rcvctrl |= (mask << SYM_LSB(RcvCtrl, PortEnable));
2157		if (!(dd->flags & QIB_NODMA_RTAIL))
2158			dd->rcvctrl |= 1ULL << QLOGIC_IB_R_TAILUPD_SHIFT;
2159		/* Write these registers before the context is enabled. */
2160		qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr, ctxt,
2161			dd->rcd[ctxt]->rcvhdrqtailaddr_phys);
2162		qib_write_kreg_ctxt(dd, kr_rcvhdraddr, ctxt,
2163			dd->rcd[ctxt]->rcvhdrq_phys);
2164
2165		if (ctxt == 0 && !dd->cspec->dummy_hdrq)
2166			alloc_dummy_hdrq(dd);
2167	}
2168	if (op & QIB_RCVCTRL_CTXT_DIS)
2169		dd->rcvctrl &= ~(mask << SYM_LSB(RcvCtrl, PortEnable));
2170	if (op & QIB_RCVCTRL_INTRAVAIL_ENB)
2171		dd->rcvctrl |= (mask << QLOGIC_IB_R_INTRAVAIL_SHIFT);
2172	if (op & QIB_RCVCTRL_INTRAVAIL_DIS)
2173		dd->rcvctrl &= ~(mask << QLOGIC_IB_R_INTRAVAIL_SHIFT);
2174	qib_write_kreg(dd, kr_rcvctrl, dd->rcvctrl);
2175	if ((op & QIB_RCVCTRL_INTRAVAIL_ENB) && dd->rhdrhead_intr_off) {
2176		/* arm rcv interrupt */
2177		val = qib_read_ureg32(dd, ur_rcvhdrhead, ctxt) |
2178			dd->rhdrhead_intr_off;
2179		qib_write_ureg(dd, ur_rcvhdrhead, val, ctxt);
2180	}
2181	if (op & QIB_RCVCTRL_CTXT_ENB) {
2182		/*
2183		 * Init the context registers also; if we were
2184		 * disabled, tail and head should both be zero
2185		 * already from the enable, but since we don't
2186		 * know, we have to do it explicitly.
2187		 */
2188		val = qib_read_ureg32(dd, ur_rcvegrindextail, ctxt);
2189		qib_write_ureg(dd, ur_rcvegrindexhead, val, ctxt);
2190
2191		val = qib_read_ureg32(dd, ur_rcvhdrtail, ctxt);
2192		dd->rcd[ctxt]->head = val;
2193		/* If kctxt, interrupt on next receive. */
2194		if (ctxt < dd->first_user_ctxt)
2195			val |= dd->rhdrhead_intr_off;
2196		qib_write_ureg(dd, ur_rcvhdrhead, val, ctxt);
2197	}
2198	if (op & QIB_RCVCTRL_CTXT_DIS) {
2199		/*
2200		 * Be paranoid, and never write 0's to these, just use an
2201		 * unused page.  Of course,
2202		 * rcvhdraddr points to a large chunk of memory, so this
2203		 * could still trash things, but at least it won't trash
2204		 * page 0, and by disabling the ctxt, it should stop "soon",
2205		 * even if a packet or two is in already in flight after we
2206		 * disabled the ctxt.  Only 6120 has this issue.
2207		 */
2208		if (ctxt >= 0) {
2209			qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr, ctxt,
2210					    dd->cspec->dummy_hdrq_phys);
2211			qib_write_kreg_ctxt(dd, kr_rcvhdraddr, ctxt,
2212					    dd->cspec->dummy_hdrq_phys);
2213		} else {
2214			unsigned i;
2215
2216			for (i = 0; i < dd->cfgctxts; i++) {
2217				qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr,
2218					    i, dd->cspec->dummy_hdrq_phys);
2219				qib_write_kreg_ctxt(dd, kr_rcvhdraddr,
2220					    i, dd->cspec->dummy_hdrq_phys);
2221			}
2222		}
2223	}
2224	spin_unlock_irqrestore(&dd->cspec->rcvmod_lock, flags);
2225}
2226
2227/*
2228 * Modify the SENDCTRL register in chip-specific way. This
2229 * is a function there may be multiple such registers with
2230 * slightly different layouts. Only operations actually used
2231 * are implemented yet.
2232 * Chip requires no back-back sendctrl writes, so write
2233 * scratch register after writing sendctrl
2234 */
2235static void sendctrl_6120_mod(struct qib_pportdata *ppd, u32 op)
2236{
2237	struct qib_devdata *dd = ppd->dd;
2238	u64 tmp_dd_sendctrl;
2239	unsigned long flags;
2240
2241	spin_lock_irqsave(&dd->sendctrl_lock, flags);
2242
2243	/* First the ones that are "sticky", saved in shadow */
2244	if (op & QIB_SENDCTRL_CLEAR)
2245		dd->sendctrl = 0;
2246	if (op & QIB_SENDCTRL_SEND_DIS)
2247		dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOEnable);
2248	else if (op & QIB_SENDCTRL_SEND_ENB)
2249		dd->sendctrl |= SYM_MASK(SendCtrl, PIOEnable);
2250	if (op & QIB_SENDCTRL_AVAIL_DIS)
2251		dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOBufAvailUpd);
2252	else if (op & QIB_SENDCTRL_AVAIL_ENB)
2253		dd->sendctrl |= SYM_MASK(SendCtrl, PIOBufAvailUpd);
2254
2255	if (op & QIB_SENDCTRL_DISARM_ALL) {
2256		u32 i, last;
2257
2258		tmp_dd_sendctrl = dd->sendctrl;
2259		/*
2260		 * disarm any that are not yet launched, disabling sends
2261		 * and updates until done.
2262		 */
2263		last = dd->piobcnt2k + dd->piobcnt4k;
2264		tmp_dd_sendctrl &=
2265			~(SYM_MASK(SendCtrl, PIOEnable) |
2266			  SYM_MASK(SendCtrl, PIOBufAvailUpd));
2267		for (i = 0; i < last; i++) {
2268			qib_write_kreg(dd, kr_sendctrl, tmp_dd_sendctrl |
2269				       SYM_MASK(SendCtrl, Disarm) | i);
2270			qib_write_kreg(dd, kr_scratch, 0);
2271		}
2272	}
2273
2274	tmp_dd_sendctrl = dd->sendctrl;
2275
2276	if (op & QIB_SENDCTRL_FLUSH)
2277		tmp_dd_sendctrl |= SYM_MASK(SendCtrl, Abort);
2278	if (op & QIB_SENDCTRL_DISARM)
2279		tmp_dd_sendctrl |= SYM_MASK(SendCtrl, Disarm) |
2280			((op & QIB_6120_SendCtrl_DisarmPIOBuf_RMASK) <<
2281			 SYM_LSB(SendCtrl, DisarmPIOBuf));
2282	if (op & QIB_SENDCTRL_AVAIL_BLIP)
2283		tmp_dd_sendctrl &= ~SYM_MASK(SendCtrl, PIOBufAvailUpd);
2284
2285	qib_write_kreg(dd, kr_sendctrl, tmp_dd_sendctrl);
2286	qib_write_kreg(dd, kr_scratch, 0);
2287
2288	if (op & QIB_SENDCTRL_AVAIL_BLIP) {
2289		qib_write_kreg(dd, kr_sendctrl, dd->sendctrl);
2290		qib_write_kreg(dd, kr_scratch, 0);
2291	}
2292
2293	spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
2294
2295	if (op & QIB_SENDCTRL_FLUSH) {
2296		u32 v;
2297		/*
2298		 * ensure writes have hit chip, then do a few
2299		 * more reads, to allow DMA of pioavail registers
2300		 * to occur, so in-memory copy is in sync with
2301		 * the chip.  Not always safe to sleep.
2302		 */
2303		v = qib_read_kreg32(dd, kr_scratch);
2304		qib_write_kreg(dd, kr_scratch, v);
2305		v = qib_read_kreg32(dd, kr_scratch);
2306		qib_write_kreg(dd, kr_scratch, v);
2307		qib_read_kreg32(dd, kr_scratch);
2308	}
2309}
2310
2311/**
2312 * qib_portcntr_6120 - read a per-port counter
2313 * @dd: the qlogic_ib device
2314 * @creg: the counter to snapshot
2315 */
2316static u64 qib_portcntr_6120(struct qib_pportdata *ppd, u32 reg)
2317{
2318	u64 ret = 0ULL;
2319	struct qib_devdata *dd = ppd->dd;
2320	u16 creg;
2321	/* 0xffff for unimplemented or synthesized counters */
2322	static const u16 xlator[] = {
2323		[QIBPORTCNTR_PKTSEND] = cr_pktsend,
2324		[QIBPORTCNTR_WORDSEND] = cr_wordsend,
2325		[QIBPORTCNTR_PSXMITDATA] = 0xffff,
2326		[QIBPORTCNTR_PSXMITPKTS] = 0xffff,
2327		[QIBPORTCNTR_PSXMITWAIT] = 0xffff,
2328		[QIBPORTCNTR_SENDSTALL] = cr_sendstall,
2329		[QIBPORTCNTR_PKTRCV] = cr_pktrcv,
2330		[QIBPORTCNTR_PSRCVDATA] = 0xffff,
2331		[QIBPORTCNTR_PSRCVPKTS] = 0xffff,
2332		[QIBPORTCNTR_RCVEBP] = cr_rcvebp,
2333		[QIBPORTCNTR_RCVOVFL] = cr_rcvovfl,
2334		[QIBPORTCNTR_WORDRCV] = cr_wordrcv,
2335		[QIBPORTCNTR_RXDROPPKT] = cr_rxdroppkt,
2336		[QIBPORTCNTR_RXLOCALPHYERR] = 0xffff,
2337		[QIBPORTCNTR_RXVLERR] = 0xffff,
2338		[QIBPORTCNTR_ERRICRC] = cr_erricrc,
2339		[QIBPORTCNTR_ERRVCRC] = cr_errvcrc,
2340		[QIBPORTCNTR_ERRLPCRC] = cr_errlpcrc,
2341		[QIBPORTCNTR_BADFORMAT] = cr_badformat,
2342		[QIBPORTCNTR_ERR_RLEN] = cr_err_rlen,
2343		[QIBPORTCNTR_IBSYMBOLERR] = cr_ibsymbolerr,
2344		[QIBPORTCNTR_INVALIDRLEN] = cr_invalidrlen,
2345		[QIBPORTCNTR_UNSUPVL] = cr_txunsupvl,
2346		[QIBPORTCNTR_EXCESSBUFOVFL] = 0xffff,
2347		[QIBPORTCNTR_ERRLINK] = cr_errlink,
2348		[QIBPORTCNTR_IBLINKDOWN] = cr_iblinkdown,
2349		[QIBPORTCNTR_IBLINKERRRECOV] = cr_iblinkerrrecov,
2350		[QIBPORTCNTR_LLI] = 0xffff,
2351		[QIBPORTCNTR_PSINTERVAL] = 0xffff,
2352		[QIBPORTCNTR_PSSTART] = 0xffff,
2353		[QIBPORTCNTR_PSSTAT] = 0xffff,
2354		[QIBPORTCNTR_VL15PKTDROP] = 0xffff,
2355		[QIBPORTCNTR_ERRPKEY] = cr_errpkey,
2356		[QIBPORTCNTR_KHDROVFL] = 0xffff,
2357	};
2358
2359	if (reg >= ARRAY_SIZE(xlator)) {
2360		qib_devinfo(ppd->dd->pcidev,
2361			 "Unimplemented portcounter %u\n", reg);
2362		goto done;
2363	}
2364	creg = xlator[reg];
2365
2366	/* handle counters requests not implemented as chip counters */
2367	if (reg == QIBPORTCNTR_LLI)
2368		ret = dd->cspec->lli_errs;
2369	else if (reg == QIBPORTCNTR_EXCESSBUFOVFL)
2370		ret = dd->cspec->overrun_thresh_errs;
2371	else if (reg == QIBPORTCNTR_KHDROVFL) {
2372		int i;
2373
2374		/* sum over all kernel contexts */
2375		for (i = 0; i < dd->first_user_ctxt; i++)
2376			ret += read_6120_creg32(dd, cr_portovfl + i);
2377	} else if (reg == QIBPORTCNTR_PSSTAT)
2378		ret = dd->cspec->pma_sample_status;
2379	if (creg == 0xffff)
2380		goto done;
2381
2382	/*
2383	 * only fast incrementing counters are 64bit; use 32 bit reads to
2384	 * avoid two independent reads when on opteron
2385	 */
2386	if (creg == cr_wordsend || creg == cr_wordrcv ||
2387	    creg == cr_pktsend || creg == cr_pktrcv)
2388		ret = read_6120_creg(dd, creg);
2389	else
2390		ret = read_6120_creg32(dd, creg);
2391	if (creg == cr_ibsymbolerr) {
2392		if (dd->cspec->ibdeltainprog)
2393			ret -= ret - dd->cspec->ibsymsnap;
2394		ret -= dd->cspec->ibsymdelta;
2395	} else if (creg == cr_iblinkerrrecov) {
2396		if (dd->cspec->ibdeltainprog)
2397			ret -= ret - dd->cspec->iblnkerrsnap;
2398		ret -= dd->cspec->iblnkerrdelta;
2399	}
2400	if (reg == QIBPORTCNTR_RXDROPPKT) /* add special cased count */
2401		ret += dd->cspec->rxfc_unsupvl_errs;
2402
2403done:
2404	return ret;
2405}
2406
2407/*
2408 * Device counter names (not port-specific), one line per stat,
2409 * single string.  Used by utilities like ipathstats to print the stats
2410 * in a way which works for different versions of drivers, without changing
2411 * the utility.  Names need to be 12 chars or less (w/o newline), for proper
2412 * display by utility.
2413 * Non-error counters are first.
2414 * Start of "error" conters is indicated by a leading "E " on the first
2415 * "error" counter, and doesn't count in label length.
2416 * The EgrOvfl list needs to be last so we truncate them at the configured
2417 * context count for the device.
2418 * cntr6120indices contains the corresponding register indices.
2419 */
2420static const char cntr6120names[] =
2421	"Interrupts\n"
2422	"HostBusStall\n"
2423	"E RxTIDFull\n"
2424	"RxTIDInvalid\n"
2425	"Ctxt0EgrOvfl\n"
2426	"Ctxt1EgrOvfl\n"
2427	"Ctxt2EgrOvfl\n"
2428	"Ctxt3EgrOvfl\n"
2429	"Ctxt4EgrOvfl\n";
2430
2431static const size_t cntr6120indices[] = {
2432	cr_lbint,
2433	cr_lbflowstall,
2434	cr_errtidfull,
2435	cr_errtidvalid,
2436	cr_portovfl + 0,
2437	cr_portovfl + 1,
2438	cr_portovfl + 2,
2439	cr_portovfl + 3,
2440	cr_portovfl + 4,
2441};
2442
2443/*
2444 * same as cntr6120names and cntr6120indices, but for port-specific counters.
2445 * portcntr6120indices is somewhat complicated by some registers needing
2446 * adjustments of various kinds, and those are ORed with _PORT_VIRT_FLAG
2447 */
2448static const char portcntr6120names[] =
2449	"TxPkt\n"
2450	"TxFlowPkt\n"
2451	"TxWords\n"
2452	"RxPkt\n"
2453	"RxFlowPkt\n"
2454	"RxWords\n"
2455	"TxFlowStall\n"
2456	"E IBStatusChng\n"
2457	"IBLinkDown\n"
2458	"IBLnkRecov\n"
2459	"IBRxLinkErr\n"
2460	"IBSymbolErr\n"
2461	"RxLLIErr\n"
2462	"RxBadFormat\n"
2463	"RxBadLen\n"
2464	"RxBufOvrfl\n"
2465	"RxEBP\n"
2466	"RxFlowCtlErr\n"
2467	"RxICRCerr\n"
2468	"RxLPCRCerr\n"
2469	"RxVCRCerr\n"
2470	"RxInvalLen\n"
2471	"RxInvalPKey\n"
2472	"RxPktDropped\n"
2473	"TxBadLength\n"
2474	"TxDropped\n"
2475	"TxInvalLen\n"
2476	"TxUnderrun\n"
2477	"TxUnsupVL\n"
2478	;
2479
2480#define _PORT_VIRT_FLAG 0x8000 /* "virtual", need adjustments */
2481static const size_t portcntr6120indices[] = {
2482	QIBPORTCNTR_PKTSEND | _PORT_VIRT_FLAG,
2483	cr_pktsendflow,
2484	QIBPORTCNTR_WORDSEND | _PORT_VIRT_FLAG,
2485	QIBPORTCNTR_PKTRCV | _PORT_VIRT_FLAG,
2486	cr_pktrcvflowctrl,
2487	QIBPORTCNTR_WORDRCV | _PORT_VIRT_FLAG,
2488	QIBPORTCNTR_SENDSTALL | _PORT_VIRT_FLAG,
2489	cr_ibstatuschange,
2490	QIBPORTCNTR_IBLINKDOWN | _PORT_VIRT_FLAG,
2491	QIBPORTCNTR_IBLINKERRRECOV | _PORT_VIRT_FLAG,
2492	QIBPORTCNTR_ERRLINK | _PORT_VIRT_FLAG,
2493	QIBPORTCNTR_IBSYMBOLERR | _PORT_VIRT_FLAG,
2494	QIBPORTCNTR_LLI | _PORT_VIRT_FLAG,
2495	QIBPORTCNTR_BADFORMAT | _PORT_VIRT_FLAG,
2496	QIBPORTCNTR_ERR_RLEN | _PORT_VIRT_FLAG,
2497	QIBPORTCNTR_RCVOVFL | _PORT_VIRT_FLAG,
2498	QIBPORTCNTR_RCVEBP | _PORT_VIRT_FLAG,
2499	cr_rcvflowctrl_err,
2500	QIBPORTCNTR_ERRICRC | _PORT_VIRT_FLAG,
2501	QIBPORTCNTR_ERRLPCRC | _PORT_VIRT_FLAG,
2502	QIBPORTCNTR_ERRVCRC | _PORT_VIRT_FLAG,
2503	QIBPORTCNTR_INVALIDRLEN | _PORT_VIRT_FLAG,
2504	QIBPORTCNTR_ERRPKEY | _PORT_VIRT_FLAG,
2505	QIBPORTCNTR_RXDROPPKT | _PORT_VIRT_FLAG,
2506	cr_invalidslen,
2507	cr_senddropped,
2508	cr_errslen,
2509	cr_sendunderrun,
2510	cr_txunsupvl,
2511};
2512
2513/* do all the setup to make the counter reads efficient later */
2514static void init_6120_cntrnames(struct qib_devdata *dd)
2515{
2516	int i, j = 0;
2517	char *s;
2518
2519	for (i = 0, s = (char *)cntr6120names; s && j <= dd->cfgctxts;
2520	     i++) {
2521		/* we always have at least one counter before the egrovfl */
2522		if (!j && !strncmp("Ctxt0EgrOvfl", s + 1, 12))
2523			j = 1;
2524		s = strchr(s + 1, '\n');
2525		if (s && j)
2526			j++;
2527	}
2528	dd->cspec->ncntrs = i;
2529	if (!s)
2530		/* full list; size is without terminating null */
2531		dd->cspec->cntrnamelen = sizeof(cntr6120names) - 1;
2532	else
2533		dd->cspec->cntrnamelen = 1 + s - cntr6120names;
2534	dd->cspec->cntrs = kmalloc(dd->cspec->ncntrs
2535		* sizeof(u64), GFP_KERNEL);
2536	if (!dd->cspec->cntrs)
2537		qib_dev_err(dd, "Failed allocation for counters\n");
2538
2539	for (i = 0, s = (char *)portcntr6120names; s; i++)
2540		s = strchr(s + 1, '\n');
2541	dd->cspec->nportcntrs = i - 1;
2542	dd->cspec->portcntrnamelen = sizeof(portcntr6120names) - 1;
2543	dd->cspec->portcntrs = kmalloc(dd->cspec->nportcntrs
2544		* sizeof(u64), GFP_KERNEL);
2545	if (!dd->cspec->portcntrs)
2546		qib_dev_err(dd, "Failed allocation for portcounters\n");
2547}
2548
2549static u32 qib_read_6120cntrs(struct qib_devdata *dd, loff_t pos, char **namep,
2550			      u64 **cntrp)
2551{
2552	u32 ret;
2553
2554	if (namep) {
2555		ret = dd->cspec->cntrnamelen;
2556		if (pos >= ret)
2557			ret = 0; /* final read after getting everything */
2558		else
2559			*namep = (char *)cntr6120names;
2560	} else {
2561		u64 *cntr = dd->cspec->cntrs;
2562		int i;
2563
2564		ret = dd->cspec->ncntrs * sizeof(u64);
2565		if (!cntr || pos >= ret) {
2566			/* everything read, or couldn't get memory */
2567			ret = 0;
2568			goto done;
2569		}
2570		if (pos >= ret) {
2571			ret = 0; /* final read after getting everything */
2572			goto done;
2573		}
2574		*cntrp = cntr;
2575		for (i = 0; i < dd->cspec->ncntrs; i++)
2576			*cntr++ = read_6120_creg32(dd, cntr6120indices[i]);
2577	}
2578done:
2579	return ret;
2580}
2581
2582static u32 qib_read_6120portcntrs(struct qib_devdata *dd, loff_t pos, u32 port,
2583				  char **namep, u64 **cntrp)
2584{
2585	u32 ret;
2586
2587	if (namep) {
2588		ret = dd->cspec->portcntrnamelen;
2589		if (pos >= ret)
2590			ret = 0; /* final read after getting everything */
2591		else
2592			*namep = (char *)portcntr6120names;
2593	} else {
2594		u64 *cntr = dd->cspec->portcntrs;
2595		struct qib_pportdata *ppd = &dd->pport[port];
2596		int i;
2597
2598		ret = dd->cspec->nportcntrs * sizeof(u64);
2599		if (!cntr || pos >= ret) {
2600			/* everything read, or couldn't get memory */
2601			ret = 0;
2602			goto done;
2603		}
2604		*cntrp = cntr;
2605		for (i = 0; i < dd->cspec->nportcntrs; i++) {
2606			if (portcntr6120indices[i] & _PORT_VIRT_FLAG)
2607				*cntr++ = qib_portcntr_6120(ppd,
2608					portcntr6120indices[i] &
2609					~_PORT_VIRT_FLAG);
2610			else
2611				*cntr++ = read_6120_creg32(dd,
2612					   portcntr6120indices[i]);
2613		}
2614	}
2615done:
2616	return ret;
2617}
2618
2619static void qib_chk_6120_errormask(struct qib_devdata *dd)
2620{
2621	static u32 fixed;
2622	u32 ctrl;
2623	unsigned long errormask;
2624	unsigned long hwerrs;
2625
2626	if (!dd->cspec->errormask || !(dd->flags & QIB_INITTED))
2627		return;
2628
2629	errormask = qib_read_kreg64(dd, kr_errmask);
2630
2631	if (errormask == dd->cspec->errormask)
2632		return;
2633	fixed++;
2634
2635	hwerrs = qib_read_kreg64(dd, kr_hwerrstatus);
2636	ctrl = qib_read_kreg32(dd, kr_control);
2637
2638	qib_write_kreg(dd, kr_errmask,
2639		dd->cspec->errormask);
2640
2641	if ((hwerrs & dd->cspec->hwerrmask) ||
2642	    (ctrl & QLOGIC_IB_C_FREEZEMODE)) {
2643		qib_write_kreg(dd, kr_hwerrclear, 0ULL);
2644		qib_write_kreg(dd, kr_errclear, 0ULL);
2645		/* force re-interrupt of pending events, just in case */
2646		qib_write_kreg(dd, kr_intclear, 0ULL);
2647		qib_devinfo(dd->pcidev,
2648			 "errormask fixed(%u) %lx->%lx, ctrl %x hwerr %lx\n",
2649			 fixed, errormask, (unsigned long)dd->cspec->errormask,
2650			 ctrl, hwerrs);
2651	}
2652}
2653
2654/**
2655 * qib_get_faststats - get word counters from chip before they overflow
2656 * @opaque - contains a pointer to the qlogic_ib device qib_devdata
2657 *
2658 * This needs more work; in particular, decision on whether we really
2659 * need traffic_wds done the way it is
2660 * called from add_timer
2661 */
2662static void qib_get_6120_faststats(unsigned long opaque)
2663{
2664	struct qib_devdata *dd = (struct qib_devdata *) opaque;
2665	struct qib_pportdata *ppd = dd->pport;
2666	unsigned long flags;
2667	u64 traffic_wds;
2668
2669	/*
2670	 * don't access the chip while running diags, or memory diags can
2671	 * fail
2672	 */
2673	if (!(dd->flags & QIB_INITTED) || dd->diag_client)
2674		/* but re-arm the timer, for diags case; won't hurt other */
2675		goto done;
2676
2677	/*
2678	 * We now try to maintain an activity timer, based on traffic
2679	 * exceeding a threshold, so we need to check the word-counts
2680	 * even if they are 64-bit.
2681	 */
2682	traffic_wds = qib_portcntr_6120(ppd, cr_wordsend) +
2683		qib_portcntr_6120(ppd, cr_wordrcv);
2684	spin_lock_irqsave(&dd->eep_st_lock, flags);
2685	traffic_wds -= dd->traffic_wds;
2686	dd->traffic_wds += traffic_wds;
2687	spin_unlock_irqrestore(&dd->eep_st_lock, flags);
2688
2689	qib_chk_6120_errormask(dd);
2690done:
2691	mod_timer(&dd->stats_timer, jiffies + HZ * ACTIVITY_TIMER);
2692}
2693
2694/* no interrupt fallback for these chips */
2695static int qib_6120_nointr_fallback(struct qib_devdata *dd)
2696{
2697	return 0;
2698}
2699
2700/*
2701 * reset the XGXS (between serdes and IBC).  Slightly less intrusive
2702 * than resetting the IBC or external link state, and useful in some
2703 * cases to cause some retraining.  To do this right, we reset IBC
2704 * as well.
2705 */
2706static void qib_6120_xgxs_reset(struct qib_pportdata *ppd)
2707{
2708	u64 val, prev_val;
2709	struct qib_devdata *dd = ppd->dd;
2710
2711	prev_val = qib_read_kreg64(dd, kr_xgxs_cfg);
2712	val = prev_val | QLOGIC_IB_XGXS_RESET;
2713	prev_val &= ~QLOGIC_IB_XGXS_RESET; /* be sure */
2714	qib_write_kreg(dd, kr_control,
2715		       dd->control & ~QLOGIC_IB_C_LINKENABLE);
2716	qib_write_kreg(dd, kr_xgxs_cfg, val);
2717	qib_read_kreg32(dd, kr_scratch);
2718	qib_write_kreg(dd, kr_xgxs_cfg, prev_val);
2719	qib_write_kreg(dd, kr_control, dd->control);
2720}
2721
2722static int qib_6120_get_ib_cfg(struct qib_pportdata *ppd, int which)
2723{
2724	int ret;
2725
2726	switch (which) {
2727	case QIB_IB_CFG_LWID:
2728		ret = ppd->link_width_active;
2729		break;
2730
2731	case QIB_IB_CFG_SPD:
2732		ret = ppd->link_speed_active;
2733		break;
2734
2735	case QIB_IB_CFG_LWID_ENB:
2736		ret = ppd->link_width_enabled;
2737		break;
2738
2739	case QIB_IB_CFG_SPD_ENB:
2740		ret = ppd->link_speed_enabled;
2741		break;
2742
2743	case QIB_IB_CFG_OP_VLS:
2744		ret = ppd->vls_operational;
2745		break;
2746
2747	case QIB_IB_CFG_VL_HIGH_CAP:
2748		ret = 0;
2749		break;
2750
2751	case QIB_IB_CFG_VL_LOW_CAP:
2752		ret = 0;
2753		break;
2754
2755	case QIB_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
2756		ret = SYM_FIELD(ppd->dd->cspec->ibcctrl, IBCCtrl,
2757				OverrunThreshold);
2758		break;
2759
2760	case QIB_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
2761		ret = SYM_FIELD(ppd->dd->cspec->ibcctrl, IBCCtrl,
2762				PhyerrThreshold);
2763		break;
2764
2765	case QIB_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
2766		/* will only take effect when the link state changes */
2767		ret = (ppd->dd->cspec->ibcctrl &
2768		       SYM_MASK(IBCCtrl, LinkDownDefaultState)) ?
2769			IB_LINKINITCMD_SLEEP : IB_LINKINITCMD_POLL;
2770		break;
2771
2772	case QIB_IB_CFG_HRTBT: /* Get Heartbeat off/enable/auto */
2773		ret = 0; /* no heartbeat on this chip */
2774		break;
2775
2776	case QIB_IB_CFG_PMA_TICKS:
2777		ret = 250; /* 1 usec. */
2778		break;
2779
2780	default:
2781		ret =  -EINVAL;
2782		break;
2783	}
2784	return ret;
2785}
2786
2787/*
2788 * We assume range checking is already done, if needed.
2789 */
2790static int qib_6120_set_ib_cfg(struct qib_pportdata *ppd, int which, u32 val)
2791{
2792	struct qib_devdata *dd = ppd->dd;
2793	int ret = 0;
2794	u64 val64;
2795	u16 lcmd, licmd;
2796
2797	switch (which) {
2798	case QIB_IB_CFG_LWID_ENB:
2799		ppd->link_width_enabled = val;
2800		break;
2801
2802	case QIB_IB_CFG_SPD_ENB:
2803		ppd->link_speed_enabled = val;
2804		break;
2805
2806	case QIB_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
2807		val64 = SYM_FIELD(dd->cspec->ibcctrl, IBCCtrl,
2808				  OverrunThreshold);
2809		if (val64 != val) {
2810			dd->cspec->ibcctrl &=
2811				~SYM_MASK(IBCCtrl, OverrunThreshold);
2812			dd->cspec->ibcctrl |= (u64) val <<
2813				SYM_LSB(IBCCtrl, OverrunThreshold);
2814			qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2815			qib_write_kreg(dd, kr_scratch, 0);
2816		}
2817		break;
2818
2819	case QIB_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
2820		val64 = SYM_FIELD(dd->cspec->ibcctrl, IBCCtrl,
2821				  PhyerrThreshold);
2822		if (val64 != val) {
2823			dd->cspec->ibcctrl &=
2824				~SYM_MASK(IBCCtrl, PhyerrThreshold);
2825			dd->cspec->ibcctrl |= (u64) val <<
2826				SYM_LSB(IBCCtrl, PhyerrThreshold);
2827			qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2828			qib_write_kreg(dd, kr_scratch, 0);
2829		}
2830		break;
2831
2832	case QIB_IB_CFG_PKEYS: /* update pkeys */
2833		val64 = (u64) ppd->pkeys[0] | ((u64) ppd->pkeys[1] << 16) |
2834			((u64) ppd->pkeys[2] << 32) |
2835			((u64) ppd->pkeys[3] << 48);
2836		qib_write_kreg(dd, kr_partitionkey, val64);
2837		break;
2838
2839	case QIB_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
2840		/* will only take effect when the link state changes */
2841		if (val == IB_LINKINITCMD_POLL)
2842			dd->cspec->ibcctrl &=
2843				~SYM_MASK(IBCCtrl, LinkDownDefaultState);
2844		else /* SLEEP */
2845			dd->cspec->ibcctrl |=
2846				SYM_MASK(IBCCtrl, LinkDownDefaultState);
2847		qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2848		qib_write_kreg(dd, kr_scratch, 0);
2849		break;
2850
2851	case QIB_IB_CFG_MTU: /* update the MTU in IBC */
2852		/*
2853		 * Update our housekeeping variables, and set IBC max
2854		 * size, same as init code; max IBC is max we allow in
2855		 * buffer, less the qword pbc, plus 1 for ICRC, in dwords
2856		 * Set even if it's unchanged, print debug message only
2857		 * on changes.
2858		 */
2859		val = (ppd->ibmaxlen >> 2) + 1;
2860		dd->cspec->ibcctrl &= ~SYM_MASK(IBCCtrl, MaxPktLen);
2861		dd->cspec->ibcctrl |= (u64)val <<
2862			SYM_LSB(IBCCtrl, MaxPktLen);
2863		qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2864		qib_write_kreg(dd, kr_scratch, 0);
2865		break;
2866
2867	case QIB_IB_CFG_LSTATE: /* set the IB link state */
2868		switch (val & 0xffff0000) {
2869		case IB_LINKCMD_DOWN:
2870			lcmd = QLOGIC_IB_IBCC_LINKCMD_DOWN;
2871			if (!dd->cspec->ibdeltainprog) {
2872				dd->cspec->ibdeltainprog = 1;
2873				dd->cspec->ibsymsnap =
2874					read_6120_creg32(dd, cr_ibsymbolerr);
2875				dd->cspec->iblnkerrsnap =
2876					read_6120_creg32(dd, cr_iblinkerrrecov);
2877			}
2878			break;
2879
2880		case IB_LINKCMD_ARMED:
2881			lcmd = QLOGIC_IB_IBCC_LINKCMD_ARMED;
2882			break;
2883
2884		case IB_LINKCMD_ACTIVE:
2885			lcmd = QLOGIC_IB_IBCC_LINKCMD_ACTIVE;
2886			break;
2887
2888		default:
2889			ret = -EINVAL;
2890			qib_dev_err(dd, "bad linkcmd req 0x%x\n", val >> 16);
2891			goto bail;
2892		}
2893		switch (val & 0xffff) {
2894		case IB_LINKINITCMD_NOP:
2895			licmd = 0;
2896			break;
2897
2898		case IB_LINKINITCMD_POLL:
2899			licmd = QLOGIC_IB_IBCC_LINKINITCMD_POLL;
2900			break;
2901
2902		case IB_LINKINITCMD_SLEEP:
2903			licmd = QLOGIC_IB_IBCC_LINKINITCMD_SLEEP;
2904			break;
2905
2906		case IB_LINKINITCMD_DISABLE:
2907			licmd = QLOGIC_IB_IBCC_LINKINITCMD_DISABLE;
2908			break;
2909
2910		default:
2911			ret = -EINVAL;
2912			qib_dev_err(dd, "bad linkinitcmd req 0x%x\n",
2913				    val & 0xffff);
2914			goto bail;
2915		}
2916		qib_set_ib_6120_lstate(ppd, lcmd, licmd);
2917		goto bail;
2918
2919	case QIB_IB_CFG_HRTBT:
2920		ret = -EINVAL;
2921		break;
2922
2923	default:
2924		ret = -EINVAL;
2925	}
2926bail:
2927	return ret;
2928}
2929
2930static int qib_6120_set_loopback(struct qib_pportdata *ppd, const char *what)
2931{
2932	int ret = 0;
2933
2934	if (!strncmp(what, "ibc", 3)) {
2935		ppd->dd->cspec->ibcctrl |= SYM_MASK(IBCCtrl, Loopback);
2936		qib_devinfo(ppd->dd->pcidev, "Enabling IB%u:%u IBC loopback\n",
2937			 ppd->dd->unit, ppd->port);
2938	} else if (!strncmp(what, "off", 3)) {
2939		ppd->dd->cspec->ibcctrl &= ~SYM_MASK(IBCCtrl, Loopback);
2940		qib_devinfo(ppd->dd->pcidev,
2941			"Disabling IB%u:%u IBC loopback (normal)\n",
2942			ppd->dd->unit, ppd->port);
2943	} else
2944		ret = -EINVAL;
2945	if (!ret) {
2946		qib_write_kreg(ppd->dd, kr_ibcctrl, ppd->dd->cspec->ibcctrl);
2947		qib_write_kreg(ppd->dd, kr_scratch, 0);
2948	}
2949	return ret;
2950}
2951
2952static void pma_6120_timer(unsigned long data)
2953{
2954	struct qib_pportdata *ppd = (struct qib_pportdata *)data;
2955	struct qib_chip_specific *cs = ppd->dd->cspec;
2956	struct qib_ibport *ibp = &ppd->ibport_data;
2957	unsigned long flags;
2958
2959	spin_lock_irqsave(&ibp->lock, flags);
2960	if (cs->pma_sample_status == IB_PMA_SAMPLE_STATUS_STARTED) {
2961		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_RUNNING;
2962		qib_snapshot_counters(ppd, &cs->sword, &cs->rword,
2963				      &cs->spkts, &cs->rpkts, &cs->xmit_wait);
2964		mod_timer(&cs->pma_timer,
2965			  jiffies + usecs_to_jiffies(ibp->pma_sample_interval));
2966	} else if (cs->pma_sample_status == IB_PMA_SAMPLE_STATUS_RUNNING) {
2967		u64 ta, tb, tc, td, te;
2968
2969		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_DONE;
2970		qib_snapshot_counters(ppd, &ta, &tb, &tc, &td, &te);
2971
2972		cs->sword = ta - cs->sword;
2973		cs->rword = tb - cs->rword;
2974		cs->spkts = tc - cs->spkts;
2975		cs->rpkts = td - cs->rpkts;
2976		cs->xmit_wait = te - cs->xmit_wait;
2977	}
2978	spin_unlock_irqrestore(&ibp->lock, flags);
2979}
2980
2981/*
2982 * Note that the caller has the ibp->lock held.
2983 */
2984static void qib_set_cntr_6120_sample(struct qib_pportdata *ppd, u32 intv,
2985				     u32 start)
2986{
2987	struct qib_chip_specific *cs = ppd->dd->cspec;
2988
2989	if (start && intv) {
2990		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_STARTED;
2991		mod_timer(&cs->pma_timer, jiffies + usecs_to_jiffies(start));
2992	} else if (intv) {
2993		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_RUNNING;
2994		qib_snapshot_counters(ppd, &cs->sword, &cs->rword,
2995				      &cs->spkts, &cs->rpkts, &cs->xmit_wait);
2996		mod_timer(&cs->pma_timer, jiffies + usecs_to_jiffies(intv));
2997	} else {
2998		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_DONE;
2999		cs->sword = 0;
3000		cs->rword = 0;
3001		cs->spkts = 0;
3002		cs->rpkts = 0;
3003		cs->xmit_wait = 0;
3004	}
3005}
3006
3007static u32 qib_6120_iblink_state(u64 ibcs)
3008{
3009	u32 state = (u32)SYM_FIELD(ibcs, IBCStatus, LinkState);
3010
3011	switch (state) {
3012	case IB_6120_L_STATE_INIT:
3013		state = IB_PORT_INIT;
3014		break;
3015	case IB_6120_L_STATE_ARM:
3016		state = IB_PORT_ARMED;
3017		break;
3018	case IB_6120_L_STATE_ACTIVE:
3019		/* fall through */
3020	case IB_6120_L_STATE_ACT_DEFER:
3021		state = IB_PORT_ACTIVE;
3022		break;
3023	default: /* fall through */
3024	case IB_6120_L_STATE_DOWN:
3025		state = IB_PORT_DOWN;
3026		break;
3027	}
3028	return state;
3029}
3030
3031/* returns the IBTA port state, rather than the IBC link training state */
3032static u8 qib_6120_phys_portstate(u64 ibcs)
3033{
3034	u8 state = (u8)SYM_FIELD(ibcs, IBCStatus, LinkTrainingState);
3035	return qib_6120_physportstate[state];
3036}
3037
3038static int qib_6120_ib_updown(struct qib_pportdata *ppd, int ibup, u64 ibcs)
3039{
3040	unsigned long flags;
3041
3042	spin_lock_irqsave(&ppd->lflags_lock, flags);
3043	ppd->lflags &= ~QIBL_IB_FORCE_NOTIFY;
3044	spin_unlock_irqrestore(&ppd->lflags_lock, flags);
3045
3046	if (ibup) {
3047		if (ppd->dd->cspec->ibdeltainprog) {
3048			ppd->dd->cspec->ibdeltainprog = 0;
3049			ppd->dd->cspec->ibsymdelta +=
3050				read_6120_creg32(ppd->dd, cr_ibsymbolerr) -
3051					ppd->dd->cspec->ibsymsnap;
3052			ppd->dd->cspec->iblnkerrdelta +=
3053				read_6120_creg32(ppd->dd, cr_iblinkerrrecov) -
3054					ppd->dd->cspec->iblnkerrsnap;
3055		}
3056		qib_hol_init(ppd);
3057	} else {
3058		ppd->dd->cspec->lli_counter = 0;
3059		if (!ppd->dd->cspec->ibdeltainprog) {
3060			ppd->dd->cspec->ibdeltainprog = 1;
3061			ppd->dd->cspec->ibsymsnap =
3062				read_6120_creg32(ppd->dd, cr_ibsymbolerr);
3063			ppd->dd->cspec->iblnkerrsnap =
3064				read_6120_creg32(ppd->dd, cr_iblinkerrrecov);
3065		}
3066		qib_hol_down(ppd);
3067	}
3068
3069	qib_6120_setup_setextled(ppd, ibup);
3070
3071	return 0;
3072}
3073
3074/* Does read/modify/write to appropriate registers to
3075 * set output and direction bits selected by mask.
3076 * these are in their canonical postions (e.g. lsb of
3077 * dir will end up in D48 of extctrl on existing chips).
3078 * returns contents of GP Inputs.
3079 */
3080static int gpio_6120_mod(struct qib_devdata *dd, u32 out, u32 dir, u32 mask)
3081{
3082	u64 read_val, new_out;
3083	unsigned long flags;
3084
3085	if (mask) {
3086		/* some bits being written, lock access to GPIO */
3087		dir &= mask;
3088		out &= mask;
3089		spin_lock_irqsave(&dd->cspec->gpio_lock, flags);
3090		dd->cspec->extctrl &= ~((u64)mask << SYM_LSB(EXTCtrl, GPIOOe));
3091		dd->cspec->extctrl |= ((u64) dir << SYM_LSB(EXTCtrl, GPIOOe));
3092		new_out = (dd->cspec->gpio_out & ~mask) | out;
3093
3094		qib_write_kreg(dd, kr_extctrl, dd->cspec->extctrl);
3095		qib_write_kreg(dd, kr_gpio_out, new_out);
3096		dd->cspec->gpio_out = new_out;
3097		spin_unlock_irqrestore(&dd->cspec->gpio_lock, flags);
3098	}
3099	/*
3100	 * It is unlikely that a read at this time would get valid
3101	 * data on a pin whose direction line was set in the same
3102	 * call to this function. We include the read here because
3103	 * that allows us to potentially combine a change on one pin with
3104	 * a read on another, and because the old code did something like
3105	 * this.
3106	 */
3107	read_val = qib_read_kreg64(dd, kr_extstatus);
3108	return SYM_FIELD(read_val, EXTStatus, GPIOIn);
3109}
3110
3111/*
3112 * Read fundamental info we need to use the chip.  These are
3113 * the registers that describe chip capabilities, and are
3114 * saved in shadow registers.
3115 */
3116static void get_6120_chip_params(struct qib_devdata *dd)
3117{
3118	u64 val;
3119	u32 piobufs;
3120	int mtu;
3121
3122	dd->uregbase = qib_read_kreg32(dd, kr_userregbase);
3123
3124	dd->rcvtidcnt = qib_read_kreg32(dd, kr_rcvtidcnt);
3125	dd->rcvtidbase = qib_read_kreg32(dd, kr_rcvtidbase);
3126	dd->rcvegrbase = qib_read_kreg32(dd, kr_rcvegrbase);
3127	dd->palign = qib_read_kreg32(dd, kr_palign);
3128	dd->piobufbase = qib_read_kreg64(dd, kr_sendpiobufbase);
3129	dd->pio2k_bufbase = dd->piobufbase & 0xffffffff;
3130
3131	dd->rcvhdrcnt = qib_read_kreg32(dd, kr_rcvegrcnt);
3132
3133	val = qib_read_kreg64(dd, kr_sendpiosize);
3134	dd->piosize2k = val & ~0U;
3135	dd->piosize4k = val >> 32;
3136
3137	mtu = ib_mtu_enum_to_int(qib_ibmtu);
3138	if (mtu == -1)
3139		mtu = QIB_DEFAULT_MTU;
3140	dd->pport->ibmtu = (u32)mtu;
3141
3142	val = qib_read_kreg64(dd, kr_sendpiobufcnt);
3143	dd->piobcnt2k = val & ~0U;
3144	dd->piobcnt4k = val >> 32;
3145	dd->last_pio = dd->piobcnt4k + dd->piobcnt2k - 1;
3146	/* these may be adjusted in init_chip_wc_pat() */
3147	dd->pio2kbase = (u32 __iomem *)
3148		(((char __iomem *)dd->kregbase) + dd->pio2k_bufbase);
3149	if (dd->piobcnt4k) {
3150		dd->pio4kbase = (u32 __iomem *)
3151			(((char __iomem *) dd->kregbase) +
3152			 (dd->piobufbase >> 32));
3153		/*
3154		 * 4K buffers take 2 pages; we use roundup just to be
3155		 * paranoid; we calculate it once here, rather than on
3156		 * ever buf allocate
3157		 */
3158		dd->align4k = ALIGN(dd->piosize4k, dd->palign);
3159	}
3160
3161	piobufs = dd->piobcnt4k + dd->piobcnt2k;
3162
3163	dd->pioavregs = ALIGN(piobufs, sizeof(u64) * BITS_PER_BYTE / 2) /
3164		(sizeof(u64) * BITS_PER_BYTE / 2);
3165}
3166
3167/*
3168 * The chip base addresses in cspec and cpspec have to be set
3169 * after possible init_chip_wc_pat(), rather than in
3170 * get_6120_chip_params(), so split out as separate function
3171 */
3172static void set_6120_baseaddrs(struct qib_devdata *dd)
3173{
3174	u32 cregbase;
3175
3176	cregbase = qib_read_kreg32(dd, kr_counterregbase);
3177	dd->cspec->cregbase = (u64 __iomem *)
3178		((char __iomem *) dd->kregbase + cregbase);
3179
3180	dd->egrtidbase = (u64 __iomem *)
3181		((char __iomem *) dd->kregbase + dd->rcvegrbase);
3182}
3183
3184/*
3185 * Write the final few registers that depend on some of the
3186 * init setup.  Done late in init, just before bringing up
3187 * the serdes.
3188 */
3189static int qib_late_6120_initreg(struct qib_devdata *dd)
3190{
3191	int ret = 0;
3192	u64 val;
3193
3194	qib_write_kreg(dd, kr_rcvhdrentsize, dd->rcvhdrentsize);
3195	qib_write_kreg(dd, kr_rcvhdrsize, dd->rcvhdrsize);
3196	qib_write_kreg(dd, kr_rcvhdrcnt, dd->rcvhdrcnt);
3197	qib_write_kreg(dd, kr_sendpioavailaddr, dd->pioavailregs_phys);
3198	val = qib_read_kreg64(dd, kr_sendpioavailaddr);
3199	if (val != dd->pioavailregs_phys) {
3200		qib_dev_err(dd,
3201			"Catastrophic software error, SendPIOAvailAddr written as %lx, read back as %llx\n",
3202			(unsigned long) dd->pioavailregs_phys,
3203			(unsigned long long) val);
3204		ret = -EINVAL;
3205	}
3206	return ret;
3207}
3208
3209static int init_6120_variables(struct qib_devdata *dd)
3210{
3211	int ret = 0;
3212	struct qib_pportdata *ppd;
3213	u32 sbufs;
3214
3215	ppd = (struct qib_pportdata *)(dd + 1);
3216	dd->pport = ppd;
3217	dd->num_pports = 1;
3218
3219	dd->cspec = (struct qib_chip_specific *)(ppd + dd->num_pports);
3220	ppd->cpspec = NULL; /* not used in this chip */
3221
3222	spin_lock_init(&dd->cspec->kernel_tid_lock);
3223	spin_lock_init(&dd->cspec->user_tid_lock);
3224	spin_lock_init(&dd->cspec->rcvmod_lock);
3225	spin_lock_init(&dd->cspec->gpio_lock);
3226
3227	/* we haven't yet set QIB_PRESENT, so use read directly */
3228	dd->revision = readq(&dd->kregbase[kr_revision]);
3229
3230	if ((dd->revision & 0xffffffffU) == 0xffffffffU) {
3231		qib_dev_err(dd,
3232			"Revision register read failure, giving up initialization\n");
3233		ret = -ENODEV;
3234		goto bail;
3235	}
3236	dd->flags |= QIB_PRESENT;  /* now register routines work */
3237
3238	dd->majrev = (u8) SYM_FIELD(dd->revision, Revision_R,
3239				    ChipRevMajor);
3240	dd->minrev = (u8) SYM_FIELD(dd->revision, Revision_R,
3241				    ChipRevMinor);
3242
3243	get_6120_chip_params(dd);
3244	pe_boardname(dd); /* fill in boardname */
3245
3246	/*
3247	 * GPIO bits for TWSI data and clock,
3248	 * used for serial EEPROM.
3249	 */
3250	dd->gpio_sda_num = _QIB_GPIO_SDA_NUM;
3251	dd->gpio_scl_num = _QIB_GPIO_SCL_NUM;
3252	dd->twsi_eeprom_dev = QIB_TWSI_NO_DEV;
3253
3254	if (qib_unordered_wc())
3255		dd->flags |= QIB_PIO_FLUSH_WC;
3256
3257	/*
3258	 * EEPROM error log 0 is TXE Parity errors. 1 is RXE Parity.
3259	 * 2 is Some Misc, 3 is reserved for future.
3260	 */
3261	dd->eep_st_masks[0].hwerrs_to_log = HWE_MASK(TXEMemParityErr);
3262
3263	/* Ignore errors in PIO/PBC on systems with unordered write-combining */
3264	if (qib_unordered_wc())
3265		dd->eep_st_masks[0].hwerrs_to_log &= ~TXE_PIO_PARITY;
3266
3267	dd->eep_st_masks[1].hwerrs_to_log = HWE_MASK(RXEMemParityErr);
3268
3269	dd->eep_st_masks[2].errs_to_log = ERR_MASK(ResetNegated);
3270
3271	ret = qib_init_pportdata(ppd, dd, 0, 1);
3272	if (ret)
3273		goto bail;
3274	ppd->link_width_supported = IB_WIDTH_1X | IB_WIDTH_4X;
3275	ppd->link_speed_supported = QIB_IB_SDR;
3276	ppd->link_width_enabled = IB_WIDTH_4X;
3277	ppd->link_speed_enabled = ppd->link_speed_supported;
3278	/* these can't change for this chip, so set once */
3279	ppd->link_width_active = ppd->link_width_enabled;
3280	ppd->link_speed_active = ppd->link_speed_enabled;
3281	ppd->vls_supported = IB_VL_VL0;
3282	ppd->vls_operational = ppd->vls_supported;
3283
3284	dd->rcvhdrentsize = QIB_RCVHDR_ENTSIZE;
3285	dd->rcvhdrsize = QIB_DFLT_RCVHDRSIZE;
3286	dd->rhf_offset = 0;
3287
3288	/* we always allocate at least 2048 bytes for eager buffers */
3289	ret = ib_mtu_enum_to_int(qib_ibmtu);
3290	dd->rcvegrbufsize = ret != -1 ? max(ret, 2048) : QIB_DEFAULT_MTU;
3291	BUG_ON(!is_power_of_2(dd->rcvegrbufsize));
3292	dd->rcvegrbufsize_shift = ilog2(dd->rcvegrbufsize);
3293
3294	qib_6120_tidtemplate(dd);
3295
3296	/*
3297	 * We can request a receive interrupt for 1 or
3298	 * more packets from current offset.  For now, we set this
3299	 * up for a single packet.
3300	 */
3301	dd->rhdrhead_intr_off = 1ULL << 32;
3302
3303	/* setup the stats timer; the add_timer is done at end of init */
3304	init_timer(&dd->stats_timer);
3305	dd->stats_timer.function = qib_get_6120_faststats;
3306	dd->stats_timer.data = (unsigned long) dd;
3307
3308	init_timer(&dd->cspec->pma_timer);
3309	dd->cspec->pma_timer.function = pma_6120_timer;
3310	dd->cspec->pma_timer.data = (unsigned long) ppd;
3311
3312	dd->ureg_align = qib_read_kreg32(dd, kr_palign);
3313
3314	dd->piosize2kmax_dwords = dd->piosize2k >> 2;
3315	qib_6120_config_ctxts(dd);
3316	qib_set_ctxtcnt(dd);
3317
3318	ret = init_chip_wc_pat(dd, 0);
3319	if (ret)
3320		goto bail;
3321	set_6120_baseaddrs(dd); /* set chip access pointers now */
3322
3323	ret = 0;
3324	if (qib_mini_init)
3325		goto bail;
3326
3327	qib_num_cfg_vls = 1; /* if any 6120's, only one VL */
3328
3329	ret = qib_create_ctxts(dd);
3330	init_6120_cntrnames(dd);
3331
3332	/* use all of 4KB buffers for the kernel, otherwise 16 */
3333	sbufs = dd->piobcnt4k ?  dd->piobcnt4k : 16;
3334
3335	dd->lastctxt_piobuf = dd->piobcnt2k + dd->piobcnt4k - sbufs;
3336	dd->pbufsctxt = dd->lastctxt_piobuf /
3337		(dd->cfgctxts - dd->first_user_ctxt);
3338
3339	if (ret)
3340		goto bail;
3341bail:
3342	return ret;
3343}
3344
3345/*
3346 * For this chip, we want to use the same buffer every time
3347 * when we are trying to bring the link up (they are always VL15
3348 * packets).  At that link state the packet should always go out immediately
3349 * (or at least be discarded at the tx interface if the link is down).
3350 * If it doesn't, and the buffer isn't available, that means some other
3351 * sender has gotten ahead of us, and is preventing our packet from going
3352 * out.  In that case, we flush all packets, and try again.  If that still
3353 * fails, we fail the request, and hope things work the next time around.
3354 *
3355 * We don't need very complicated heuristics on whether the packet had
3356 * time to go out or not, since even at SDR 1X, it goes out in very short
3357 * time periods, covered by the chip reads done here and as part of the
3358 * flush.
3359 */
3360static u32 __iomem *get_6120_link_buf(struct qib_pportdata *ppd, u32 *bnum)
3361{
3362	u32 __iomem *buf;
3363	u32 lbuf = ppd->dd->piobcnt2k + ppd->dd->piobcnt4k - 1;
3364
3365	/*
3366	 * always blip to get avail list updated, since it's almost
3367	 * always needed, and is fairly cheap.
3368	 */
3369	sendctrl_6120_mod(ppd->dd->pport, QIB_SENDCTRL_AVAIL_BLIP);
3370	qib_read_kreg64(ppd->dd, kr_scratch); /* extra chip flush */
3371	buf = qib_getsendbuf_range(ppd->dd, bnum, lbuf, lbuf);
3372	if (buf)
3373		goto done;
3374
3375	sendctrl_6120_mod(ppd, QIB_SENDCTRL_DISARM_ALL | QIB_SENDCTRL_FLUSH |
3376			  QIB_SENDCTRL_AVAIL_BLIP);
3377	ppd->dd->upd_pio_shadow  = 1; /* update our idea of what's busy */
3378	qib_read_kreg64(ppd->dd, kr_scratch); /* extra chip flush */
3379	buf = qib_getsendbuf_range(ppd->dd, bnum, lbuf, lbuf);
3380done:
3381	return buf;
3382}
3383
3384static u32 __iomem *qib_6120_getsendbuf(struct qib_pportdata *ppd, u64 pbc,
3385					u32 *pbufnum)
3386{
3387	u32 first, last, plen = pbc & QIB_PBC_LENGTH_MASK;
3388	struct qib_devdata *dd = ppd->dd;
3389	u32 __iomem *buf;
3390
3391	if (((pbc >> 32) & PBC_6120_VL15_SEND_CTRL) &&
3392		!(ppd->lflags & (QIBL_IB_AUTONEG_INPROG | QIBL_LINKACTIVE)))
3393		buf = get_6120_link_buf(ppd, pbufnum);
3394	else {
3395
3396		if ((plen + 1) > dd->piosize2kmax_dwords)
3397			first = dd->piobcnt2k;
3398		else
3399			first = 0;
3400		/* try 4k if all 2k busy, so same last for both sizes */
3401		last = dd->piobcnt2k + dd->piobcnt4k - 1;
3402		buf = qib_getsendbuf_range(dd, pbufnum, first, last);
3403	}
3404	return buf;
3405}
3406
3407static int init_sdma_6120_regs(struct qib_pportdata *ppd)
3408{
3409	return -ENODEV;
3410}
3411
3412static u16 qib_sdma_6120_gethead(struct qib_pportdata *ppd)
3413{
3414	return 0;
3415}
3416
3417static int qib_sdma_6120_busy(struct qib_pportdata *ppd)
3418{
3419	return 0;
3420}
3421
3422static void qib_sdma_update_6120_tail(struct qib_pportdata *ppd, u16 tail)
3423{
3424}
3425
3426static void qib_6120_sdma_sendctrl(struct qib_pportdata *ppd, unsigned op)
3427{
3428}
3429
3430static void qib_sdma_set_6120_desc_cnt(struct qib_pportdata *ppd, unsigned cnt)
3431{
3432}
3433
3434/*
3435 * the pbc doesn't need a VL15 indicator, but we need it for link_buf.
3436 * The chip ignores the bit if set.
3437 */
3438static u32 qib_6120_setpbc_control(struct qib_pportdata *ppd, u32 plen,
3439				   u8 srate, u8 vl)
3440{
3441	return vl == 15 ? PBC_6120_VL15_SEND_CTRL : 0;
3442}
3443
3444static void qib_6120_initvl15_bufs(struct qib_devdata *dd)
3445{
3446}
3447
3448static void qib_6120_init_ctxt(struct qib_ctxtdata *rcd)
3449{
3450	rcd->rcvegrcnt = rcd->dd->rcvhdrcnt;
3451	rcd->rcvegr_tid_base = rcd->ctxt * rcd->rcvegrcnt;
3452}
3453
3454static void qib_6120_txchk_change(struct qib_devdata *dd, u32 start,
3455	u32 len, u32 avail, struct qib_ctxtdata *rcd)
3456{
3457}
3458
3459static void writescratch(struct qib_devdata *dd, u32 val)
3460{
3461	(void) qib_write_kreg(dd, kr_scratch, val);
3462}
3463
3464static int qib_6120_tempsense_rd(struct qib_devdata *dd, int regnum)
3465{
3466	return -ENXIO;
3467}
3468
3469#ifdef CONFIG_INFINIBAND_QIB_DCA
3470static int qib_6120_notify_dca(struct qib_devdata *dd, unsigned long event)
3471{
3472	return 0;
3473}
3474#endif
3475
3476/* Dummy function, as 6120 boards never disable EEPROM Write */
3477static int qib_6120_eeprom_wen(struct qib_devdata *dd, int wen)
3478{
3479	return 1;
3480}
3481
3482/**
3483 * qib_init_iba6120_funcs - set up the chip-specific function pointers
3484 * @pdev: pci_dev of the qlogic_ib device
3485 * @ent: pci_device_id matching this chip
3486 *
3487 * This is global, and is called directly at init to set up the
3488 * chip-specific function pointers for later use.
3489 *
3490 * It also allocates/partially-inits the qib_devdata struct for
3491 * this device.
3492 */
3493struct qib_devdata *qib_init_iba6120_funcs(struct pci_dev *pdev,
3494					   const struct pci_device_id *ent)
3495{
3496	struct qib_devdata *dd;
3497	int ret;
3498
3499	dd = qib_alloc_devdata(pdev, sizeof(struct qib_pportdata) +
3500			       sizeof(struct qib_chip_specific));
3501	if (IS_ERR(dd))
3502		goto bail;
3503
3504	dd->f_bringup_serdes    = qib_6120_bringup_serdes;
3505	dd->f_cleanup           = qib_6120_setup_cleanup;
3506	dd->f_clear_tids        = qib_6120_clear_tids;
3507	dd->f_free_irq          = qib_6120_free_irq;
3508	dd->f_get_base_info     = qib_6120_get_base_info;
3509	dd->f_get_msgheader     = qib_6120_get_msgheader;
3510	dd->f_getsendbuf        = qib_6120_getsendbuf;
3511	dd->f_gpio_mod          = gpio_6120_mod;
3512	dd->f_eeprom_wen	= qib_6120_eeprom_wen;
3513	dd->f_hdrqempty         = qib_6120_hdrqempty;
3514	dd->f_ib_updown         = qib_6120_ib_updown;
3515	dd->f_init_ctxt         = qib_6120_init_ctxt;
3516	dd->f_initvl15_bufs     = qib_6120_initvl15_bufs;
3517	dd->f_intr_fallback     = qib_6120_nointr_fallback;
3518	dd->f_late_initreg      = qib_late_6120_initreg;
3519	dd->f_setpbc_control    = qib_6120_setpbc_control;
3520	dd->f_portcntr          = qib_portcntr_6120;
3521	dd->f_put_tid           = (dd->minrev >= 2) ?
3522				      qib_6120_put_tid_2 :
3523				      qib_6120_put_tid;
3524	dd->f_quiet_serdes      = qib_6120_quiet_serdes;
3525	dd->f_rcvctrl           = rcvctrl_6120_mod;
3526	dd->f_read_cntrs        = qib_read_6120cntrs;
3527	dd->f_read_portcntrs    = qib_read_6120portcntrs;
3528	dd->f_reset             = qib_6120_setup_reset;
3529	dd->f_init_sdma_regs    = init_sdma_6120_regs;
3530	dd->f_sdma_busy         = qib_sdma_6120_busy;
3531	dd->f_sdma_gethead      = qib_sdma_6120_gethead;
3532	dd->f_sdma_sendctrl     = qib_6120_sdma_sendctrl;
3533	dd->f_sdma_set_desc_cnt = qib_sdma_set_6120_desc_cnt;
3534	dd->f_sdma_update_tail  = qib_sdma_update_6120_tail;
3535	dd->f_sendctrl          = sendctrl_6120_mod;
3536	dd->f_set_armlaunch     = qib_set_6120_armlaunch;
3537	dd->f_set_cntr_sample   = qib_set_cntr_6120_sample;
3538	dd->f_iblink_state      = qib_6120_iblink_state;
3539	dd->f_ibphys_portstate  = qib_6120_phys_portstate;
3540	dd->f_get_ib_cfg        = qib_6120_get_ib_cfg;
3541	dd->f_set_ib_cfg        = qib_6120_set_ib_cfg;
3542	dd->f_set_ib_loopback   = qib_6120_set_loopback;
3543	dd->f_set_intr_state    = qib_6120_set_intr_state;
3544	dd->f_setextled         = qib_6120_setup_setextled;
3545	dd->f_txchk_change      = qib_6120_txchk_change;
3546	dd->f_update_usrhead    = qib_update_6120_usrhead;
3547	dd->f_wantpiobuf_intr   = qib_wantpiobuf_6120_intr;
3548	dd->f_xgxs_reset        = qib_6120_xgxs_reset;
3549	dd->f_writescratch      = writescratch;
3550	dd->f_tempsense_rd	= qib_6120_tempsense_rd;
3551#ifdef CONFIG_INFINIBAND_QIB_DCA
3552	dd->f_notify_dca = qib_6120_notify_dca;
3553#endif
3554	/*
3555	 * Do remaining pcie setup and save pcie values in dd.
3556	 * Any error printing is already done by the init code.
3557	 * On return, we have the chip mapped and accessible,
3558	 * but chip registers are not set up until start of
3559	 * init_6120_variables.
3560	 */
3561	ret = qib_pcie_ddinit(dd, pdev, ent);
3562	if (ret < 0)
3563		goto bail_free;
3564
3565	/* initialize chip-specific variables */
3566	ret = init_6120_variables(dd);
3567	if (ret)
3568		goto bail_cleanup;
3569
3570	if (qib_mini_init)
3571		goto bail;
3572
3573	if (qib_pcie_params(dd, 8, NULL, NULL))
3574		qib_dev_err(dd,
3575			"Failed to setup PCIe or interrupts; continuing anyway\n");
3576	dd->cspec->irq = pdev->irq; /* save IRQ */
3577
3578	/* clear diagctrl register, in case diags were running and crashed */
3579	qib_write_kreg(dd, kr_hwdiagctrl, 0);
3580
3581	if (qib_read_kreg64(dd, kr_hwerrstatus) &
3582	    QLOGIC_IB_HWE_SERDESPLLFAILED)
3583		qib_write_kreg(dd, kr_hwerrclear,
3584			       QLOGIC_IB_HWE_SERDESPLLFAILED);
3585
3586	/* setup interrupt handler (interrupt type handled above) */
3587	qib_setup_6120_interrupt(dd);
3588	/* Note that qpn_mask is set by qib_6120_config_ctxts() first */
3589	qib_6120_init_hwerrors(dd);
3590
3591	goto bail;
3592
3593bail_cleanup:
3594	qib_pcie_ddcleanup(dd);
3595bail_free:
3596	qib_free_devdata(dd);
3597	dd = ERR_PTR(ret);
3598bail:
3599	return dd;
3600}
3601